Source code

001
002package gudusoft.gsqlparser.dlineage;
003
004import gudusoft.gsqlparser.*;
005import gudusoft.gsqlparser.common.structured.StructuredAdapterContext;
006import gudusoft.gsqlparser.common.structured.StructuredDataflowDescriptor;
007import gudusoft.gsqlparser.common.structured.StructuredDataflowRegistry;
008import gudusoft.gsqlparser.common.structured.StructuredFieldBinding;
009import gudusoft.gsqlparser.common.structured.StructuredValueSource;
010import gudusoft.gsqlparser.dlineage.dataflow.listener.DataFlowHandleListener;
011import gudusoft.gsqlparser.dlineage.dataflow.metadata.MetadataReader;
012import gudusoft.gsqlparser.dlineage.dataflow.metadata.grabit.GrabitMetadataAnalyzer;
013import gudusoft.gsqlparser.dlineage.dataflow.metadata.sqldep.SQLDepMetadataAnalyzer;
014import gudusoft.gsqlparser.dlineage.dataflow.metadata.sqlflow.SqlflowMetadataAnalyzer;
015import gudusoft.gsqlparser.dlineage.impl.powerquery.PowerQueryLineageResult;
016import gudusoft.gsqlparser.dlineage.impl.powerquery.TPowerQueryAnalyzer;
017import gudusoft.gsqlparser.dlineage.dataflow.model.*;
018import gudusoft.gsqlparser.dlineage.dataflow.model.Process;
019import gudusoft.gsqlparser.dlineage.dataflow.model.JoinRelationship.JoinClauseType;
020import gudusoft.gsqlparser.dlineage.dataflow.model.json.Coordinate;
021import gudusoft.gsqlparser.dlineage.dataflow.model.json.Dataflow;
022import gudusoft.gsqlparser.dlineage.dataflow.model.json.Error;
023import gudusoft.gsqlparser.dlineage.dataflow.model.xml.*;
024import gudusoft.gsqlparser.dlineage.dataflow.sqlenv.SQLEnvParser;
025import gudusoft.gsqlparser.dlineage.metadata.MetadataUtil;
026import gudusoft.gsqlparser.dlineage.metadata.Sqlflow;
027import gudusoft.gsqlparser.dlineage.util.*;
028import gudusoft.gsqlparser.nodes.TTable;
029import gudusoft.gsqlparser.nodes.*;
030import gudusoft.gsqlparser.nodes.couchbase.TObjectConstruct;
031import gudusoft.gsqlparser.nodes.couchbase.TPair;
032import gudusoft.gsqlparser.nodes.functions.TJsonObjectFunction;
033import gudusoft.gsqlparser.nodes.hive.THiveTransformClause;
034import gudusoft.gsqlparser.nodes.hive.THiveTransformClause.ETransformType;
035import gudusoft.gsqlparser.resolver2.ScopeBuildResult;
036import gudusoft.gsqlparser.resolver2.TSQLResolver2;
037import gudusoft.gsqlparser.sqlenv.ESQLDataObjectType;
038import gudusoft.gsqlparser.sqlenv.TSQLEnv;
039import gudusoft.gsqlparser.sqlenv.TSQLSchema;
040import gudusoft.gsqlparser.sqlenv.TSQLTable;
041import gudusoft.gsqlparser.sqlenv.parser.TJSONSQLEnvParser;
042import gudusoft.gsqlparser.stmt.*;
043import gudusoft.gsqlparser.stmt.db2.TDb2ReturnStmt;
044import gudusoft.gsqlparser.stmt.db2.TDb2SqlVariableDeclaration;
045import gudusoft.gsqlparser.stmt.hive.THiveLoad;
046import gudusoft.gsqlparser.stmt.mssql.*;
047import gudusoft.gsqlparser.stmt.mysql.TLoadDataStmt;
048import gudusoft.gsqlparser.stmt.oracle.*;
049import gudusoft.gsqlparser.stmt.powerquery.TPowerQueryDocumentStmt;
050import gudusoft.gsqlparser.stmt.redshift.TRedshiftCopy;
051import gudusoft.gsqlparser.stmt.redshift.TRedshiftDeclare;
052import gudusoft.gsqlparser.stmt.snowflake.*;
053import gudusoft.gsqlparser.stmt.teradata.TTeradataCreateProcedure;
054import gudusoft.gsqlparser.util.*;
055import gudusoft.gsqlparser.util.json.JSON;
056
057import javax.script.ScriptEngine;
058import javax.script.ScriptEngineManager;
059import javax.script.ScriptException;
060import java.io.*;
061import java.util.*;
062import java.util.concurrent.atomic.AtomicInteger;
063import java.util.regex.Matcher;
064import java.util.regex.Pattern;
065import java.util.stream.Collectors;
066
067import static gudusoft.gsqlparser.EJoinType.right;
068
069@SuppressWarnings("rawtypes")
070public class DataFlowAnalyzer implements IDataFlowAnalyzer {
071
072        private static final Logger logger = LoggerFactory.getLogger(DataFlowAnalyzer.class);
073
074        private static final List<String> TERADATA_BUILTIN_FUNCTIONS = Arrays
075                        .asList(new String[] { "ACCOUNT", "CURRENT_DATE", "CURRENT_ROLE", "CURRENT_TIME", "CURRENT_TIMESTAMP",
076                                        "CURRENT_USER", "DATABASE", "DATE", "PROFILE", "ROLE", "SESSION", "TIME", "USER", "SYSDATE", });
077
078        private static final List<String> CONSTANT_BUILTIN_FUNCTIONS = Arrays.asList(new String[] { "ACCOUNT",
079                        "CURRENT_DATE", "CURRENT_ROLE", "CURRENT_TIME", "CURRENT_TIMESTAMP", "CURRENT_USER", "DATABASE", "DATE",
080                        "PROFILE", "ROLE", "SESSION", "TIME", "USER", "SYSDATE", "GETDATE" });
081
082        private Stack<TCustomSqlStatement> stmtStack = new Stack<TCustomSqlStatement>();
083        private List<ResultSet> appendResultSets = new ArrayList<ResultSet>();
084        private Set<TCustomSqlStatement> accessedStatements = new HashSet<TCustomSqlStatement>();
085        private Set<TSelectSqlStatement> accessedSubqueries = new HashSet<TSelectSqlStatement>();
086        private Map<String, TCustomSqlStatement> viewDDLMap = new HashMap<String, TCustomSqlStatement>();
087        private Map<String, TCustomSqlStatement> procedureDDLMap = new HashMap<String, TCustomSqlStatement>();
088        private Map<TTable, TObjectNameList> structObjectMap = new HashMap<TTable, TObjectNameList>();
089
090        private SqlInfo[] sqlInfos;
091        private List<ErrorInfo> errorInfos = new ArrayList<ErrorInfo>();
092        private IndexedLinkedHashMap<String, List<SqlInfo>> sqlInfoMap = new IndexedLinkedHashMap<String, List<SqlInfo>>();
093        private TSQLEnv sqlenv = null;
094        private ModelBindingManager modelManager = new ModelBindingManager();
095        private ModelFactory modelFactory = new ModelFactory(modelManager);
096        private PipelinedFunctionAnalyzer pipelinedAnalyzer;
097        private List<Long> tableIds = new ArrayList<Long>();
098        private TTableList hiveFromTables;
099        private dataflow dataflow;
100        private String dataflowString;
101        private Option option = new Option();
102
103        {
104                modelManager.TABLE_COLUMN_ID = option.getStartId();
105                modelManager.RELATION_ID = option.getStartId();
106                ModelBindingManager.set(modelManager);
107                ModelBindingManager.setGlobalStmtStack(stmtStack);
108                ModelBindingManager.setGlobalOption(option);
109                ModelBindingManager.setGlobalSqlInfo(sqlInfoMap);
110        }
111
112    private void initPipelinedFunctionAnalyzer() {
113        // Only initialize pipelinedAnalyzer for Oracle
114        if (option.getVendor() == EDbVendor.dbvoracle)
115        {
116            pipelinedAnalyzer = new PipelinedFunctionAnalyzer(modelManager, modelFactory, option);
117        }
118        else{
119            pipelinedAnalyzer = null;
120        }
121    }
122
123        public DataFlowAnalyzer(String sqlContent, Option option) {
124                SqlInfo[] sqlInfos = new SqlInfo[1];
125                SqlInfo info = new SqlInfo();
126                info.setSql(sqlContent);
127                info.setOriginIndex(0);
128                sqlInfos[0] = info;
129                this.option = option;
130                ModelBindingManager.setGlobalOption(this.option);
131                this.sqlInfos = convertSQL(option.getVendor(), JSON.toJSONString(sqlInfos)).toArray(new SqlInfo[0]);
132    }
133
134    public DataFlowAnalyzer(String sqlContent, EDbVendor dbVendor, boolean simpleOutput, String defaultServer,
135                        String defaultDatabase, String defaltSchema) {
136                SqlInfo[] sqlInfos = new SqlInfo[1];
137                SqlInfo info = new SqlInfo();
138                info.setSql(sqlContent);
139                info.setOriginIndex(0);
140                sqlInfos[0] = info;
141                option.setVendor(dbVendor);
142                option.setSimpleOutput(simpleOutput);
143                option.setDefaultServer(defaultServer);
144                option.setDefaultDatabase(defaultDatabase);
145                option.setDefaultSchema(defaltSchema);
146                this.sqlInfos = convertSQL(dbVendor, JSON.toJSONString(sqlInfos)).toArray(new SqlInfo[0]);
147        }
148
149        public DataFlowAnalyzer(String sqlContent, EDbVendor dbVendor, boolean simpleOutput) {
150                SqlInfo[] sqlInfos = new SqlInfo[1];
151                SqlInfo info = new SqlInfo();
152                info.setSql(sqlContent);
153                info.setOriginIndex(0);
154                sqlInfos[0] = info;
155                option.setVendor(dbVendor);
156                option.setSimpleOutput(simpleOutput);
157                this.sqlInfos = convertSQL(dbVendor, JSON.toJSONString(sqlInfos)).toArray(new SqlInfo[0]);
158    }
159
160        public DataFlowAnalyzer(String[] sqlContents, Option option) {
161                SqlInfo[] sqlInfos = new SqlInfo[sqlContents.length];
162                for (int i = 0; i < sqlContents.length; i++) {
163                        SqlInfo info = new SqlInfo();
164                        info.setSql(sqlContents[i]);
165                        info.setOriginIndex(0);
166                        sqlInfos[i] = info;
167                }
168                this.option = option;
169                ModelBindingManager.setGlobalOption(this.option);
170                this.sqlInfos = convertSQL(option.getVendor(), JSON.toJSONString(sqlInfos)).toArray(new SqlInfo[0]);
171        }
172
173        public DataFlowAnalyzer(String[] sqlContents, EDbVendor dbVendor, boolean simpleOutput, String defaultServer,
174                        String defaultDatabase, String defaltSchema) {
175                SqlInfo[] sqlInfos = new SqlInfo[sqlContents.length];
176                for (int i = 0; i < sqlContents.length; i++) {
177                        SqlInfo info = new SqlInfo();
178                        info.setSql(sqlContents[i]);
179                        info.setOriginIndex(0);
180                        sqlInfos[i] = info;
181                }
182                option.setVendor(dbVendor);
183                option.setSimpleOutput(simpleOutput);
184                option.setDefaultServer(defaultServer);
185                option.setDefaultDatabase(defaultDatabase);
186                option.setDefaultSchema(defaltSchema);
187                this.sqlInfos = convertSQL(dbVendor, JSON.toJSONString(sqlInfos)).toArray(new SqlInfo[0]);
188        }
189
190        public DataFlowAnalyzer(String[] sqlContents, EDbVendor dbVendor, boolean simpleOutput) {
191                SqlInfo[] sqlInfos = new SqlInfo[sqlContents.length];
192                for (int i = 0; i < sqlContents.length; i++) {
193                        SqlInfo info = new SqlInfo();
194                        info.setSql(sqlContents[i]);
195                        info.setOriginIndex(0);
196                        sqlInfos[i] = info;
197                }
198                option.setVendor(dbVendor);
199                option.setSimpleOutput(simpleOutput);
200                this.sqlInfos = convertSQL(dbVendor, JSON.toJSONString(sqlInfos)).toArray(new SqlInfo[0]);
201        }
202
203        public DataFlowAnalyzer(SqlInfo[] sqlInfos, Option option) {
204                this.sqlInfos = sqlInfos;
205                this.option = option;
206                ModelBindingManager.setGlobalOption(this.option);
207        }
208
209        public DataFlowAnalyzer(SqlInfo[] sqlInfos, EDbVendor dbVendor, boolean simpleOutput) {
210                this.sqlInfos = sqlInfos;
211                option.setVendor(dbVendor);
212                option.setSimpleOutput(simpleOutput);
213        }
214
215        public DataFlowAnalyzer(File[] sqlFiles, Option option) {
216                SqlInfo[] sqlInfos = new SqlInfo[sqlFiles.length];
217                for (int i = 0; i < sqlFiles.length; i++) {
218                        SqlInfo info = new SqlInfo();
219                        info.setSql(SQLUtil.getFileContent(sqlFiles[i]));
220                        info.setFileName(sqlFiles[i].getName());
221                        info.setFilePath(sqlFiles[i].getAbsolutePath());
222                        info.setOriginIndex(0);
223                        sqlInfos[i] = info;
224                }
225                this.sqlInfos = sqlInfos;
226                this.option = option;
227                ModelBindingManager.setGlobalOption(this.option);
228        }
229
230        public DataFlowAnalyzer(File[] sqlFiles, EDbVendor dbVendor, boolean simpleOutput) {
231                SqlInfo[] sqlInfos = new SqlInfo[sqlFiles.length];
232                for (int i = 0; i < sqlFiles.length; i++) {
233                        SqlInfo info = new SqlInfo();
234                        info.setSql(SQLUtil.getFileContent(sqlFiles[i]));
235                        info.setFileName(sqlFiles[i].getName());
236                        info.setFilePath(sqlFiles[i].getAbsolutePath());
237                        info.setOriginIndex(0);
238                        sqlInfos[i] = info;
239                }
240                this.sqlInfos = sqlInfos;
241                option.setVendor(dbVendor);
242                option.setSimpleOutput(simpleOutput);
243        }
244
245        public DataFlowAnalyzer(File sqlFile, Option option) {
246                File[] children = SQLUtil.listFiles(sqlFile);
247                SqlInfo[] sqlInfos = new SqlInfo[children.length];
248                for (int i = 0; i < children.length; i++) {
249                        SqlInfo info = new SqlInfo();
250                        info.setSql(SQLUtil.getFileContent(children[i]));
251                        info.setFileName(children[i].getName());
252                        info.setFilePath(children[i].getAbsolutePath());
253                        info.setOriginIndex(0);
254                        sqlInfos[i] = info;
255                }
256                this.sqlInfos = sqlInfos;
257                this.option = option;
258                ModelBindingManager.setGlobalOption(this.option);
259        }
260
261        public DataFlowAnalyzer(File sqlFile, EDbVendor dbVendor, boolean simpleOutput) {
262                File[] children = SQLUtil.listFiles(sqlFile);
263                List<SqlInfo> sqlInfos = new ArrayList<>();
264                for (int i = 0; i < children.length; i++) {
265                        SqlInfo info = new SqlInfo();
266                        info.setSql(SQLUtil.getFileContent(children[i]));
267            if(children[i].getName().toLowerCase().endsWith(".csv")){
268                if(!MetadataReader.isMetadata(info.getSql())){
269                    continue;
270                }
271            }
272            else if(children[i].getName().toLowerCase().endsWith(".json")){
273                if (!(MetadataReader.isGrabit(info.getSql()) || MetadataReader.isSqlflow(info.getSql()))){
274                    continue;
275                }
276            }
277                        info.setFileName(children[i].getName());
278                        info.setFilePath(children[i].getAbsolutePath());
279                        info.setOriginIndex(0);
280                        sqlInfos.add(info);
281                }
282                this.sqlInfos = sqlInfos.toArray(new SqlInfo[0]);
283                option.setVendor(dbVendor);
284                option.setSimpleOutput(simpleOutput);
285        }
286
287        public boolean isIgnoreRecordSet() {
288                return option.isIgnoreRecordSet();
289        }
290
291        public void setIgnoreRecordSet(boolean ignoreRecordSet) {
292                option.setIgnoreRecordSet(ignoreRecordSet);
293        }
294
295        public boolean isSimpleShowTopSelectResultSet() {
296                return option.isSimpleShowTopSelectResultSet();
297        }
298
299        public void setSimpleShowTopSelectResultSet(boolean simpleShowTopSelectResultSet) {
300                option.setSimpleShowTopSelectResultSet(simpleShowTopSelectResultSet);
301        }
302
303        public boolean isSimpleShowFunction() {
304                return option.isSimpleShowFunction();
305        }
306
307        public void setSimpleShowFunction(boolean simpleShowFunction) {
308                option.setSimpleShowFunction(simpleShowFunction);
309        }
310
311        public boolean isShowJoin() {
312                return option.isShowJoin();
313        }
314
315        public void setShowJoin(boolean showJoin) {
316                option.setShowJoin(showJoin);
317        }
318
319        public void setShowCallRelation(boolean showCallRelation) {
320                option.setShowCallRelation(showCallRelation);
321        }
322
323        public boolean isShowCallRelation() {
324                return option.isShowCallRelation();
325        }
326
327        public boolean isShowImplicitSchema() {
328                return option.isShowImplicitSchema();
329        }
330
331        public void setShowImplicitSchema(boolean showImplicitSchema) {
332                option.setShowImplicitSchema(showImplicitSchema);
333        }
334
335        public boolean isShowConstantTable() {
336                return option.isShowConstantTable();
337        }
338
339        public void setShowConstantTable(boolean showConstantTable) {
340                option.setShowConstantTable(showConstantTable);
341        }
342
343        public boolean isShowCountTableColumn() {
344                return option.isShowCountTableColumn();
345        }
346
347        public void setShowCountTableColumn(boolean showCountTableColumn) {
348                option.setShowCountTableColumn(showCountTableColumn);
349        }
350
351        public boolean isTransform() {
352                return option.isTransform();
353        }
354
355        public void setTransform(boolean transform) {
356                option.setTransform(transform);
357                if (option.isTransformCoordinate()) {
358                        option.setTransform(true);
359                }
360        }
361
362        public boolean isTransformCoordinate() {
363                return option.isTransformCoordinate();
364        }
365
366        public void setTransformCoordinate(boolean transformCoordinate) {
367                option.setTransformCoordinate(transformCoordinate);
368                if (transformCoordinate) {
369                        option.setTransform(true);
370                }
371        }
372
373        public boolean isLinkOrphanColumnToFirstTable() {
374                return option.isLinkOrphanColumnToFirstTable();
375        }
376
377        public void setLinkOrphanColumnToFirstTable(boolean linkOrphanColumnToFirstTable) {
378                option.setLinkOrphanColumnToFirstTable(linkOrphanColumnToFirstTable);
379        }
380
381        public boolean isIgnoreTemporaryTable() {
382                return option.isIgnoreTemporaryTable();
383        }
384
385        public void setIgnoreTemporaryTable(boolean ignoreTemporaryTable) {
386                option.setIgnoreTemporaryTable(ignoreTemporaryTable);
387        }
388        
389        public boolean isIgnoreCoordinate() {
390                return option.isIgnoreCoordinate();
391        }
392
393        public void setIgnoreCoordinate(boolean ignoreCoordinate) {
394                option.setIgnoreCoordinate(ignoreCoordinate);
395        }
396
397        public void setHandleListener(DataFlowHandleListener listener) {
398                option.setHandleListener(listener);
399        }
400
401        public void setSqlEnv(TSQLEnv sqlenv) {
402                this.sqlenv = sqlenv;
403        }
404
405        public void setOption(Option option) {
406                this.option = option;
407                ModelBindingManager.setGlobalOption(this.option);
408        }
409
410        public Option getOption() {
411                return option;
412        }
413
414        public synchronized String chechSyntax() {
415                StringBuilder builder = new StringBuilder();
416                if (sqlInfos != null) {
417                        for (SqlInfo sqlInfo : sqlInfos) {
418                                String content = sqlInfo.getSql();
419                                if (content != null && content.indexOf("<dlineage") != -1) {
420                                        try {
421                                                XML2Model.loadXML(dataflow.class, content);
422                                                continue;
423                                        } catch (Exception e) {
424                                                builder.append("Parsing dataflow ").append("occurs errors.\n").append(e.getMessage())
425                                                                .append("\n");
426                                        }
427                                }
428                                if (content != null && content.trim().startsWith("{")) {
429                                        Map queryObject = (Map) JSON.parseObject(content);
430                                        content = (String) queryObject.get("sourceCode");
431                                }
432                                if (MetadataReader.isMetadata(content)) {
433                                        continue;
434                                }
435                                TGSqlParser sqlparser = new TGSqlParser(option.getVendor());
436                                sqlparser.sqltext = content;
437                                int result = sqlparser.parse();
438                                if (result != 0) {
439                                        builder.append("Parsing sql ").append("occurs errors.\n").append(sqlparser.getErrormessage())
440                                                        .append("\n");
441                                }
442                        }
443                }
444                return builder.toString();
445        }
446
447        public synchronized String generateDataFlow(boolean withExtraInfo) {
448                if (ModelBindingManager.get() == null) {
449                        ModelBindingManager.set(modelManager);
450                }
451
452        initPipelinedFunctionAnalyzer();
453
454                dataflow = analyzeSqlScript();
455
456                if (dataflow != null && !withExtraInfo && dataflow.getResultsets() != null) {
457                        for (table t : dataflow.getResultsets()) {
458                                t.setIsTarget(null);
459                                if (t.getColumns() != null) {
460                                        for (column t1 : t.getColumns()) {
461                                                t1.setIsFunction(null);
462                                        }
463                                }
464                        }
465                }
466                
467                if(dataflow!=null && dataflow.getRelationships()!=null && option.getFilterRelationTypes()!=null && !option.getFilterRelationTypes().isEmpty()) {
468                        List<relationship> relationships = new ArrayList<relationship>();
469                        for(relationship relationship: dataflow.getRelationships()) {
470                                if(option.getFilterRelationTypes().contains(relationship.getType())) {
471                                        relationships.add(relationship);
472                                }
473                        }
474                        dataflow.setRelationships(relationships);
475                }
476
477                if (option.getHandleListener() != null) {
478                        option.getHandleListener().endAnalyze(dataflow);
479                }
480
481                if (option.isOutput()) {
482                        if (option.getHandleListener() != null) {
483                                option.getHandleListener().startOutputDataFlowXML();
484                        }
485                        if (dataflow != null) {
486                                if (option.isTextFormat()) {
487                                        dataflowString = getTextOutput(dataflow);
488                                } else {
489                                        try {
490                                                dataflowString = XML2Model.saveXML(dataflow);
491                                        }catch (Exception e){
492                                                logger.error("Output dataflow to xml failed.", e);
493                                                dataflowString = null;
494                                        }
495                                }
496                        }
497                        if (option.getHandleListener() != null) {
498                                option.getHandleListener().endOutputDataFlowXML(dataflowString == null ? 0 : dataflowString.length());
499                        }
500                }
501
502                return dataflowString;
503        }
504
505        private dataflow removeDuplicateColumns(dataflow dataflow) {
506                List<table> tables = new ArrayList<table>();
507                if (dataflow.getTables() != null) {
508                        tables.addAll(dataflow.getTables());
509                }
510                if (dataflow.getViews() != null) {
511                        tables.addAll(dataflow.getViews());
512                }
513                if (dataflow.getStages() != null) {
514                        tables.addAll(dataflow.getStages());
515                }
516                if (dataflow.getDatasources() != null) {
517                        tables.addAll(dataflow.getDatasources());
518                }
519                if (dataflow.getStreams() != null) {
520                        tables.addAll(dataflow.getStreams());
521                }
522                if (dataflow.getPaths() != null) {
523                        tables.addAll(dataflow.getPaths());
524                }
525                if (dataflow.getVariables() != null) {
526                        tables.addAll(dataflow.getVariables());
527                }
528                if (dataflow.getResultsets() != null) {
529                        tables.addAll(dataflow.getResultsets());
530                }
531                for (table table : tables) {
532                        if (table.getColumns() == null) {
533                                continue;
534                        }
535                        Set<String> columnIds = new HashSet<String>();
536                        Iterator<column> iter = table.getColumns().iterator();
537                        while(iter.hasNext()) {
538                                column column = iter.next();
539                                String id = column.getId();
540                                if (columnIds.contains(id)) {
541                                        iter.remove();
542                                } else {
543                                        columnIds.add(id);
544                                }
545                        }
546                }
547                return dataflow;
548        }
549
550        public synchronized String generateDataFlow() {
551                return generateDataFlow(false);
552        }
553
554        public synchronized String generateSqlInfos() {
555                return JSON.toJSONString(sqlInfoMap);
556        }
557
558        public Map<String, List<SqlInfo>> getSqlInfos() {
559                return sqlInfoMap;
560        }
561
562        public Map getHashSQLMap() {
563                return modelManager.getHashSQLMap();
564        }
565        
566        public Map getDynamicSQLMap() {
567                return modelManager.getDynamicSQLMap();
568        }
569
570        /**
571         * @deprecated please use SqlInfoHelper.getSelectedDbObjectInfo
572         */
573        public DbObjectPosition getSelectedDbObjectInfo(Coordinate start, Coordinate end) {
574                if (start == null || end == null) {
575                        throw new IllegalArgumentException("Coordinate can't be null.");
576                }
577
578                String hashCode = start.getHashCode();
579
580                if (hashCode == null) {
581                        throw new IllegalArgumentException("Coordinate hashcode can't be null.");
582                }
583
584                int dbObjectStartLine = (int) start.getX() - 1;
585                int dbObjectStarColumn = (int) start.getY() - 1;
586                int dbObjectEndLine = (int) end.getX() - 1;
587                int dbObjectEndColumn = (int) end.getY() - 1;
588                List<SqlInfo> sqlInfoList;
589                if (hashCode.matches("\\d+")) {
590                        sqlInfoList = sqlInfoMap.getValueAtIndex(Integer.valueOf(hashCode));
591                } else {
592                        sqlInfoList = sqlInfoMap.get(hashCode);
593                }
594                for (int j = 0; j < sqlInfoList.size(); j++) {
595                        SqlInfo sqlInfo = sqlInfoList.get(j);
596                        int startLine = sqlInfo.getLineStart();
597                        int endLine = sqlInfo.getLineEnd();
598                        if (dbObjectStartLine >= startLine && dbObjectStartLine <= endLine) {
599                                DbObjectPosition position = new DbObjectPosition();
600                                position.setFile(sqlInfo.getFileName());
601                                position.setFilePath(sqlInfo.getFilePath());
602                                position.setSql(sqlInfo.getSql());
603                                position.setIndex(sqlInfo.getOriginIndex());
604                                List<Pair<Integer, Integer>> positions = position.getPositions();
605                                positions.add(new Pair<Integer, Integer>(
606                                                dbObjectStartLine - startLine + sqlInfo.getOriginLineStart() + 1, dbObjectStarColumn + 1));
607                                positions.add(new Pair<Integer, Integer>(dbObjectEndLine - startLine + sqlInfo.getOriginLineStart() + 1,
608                                                dbObjectEndColumn + 1));
609                                return position;
610                        }
611                }
612                return null;
613        }
614
615        public static dataflow mergeTables(dataflow dataflow, Long startId) {
616                return mergeTables(dataflow, startId, new Option());
617        }
618
619        public static dataflow mergeTables(dataflow dataflow, Long startId, Option option) {
620                List<table> tableCopy = new ArrayList<table>();
621                List<table> viewCopy = new ArrayList<table>();
622                List<table> databaseCopy = new ArrayList<table>();
623                List<table> schemaCopy = new ArrayList<table>();
624                List<table> stageCopy = new ArrayList<table>();
625                List<table> dataSourceCopy = new ArrayList<table>();
626                List<table> streamCopy = new ArrayList<table>();
627                List<table> fileCopy = new ArrayList<table>();
628                List<table> variableCopy = new ArrayList<table>();
629                List<table> cursorCopy = new ArrayList<table>();
630                List<table> resultSetCopy = new ArrayList<table>();
631                if (dataflow.getTables() != null) {
632                        tableCopy.addAll(dataflow.getTables());
633                }
634                dataflow.setTables(tableCopy);
635                if (dataflow.getViews() != null) {
636                        viewCopy.addAll(dataflow.getViews());
637                }
638                dataflow.setViews(viewCopy);
639                if (dataflow.getDatabases() != null) {
640                        databaseCopy.addAll(dataflow.getDatabases());
641                }
642                dataflow.setDatabases(databaseCopy);
643                if (dataflow.getSchemas() != null) {
644                        schemaCopy.addAll(dataflow.getSchemas());
645                }
646                dataflow.setSchemas(schemaCopy);
647                if (dataflow.getStages() != null) {
648                        stageCopy.addAll(dataflow.getStages());
649                }
650                dataflow.setStages(stageCopy);
651                if (dataflow.getDatasources() != null) {
652                        dataSourceCopy.addAll(dataflow.getDatasources());
653                }
654                dataflow.setDatasources(dataSourceCopy);
655                if (dataflow.getStreams() != null) {
656                        streamCopy.addAll(dataflow.getStreams());
657                }
658                dataflow.setStreams(streamCopy);
659                if (dataflow.getPaths() != null) {
660                        fileCopy.addAll(dataflow.getPaths());
661                }
662                dataflow.setPaths(fileCopy);
663                if (dataflow.getVariables() != null) {
664                        variableCopy.addAll(dataflow.getVariables());
665                }
666                dataflow.setVariables(variableCopy);
667                if (dataflow.getResultsets() != null) {
668                        resultSetCopy.addAll(dataflow.getResultsets());
669                }
670                dataflow.setResultsets(resultSetCopy);
671
672                Map<String, List<table>> tableMap = new HashMap<String, List<table>>();
673                Map<String, String> tableTypeMap = new HashMap<String, String>();
674                Map<String, TMssqlCreateType> mssqlTypeMap = new HashMap<String, TMssqlCreateType>();
675                Map<String, String> tableIdMap = new HashMap<String, String>();
676
677                Map<String, List<column>> columnMap = new HashMap<String, List<column>>();
678                Map<String, Set<String>> tableColumnMap = new HashMap<String, Set<String>>();
679                Map<String, String> columnIdMap = new HashMap<String, String>();
680                Map<String, column> columnMergeIdMap = new HashMap<String, column>();
681
682                List<table> tables = new ArrayList<table>();
683                tables.addAll(dataflow.getTables());
684                tables.addAll(dataflow.getViews());
685                tables.addAll(dataflow.getDatabases());
686                tables.addAll(dataflow.getSchemas());
687                tables.addAll(dataflow.getStages());
688                tables.addAll(dataflow.getDatasources());
689                tables.addAll(dataflow.getStreams());
690                tables.addAll(dataflow.getPaths());
691                tables.addAll(dataflow.getResultsets());
692                tables.addAll(dataflow.getVariables());
693                
694                Set<String> columnIds = new HashSet<String>();
695
696                for (table table : tables) {
697                        String qualifiedTableName = DlineageUtil.getQualifiedTableName(table);
698                        String tableFullName = DlineageUtil.getIdentifierNormalTableName(qualifiedTableName);
699                        if ("variable".endsWith(table.getType()) && !SQLUtil.isEmpty(table.getParent())) {
700                                tableFullName = table.getParent() + "." + tableFullName;
701                        }
702                        
703                        if (!tableMap.containsKey(tableFullName)) {
704                                tableMap.put(tableFullName, new ArrayList<table>());
705                        }
706
707                        tableMap.get(tableFullName).add(table);
708
709                        if (!tableTypeMap.containsKey(tableFullName)) {
710                                tableTypeMap.put(tableFullName, table.getType());
711                        } else if ("view".equals(table.getSubType())) {
712                                tableTypeMap.put(tableFullName, table.getType());
713                        } else if ("database".equals(table.getSubType())) {
714                                tableTypeMap.put(tableFullName, table.getType());
715                        } else if ("schema".equals(table.getSubType())) {
716                                tableTypeMap.put(tableFullName, table.getType());
717                        } else if ("stage".equals(table.getSubType())) {
718                                tableTypeMap.put(tableFullName, table.getType());
719                        } else if ("sequence".equals(table.getSubType())) {
720                                tableTypeMap.put(tableFullName, table.getType());
721                        } else if ("datasource".equals(table.getSubType())) {
722                                tableTypeMap.put(tableFullName, table.getType());
723                        } else if ("stream".equals(table.getSubType())) {
724                                tableTypeMap.put(tableFullName, table.getType());
725                        } else if ("file".equals(table.getSubType())) {
726                                tableTypeMap.put(tableFullName, table.getType());
727                        } else if ("table".equals(tableTypeMap.get(tableFullName))) {
728                                tableTypeMap.put(tableFullName, table.getType());
729                        } else if ("variable".equals(tableTypeMap.get(tableFullName))) {
730                                tableTypeMap.put(tableFullName, table.getType());
731                        }
732
733                        if (table.getColumns() != null) {
734                                if (!tableColumnMap.containsKey(tableFullName)) {
735                                        tableColumnMap.put(tableFullName, new LinkedHashSet<String>());
736                                }
737                                for (column column : table.getColumns()) {
738                                        String columnFullName = tableFullName + "."
739                                                        + (column.getQualifiedTable() != null
740                                                                        ? (DlineageUtil.getIdentifierNormalTableName(column.getQualifiedTable()) + ".")
741                                                                        : "")
742                                                        + ("false".equals(table.getIsTarget()) ? DlineageUtil.normalizeColumnName(column.getName())
743                                                                        : DlineageUtil.getIdentifierNormalColumnName(column.getName()));
744
745                                        if (!columnMap.containsKey(columnFullName)) {
746                                                columnMap.put(columnFullName, new ArrayList<column>());
747                                                tableColumnMap.get(tableFullName).add(columnFullName);
748                                        }
749
750                                        columnMap.get(columnFullName).add(column);
751                                        columnIds.add(column.getId());
752                                }
753                        }
754                }
755
756                Set<String> relationSourceParentIds = new HashSet<String>();
757                if (dataflow.getRelationships() != null) {
758                        for (relationship rel : dataflow.getRelationships()) {
759                                if (rel.getSources() != null) {
760                                        for (sourceColumn src : rel.getSources()) {
761                                                if (src.getParent_id() != null) {
762                                                        relationSourceParentIds.add(src.getParent_id());
763                                                }
764                                        }
765                                }
766                        }
767                }
768
769                Iterator<String> tableNameIter = tableMap.keySet().iterator();
770                while (tableNameIter.hasNext()) {
771                        String tableName = tableNameIter.next();
772                        List<table> tableList = tableMap.get(tableName);
773                        table table;
774                        if (tableList.size() > 1) {
775                                table standardTable = tableList.get(0);
776                                // Function允许重名，不做合并处理
777                                if (standardTable.isFunction()) {
778                                        continue;
779                                }
780                                
781                                // Variable允许重名，不做合并处理
782                                if (standardTable.isVariable()) {
783                                        continue;
784                                }
785                                
786                                // 临时表不做合并处理
787                                if(SQLUtil.isTempTable(standardTable)) {
788                                        continue;
789                                }
790
791                                String type = tableTypeMap.get(tableName);
792                                table = new table();
793                                table.setId(String.valueOf(++startId));
794                                table.setServer(standardTable.getServer());
795                                table.setDatabase(standardTable.getDatabase());
796                                table.setSchema(standardTable.getSchema());
797                                table.setName(standardTable.getName());
798                                table.setDisplayName(standardTable.getDisplayName());
799                                table.setParent(standardTable.getParent());
800                                table.setMore(standardTable.getMore());
801                                table.setColumns(new ArrayList<column>());
802                                table.setSubType(standardTable.getSubType());
803                                Set<String> processIds = new LinkedHashSet<String>();
804                                for (int k = 0; k < tableList.size(); k++) {
805                                        if (tableList.get(k).getProcessIds() != null) {
806                                                processIds.addAll(tableList.get(k).getProcessIds());
807                                        }
808                                }
809                                if (!processIds.isEmpty()) {
810                                        table.setProcessIds(new ArrayList<String>(processIds));
811                                }
812                                Set<String> alias = new LinkedHashSet<String>();
813                                for (int k = 0; k < tableList.size(); k++) {
814                                        if (tableList.get(k).getAlias() != null) {
815                                                alias.addAll(Arrays.asList(tableList.get(k).getAlias().split("\\s*,\\s*")));
816                                        }
817                                }
818                                if (!alias.isEmpty()) {
819                                        String aliasString = Arrays.toString(alias.toArray(new String[0]));
820                                        table.setAlias(aliasString.substring(1, aliasString.length() - 1));
821                                }
822                                table.setType(type);
823                                for (table item : tableList) {
824                                        if (!SQLUtil.isEmpty(table.getCoordinate()) && !SQLUtil.isEmpty(item.getCoordinate())) {
825                                                if (table.getCoordinate().indexOf(item.getCoordinate()) == -1) {
826                                                        table.appendCoordinate(item.getCoordinate());
827                                                }
828                                        } else if (!SQLUtil.isEmpty(item.getCoordinate())) {
829                                                table.setCoordinate(item.getCoordinate());
830                                        }
831
832                                        if (item.getStarStmt() != null) {
833                                                table.setStarStmt(item.getStarStmt());
834                                        }
835
836                                        tableIdMap.put(item.getId(), table.getId());
837
838                                        if (item.isView()) {
839                                                dataflow.getViews().remove(item);
840                                        } else if (item.isDatabaseType()) {
841                                                dataflow.getDatabases().remove(item);
842                                        } else if (item.isSchemaType()) {
843                                                dataflow.getSchemas().remove(item);
844                                        } else if (item.isStage()) {
845                                                dataflow.getStages().remove(item);
846                                        } else if (item.isDataSource()) {
847                                                dataflow.getDatasources().remove(item);
848                                        } else if (item.isStream()) {
849                                                dataflow.getStreams().remove(item);
850                                        } else if (item.isFile()) {
851                                                dataflow.getPaths().remove(item);
852                                        } else if (item.isVariable()) {
853                                                dataflow.getVariables().remove(item);
854                                        } else if (item.isTable()) {
855                                                dataflow.getTables().remove(item);
856                                        } else if (item.isResultSet()) {
857                                                dataflow.getResultsets().remove(item);
858                                        }
859                                }
860
861                                if (table.isView()) {
862                                        dataflow.getViews().add(table);
863                                } else if (table.isDatabaseType()) {
864                                        dataflow.getDatabases().add(table);
865                                } else if (table.isSchemaType()) {
866                                        dataflow.getSchemas().add(table);
867                                } else if (table.isStage()) {
868                                        dataflow.getStages().add(table);
869                                } else if (table.isDataSource()) {
870                                        dataflow.getDatasources().add(table);
871                                } else if (table.isStream()) {
872                                        dataflow.getStreams().add(table);
873                                } else if (table.isFile()) {
874                                        dataflow.getPaths().add(table);
875                                } else if (table.isVariable()) {
876                                        dataflow.getVariables().add(table);
877                                } else if (table.isResultSet()) {
878                                        dataflow.getResultsets().add(table);
879                                } else {
880                                        dataflow.getTables().add(table);
881                                }
882                        } else {
883                                table = tableList.get(0);
884                                if(Boolean.TRUE.toString().equals(table.getIsDetermined())){
885                                        continue;
886                                }
887                                
888                                if (option.isIgnoreUnusedSynonym() && SubType.synonym.name().equals(table.getSubType())) {
889                                        boolean hasSourceRelation = relationSourceParentIds.contains(table.getId());
890                                        if (!hasSourceRelation) {
891                                                dataflow.getTables().remove(table);
892                                                tableColumnMap.get(tableName).clear();
893                                                continue;
894                                        }
895                                }
896                        }
897
898                        Set<String> columns = tableColumnMap.get(tableName);
899                        Iterator<String> columnIter = columns.iterator();
900                        List<column> mergeColumns = new ArrayList<column>();
901                        while (columnIter.hasNext()) {
902                                String columnName = columnIter.next();
903                                List<column> columnList = columnMap.get(columnName);
904                                List<column> functions = new ArrayList<column>();
905                                for (column t : columnList) {
906                                        if (Boolean.TRUE.toString().equals(t.getIsFunction())) {
907                                                functions.add(t);
908                                        }
909                                }
910                                if (functions != null && !functions.isEmpty()) {
911                                        for (column function : functions) {
912                                                mergeColumns.add(function);
913                                                columnIdMap.put(function.getId(), function.getId());
914                                                columnMergeIdMap.put(function.getId(), function);
915                                        }
916
917                                        columnList.removeAll(functions);
918                                }
919                                if (!columnList.isEmpty()) {
920                                        column firstColumn = columnList.iterator().next();
921                                        if (columnList.size() > 1) {
922                                                column mergeColumn = new column();
923                                                mergeColumn.setId(String.valueOf(++startId));
924                                                mergeColumn.setName(firstColumn.getName());
925                                                mergeColumn.setDisplayName(firstColumn.getDisplayName());
926                                                mergeColumn.setSource(firstColumn.getSource());
927                                                mergeColumn.setQualifiedTable(firstColumn.getQualifiedTable());
928                                                mergeColumn.setDataType(firstColumn.getDataType());
929                                                mergeColumn.setForeignKey(firstColumn.isForeignKey());
930                                                mergeColumn.setPrimaryKey(firstColumn.isPrimaryKey());
931                                                mergeColumn.setUnqiueKey(firstColumn.isUnqiueKey());
932                                                mergeColumn.setIndexKey(firstColumn.isIndexKey());
933                                                mergeColumns.add(mergeColumn);
934                                                for (column item : columnList) {
935                                                        mergeColumn.appendCoordinate(item.getCoordinate());
936                                                        columnIdMap.put(item.getId(), mergeColumn.getId());
937                                                }
938                                                columnMergeIdMap.put(mergeColumn.getId(), mergeColumn);
939                                                columnIds.add(mergeColumn.getId());
940                                        } else {
941                                                mergeColumns.add(firstColumn);
942                                                columnIdMap.put(firstColumn.getId(), firstColumn.getId());
943                                                columnMergeIdMap.put(firstColumn.getId(), firstColumn);
944                                        }
945                                }
946                        }
947                        table.setColumns(mergeColumns);
948                }
949
950                if (dataflow.getRelationships() != null) {
951                        Map<String, relationship> mergeRelations = new LinkedHashMap<String, relationship>();
952                        for (int i = 0; i < dataflow.getRelationships().size(); i++) {
953                                relationship relation = dataflow.getRelationships().get(i);
954                                
955                                if("crud".equals(relation.getType()) && option.getAnalyzeMode() == AnalyzeMode.crud) {
956                                        String jsonString = JSON.toJSONString(relation, true);
957                                        String key = SHA256.getMd5(jsonString);
958                                        if (!mergeRelations.containsKey(key)) {
959                                                mergeRelations.put(key, relation);
960                                        }
961                                        continue;
962                                }
963                                
964                                targetColumn target = relation.getTarget();
965                                if ("call".equals(relation.getType())) {
966                                        target = relation.getCaller();
967                                }
968                                if (target != null && tableIdMap.containsKey(target.getParent_id())) {
969                                        target.setParent_id(tableIdMap.get(target.getParent_id()));
970                                }
971
972                                if (columnIdMap.containsKey(target.getId())) {
973                                        target.setId(columnIdMap.get(target.getId()));
974                                        target.setCoordinate(columnMergeIdMap.get(target.getId()).getCoordinate());
975                                }
976                                else if(option.isIgnoreUnusedSynonym() && EffectType.synonym.name().equals(relation.getEffectType())){
977                                        continue;
978                                }
979                                
980                                if (!"call".equals(relation.getType()) && !columnIds.contains(target.getId())) {
981                                        continue;
982                                }
983
984                                List<sourceColumn> sources = relation.getSources();
985                                if ("call".equals(relation.getType())) {
986                                        sources = relation.getCallees();
987                                }
988                                Set<sourceColumn> sourceSet = new LinkedHashSet<sourceColumn>();
989                                if (sources != null) {
990                                        for (sourceColumn source : sources) {
991                                                if (!"call".equals(relation.getType()) && !columnIds.contains(source.getId())) {
992                                                        continue;
993                                                }
994                                                if (tableIdMap.containsKey(source.getParent_id())) {
995                                                        source.setParent_id(tableIdMap.get(source.getParent_id()));
996                                                }
997                                                if (tableIdMap.containsKey(source.getSource_id())) {
998                                                        source.setSource_id(tableIdMap.get(source.getSource_id()));
999                                                }
1000                                                if (columnIdMap.containsKey(source.getId())) {
1001                                                        source.setId(columnIdMap.get(source.getId()));
1002                                                        source.setCoordinate(columnMergeIdMap.get(source.getId()).getCoordinate());
1003                                                }
1004                                        }
1005
1006                                        sourceSet.addAll(sources);
1007                                        if ("call".equals(relation.getType())) {
1008                                                relation.setCallees(new ArrayList<sourceColumn>(sourceSet));
1009                                        } else {
1010                                                relation.setSources(new ArrayList<sourceColumn>(sourceSet));
1011                                        }
1012                                }
1013
1014                                String jsonString = JSON.toJSONString(relation, true);
1015                String key = SHA256.getMd5(jsonString);
1016                                if (!mergeRelations.containsKey(key)) {
1017                                        mergeRelations.put(key, relation);
1018                                }
1019                        }
1020
1021                        dataflow.setRelationships(new ArrayList<relationship>(mergeRelations.values()));
1022                }
1023
1024                tableMap.clear();
1025                tableTypeMap.clear();
1026                tableIdMap.clear();
1027                columnMap.clear();
1028                tableColumnMap.clear();
1029                columnIdMap.clear();
1030                columnMergeIdMap.clear();
1031                tables.clear();
1032                return dataflow;
1033        }
1034
1035        public synchronized dataflow getDataFlow() {
1036                if (dataflow != null) {
1037                        return dataflow;
1038                } else if (dataflowString != null) {
1039                        return XML2Model.loadXML(dataflow.class, dataflowString);
1040                }
1041                return null;
1042        }
1043
1044        List<ErrorInfo> metadataErrors = new ArrayList<>();
1045        
1046        private synchronized dataflow analyzeSqlScript() {
1047                init();
1048
1049                try {
1050                        dataflow dataflow = new dataflow();
1051                        
1052                        if (sqlInfos != null) {
1053                                if (option.getHandleListener() != null) {
1054                                        if (sqlInfos.length == 1) {
1055                                                option.getHandleListener().startAnalyze(null, sqlInfos[0].getSql().length(), false);
1056                                        } else {
1057                                                option.getHandleListener().startAnalyze(null, sqlInfos.length, true);
1058                                        }
1059                                }
1060
1061                                if (sqlenv == null) {
1062                                        if (option.getHandleListener() != null) {
1063                                                option.getHandleListener().startParseSQLEnv();
1064                                        }
1065                                        TSQLEnv[] sqlenvs = new SQLEnvParser(option.getDefaultServer(), option.getDefaultDatabase(),
1066                                                        option.getDefaultSchema()).parseSQLEnv(option.getVendor(), sqlInfos);
1067                                        if (sqlenvs != null && sqlenvs.length > 0) {
1068                                                sqlenv = sqlenvs[0];
1069                                        }
1070                                        if (option.getHandleListener() != null) {
1071                                                option.getHandleListener().endParseSQLEnv();
1072                                        }
1073                                }
1074                                TGSqlParser sqlparser = new TGSqlParser(option.getVendor());
1075                                Map<String, Pair3<StringBuilder, AtomicInteger, String>> databaseMap = new LinkedHashMap<String, Pair3<StringBuilder, AtomicInteger, String>>();
1076                                for (int i = 0; i < sqlInfos.length; i++) {
1077                                        SqlInfo sqlInfo = sqlInfos[i];
1078                                        if (sqlInfo == null) {
1079                                                sqlInfoMap.put(String.valueOf(i), new ArrayList<SqlInfo>());
1080                                                continue;
1081                                        }
1082                                        String sql = sqlInfo.getSql();
1083                                        if (SQLUtil.isEmpty(sql) && sqlInfo.getFileName() != null
1084                                                        && new File(sqlInfo.getFileName()).exists()) {
1085                                                sql = SQLUtil.getFileContent(sqlInfo.getFileName());
1086                                        }
1087                                        if (SQLUtil.isEmpty(sql) && sqlInfo.getFilePath() != null
1088                                                        && new File(sqlInfo.getFilePath()).exists()) {
1089                                                sql = SQLUtil.getFileContent(sqlInfo.getFilePath());
1090                                        }
1091                                        String sqlTrim = null;
1092                                        if (sql != null) {
1093                                                sqlTrim = sql.substring(0, Math.min(sql.length(), 512)).trim();
1094                                        }
1095                                        if(sql!=null && sqlTrim.startsWith("<") && sqlTrim.indexOf("<dlineage")!=-1){
1096                                                dataflow temp = XML2Model.loadXML(dataflow.class, sql);
1097                                                if(sqlInfos.length == 1){
1098                                                        dataflow = temp;
1099                                                }
1100                                                else {
1101                                                        if (temp.getTables() != null) {
1102                                                                dataflow.getTables().addAll(temp.getTables());
1103                                                        }
1104                                                        if (temp.getViews() != null) {
1105                                                                dataflow.getViews().addAll(temp.getViews());
1106                                                        }
1107                                                        if (temp.getResultsets() != null) {
1108                                                                dataflow.getResultsets().addAll(temp.getResultsets());
1109                                                        }
1110                                                        if (temp.getRelationships() != null) {
1111                                                                dataflow.getRelationships().addAll(temp.getRelationships());
1112                                                        }
1113                                                        if (temp.getErrors() != null) {
1114                                                                dataflow.getErrors().addAll(temp.getErrors());
1115                                                        }
1116                                                }
1117                                        }
1118                                        else if (sql != null && sqlTrim.startsWith("{")) {
1119                                                EDbVendor vendor = SQLUtil.isEmpty(sqlInfo.getDbVendor()) ? option.getVendor()
1120                                                                : EDbVendor.valueOf(sqlInfo.getDbVendor());
1121                                                TSQLEnv[] sqlenvs = new TJSONSQLEnvParser(option.getDefaultServer(),
1122                                                                option.getDefaultDatabase(), option.getDefaultSchema()).parseSQLEnv(vendor, sql);
1123                                                if (sqlenvs != null && sqlenvs.length > 0) {
1124                                                        if (sqlenv == null) {
1125                                                                sqlenv = sqlenvs[0];
1126                                                        } else {
1127                                                                sqlenv = SQLEnvParser.mergeSQLEnv(Arrays.asList(sqlenv, sqlenvs[0]));
1128                                                        }
1129                                                }
1130                                                if (sqlenv != null) {
1131                                                        if (MetadataReader.isGrabit(sql) || MetadataReader.isSqlflow(sql)) {
1132                                                                String hash = SHA256.getMd5(sql);
1133                                                                String fileHash = SHA256.getMd5(hash);
1134                                                                if (!sqlInfoMap.containsKey(fileHash)) {
1135                                                                        sqlInfoMap.put(fileHash, new ArrayList<SqlInfo>());
1136                                                                        sqlInfoMap.get(fileHash).add(sqlInfo);
1137                                                                }
1138                                                                ModelBindingManager.setGlobalHash(fileHash);
1139                                                                dataflow temp = null;
1140
1141                                                                if (MetadataReader.isGrabit(sql)) {
1142                                                                        temp = new GrabitMetadataAnalyzer().analyzeMetadata(option.getVendor(), sql);
1143                                                                } else {
1144                                                                        temp = new SqlflowMetadataAnalyzer(sqlenv).analyzeMetadata(option.getVendor(), sql);
1145                                                                }
1146//                                                              if (temp.getPackages() != null) {
1147//                                                                      dataflow.getPackages().addAll(temp.getPackages());
1148//                                                              }
1149//                                                              if (temp.getProcedures() != null) {
1150//                                                                      dataflow.getProcedures().addAll(temp.getProcedures());
1151//                                                              }
1152                                                                if (temp.getTables() != null) {
1153                                                                        dataflow.getTables().addAll(temp.getTables());
1154                                                                }
1155                                                                if (temp.getViews() != null) {
1156                                                                        dataflow.getViews().addAll(temp.getViews());
1157                                                                }
1158                                                                if (temp.getResultsets() != null) {
1159                                                                        dataflow.getResultsets().addAll(temp.getResultsets());
1160                                                                }
1161                                                                if (temp.getRelationships() != null) {
1162                                                                        dataflow.getRelationships().addAll(temp.getRelationships());
1163                                                                }
1164                                                                if (temp.getErrors() != null) {
1165                                                                        dataflow.getErrors().addAll(temp.getErrors());
1166                                                                }
1167                                                                if (sql.indexOf("createdBy") != -1) {
1168                                                                        if (sql.toLowerCase().indexOf("sqldep") != -1
1169                                                                                        || sql.toLowerCase().indexOf("grabit") != -1) {
1170                                                                                Map jsonObject = (Map) JSON.parseObject(sql);
1171                                                                                List<Map> queries = (List<Map>) jsonObject.get("queries");
1172                                                                                if (queries != null) {
1173                                                                                        for (int j = 0; j < queries.size(); j++) {
1174                                                                                                Map queryObject = queries.get(j);
1175                                                                                                appendSqlInfo(databaseMap, j, sqlInfo, queryObject);
1176                                                                                        }
1177                                                                                }
1178                                                                        } else if (sql.toLowerCase().indexOf("sqlflow") != -1) {
1179                                                                                Map sqlflow = (Map) JSON.parseObject(sql);
1180                                                                                List<Map> servers = (List<Map>) sqlflow.get("servers");
1181                                                                                if (servers != null) {
1182                                                                                        for (Map serverObject : servers) {
1183                                                                                                List<Map> queries = (List<Map>) serverObject.get("queries");
1184                                                                                                if (queries != null) {
1185                                                                                                        for (int j = 0; j < queries.size(); j++) {
1186                                                                                                                Map queryObject = queries.get(j);
1187                                                                                                                appendSqlInfo(databaseMap, j, sqlInfo, queryObject);
1188                                                                                                        }
1189                                                                                                }
1190                                                                                        }
1191                                                                                }
1192                                                                                List<Map> errorMessages =  (List<Map>) sqlflow.get("errorMessages");
1193                                                                                if(errorMessages!=null && !errorMessages.isEmpty()) {
1194                                                                                        for(Map error: errorMessages){
1195                                                                                                ErrorInfo errorInfo = new ErrorInfo();
1196                                                                                                errorInfo.setErrorType(ErrorInfo.METADATA_ERROR);
1197                                                                                                errorInfo.setErrorMessage((String)error.get("errorMessage"));
1198                                                                                                errorInfo.setFileName(sqlInfo.getFileName());
1199                                                                                                errorInfo.setFilePath(sqlInfo.getFilePath());
1200                                                                                                errorInfo.setStartPosition(new Pair3<Long, Long, String>(-1L, -1L,
1201                                                                                                                ModelBindingManager.getGlobalHash()));
1202                                                                                                errorInfo.setEndPosition(new Pair3<Long, Long, String>(-1L, -1L,
1203                                                                                                                ModelBindingManager.getGlobalHash()));
1204                                                                                                errorInfo.setOriginStartPosition(new Pair<Long, Long>(-1L, -1L));
1205                                                                                                errorInfo.setOriginEndPosition(new Pair<Long, Long>(-1L, -1L));
1206                                                                                                metadataErrors.add(errorInfo);
1207                                                                                        }
1208                                                                                }
1209                                                                        }
1210                                                                }
1211                                                        } else {
1212                                                                Map queryObject = (Map) JSON.parseObject(sql);
1213                                                                appendSqlInfo(databaseMap, i, sqlInfo, queryObject);
1214                                                        }
1215                                                } else {
1216                                                        Map queryObject = (Map) JSON.parseObject(sql);
1217                                                        appendSqlInfo(databaseMap, i, sqlInfo, queryObject);
1218                                                }
1219                                        } else {
1220                                                ModelBindingManager.removeGlobalDatabase();
1221                                                ModelBindingManager.removeGlobalSchema();
1222                                                ModelBindingManager.removeGlobalHash();
1223
1224                                                String content = sql;
1225
1226                                                if (content == null) {
1227                                                        continue;
1228                                                }
1229
1230                                                String delimiterChar = String.valueOf(sqlparser.getDelimiterChar());
1231
1232                                                if (sqlInfos.length > 1) {
1233                                                        String endTrim = SQLUtil.endTrim(content);
1234                                                        if (endTrim.endsWith(delimiterChar) || endTrim.endsWith(";")) {
1235                                                                content += "\n";
1236                                                        } else if (option.getVendor() == EDbVendor.dbvredshift
1237                                                                        || option.getVendor() == EDbVendor.dbvgaussdb
1238                                                                        || option.getVendor() == EDbVendor.dbvedb
1239                                                                        || option.getVendor() == EDbVendor.dbvpostgresql
1240                                                                        || option.getVendor() == EDbVendor.dbvmysql
1241                                                                        || option.getVendor() == EDbVendor.dbvteradata) {
1242                                                                content += ("\n\n-- " + TBaseType.sqlflow_stmt_delimiter_str + "\n\n");
1243                                                        } else {
1244                                                                content = endTrim + ";" + "\n";
1245                                                        }
1246                                                }
1247
1248                                                sqlInfo.setSql(content);
1249
1250                                                if (MetadataReader.isMetadata(content)) {
1251                                                        String hash = SHA256.getMd5(content);
1252                                                        ModelBindingManager.setGlobalHash(hash);
1253                                                        dataflow temp = new SQLDepMetadataAnalyzer().analyzeMetadata(option.getVendor(), content);
1254                                                        if (temp.getProcedures() != null) {
1255                                                                dataflow.getProcedures().addAll(temp.getProcedures());
1256                                                        }
1257                                                        if (temp.getTables() != null) {
1258                                                                dataflow.getTables().addAll(temp.getTables());
1259                                                        }
1260                                                        if (temp.getViews() != null) {
1261                                                                dataflow.getViews().addAll(temp.getViews());
1262                                                        }
1263                                                        if (temp.getResultsets() != null) {
1264                                                                dataflow.getResultsets().addAll(temp.getResultsets());
1265                                                        }
1266                                                        if (temp.getRelationships() != null) {
1267                                                                dataflow.getRelationships().addAll(temp.getRelationships());
1268                                                        }
1269                                                        if (temp.getErrors() != null) {
1270                                                                dataflow.getErrors().addAll(temp.getErrors());
1271                                                        }
1272                                                        String fileHash = SHA256.getMd5(hash);
1273                                                        if (!sqlInfoMap.containsKey(fileHash)) {
1274                                                                sqlInfoMap.put(fileHash, new ArrayList<SqlInfo>());
1275                                                                sqlInfoMap.get(fileHash).add(sqlInfo);
1276                                                        }
1277                                                } else {
1278                                                        String sqlHash = SHA256.getMd5(content);
1279                                                        String fileHash = SHA256.getMd5(sqlHash);
1280                                                        if (!sqlInfoMap.containsKey(fileHash)) {
1281                                                                sqlInfoMap.put(fileHash, new ArrayList<SqlInfo>());
1282                                                        }
1283
1284                                                        String database = TSQLEnv.DEFAULT_DB_NAME;
1285                                                        String schema = TSQLEnv.DEFAULT_SCHEMA_NAME;
1286                                                        if (sqlenv != null) {
1287                                                                database = sqlenv.getDefaultCatalogName();
1288                                                                if (database == null) {
1289                                                                        database = TSQLEnv.DEFAULT_DB_NAME;
1290                                                                }
1291                                                                schema = sqlenv.getDefaultSchemaName();
1292                                                                if (schema == null) {
1293                                                                        schema = TSQLEnv.DEFAULT_SCHEMA_NAME;
1294                                                                }
1295                                                        }
1296
1297                                                        boolean supportCatalog = TSQLEnv.supportCatalog(option.getVendor());
1298                                                        boolean supportSchema = TSQLEnv.supportSchema(option.getVendor());
1299                                                        StringBuilder builder = new StringBuilder();
1300                                                        if (supportCatalog) {
1301                                                                builder.append(database);
1302                                                        }
1303                                                        if (supportSchema) {
1304                                                                if (builder.length() > 0) {
1305                                                                        builder.append(".");
1306                                                                }
1307                                                                builder.append(schema);
1308                                                        }
1309                                                        String group = builder.toString();
1310                                                        SqlInfo sqlInfoItem = new SqlInfo();
1311                                                        sqlInfoItem.setFileName(sqlInfo.getFileName());
1312                                                        sqlInfoItem.setFilePath(sqlInfo.getFilePath());
1313                                                        sqlInfoItem.setSql(sqlInfo.getSql());
1314                                                        sqlInfoItem.setOriginIndex(0);
1315                                                        sqlInfoItem.setOriginLineStart(0);
1316                                                        int lineEnd = sqlInfo.getSql().split("\n").length - 1;
1317                                                        sqlInfoItem.setOriginLineEnd(lineEnd);
1318                                                        sqlInfoItem.setIndex(0);
1319                                                        sqlInfoItem.setLineStart(0);
1320                                                        sqlInfoItem.setLineEnd(lineEnd);
1321                                                        sqlInfoItem.setHash(SHA256.getMd5(sqlHash));
1322                                                        sqlInfoItem.setGroup(group);
1323                                                        sqlInfoMap.get(fileHash).add(sqlInfoItem);
1324                                                        if (!databaseMap.containsKey(sqlHash)) {
1325                                                                databaseMap.put(sqlHash, new Pair3<StringBuilder, AtomicInteger, String>(
1326                                                                                new StringBuilder(), new AtomicInteger(), group));
1327                                                        }
1328                                                        databaseMap.get(sqlHash).first.append(sqlInfoItem.getSql());
1329                                                        databaseMap.get(sqlHash).second.incrementAndGet();
1330                                                }
1331                                        }
1332                                }
1333
1334                                boolean supportCatalog = TSQLEnv.supportCatalog(option.getVendor());
1335                                boolean supportSchema = TSQLEnv.supportSchema(option.getVendor());
1336
1337                                Iterator<String> schemaIter = databaseMap.keySet().iterator();
1338                                while (schemaIter.hasNext()) {
1339                                        if (option.getHandleListener() != null && option.getHandleListener().isCanceled()) {
1340                                                break;
1341                                        }
1342                                        String key = schemaIter.next();
1343                                        String group = databaseMap.get(key).third;
1344                                        String[] split = SQLUtil.parseNames(group).toArray(new String[0]);
1345
1346                                        ModelBindingManager.removeGlobalDatabase();
1347                                        ModelBindingManager.removeGlobalSchema();
1348                                        ModelBindingManager.removeGlobalSQLEnv();
1349                                        ModelBindingManager.removeGlobalHash();
1350
1351                                        String defaultDatabase = null;
1352                                        String defaultSchema = null;
1353                                        if(sqlenv!=null){
1354                                                defaultDatabase = sqlenv.getDefaultCatalogName();
1355                                                defaultSchema = sqlenv.getDefaultSchemaName();
1356                                        }
1357                                        if (supportCatalog && supportSchema) {
1358                                                if (split.length >= 2) {
1359                                                        if (!TSQLEnv.DEFAULT_DB_NAME.equals(split[split.length - 2])) {
1360                                                                ModelBindingManager.setGlobalDatabase(split[split.length - 2]);
1361                                                                sqlenv.setDefaultCatalogName(ModelBindingManager.getGlobalDatabase());
1362                                                        }
1363                                                }
1364                                                if (split.length >= 1) {
1365                                                        if (!TSQLEnv.DEFAULT_SCHEMA_NAME.equals(split[split.length - 1])) {
1366                                                                ModelBindingManager.setGlobalSchema(split[split.length - 1]);
1367                                                                sqlenv.setDefaultSchemaName(ModelBindingManager.getGlobalSchema());
1368                                                        }
1369                                                }
1370                                        } else if (supportCatalog) {
1371                                                if (!TSQLEnv.DEFAULT_DB_NAME.equals(split[split.length - 1])) {
1372                                                        ModelBindingManager.setGlobalDatabase(split[split.length - 1]);
1373                                                        sqlenv.setDefaultCatalogName(ModelBindingManager.getGlobalDatabase());
1374                                                }
1375                                        } else if (supportSchema) {
1376                                                if (!TSQLEnv.DEFAULT_SCHEMA_NAME.equals(split[split.length - 1])) {
1377                                                        ModelBindingManager.setGlobalSchema(split[split.length - 1]);
1378                                                        sqlenv.setDefaultSchemaName(ModelBindingManager.getGlobalSchema());
1379                                                }
1380                                        }
1381                                        if (option.getHandleListener() != null) {
1382                                                option.getHandleListener().startParse(null, databaseMap.get(key).first.toString());
1383                                        }
1384
1385                                        if (sqlenv == null) {
1386                                                sqlenv = new TSQLEnv(option.getVendor()) {
1387
1388                                                        @Override
1389                                                        public void initSQLEnv() {
1390                                                                // TODO Auto-generated method stub
1391
1392                                                        }
1393                                                };
1394                                        }
1395                                        ModelBindingManager.setGlobalSQLEnv(sqlenv);
1396                                        sqlparser.sqltext = databaseMap.get(key).first.toString();
1397                                        ModelBindingManager.setGlobalHash(SHA256.getMd5(key));
1398                                        analyzeAndOutputResult(sqlparser);
1399                                        if(sqlenv!=null){
1400                                                sqlenv.setDefaultCatalogName(defaultDatabase);
1401                                                sqlenv.setDefaultSchemaName(defaultSchema);
1402                                        }
1403                                }
1404
1405                                appendProcesses(dataflow);
1406                                appendOraclePackages(dataflow);
1407                                appendProcedures(dataflow);
1408                                appendTables(dataflow);
1409                                appendViews(dataflow);
1410                                appendResultSets(dataflow);
1411                                appendRelations(dataflow);
1412                                appendErrors(dataflow);
1413                        }
1414
1415                        dataflow = handleDataflowExecProcedure(dataflow);
1416
1417                        if (dataflow != null && option.getAnalyzeMode() != AnalyzeMode.crud) {
1418                                if (!isShowJoin()) {
1419                                        dataflow = mergeTables(dataflow, modelManager.TABLE_COLUMN_ID, option);
1420                                } else {
1421                                        dataflow = removeDuplicateColumns(dataflow);
1422                                }
1423                        }
1424
1425                        if (dataflow != null && option.isNormalizeOutput()) {
1426                                dataflow = getNormalizeDataflow(dataflow);
1427                        }
1428
1429                        if (dataflow != null && option.isIgnoreCoordinate()) {
1430                                dataflow = filterDataflowCoordinate(dataflow);
1431                        }
1432                        
1433                        if (option.isSimpleOutput() || option.isIgnoreRecordSet()) {
1434                                List<String> showTypes = new ArrayList<>();
1435                                if(option.getSimpleShowRelationTypes()!=null) {
1436                                        showTypes.addAll(option.getSimpleShowRelationTypes());
1437                                }
1438                                if(showTypes.isEmpty()) {
1439                                        showTypes.add("fdd");
1440                                }
1441                                if(option.isShowCallRelation()) {
1442                                        showTypes.add("call");
1443                                }
1444                                if(option.isShowERDiagram()) {
1445                                        showTypes.add("er");
1446                                }
1447                                dataflow simpleDataflow = getSimpleDataflow(dataflow, option.isSimpleOutput(), showTypes);
1448                                if (simpleDataflow.getResultsets() != null) {
1449                                        for (table t : simpleDataflow.getResultsets()) {
1450                                                t.setIsTarget(null);
1451                                        }
1452                                }
1453                                return simpleDataflow;
1454                        } else {
1455                                return dataflow;
1456                        }
1457                        
1458                } catch (Exception e) {
1459                        logger.error("analyze sql failed.", e);
1460                        ErrorInfo errorInfo = new ErrorInfo();
1461                        errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
1462                        if (e.getMessage() == null) {
1463                                if (e.getStackTrace() != null && e.getStackTrace().length > 0) {
1464                                        errorInfo.setErrorMessage(e.getClass().getSimpleName() + ": " + e.getStackTrace()[0].toString());
1465                                } else {
1466                                        errorInfo.setErrorMessage(e.getClass().getSimpleName());
1467                                }
1468                        } else {
1469                                errorInfo.setErrorMessage(e.getClass().getSimpleName() + ": " + e.getMessage());
1470                        }
1471                        errorInfo.fillInfo(this);
1472                        errorInfos.add(errorInfo);
1473                }
1474
1475                modelManager.reset();
1476                return null;
1477        }
1478
1479        private dataflow handleDataflowExecProcedure(dataflow dataflow) {
1480                Set<String> procedures = new HashSet<String>();
1481                for (procedure procedure : dataflow.getProcedures()) {
1482                        procedures.add(DlineageUtil.getIdentifierNormalTableName(procedure.getName()));
1483                }
1484                List<table> removeResultSets = new ArrayList<table>();
1485                
1486                Map<String, table> allMap = new HashMap<String, table>();
1487                
1488                if (dataflow.getResultsets() != null) {
1489                        for (table t : dataflow.getResultsets()) {
1490                                allMap.put(t.getId().toLowerCase(), t);
1491                        }
1492                }
1493
1494                if (dataflow.getTables() != null) {
1495                        for (table t : dataflow.getTables()) {
1496                                allMap.put(t.getId().toLowerCase(), t);
1497                        }
1498                }
1499
1500                if (dataflow.getViews() != null) {
1501                        for (table t : dataflow.getViews()) {
1502                                allMap.put(t.getId().toLowerCase(), t);
1503                        }
1504                }
1505
1506                if (dataflow.getVariables() != null) {
1507                        for (table t : dataflow.getVariables()) {
1508                                allMap.put(t.getId().toLowerCase(), t);
1509                        }
1510                }
1511
1512                
1513                for (table resultSet : dataflow.getResultsets()) {
1514                        String resultSetName = DlineageUtil.getIdentifierNormalTableName(resultSet.getName());
1515                        if (!(ResultSetType.function.name().equals(resultSet.getType())
1516                                        && modelManager.getFunctionTable(resultSetName) != null && procedures.contains(resultSetName))) {
1517                                continue;
1518                        }
1519
1520                        Set<Object> functionTables = modelManager
1521                                        .getFunctionTable(DlineageUtil.getIdentifierNormalTableName(resultSet.getName()));
1522                        List<ResultSet> functionResults = functionTables.stream().filter(t -> t instanceof ResultSet)
1523                                        .map(t -> (ResultSet) t).collect(Collectors.toList());
1524                        if (!(resultSet.getColumns().size() == 1
1525                                        && DlineageUtil.getIdentifierNormalTableName(resultSet.getColumns().get(0).getName()).equals(resultSetName))
1526                                        || functionResults.isEmpty()) {
1527                                continue;
1528                        }
1529
1530                        column column = resultSet.getColumns().get(0);
1531                        String sourceColumnId = column.getId();
1532                        boolean reserveResultSet = false;
1533                        List<relationship> appendRelations = new ArrayList<relationship>();
1534                        Set<relationship> removeRelations = new HashSet<relationship>();
1535                        
1536                        for (relationship relationship : dataflow.getRelationships()) {
1537                                targetColumn targetColumn = relationship.getTarget();
1538                                List<sourceColumn> sourceColumns = relationship.getSources();
1539                                List<sourceColumn> newSourceColumns = new ArrayList<sourceColumn>();
1540                                for (sourceColumn sourceColumn : sourceColumns) {
1541                                        if (!sourceColumn.getId().equals(sourceColumnId)) {
1542                                                continue;
1543                                        }
1544                                        for (ResultSet sourceResultSet : functionResults) {
1545                                                for (ResultColumn resultColumn : sourceResultSet.getColumns()) {
1546                                                        if (!DlineageUtil.compareColumnIdentifier(resultColumn.getName(),
1547                                                                        targetColumn.getColumn())) {
1548                                                                if (targetColumn.getColumn().endsWith("*")) {
1549                                                                        table parent = allMap.get(targetColumn.getParent_id());
1550                                                                        if (parent != null && parent.getColumns().size() > 1) {
1551                                                                                for (column item : parent.getColumns()) {
1552                                                                                        if (DlineageUtil.compareColumnIdentifier(resultColumn.getName(),
1553                                                                                                        item.getName())) {
1554                                                                                                relationship newRelation = appendStarRelation(dataflow, relationship, item, resultColumn);
1555                                                                                                appendRelations.add(newRelation);
1556                                                                                                removeRelations.add(relationship);
1557                                                                                                newSourceColumns.add(newRelation.getSources().get(0));
1558                                                                                                if (resultColumn.getResultSet() != null && resultColumn.getResultSet().isTarget()) {
1559                                                                                                        dataflow.getResultsets().stream().filter(
1560                                                                                                                        t -> t.getId().equals(String.valueOf(resultColumn.getResultSet().getId())))
1561                                                                                                                        .forEach(t -> t.setIsTarget(Boolean.FALSE.toString()));
1562                                                                                                }
1563                                                                                        }
1564                                                                                }
1565                                                                        }
1566                                                                }
1567                                                                continue;
1568                                                        }
1569                                                        
1570                                                        sourceColumn sourceColumn1 = new sourceColumn();
1571                                                        sourceColumn1.setId(String.valueOf(resultColumn.getId()));
1572                                                        sourceColumn1.setColumn(resultColumn.getName());
1573                                                        sourceColumn1.setParent_id(String.valueOf(resultColumn.getResultSet().getId()));
1574                                                        sourceColumn1.setParent_name(getResultSetName(resultColumn.getResultSet()));
1575                                                        if (resultColumn.getStartPosition() != null
1576                                                                        && resultColumn.getEndPosition() != null) {
1577                                                                sourceColumn1.setCoordinate(convertCoordinate(resultColumn.getStartPosition())
1578                                                                                + "," + convertCoordinate(resultColumn.getEndPosition()));
1579                                                        }
1580                                                        newSourceColumns.add(sourceColumn1);
1581                                                        
1582                                                        if (resultColumn.getResultSet() != null && resultColumn.getResultSet().isTarget()) {
1583                                                                dataflow.getResultsets().stream().filter(
1584                                                                                t -> t.getId().equals(String.valueOf(resultColumn.getResultSet().getId())))
1585                                                                                .forEach(t -> t.setIsTarget(Boolean.FALSE.toString()));
1586                                                        }
1587                                                }
1588                                        }
1589
1590                                        if (!newSourceColumns.isEmpty()) {
1591                                                if (removeRelations.isEmpty()) {
1592                                                        relationship.setSources(newSourceColumns);
1593                                                }
1594                                        }
1595                                        else {
1596                                                reserveResultSet = true;
1597                                        }
1598                                }
1599                        }
1600                        
1601                        dataflow.getRelationships().addAll(appendRelations);
1602                        dataflow.getRelationships().removeAll(removeRelations);
1603                        
1604                        if(!reserveResultSet) {
1605                                removeResultSets.add(resultSet);
1606                        }
1607                }
1608                dataflow.getResultsets().removeAll(removeResultSets);
1609                return dataflow;
1610        }
1611
1612        private relationship appendStarRelation(dataflow dataflow,
1613                        relationship relationship, column item, ResultColumn resultColumn) {
1614                relationship relationElement = new relationship();
1615                relationElement.setType(relationship.getType());
1616                relationElement.setEffectType(relationship.getEffectType());
1617                relationElement.setSqlHash(relationship.getSqlHash());
1618                relationElement.setSqlComment(relationship.getSqlComment());
1619                relationElement.setProcedureId(relationship.getProcedureId());
1620                relationElement.setId(String.valueOf(++ModelBindingManager.get().RELATION_ID));
1621                relationElement.setProcessId(relationship.getProcessId());
1622                relationElement.setProcessType(relationship.getProcessType());
1623                
1624                targetColumn targetColumn1 = new targetColumn();
1625                targetColumn1.setId(String.valueOf(item.getId()));
1626                targetColumn1.setColumn(item.getName());
1627                targetColumn1.setParent_id(relationship.getTarget().getParent_id());
1628                targetColumn1.setParent_name(relationship.getTarget().getParent_name());
1629                targetColumn1.setParent_alias(relationship.getTarget().getParent_alias());
1630                if (relationship.getTarget().getCoordinate() != null) {
1631                        targetColumn1.setCoordinate(item.getCoordinate());
1632                }
1633                relationElement.setTarget(targetColumn1);       
1634                
1635                sourceColumn sourceColumn1 = new sourceColumn();
1636                sourceColumn1.setId(String.valueOf(resultColumn.getId()));
1637                sourceColumn1.setColumn(resultColumn.getName());
1638                sourceColumn1.setParent_id(String.valueOf(resultColumn.getResultSet().getId()));
1639                sourceColumn1.setParent_name(getResultSetName(resultColumn.getResultSet()));
1640                if (resultColumn.getStartPosition() != null
1641                                && resultColumn.getEndPosition() != null) {
1642                        sourceColumn1.setCoordinate(convertCoordinate(resultColumn.getStartPosition())
1643                                        + "," + convertCoordinate(resultColumn.getEndPosition()));
1644                }
1645                relationElement.addSource(sourceColumn1);
1646                return relationElement;
1647        }
1648
1649        private dataflow getNormalizeDataflow(dataflow instance) {
1650                List<table> tables = new ArrayList<>();
1651                if (instance.getResultsets() != null) {
1652                        for (table t : instance.getResultsets()) {
1653                                tables.add(t);
1654                        }
1655                }
1656
1657                if (instance.getTables() != null) {
1658                        for (table t : instance.getTables()) {
1659                                tables.add(t);
1660                        }
1661                }
1662
1663                if (instance.getViews() != null) {
1664                        for (table t : instance.getViews()) {
1665                                tables.add(t);
1666                        }
1667                }
1668
1669                if (instance.getPaths() != null) {
1670                        for (table t : instance.getPaths()) {
1671                                tables.add(t);
1672                        }
1673                }
1674
1675                if (instance.getStages() != null) {
1676                        for (table t : instance.getStages()) {
1677                                tables.add(t);
1678                        }
1679                }
1680
1681                if (instance.getSequences() != null) {
1682                        for (table t : instance.getSequences()) {
1683                                tables.add(t);
1684                        }
1685                }
1686
1687                if (instance.getDatasources() != null) {
1688                        for (table t : instance.getDatasources()) {
1689                                tables.add(t);
1690                        }
1691                }
1692
1693                if (instance.getDatabases() != null) {
1694                        for (table t : instance.getDatabases()) {
1695                                tables.add(t);
1696                        }
1697                }
1698
1699                if (instance.getSchemas() != null) {
1700                        for (table t : instance.getSchemas()) {
1701                                tables.add(t);
1702                        }
1703                }
1704
1705                if (instance.getStreams() != null) {
1706                        for (table t : instance.getStreams()) {
1707                                tables.add(t);
1708                        }
1709                }
1710
1711                if (instance.getVariables() != null) {
1712                        for (table t : instance.getVariables()) {
1713                                tables.add(t);
1714                        }
1715                }
1716
1717                Map<String, String> idNameMap = new HashMap();
1718
1719                for(table table: tables){
1720                        if(table.getDatabase()!=null) {
1721                                table.setDatabase(DlineageUtil.getIdentifierNormalName(table.getDatabase(), ESQLDataObjectType.dotCatalog));
1722                        }
1723                        if(table.getSchema()!=null) {
1724                                table.setSchema(DlineageUtil.getIdentifierNormalName(table.getSchema(), ESQLDataObjectType.dotSchema));
1725                        }
1726                        if(table.getName()!=null) {
1727                                table.setName(DlineageUtil.getIdentifierNormalName(table.getName(), ESQLDataObjectType.dotTable));
1728                                idNameMap.put(table.getId(), table.getName());
1729                        }
1730                        if(table.getColumns()!=null){
1731                                for(column column: table.getColumns()){
1732                                        column.setName(DlineageUtil.getIdentifierNormalName(column.getName(), ESQLDataObjectType.dotColumn));
1733                                        idNameMap.put(column.getId(), column.getName());
1734                                }
1735                        }
1736                }
1737                if(instance.getPackages()!=null){
1738                        for(oraclePackage oraclePackage: instance.getPackages()){
1739                                if(oraclePackage.getDatabase()!=null) {
1740                                        oraclePackage.setDatabase(DlineageUtil.getIdentifierNormalName(oraclePackage.getDatabase(), ESQLDataObjectType.dotCatalog));
1741                                }
1742                                if(oraclePackage.getSchema()!=null) {
1743                                        oraclePackage.setSchema(DlineageUtil.getIdentifierNormalName(oraclePackage.getSchema(), ESQLDataObjectType.dotSchema));
1744                                }
1745                                if(oraclePackage.getName()!=null) {
1746                                        oraclePackage.setName(DlineageUtil.getIdentifierNormalName(oraclePackage.getName(), ESQLDataObjectType.dotTable));
1747                                        idNameMap.put(oraclePackage.getId(), oraclePackage.getName());
1748                                }
1749                                if(oraclePackage.getArguments()!=null){
1750                                        for(argument argument: oraclePackage.getArguments()){
1751                                                argument.setName(DlineageUtil.getIdentifierNormalName(argument.getName(), ESQLDataObjectType.dotColumn));
1752                                                idNameMap.put(argument.getId(), argument.getName());
1753                                        }
1754                                }
1755                                if(oraclePackage.getProcedures()!=null){
1756                                        for(procedure procedure: oraclePackage.getProcedures()){
1757                                                if(procedure.getDatabase()!=null) {
1758                                                        procedure.setDatabase(DlineageUtil.getIdentifierNormalName(procedure.getDatabase(), ESQLDataObjectType.dotCatalog));
1759                                                }
1760                                                if(procedure.getSchema()!=null) {
1761                                                        procedure.setSchema(DlineageUtil.getIdentifierNormalName(procedure.getSchema(), ESQLDataObjectType.dotSchema));
1762                                                }
1763                                                if(procedure.getName()!=null) {
1764                                                        procedure.setName(DlineageUtil.getIdentifierNormalName(procedure.getName(), ESQLDataObjectType.dotTable));
1765                                                        idNameMap.put(procedure.getId(), procedure.getName());
1766                                                }
1767                                                for(argument argument: procedure.getArguments()){
1768                                                        argument.setName(DlineageUtil.getIdentifierNormalName(argument.getName(), ESQLDataObjectType.dotColumn));
1769                                                        idNameMap.put(argument.getId(), argument.getName());
1770                                                }
1771                                        }
1772                                }
1773                        }
1774                }
1775                if(instance.getProcedures()!=null){
1776                        for(procedure procedure: instance.getProcedures()){
1777                                if(procedure.getDatabase()!=null) {
1778                                        procedure.setDatabase(DlineageUtil.getIdentifierNormalName(procedure.getDatabase(), ESQLDataObjectType.dotCatalog));
1779                                }
1780                                if(procedure.getSchema()!=null) {
1781                                        procedure.setSchema(DlineageUtil.getIdentifierNormalName(procedure.getSchema(), ESQLDataObjectType.dotSchema));
1782                                }
1783                                if(procedure.getName()!=null) {
1784                                        procedure.setName(DlineageUtil.getIdentifierNormalName(procedure.getName(), ESQLDataObjectType.dotTable));
1785                                        idNameMap.put(procedure.getId(), procedure.getName());
1786                                }
1787                                for(argument argument: procedure.getArguments()){
1788                                        argument.setName(DlineageUtil.getIdentifierNormalName(argument.getName(), ESQLDataObjectType.dotColumn));
1789                                        idNameMap.put(argument.getId(), argument.getName());
1790                                }
1791                        }
1792                }
1793                if (instance.getProcesses() != null) {
1794                        for (process process : instance.getProcesses()) {
1795                                if(process.getDatabase()!=null) {
1796                                        process.setDatabase(DlineageUtil.getIdentifierNormalName(process.getDatabase(), ESQLDataObjectType.dotCatalog));
1797                                }
1798                                if(process.getSchema()!=null) {
1799                                        process.setSchema(DlineageUtil.getIdentifierNormalName(process.getSchema(), ESQLDataObjectType.dotSchema));
1800                                }
1801                                if (process.getProcedureId() != null) {
1802                                        process.setProcedureName(DlineageUtil.getIdentifierNormalName(process.getProcedureName(), ESQLDataObjectType.dotTable));
1803                                        idNameMap.put(process.getId(), process.getName());
1804                                }
1805                        }
1806                }
1807
1808                if(instance.getRelationships()!=null){
1809                        for(relationship relation: instance.getRelationships()){
1810                                targetColumn targetColumn = relation.getTarget();
1811                                if(targetColumn != null) {
1812                                        targetColumn.setColumn(idNameMap.get(targetColumn.getId()));
1813                                        targetColumn.setTarget_name(idNameMap.get(targetColumn.getTarget_id()));
1814                                        targetColumn.setParent_name(idNameMap.get(targetColumn.getParent_id()));
1815                                }
1816                                List<sourceColumn> sourceColumns = relation.getSources();
1817                                if(sourceColumns!=null){
1818                                        for(sourceColumn sourceColumn: sourceColumns){
1819                                                sourceColumn.setColumn(idNameMap.get(sourceColumn.getId()));
1820                                                sourceColumn.setSource_name(idNameMap.get(sourceColumn.getSource_id()));
1821                                                sourceColumn.setParent_name(idNameMap.get(sourceColumn.getParent_id()));
1822                                        }
1823                                }
1824                                targetColumn = relation.getCaller();
1825                                if(targetColumn != null) {
1826                                        targetColumn.setName(idNameMap.get(targetColumn.getId()));
1827                                }
1828                                sourceColumns = relation.getCallees();
1829                                if(sourceColumns!=null){
1830                                        for(sourceColumn sourceColumn: sourceColumns){
1831                                                sourceColumn.setName(idNameMap.get(sourceColumn.getId()));
1832                                        }
1833                                }
1834                        }
1835                }
1836
1837
1838                return instance;
1839        }
1840
1841        private dataflow filterDataflowCoordinate(dataflow instance) {
1842                List<table> tables = new ArrayList<>();
1843                if (instance.getResultsets() != null) {
1844                        for (table t : instance.getResultsets()) {
1845                                tables.add(t);
1846                        }
1847                }
1848
1849                if (instance.getTables() != null) {
1850                        for (table t : instance.getTables()) {
1851                                tables.add(t);
1852                        }
1853                }
1854
1855                if (instance.getViews() != null) {
1856                        for (table t : instance.getViews()) {
1857                                tables.add(t);
1858                        }
1859                }
1860
1861                if (instance.getPaths() != null) {
1862                        for (table t : instance.getPaths()) {
1863                                tables.add(t);
1864                        }
1865                }
1866
1867                if (instance.getStages() != null) {
1868                        for (table t : instance.getStages()) {
1869                                tables.add(t);
1870                        }
1871                }
1872
1873                if (instance.getSequences() != null) {
1874                        for (table t : instance.getSequences()) {
1875                                tables.add(t);
1876                        }
1877                }
1878
1879                if (instance.getDatasources() != null) {
1880                        for (table t : instance.getDatasources()) {
1881                                tables.add(t);
1882                        }
1883                }
1884
1885                if (instance.getDatabases() != null) {
1886                        for (table t : instance.getDatabases()) {
1887                                tables.add(t);
1888                        }
1889                }
1890
1891                if (instance.getSchemas() != null) {
1892                        for (table t : instance.getSchemas()) {
1893                                tables.add(t);
1894                        }
1895                }
1896
1897                if (instance.getStreams() != null) {
1898                        for (table t : instance.getStreams()) {
1899                                tables.add(t);
1900                        }
1901                }
1902
1903                if (instance.getVariables() != null) {
1904                        for (table t : instance.getVariables()) {
1905                                tables.add(t);
1906                        }
1907                }
1908
1909                for(table table: tables){
1910                        table.clearCoordinate();
1911                        if(table.getColumns()!=null){
1912                                for(column column: table.getColumns()){
1913                                        column.clearCoordinate();
1914                                }
1915                        }
1916                }
1917                if(instance.getPackages()!=null){
1918                        for(oraclePackage oraclePackage: instance.getPackages()){
1919                                oraclePackage.setCoordinate(null);
1920                                if(oraclePackage.getProcedures()!=null){
1921                                        for(procedure procedure: oraclePackage.getProcedures()){
1922                                                procedure.setCoordinate(null);
1923                                                for(argument argument: procedure.getArguments()){
1924                                                        argument.setCoordinate(null);
1925                                                }
1926                                        }
1927                                }
1928                        }
1929                }
1930                if(instance.getProcedures()!=null){
1931                        for(procedure procedure: instance.getProcedures()){
1932                                procedure.setCoordinate(null);
1933                                for(argument argument: procedure.getArguments()){
1934                                        argument.setCoordinate(null);
1935                                }
1936                        }
1937                }
1938                if (instance.getProcesses() != null) {
1939                        for (process process : instance.getProcesses()) {
1940                                process.setCoordinate(null);
1941                        }
1942                }
1943
1944                if(instance.getRelationships()!=null){
1945                        for(relationship relation: instance.getRelationships()){
1946                                targetColumn targetColumn = relation.getTarget();
1947                                if(targetColumn != null) {
1948                                        targetColumn.setCoordinate(null);
1949                                }
1950                                List<sourceColumn> sourceColumns = relation.getSources();
1951                                if(sourceColumns!=null){
1952                                        for(sourceColumn sourceColumn: sourceColumns){
1953                                                sourceColumn.setCoordinate(null);
1954                                        }
1955                                }
1956                                targetColumn = relation.getCaller();
1957                                if(targetColumn != null) {
1958                                        targetColumn.setCoordinate(null);
1959                                }
1960                                sourceColumns = relation.getCallees();
1961                                if(sourceColumns!=null){
1962                                        for(sourceColumn sourceColumn: sourceColumns){
1963                                                sourceColumn.setCoordinate(null);
1964                                        }
1965                                }
1966                        }
1967                }
1968
1969                return instance;
1970        }
1971
1972        private void appendSqlInfo(Map<String, Pair3<StringBuilder, AtomicInteger, String>> databaseMap, int index,
1973                        SqlInfo sqlInfo, Map queryObject) {
1974                EDbVendor vendor = option.getVendor();
1975                if (!SQLUtil.isEmpty(sqlInfo.getDbVendor())) {
1976                        vendor = EDbVendor.valueOf(sqlInfo.getDbVendor());
1977                }
1978
1979                boolean supportCatalog = TSQLEnv.supportCatalog(vendor);
1980                boolean supportSchema = TSQLEnv.supportSchema(vendor);
1981
1982                String groupName = (String) queryObject.get("groupName");
1983                if (DlineageUtil.isProcedureExcluded(groupName)) {
1984                        return;
1985                }
1986
1987                String content = (String) queryObject.get("sourceCode");
1988                if (SQLUtil.isEmpty(content)) {
1989                        return;
1990                }
1991                StringBuilder builder = new StringBuilder();
1992                if (supportCatalog) {
1993                        String database = (String) queryObject.get("database");
1994                        if (database.indexOf(".") != -1) {
1995                                String delimitedChar = TSQLEnv.delimitedChar(vendor);
1996                                database = delimitedChar + SQLUtil.trimColumnStringQuote(database) + delimitedChar;
1997                        }
1998                        builder.append(database);
1999                }
2000                if (supportSchema) {
2001                        String schema = (String) queryObject.get("schema");
2002                        if (schema.indexOf(".") != -1) {
2003                                String delimitedChar = TSQLEnv.delimitedChar(vendor);
2004                                schema = delimitedChar + SQLUtil.trimColumnStringQuote(schema) + delimitedChar;
2005                        }
2006                        if (builder.length() > 0) {
2007                                builder.append(".");
2008                        }
2009                        builder.append(schema);
2010                }
2011                String group = builder.toString();
2012                String sqlHash = SHA256.getMd5(content);
2013                String hash = SHA256.getMd5(sqlHash);
2014                if (!databaseMap.containsKey(sqlHash)) {
2015                        databaseMap.put(sqlHash,
2016                                        new Pair3<StringBuilder, AtomicInteger, String>(new StringBuilder(), new AtomicInteger(), group));
2017                }
2018                TGSqlParser parser = new TGSqlParser(option.getVendor());
2019                String delimiterChar = String.valueOf(parser.getDelimiterChar());
2020                StringBuilder buffer = new StringBuilder(content);
2021                if (content.trim().endsWith(delimiterChar) || content.trim().endsWith(";")) {
2022                        buffer.append("\n");
2023                } else if(vendor == EDbVendor.dbvredshift
2024                                || vendor == EDbVendor.dbvgaussdb
2025                                || vendor == EDbVendor.dbvedb
2026                                || vendor == EDbVendor.dbvpostgresql
2027                                || vendor == EDbVendor.dbvmysql
2028                                || vendor == EDbVendor.dbvteradata){
2029                        buffer.append("\n\n-- " + TBaseType.sqlflow_stmt_delimiter_str + "\n\n");
2030                } else{
2031                        SQLUtil.endTrim(buffer);
2032                        buffer.append(";").append("\n");
2033                }
2034
2035                int lineStart = databaseMap.get(sqlHash).first.toString().split("\n", -1).length - 1;
2036                if (databaseMap.get(sqlHash).first.toString().length() == 0) {
2037                        lineStart = 0;
2038                }
2039                databaseMap.get(sqlHash).first.append(buffer.toString());
2040                SqlInfo sqlInfoItem = new SqlInfo();
2041                sqlInfoItem.setServer(sqlInfo.getServer());
2042                sqlInfoItem.setDbVendor(sqlInfo.getDbVendor());
2043                sqlInfoItem.setFileName(sqlInfo.getFileName());
2044                sqlInfoItem.setFilePath(sqlInfo.getFilePath());
2045                sqlInfoItem.setSql(buffer.toString());
2046                sqlInfoItem.setOriginIndex(index);
2047                sqlInfoItem.setOriginLineStart(0);
2048                sqlInfoItem.setOriginLineEnd(buffer.toString().split("\n", -1).length - 1);
2049                sqlInfoItem.setIndex(databaseMap.get(sqlHash).second.getAndIncrement());
2050                sqlInfoItem.setLineStart(lineStart);
2051                sqlInfoItem.setLineEnd(databaseMap.get(sqlHash).first.toString().split("\n", -1).length - 1);
2052                sqlInfoItem.setGroup(group);
2053                sqlInfoItem.setHash(hash);
2054
2055                if (!sqlInfoMap.containsKey(hash)) {
2056                        sqlInfoMap.put(hash, new ArrayList<SqlInfo>());
2057                }
2058                sqlInfoMap.get(hash).add(sqlInfoItem);
2059        }
2060
2061        static String getTextOutput(dataflow dataflow) {
2062                StringBuffer buffer = new StringBuffer();
2063                List<relationship> relations = dataflow.getRelationships();
2064                if (relations != null) {
2065                        for (int i = 0; i < relations.size(); i++) {
2066                                relationship relation = relations.get(i);
2067                                targetColumn target = relation.getTarget();
2068                                List<sourceColumn> sources = relation.getSources();
2069                                if (target != null && sources != null && sources.size() > 0) {
2070                                        buffer.append(target.getColumn()).append(" depends on: ");
2071                                        Set<String> columnSet = new LinkedHashSet<String>();
2072                                        for (int j = 0; j < sources.size(); j++) {
2073                                                sourceColumn sourceColumn = sources.get(j);
2074                                                String columnName = sourceColumn.getColumn();
2075                                                if (sourceColumn.getParent_name() != null && sourceColumn.getParent_name().length() > 0) {
2076                                                        columnName = sourceColumn.getParent_name() + "." + columnName;
2077                                                }
2078                                                columnSet.add(columnName);
2079                                        }
2080                                        String[] columns = columnSet.toArray(new String[0]);
2081                                        for (int j = 0; j < columns.length; j++) {
2082                                                buffer.append(columns[j]);
2083                                                if (j == columns.length - 1) {
2084                                                        buffer.append("\n");
2085                                                } else
2086                                                        buffer.append(", ");
2087                                        }
2088                                }
2089                        }
2090                }
2091                return buffer.toString();
2092        }
2093
2094        private String mergeRelationType(List<Pair<sourceColumn, List<String>>> typePaths) {
2095                RelationshipType relationType = RelationshipType.join;
2096                for (int i = 0; i < typePaths.size(); i++) {
2097                        List<String> path = typePaths.get(i).second;
2098                        RelationshipType type = RelationshipType.valueOf(getRelationType(path));
2099                        if (type.ordinal() < relationType.ordinal()) {
2100                                relationType = type;
2101                        }
2102                }
2103                return relationType.name();
2104        }
2105
2106        private String getRelationType(List<String> typePaths) {
2107                if (typePaths.contains("join"))
2108                        return "join";
2109                if (typePaths.contains("fdr"))
2110                        return "fdr";
2111                if (typePaths.contains("frd"))
2112                        return "frd";
2113                if (typePaths.contains("fddi"))
2114                        return "fddi";
2115                return "fdd";
2116        }
2117
2118        public dataflow getSimpleDataflow(dataflow instance, boolean simpleOutput) throws Exception {
2119                return getSimpleDataflow(instance, simpleOutput, Arrays.asList("fdd"));
2120        }
2121        
2122        public dataflow getSimpleDataflow(dataflow instance, boolean simpleOutput, List<String> types) throws Exception {
2123                ModelBindingManager.setGlobalVendor(option.getVendor());
2124                allMap.clear();
2125                targetTables.clear();
2126                resultSetMap.clear();
2127                tableMap.clear();
2128                viewMap.clear();
2129                cursorMap.clear();
2130                variableMap.clear();
2131                fileMap.clear();
2132                stageMap.clear();
2133                sequenceMap.clear();
2134                dataSourceMap.clear();
2135                databaseMap.clear();
2136                schemaMap.clear();
2137                streamMap.clear();
2138                dataflow simple = new dataflow();
2139                List<relationship> simpleRelations = new ArrayList<relationship>();
2140                List<relationship> relations = instance.getRelationships();
2141                if (instance.getResultsets() != null) {
2142                        for (table t : instance.getResultsets()) {
2143                                resultSetMap.put(t.getId().toLowerCase(), t);
2144                                allMap.put(t.getId().toLowerCase(), t);
2145                        }
2146                }
2147
2148                if (instance.getTables() != null) {
2149                        for (table t : instance.getTables()) {
2150                                tableMap.put(t.getId().toLowerCase(), t);
2151                                allMap.put(t.getId().toLowerCase(), t);
2152                        }
2153                }
2154
2155                if (instance.getViews() != null) {
2156                        for (table t : instance.getViews()) {
2157                                viewMap.put(t.getId().toLowerCase(), t);
2158                                allMap.put(t.getId().toLowerCase(), t);
2159                        }
2160                }
2161
2162                if (instance.getPaths() != null) {
2163                        for (table t : instance.getPaths()) {
2164                                fileMap.put(t.getId().toLowerCase(), t);
2165                                allMap.put(t.getId().toLowerCase(), t);
2166                        }
2167                }
2168
2169                if (instance.getStages() != null) {
2170                        for (table t : instance.getStages()) {
2171                                stageMap.put(t.getId().toLowerCase(), t);
2172                                allMap.put(t.getId().toLowerCase(), t);
2173                        }
2174                }
2175                
2176                if (instance.getSequences() != null) {
2177                        for (table t : instance.getSequences()) {
2178                                sequenceMap.put(t.getId().toLowerCase(), t);
2179                                allMap.put(t.getId().toLowerCase(), t);
2180                        }
2181                }
2182
2183                if (instance.getDatasources() != null) {
2184                        for (table t : instance.getDatasources()) {
2185                                dataSourceMap.put(t.getId().toLowerCase(), t);
2186                                allMap.put(t.getId().toLowerCase(), t);
2187                        }
2188                }
2189
2190                if (instance.getDatabases() != null) {
2191                        for (table t : instance.getDatabases()) {
2192                                databaseMap.put(t.getId().toLowerCase(), t);
2193                                allMap.put(t.getId().toLowerCase(), t);
2194                        }
2195                }
2196
2197                if (instance.getSchemas() != null) {
2198                        for (table t : instance.getSchemas()) {
2199                                schemaMap.put(t.getId().toLowerCase(), t);
2200                                allMap.put(t.getId().toLowerCase(), t);
2201                        }
2202                }
2203
2204                if (instance.getStreams() != null) {
2205                        for (table t : instance.getStreams()) {
2206                                streamMap.put(t.getId().toLowerCase(), t);
2207                                allMap.put(t.getId().toLowerCase(), t);
2208                        }
2209                }
2210
2211                if (instance.getVariables() != null) {
2212                        for (table t : instance.getVariables()) {
2213                                if(SubType.cursor.name().equals(t.getSubType())){
2214                                        cursorMap.put(t.getId().toLowerCase(), t);
2215                                }
2216                                else {
2217                                        variableMap.put(t.getId().toLowerCase(), t);
2218                                }
2219                                allMap.put(t.getId().toLowerCase(), t);
2220                        }
2221                }
2222
2223                if (relations != null) {
2224                        
2225                        List<relationship> filterRelations = new ArrayList<>();
2226                        for (relationship relationElem : relations) {
2227                                if (!types.contains(relationElem.getType()))
2228                                        continue;
2229                                else {
2230                                        filterRelations.add(relationElem);
2231                                }
2232                        }
2233                        
2234                        relations = filterRelations;
2235                        
2236                        Map<String, Set<relationship>> targetIdRelationMap = new HashMap<String, Set<relationship>>();
2237                        for (relationship relation : relations) {
2238                                if (relation.getTarget() != null) {
2239                                        String key = relation.getTarget().getParent_id() + "." + relation.getTarget().getId();
2240                                        if (!targetIdRelationMap.containsKey(key)) {
2241                                                targetIdRelationMap.put(key, new TreeSet<relationship>(new Comparator<relationship>() {
2242                                                        @Override
2243                                                        public int compare(relationship o1, relationship o2) {
2244                                                                return o1.getId().compareTo(o2.getId());
2245                                                        }
2246                                                }));
2247                                        }
2248                                        targetIdRelationMap.get(key).add(relation);
2249                                }
2250                        }
2251
2252                        Iterator<String> keys = targetIdRelationMap.keySet().iterator();
2253                        while (keys.hasNext()) {
2254                                String key = keys.next();
2255                                if (targetIdRelationMap.get(key).size() > 500) {
2256                                        keys.remove();
2257                                }
2258                        }
2259
2260                        for (relationship relationElem : relations) {
2261                                if (RelationshipType.call.name().equals(relationElem.getType())) {
2262                                        continue;
2263                                }
2264                                if (RelationshipType.er.name().equals(relationElem.getType())) {
2265                                        continue;
2266                                }
2267                                targetColumn target = relationElem.getTarget();
2268                                String targetParent = target.getParent_id();
2269                                if (isTarget(instance, targetParent, simpleOutput)) {
2270                                        List<Pair<sourceColumn, List<String>>> relationSources = new ArrayList<Pair<sourceColumn, List<String>>>();
2271                                        findSourceRelations(target, instance, targetIdRelationMap, relationElem, relationSources,
2272                                                        new String[] { relationElem.getType() }, simpleOutput);
2273                                        if (relationSources.size() > 0) {
2274                                                Map<sourceColumn, List<Pair<sourceColumn, List<String>>>> columnMap = new LinkedHashMap<sourceColumn, List<Pair<sourceColumn, List<String>>>>();
2275                                                for (Pair<sourceColumn, List<String>> t : relationSources) {
2276                                                        sourceColumn key = ((Pair<sourceColumn, List<String>>) t).first;
2277                                                        if (!columnMap.containsKey(key)) {
2278                                                                columnMap.put(key, new ArrayList<Pair<sourceColumn, List<String>>>());
2279                                                        }
2280                                                        columnMap.get(key).add(t);
2281                                                }
2282                                                Iterator<sourceColumn> iter = columnMap.keySet().iterator();
2283                                                Map<String, List<sourceColumn>> relationSourceMap = new HashMap<String, List<sourceColumn>>();
2284                                                while (iter.hasNext()) {
2285                                                        sourceColumn column = iter.next();
2286                                                        String relationType = mergeRelationType(columnMap.get(column));
2287                                                        if (!relationSourceMap.containsKey(relationType)) {
2288                                                                relationSourceMap.put(relationType, new ArrayList<sourceColumn>());
2289                                                        }
2290                                                        relationSourceMap.get(relationType).add(column);
2291                                                }
2292
2293                                                Iterator<String> sourceIter = relationSourceMap.keySet().iterator();
2294                                                while (sourceIter.hasNext()) {
2295                                                        String relationType = sourceIter.next();
2296                                                        relationship simpleRelation = (relationship) relationElem.clone();
2297                                                        simpleRelation.setSources(relationSourceMap.get(relationType));
2298                                                        simpleRelation.setType(relationType);
2299                                                        simpleRelation.setId(String.valueOf(++ModelBindingManager.get().RELATION_ID));
2300                                                        simpleRelations.add(simpleRelation);
2301                                                }
2302                                        }
2303                                }
2304                        }
2305                }
2306
2307                simple.setProcedures(instance.getProcedures());
2308                simple.setPackages(instance.getPackages());
2309                simple.setProcesses(instance.getProcesses());
2310                simple.setErrors(instance.getErrors());
2311                List<table> tables = new ArrayList<table>();
2312                for (table t : instance.getTables()) {
2313                        if (!SQLUtil.isTempTable(t)) {
2314                                tables.add(t);
2315                        }
2316                        else {
2317                                if (option.isIgnoreTemporaryTable()) {
2318                                        continue;
2319                                }
2320                                else {
2321                                        tables.add(t);
2322                                }
2323                        }
2324                }
2325                simple.setStages(instance.getStages());
2326                simple.setSequences(instance.getSequences());
2327                simple.setDatasources(instance.getDatasources());
2328                simple.setStreams(instance.getStreams());
2329                simple.setPaths(instance.getPaths());
2330                simple.setTables(tables);
2331                simple.setViews(instance.getViews());
2332                if(option.isSimpleShowVariable()) {
2333                        simple.setVariables(instance.getVariables());
2334                }
2335                else if(option.isSimpleShowCursor()) {
2336                        simple.setVariables(instance.getVariables().stream().filter(t->SubType.cursor.name().equals(t.getSubType())).collect(Collectors.toList()));
2337                }
2338                if (instance.getResultsets() != null) {
2339                        List<table> resultSets = new ArrayList<table>();
2340                        for (int i = 0; i < instance.getResultsets().size(); i++) {
2341                                table resultSet = instance.getResultsets().get(i);
2342                                if (isTargetResultSet(instance, resultSet.getId(), simpleOutput)) {
2343                                        // special handle function #524 #296
2344                                        resultSets.add(resultSet);
2345                                }
2346                        }
2347                        simple.setResultsets(resultSets);
2348                }
2349
2350                List<table> functions = new ArrayList<table>();
2351                if (option.isShowCallRelation()) {
2352                        for (int i = 0; i < relations.size(); i++) {
2353                                relationship relationElem = relations.get(i);
2354                                if (!RelationshipType.call.name().equals(relationElem.getType())) {
2355                                        continue;
2356                                }
2357                                simpleRelations.add(relationElem);
2358                                for (sourceColumn callee : relationElem.getCallees()) {
2359                                        String calleeId = callee.getId();
2360                                        if (resultSetMap.containsKey(calleeId)) {
2361                                                table function = resultSetMap.get(calleeId);
2362                                                function.setIsTarget("true");
2363                                                functions.add(function);
2364                                        }
2365                                }
2366                        }
2367                }
2368
2369                if (option.isShowERDiagram()) {
2370                        for (int i = 0; i < relations.size(); i++) {
2371                                relationship relationElem = relations.get(i);
2372                                if (!RelationshipType.er.name().equals(relationElem.getType())) {
2373                                        continue;
2374                                }
2375                                simpleRelations.add(relationElem);
2376                                for (sourceColumn callee : relationElem.getCallees()) {
2377                                        String calleeId = callee.getId();
2378                                        if (resultSetMap.containsKey(calleeId)) {
2379                                                table function = resultSetMap.get(calleeId);
2380                                                function.setIsTarget("true");
2381                                                functions.add(function);
2382                                        }
2383                                }
2384                        }
2385                }
2386                
2387                if (!functions.isEmpty()) {
2388                        if (simple.getResultsets() == null) {
2389                                simple.setResultsets(functions);
2390                        } else {
2391                                simple.getResultsets().addAll(functions);
2392                        }
2393                }
2394
2395                simple.setRelationships(simpleRelations);
2396                simple.setOrientation(instance.getOrientation());
2397                targetTables.clear();
2398                resultSetMap.clear();
2399                tableMap.clear();
2400                viewMap.clear();
2401                cursorMap.clear();
2402                variableMap.clear();
2403                fileMap.clear();
2404                stageMap.clear();
2405                dataSourceMap.clear();
2406                databaseMap.clear();
2407                schemaMap.clear();
2408                streamMap.clear();
2409                return simple;
2410        }
2411
2412        private void findSourceRelations(targetColumn target, dataflow instance, Map<String, Set<relationship>> sourceIdRelationMap,
2413                        relationship targetRelation, List<Pair<sourceColumn, List<String>>> relationSources, String[] pathTypes, boolean simpleOutput) {
2414                findStarSourceRelations(target, instance, null, sourceIdRelationMap, targetRelation, relationSources, pathTypes,
2415                                new HashSet<String>(), new LinkedHashSet<transform>(), new LinkedHashSet<candidateTable>(), 0, simpleOutput);
2416        }
2417
2418        private void findStarSourceRelations(targetColumn target, dataflow instance, targetColumn starRelationTarget,
2419                        Map<String, Set<relationship>> sourceIdRelationMap, relationship targetRelation,
2420                        List<Pair<sourceColumn, List<String>>> relationSources, String[] pathTypes, Set<String> paths,
2421                        Set<transform> transforms, Set<candidateTable> candidateTables, int level, boolean simpleOutput) {
2422                if (targetRelation != null && targetRelation.getSources() != null) {
2423                        
2424                        //获取source为*的Column Parent
2425                        String starParentId = null;
2426                        for (int i = 0; i < targetRelation.getSources().size(); i++) {
2427                                sourceColumn source = targetRelation.getSources().get(i);
2428                                if (starRelationTarget != null && "*".equals(source.getColumn())) {
2429                                        starParentId = source.getParent_id();
2430                                }
2431                        }
2432                        
2433                        for (int i = 0; i < targetRelation.getSources().size(); i++) {
2434                                sourceColumn source = targetRelation.getSources().get(i);
2435                                if (starRelationTarget != null && !"*".equals(source.getColumn())
2436                                                && !DlineageUtil.getIdentifierNormalColumnName(starRelationTarget.getColumn())
2437                                                                .equals(DlineageUtil.getIdentifierNormalColumnName(source.getColumn()))) {
2438                                        table parent = allMap.get(source.getParent_id());
2439                                        if (parent != null && isFunction(parent)) {
2440                                                // function返回值未知，不对星号做处理
2441                                        } 
2442                                        else if (parent == null) {
2443                                                continue;
2444                                        } else if(starParentId!=null && starParentId.equals(parent.getId())){
2445                                                //如果source和 * column的parent相同，则跳过
2446                                                continue;
2447                                        }
2448                                }
2449
2450                                String sourceColumnId = source.getId();
2451                                String sourceParentId = source.getParent_id();
2452                                if (sourceParentId == null || sourceColumnId == null) {
2453                                        continue;
2454                                }
2455                                if (isTarget(instance, sourceParentId, simpleOutput)) {
2456                                        List<transform> transforms2 = new ArrayList<transform>(transforms.size());
2457                                        transforms2.addAll(transforms);
2458                                        Collections.reverse(transforms2);
2459                                        
2460                                        List<candidateTable> candidateTables2 = new ArrayList<candidateTable>(candidateTables.size());
2461                                        candidateTables2.addAll(candidateTables);
2462                                        
2463                                        sourceColumn sourceColumnCopy = DlineageUtil.copySourceColumn(source);
2464                                        for (transform t : transforms2) {
2465                                                sourceColumnCopy.addTransform(t);
2466                                        }
2467                                        
2468                                        for (candidateTable t : candidateTables2) {
2469                                                sourceColumnCopy.addCandidateParent(t);
2470                                        }
2471                                        
2472                                        if(Boolean.TRUE.equals(target.isStruct()) && Boolean.TRUE.equals(source.isStruct())) {
2473                                                List<String> targetColumns = SQLUtil.parseNames(target.getColumn());
2474                                                List<String> sourceColumns = SQLUtil.parseNames(source.getColumn());
2475                                                if(!DlineageUtil.getIdentifierNormalColumnName(targetColumns.get(targetColumns.size()-1))
2476                                                                .equals(DlineageUtil.getIdentifierNormalColumnName(sourceColumns.get(sourceColumns.size()-1)))) {
2477                                                        continue;
2478                                                }
2479                                        }
2480                                        relationSources.add(new Pair<sourceColumn, List<String>>(sourceColumnCopy, Arrays.asList(pathTypes)));
2481                                } else {
2482                                        Set<relationship> sourceRelations = sourceIdRelationMap
2483                                                        .get(source.getParent_id() + "." + source.getId());
2484                                        if (sourceRelations != null) {
2485                                                if (paths.contains(source.getParent_id() + "." + source.getId())) {
2486                                                        continue;
2487                                                } else {
2488                                                        paths.add(source.getParent_id() + "." + source.getId());
2489                                                        if (source.getTransforms() != null) {
2490                                                                transforms.addAll(source.getTransforms());
2491                                                        }
2492                                                        if (source.getCandidateParents() != null) {
2493                                                                candidateTables.addAll(source.getCandidateParents());
2494                                                        }
2495                                                }
2496                                                for (relationship relation : sourceRelations) {
2497                                                        LinkedHashSet<transform> transforms2 = new LinkedHashSet<transform>(transforms.size());
2498                                                        transforms2.addAll(transforms);
2499                                                        LinkedHashSet<candidateTable> candidateTables2 = new LinkedHashSet<candidateTable>(candidateTables.size());
2500                                                        candidateTables2.addAll(candidateTables);
2501                                                        String[] types = new String[pathTypes.length + 1];
2502                                                        types[0] = relation.getType();
2503                                                        System.arraycopy(pathTypes, 0, types, 1, pathTypes.length);
2504                                                        if (!"*".equals(source.getColumn())) {
2505                                                                findStarSourceRelations(target, instance, null, sourceIdRelationMap, relation, relationSources,
2506                                                                                types, paths, transforms2, candidateTables2, level + 1, simpleOutput);
2507                                                        } else {
2508                                                                findStarSourceRelations(target, instance,
2509                                                                                starRelationTarget == null ? targetRelation.getTarget() : starRelationTarget,
2510                                                                                sourceIdRelationMap, relation, relationSources, types, paths, transforms, candidateTables2,
2511                                                                                level + 1, simpleOutput);
2512                                                        }
2513                                                }
2514                                        }
2515                                }
2516                        }
2517                }
2518        }
2519
2520        private Map<String, Boolean> targetTables = new HashMap<String, Boolean>();
2521        private Map<String, table> resultSetMap = new HashMap<String, table>();
2522        private Map<String, table> tableMap = new HashMap<String, table>();
2523        private Map<String, table> viewMap = new HashMap<String, table>();
2524        private Map<String, table> cursorMap = new HashMap<String, table>();
2525        private Map<String, table> variableMap = new HashMap<String, table>();
2526        private Map<String, table> fileMap = new HashMap<String, table>();
2527        private Map<String, table> stageMap = new HashMap<String, table>();
2528        private Map<String, table> sequenceMap = new HashMap<String, table>();
2529        private Map<String, table> dataSourceMap = new HashMap<String, table>();
2530        private Map<String, table> databaseMap = new HashMap<String, table>();
2531        private Map<String, table> schemaMap = new HashMap<String, table>();
2532        private Map<String, table> streamMap = new HashMap<String, table>();
2533        private Map<String, table> allMap = new HashMap<String, table>();
2534
2535        private boolean isTarget(dataflow instance, String targetParentId, boolean simpleOutput) {
2536                if (targetTables.containsKey(targetParentId))
2537                        return targetTables.get(targetParentId);
2538                if (isTable(instance, targetParentId)) {
2539                        targetTables.put(targetParentId, true);
2540                        return true;
2541                } else if (isView(instance, targetParentId)) {
2542                        targetTables.put(targetParentId, true);
2543                        return true;
2544                } else if (isFile(instance, targetParentId)) {
2545                        targetTables.put(targetParentId, true);
2546                        return true;
2547                } else if (isDatabase(instance, targetParentId)) {
2548                        targetTables.put(targetParentId, true);
2549                        return true;
2550                } else if (isSchema(instance, targetParentId)) {
2551                        targetTables.put(targetParentId, true);
2552                        return true;
2553                } else if (isStage(instance, targetParentId)) {
2554                        targetTables.put(targetParentId, true);
2555                        return true;
2556                } else if (isSequence(instance, targetParentId)) {
2557                        targetTables.put(targetParentId, true);
2558                        return true;
2559                } else if (isDataSource(instance, targetParentId)) {
2560                        targetTables.put(targetParentId, true);
2561                        return true;
2562                } else if (isStream(instance, targetParentId)) {
2563                        targetTables.put(targetParentId, true);
2564                        return true;
2565                } else if (isCursor(instance, targetParentId) && option.isSimpleShowCursor()) {
2566                        targetTables.put(targetParentId, true);
2567                        return true;
2568                } else if ((isVariable(instance, targetParentId) || isCursor(instance, targetParentId)) && option.isSimpleShowVariable()) {
2569                        targetTables.put(targetParentId, true);
2570                        return true;
2571                } else if (isTargetResultSet(instance, targetParentId, simpleOutput)) {
2572                        targetTables.put(targetParentId, true);
2573                        return true;
2574                }
2575                targetTables.put(targetParentId, false);
2576                return false;
2577        }
2578
2579        private boolean isTargetResultSet(dataflow instance, String targetParent, boolean simpleOutput) {
2580                if (resultSetMap.containsKey(targetParent.toLowerCase())) {
2581                        table result = resultSetMap.get(targetParent.toLowerCase());
2582                        boolean isTarget = result.isTarget();
2583                        Option option = ModelBindingManager.getGlobalOption();
2584                        if (option != null && option.isSqlflowIgnoreFunction() && isFunction(result)) {
2585                                return false;
2586                        }
2587                        if (isTarget && simpleOutput) {
2588                                if (option != null && option.isSimpleShowFunction() && isFunction(result)) {
2589                                        return true;
2590
2591                                } else if (option != null && option.isSimpleShowTopSelectResultSet()) {
2592                                        return true;
2593                                }
2594                                if (ResultSetType.of(result.getType()) != null && option.containsResultSetType(ResultSetType.of(result.getType()))) {
2595                                        return true;
2596                                }
2597                        } else
2598                                return isTarget;
2599                }
2600                return false;
2601        }
2602
2603        private boolean isFunction(table resultSet) {
2604                if("function".equals(resultSet.getType())){
2605                        return true;
2606                }
2607                else if("resultset".equals(resultSet.getType()) && "function".equals(resultSet.getSubType())){
2608                        return true;
2609                }
2610                return false;
2611        }
2612
2613        private boolean isView(dataflow instance, String targetParent) {
2614                if (viewMap.containsKey(targetParent.toLowerCase())) {
2615                        return true;
2616                }
2617                return false;
2618        }
2619
2620        private boolean isCursor(dataflow instance, String targetParent) {
2621                if (cursorMap.containsKey(targetParent.toLowerCase())) {
2622                        return true;
2623                }
2624                return false;
2625        }
2626
2627        private boolean isVariable(dataflow instance, String targetParent) {
2628                if (variableMap.containsKey(targetParent.toLowerCase())) {
2629                        return true;
2630                }
2631                return false;
2632        }
2633
2634        private boolean isFile(dataflow instance, String targetParent) {
2635                if (fileMap.containsKey(targetParent.toLowerCase())) {
2636                        return true;
2637                }
2638                return false;
2639        }
2640
2641        private boolean isStage(dataflow instance, String targetParent) {
2642                if (stageMap.containsKey(targetParent.toLowerCase())) {
2643                        return true;
2644                }
2645                return false;
2646        }
2647        
2648        private boolean isSequence(dataflow instance, String targetParent) {
2649                if (sequenceMap.containsKey(targetParent.toLowerCase())) {
2650                        return true;
2651                }
2652                return false;
2653        }
2654
2655        private boolean isDataSource(dataflow instance, String targetParent) {
2656                if (dataSourceMap.containsKey(targetParent.toLowerCase())) {
2657                        return true;
2658                }
2659                return false;
2660        }
2661
2662        private boolean isDatabase(dataflow instance, String targetParent) {
2663                if (databaseMap.containsKey(targetParent.toLowerCase())) {
2664                        return true;
2665                }
2666                return false;
2667        }
2668
2669        private boolean isSchema(dataflow instance, String targetParent) {
2670                if (schemaMap.containsKey(targetParent.toLowerCase())) {
2671                        return true;
2672                }
2673                return false;
2674        }
2675
2676        private boolean isStream(dataflow instance, String targetParent) {
2677                if (streamMap.containsKey(targetParent.toLowerCase())) {
2678                        return true;
2679                }
2680                return false;
2681        }
2682
2683        private boolean isTable(dataflow instance, String targetParent) {
2684                if (tableMap.containsKey(targetParent.toLowerCase())) {
2685                        if (SQLUtil.isTempTable(tableMap.get(targetParent))) {
2686                                if (option.isIgnoreTemporaryTable()) {
2687                                        return false;
2688                                }
2689                        }
2690                        if (tableMap.get(targetParent).isFunction()) {
2691                                if (option != null && option.isSimpleShowFunction()) {
2692                                        return true;
2693                                } else {
2694                                        return false;
2695                                }
2696                        }
2697                        if (SubType.synonym.name().equals(tableMap.get(targetParent).getSubType())) {
2698                                if (option != null && option.isSimpleShowSynonym()) {
2699                                        return true;
2700                                } else {
2701                                        return false;
2702                                }
2703                        }
2704                        return true;
2705                } else {
2706                        return false;
2707                }
2708        }
2709
2710        private void init() {
2711                metadataErrors.clear();
2712                sqlInfoMap.clear();
2713                errorInfos.clear();
2714                dataflow = null;
2715                dataflowString = null;
2716                ModelBindingManager.removeGlobalDatabase();
2717                ModelBindingManager.removeGlobalSchema();
2718                ModelBindingManager.removeGlobalVendor();
2719                ModelBindingManager.removeGlobalSQLEnv();
2720                ModelBindingManager.removeGlobalHash();
2721                appendResultSets.clear();
2722                appendStarColumns.clear();
2723                appendTableStarColumns.clear();
2724                modelManager.TABLE_COLUMN_ID = option.getStartId();
2725                modelManager.RELATION_ID = option.getStartId();
2726                modelManager.DISPLAY_ID.clear();
2727                modelManager.DISPLAY_NAME.clear();
2728                tableIds.clear();
2729                ModelBindingManager.setGlobalVendor(option.getVendor());
2730                modelManager.reset();
2731        }
2732
2733        private String getErrorMessage(TSyntaxError error, String errorType) {
2734                String s = "", hint = "Syntax error";
2735                if (ErrorInfo.SYNTAX_HINT.equals(errorType)) {
2736                        hint = "Syntax hint";
2737                }
2738                if (error.hint.length() > 0)
2739                        hint = error.hint;
2740                s = s + hint + "(" + error.errorno + ") near: " + error.tokentext;
2741                s = s + "(" + error.lineNo;
2742                s = s + "," + error.columnNo + ")";
2743                return s;
2744        }
2745
2746//      boolean OLD_ENABLE_RESOLVER = TBaseType.isEnableResolver();
2747        private void analyzeAndOutputResult(TGSqlParser sqlparser) {
2748                try {
2749                        accessedSubqueries.clear();
2750                        accessedStatements.clear();
2751                        stmtStack.clear();
2752                        viewDDLMap.clear();
2753                        procedureDDLMap.clear();
2754                        structObjectMap.clear();
2755                        try {
2756                                if(sqlenv!=null) {
2757                                        sqlparser.setSqlEnv(sqlenv);
2758                                }
2759                                int result = sqlparser.parse();
2760                                if (result != 0) {
2761                                        ArrayList<TSyntaxError> errors = sqlparser.getSyntaxErrors();
2762                                        if (errors != null && !errors.isEmpty()) {
2763                                                for (int i = 0; i < errors.size(); i++) {
2764                                                        TSyntaxError error = errors.get(i);
2765                                                        ErrorInfo errorInfo = new ErrorInfo();
2766                                                        errorInfo.setErrorType(ErrorInfo.SYNTAX_ERROR);
2767                                                        errorInfo.setErrorMessage(getErrorMessage(error, ErrorInfo.SYNTAX_ERROR));
2768                                                        errorInfo.setStartPosition(new Pair3<Long, Long, String>(error.lineNo, error.columnNo,
2769                                                                        ModelBindingManager.getGlobalHash()));
2770                                                        String[] segments = error.tokentext.split("\n");
2771                                                        if (segments.length <= 1) {
2772                                                                errorInfo.setEndPosition(new Pair3<Long, Long, String>(error.lineNo,
2773                                                                                error.columnNo + error.tokentext.length(),
2774                                                                                ModelBindingManager.getGlobalHash()));
2775                                                        } else {
2776                                                                errorInfo.setEndPosition(
2777                                                                                new Pair3<Long, Long, String>(error.lineNo + segments.length - 1,
2778                                                                                                (long) segments[segments.length - 1].length() + 1,
2779                                                                                                ModelBindingManager.getGlobalHash()));
2780                                                        }
2781                                                        ;
2782                                                        errorInfo.fillInfo(this);
2783                                                        errorInfos.add(errorInfo);
2784                                                }
2785                                        }
2786                                }
2787
2788                                if (option.getHandleListener() != null) {
2789                                        option.getHandleListener().endParse(result == 0);
2790                                }
2791                        } catch (Exception e) {
2792                                logger.error("analyze sql failed.", e);
2793                                if (option.getHandleListener() != null) {
2794                                        option.getHandleListener().endParse(false);
2795                                }
2796                                ErrorInfo errorInfo = new ErrorInfo();
2797                                errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
2798                                if (e.getMessage() == null) {
2799                                        if (e.getStackTrace() != null && e.getStackTrace().length > 0) {
2800                                                errorInfo
2801                                                                .setErrorMessage(e.getClass().getSimpleName() + ": " + e.getStackTrace()[0].toString());
2802                                        } else {
2803                                                errorInfo.setErrorMessage(e.getClass().getSimpleName());
2804                                        }
2805                                } else {
2806                                        errorInfo.setErrorMessage(e.getClass().getSimpleName() + ": " + e.getMessage());
2807                                }
2808                                errorInfo.fillInfo(this);
2809                                errorInfos.add(errorInfo);
2810                                return;
2811                        }
2812
2813                        
2814                        TSQLResolver2 resolver = sqlparser.getResolver2();
2815                        if (resolver != null && option.getVendor() == EDbVendor.dbvbigquery) {
2816                                ScopeBuildResult buildResult = resolver.getScopeBuildResult();
2817                                List<TObjectName> columns = buildResult.getAllColumnReferences();
2818                                for (TObjectName col : columns) {
2819                                        structObjectMap.putIfAbsent(col.getSourceTable(), new TObjectNameList());
2820                                        structObjectMap.get(col.getSourceTable()).addObjectName(col);
2821                                }
2822                        }
2823                        
2824                        if (option.getHandleListener() != null) {
2825                                option.getHandleListener().startAnalyzeDataFlow(sqlparser);
2826                        }
2827
2828                        for (int i = 0; i < sqlparser.getSqlstatements().size(); i++) {
2829                                if (option.getHandleListener() != null && option.getHandleListener().isCanceled()) {
2830                                        break;
2831                                }
2832
2833                                TCustomSqlStatement stmt = sqlparser.getSqlstatements().get(i);
2834                                if (stmt.getErrorCount() == 0) {
2835                                        if (stmt.getParentStmt() == null) {
2836                                                modelManager.collectSqlHash(stmt);
2837                                                if (stmt instanceof TUseDatabase || stmt instanceof TUseSchema
2838                                                                || stmt instanceof TCreateTableSqlStatement
2839                                                                || stmt instanceof TCreateExternalDataSourceStmt || stmt instanceof TCreateStageStmt
2840                                                                || stmt instanceof TMssqlCreateType
2841                                                                || stmt instanceof TMssqlDeclare
2842                                                                || stmt instanceof TPlsqlCreateType_Placeholder
2843                                                                || stmt instanceof TPlsqlCreateType
2844                                                                || stmt instanceof TPlsqlTableTypeDefStmt
2845                                                                || (stmt instanceof TCreateFunctionStmt && hasDb2ReturnStmt((TCreateFunctionStmt)stmt))
2846                                                                || (stmt instanceof TMssqlCreateFunction
2847                                                                                && (((TMssqlCreateFunction) stmt).getReturnTableDefinitions() != null
2848                                                                                                || ((TMssqlCreateFunction) stmt).getReturnStmt() != null))) {
2849                                                        boolean listen = false;
2850                                                        if (option.getHandleListener() != null && !accessedStatements.contains(stmt)) {
2851                                                                option.getHandleListener().startAnalyzeStatment(stmt);
2852                                                                listen = true;
2853                                                        }
2854                                                        analyzeCustomSqlStmt(stmt);
2855                                                        if (listen && option.getHandleListener() != null) {
2856                                                                option.getHandleListener().endAnalyzeStatment(stmt);
2857                                                        }
2858                                                }
2859                                        }
2860                                }
2861                        }
2862
2863                        for (int i = 0; i < sqlparser.sqlstatements.size(); i++) {
2864                                if (option.getHandleListener() != null && option.getHandleListener().isCanceled()) {
2865                                        break;
2866                                }
2867
2868                                TCustomSqlStatement stmt = sqlparser.getSqlstatements().get(i);
2869                                if (stmt.getErrorCount() == 0) {
2870                                        if (stmt.getParentStmt() == null) {
2871                                                if (stmt instanceof TUseDatabase 
2872                                                                || stmt instanceof TUseSchema
2873                                                                || stmt instanceof TCreateViewSqlStatement 
2874                                                                || stmt instanceof TCreateSynonymStmt 
2875                                                                || stmt instanceof TStoredProcedureSqlStatement) {
2876                                                        boolean listen = false;
2877                                                        if (option.getHandleListener() != null && !accessedStatements.contains(stmt)) {
2878                                                                option.getHandleListener().startAnalyzeStatment(stmt);
2879                                                                listen = true;
2880                                                        }
2881                                                        if (stmt instanceof TUseDatabase || stmt instanceof TUseSchema) {
2882                                                                analyzeCustomSqlStmt(stmt);
2883                                                        } else if (stmt instanceof TCreateViewSqlStatement) {
2884                                                                TCreateViewSqlStatement view = (TCreateViewSqlStatement) stmt;
2885                                                                if(view.getViewName()!=null) {
2886                                                                        viewDDLMap.put(DlineageUtil.getTableFullName(view.getViewName().toString()), view);
2887                                                                }
2888                                                        } else if (stmt instanceof TCreateSynonymStmt) {
2889                                                                TCreateSynonymStmt synonym = (TCreateSynonymStmt) stmt;
2890                                                                if(synonym.getSynonymName()!=null) {
2891                                                                        viewDDLMap.put(DlineageUtil.getTableFullName(synonym.getSynonymName().toString()), synonym);
2892                                                                }
2893                                                        } else if (stmt instanceof TStoredProcedureSqlStatement) {
2894                                                                TStoredProcedureSqlStatement procedure = (TStoredProcedureSqlStatement) stmt;
2895                                                                if(procedure.getStoredProcedureName() == null) {
2896                                                                        continue;
2897                                                                }
2898                                                                procedureDDLMap.put(DlineageUtil.getProcedureNameWithArgNum(procedure), procedure);
2899                                                        }
2900                                                        if (listen && option.getHandleListener() != null) {
2901                                                                option.getHandleListener().endAnalyzeStatment(stmt);
2902                                                        }
2903                                                }
2904                                        }
2905                                }
2906                        }
2907
2908                        for (int i = 0; i < sqlparser.sqlstatements.size(); i++) {
2909                                if (option.getHandleListener() != null && option.getHandleListener().isCanceled()) {
2910                                        break;
2911                                }
2912
2913                                TCustomSqlStatement stmt = sqlparser.getSqlstatements().get(i);
2914                                if (stmt.getErrorCount() == 0) {
2915                                        if (stmt.getParentStmt() == null) {
2916                                                if (stmt instanceof TUseDatabase 
2917                                                                || stmt instanceof TUseSchema
2918                                                                || stmt instanceof TCreateViewSqlStatement 
2919                                                                || stmt instanceof TCreateSynonymStmt) {
2920                                                        boolean listen = false;
2921                                                        if (option.getHandleListener() != null && !accessedStatements.contains(stmt)) {
2922                                                                option.getHandleListener().startAnalyzeStatment(stmt);
2923                                                                listen = true;
2924                                                        }
2925                                                        analyzeCustomSqlStmt(stmt);
2926                                                        if (listen && option.getHandleListener() != null) {
2927                                                                option.getHandleListener().endAnalyzeStatment(stmt);
2928                                                        }
2929                                                }
2930                                        }
2931                                }
2932                        }
2933
2934                        for (int i = 0; i < sqlparser.sqlstatements.size(); i++) {
2935                                if (option.getHandleListener() != null && option.getHandleListener().isCanceled()) {
2936                                        break;
2937                                }
2938
2939                                TCustomSqlStatement stmt = sqlparser.getSqlstatements().get(i);
2940                                if (stmt.getErrorCount() == 0) {
2941                                        if (stmt.getParentStmt() == null) {
2942                                                if (stmt instanceof TUseDatabase || stmt instanceof TUseSchema
2943                                                                || stmt instanceof TStoredProcedureSqlStatement) {
2944                                                        boolean listen = false;
2945                                                        if (option.getHandleListener() != null && !accessedStatements.contains(stmt)) {
2946                                                                option.getHandleListener().startAnalyzeStatment(stmt);
2947                                                                listen = true;
2948                                                        }
2949                                                        if (stmt instanceof TPlsqlCreateTrigger)
2950                                                                continue;
2951                                                        analyzeCustomSqlStmt(stmt);
2952                                                        if (listen && option.getHandleListener() != null) {
2953                                                                option.getHandleListener().endAnalyzeStatment(stmt);
2954                                                        }
2955                                                }
2956                                        }
2957                                }
2958                        }
2959
2960                        for (int i = 0; i < sqlparser.sqlstatements.size(); i++) {
2961                                if (option.getHandleListener() != null && option.getHandleListener().isCanceled()) {
2962                                        break;
2963                                }
2964
2965                                TCustomSqlStatement stmt = sqlparser.getSqlstatements().get(i);
2966
2967                                if (option.isIgnoreTopSelect()) {
2968                                        if ((stmt instanceof TSelectSqlStatement && ((TSelectSqlStatement)stmt).getIntoClause() == null && ((TSelectSqlStatement)stmt).getIntoTableClause() == null ) || stmt instanceof TRedshiftDeclare
2969                                                        || stmt instanceof TRedshiftDeclare) {
2970                                                continue;
2971                                        }
2972                                }
2973
2974                                if (stmt.getErrorCount() == 0) {
2975                                        if (stmt.getParentStmt() == null) {
2976                                                if (!(stmt instanceof TCreateViewSqlStatement) && !(stmt instanceof TCreateStageStmt)
2977                                                                && !(stmt instanceof TCreateExternalDataSourceStmt)
2978                                                                && !(stmt instanceof TCreateViewSqlStatement) && !(stmt instanceof TMssqlDeclare)
2979                                                                && !(stmt instanceof TMssqlCreateFunction
2980                                                                                && ((TMssqlCreateFunction) stmt).getReturnTableDefinitions() != null)) {
2981                                                        boolean listen = false;
2982                                                        if (option.getHandleListener() != null && !accessedStatements.contains(stmt)) {
2983                                                                option.getHandleListener().startAnalyzeStatment(stmt);
2984                                                                listen = true;
2985                                                        }
2986                                                        analyzeCustomSqlStmt(stmt);
2987                                                        if (listen && option.getHandleListener() != null) {
2988                                                                option.getHandleListener().endAnalyzeStatment(stmt);
2989                                                        }
2990                                                }
2991                                        }
2992                                }
2993                        }
2994
2995                        if (option.getHandleListener() != null) {
2996                                option.getHandleListener().endAnalyzeDataFlow(sqlparser);
2997                        }
2998
2999                        // Finalize pipelined function stitching after all passes
3000                        if (!modelManager.getPendingPipelinedCallSites().isEmpty() && sqlparser.getSqlstatements().size() > 0 && pipelinedAnalyzer != null) {
3001                                TCustomSqlStatement lastStmt = sqlparser.getSqlstatements().get(sqlparser.getSqlstatements().size() - 1);
3002                                stmtStack.push(lastStmt);
3003                                try {
3004                                        pipelinedAnalyzer.stitchPendingCallSites();
3005                                } catch (Exception ex) {
3006                                        // Don't let pipelined stitching failure break main flow
3007                                } finally {
3008                                        stmtStack.pop();
3009                                }
3010                        }
3011                } catch (Throwable e) {
3012                        logger.error("analyze sql failed.", e);
3013                        ErrorInfo errorInfo = new ErrorInfo();
3014                        errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
3015                        if (e.getMessage() == null) {
3016                                if (e.getStackTrace() != null && e.getStackTrace().length > 0) {
3017                                        errorInfo.setErrorMessage(e.getClass().getSimpleName() + ": " + e.getStackTrace()[0].toString());
3018                                } else {
3019                                        errorInfo.setErrorMessage(e.getClass().getSimpleName());
3020                                }
3021                        } else {
3022                                errorInfo.setErrorMessage(e.getClass().getSimpleName() + ": " + e.getMessage());
3023                        }
3024                        errorInfo.fillInfo(this);
3025                        errorInfos.add(errorInfo);
3026                }
3027
3028        }
3029
3030        private boolean hasDb2ReturnStmt(TCreateFunctionStmt stmt) {
3031                if (stmt.getReturnStmt() != null) {
3032                        return true;
3033                }
3034                if (stmt.getBodyStatements() != null) {
3035                        for (int i = 0; i < stmt.getBodyStatements().size(); i++) {
3036                                if(stmt.getBodyStatements().get(i) instanceof TDb2ReturnStmt) {
3037                                        return true;
3038                                }
3039                        }
3040                }
3041                return false;
3042        }
3043
3044        private void analyzeCustomSqlStmt(TCustomSqlStatement stmt) {
3045                if (!accessedStatements.contains(stmt)) {
3046                        accessedStatements.add(stmt);
3047                } else if (!(stmt instanceof TUseDatabase || stmt instanceof TUseSchema)) {
3048                        return;
3049                }
3050
3051                ArrayList<TSyntaxError> errors = stmt.getSyntaxHints();
3052                if (errors != null && !errors.isEmpty()) {
3053                        for (int i = 0; i < errors.size(); i++) {
3054                                TSyntaxError error = errors.get(i);
3055                                ErrorInfo errorInfo = new ErrorInfo();
3056                                errorInfo.setErrorType(ErrorInfo.SYNTAX_HINT);
3057                                errorInfo.setErrorMessage(getErrorMessage(error, ErrorInfo.SYNTAX_HINT));
3058                                errorInfo.setStartPosition(new Pair3<Long, Long, String>(error.lineNo, error.columnNo,
3059                                                ModelBindingManager.getGlobalHash()));
3060                                String[] segments = error.tokentext.split("\n");
3061                                if (segments.length == 1) {
3062                                        errorInfo.setEndPosition(new Pair3<Long, Long, String>(error.lineNo,
3063                                                        error.columnNo + error.tokentext.length(), ModelBindingManager.getGlobalHash()));
3064                                } else {
3065                                        errorInfo.setEndPosition(new Pair3<Long, Long, String>(error.lineNo + segments.length - 1,
3066                                                        (long) segments[segments.length - 1].length() + 1, ModelBindingManager.getGlobalHash()));
3067                                }
3068                                errorInfo.fillInfo(this);
3069                                errorInfos.add(errorInfo);
3070                        }
3071                }
3072
3073                if (option.getAnalyzeMode() == AnalyzeMode.dynamic) {
3074                        if (!(stmt instanceof TStoredProcedureSqlStatement
3075                                        || stmt instanceof TExecuteSqlStatement
3076                                        || stmt instanceof TMssqlExecute
3077                                        || stmt instanceof TExecImmeStmt)) {
3078                                return;
3079                        }
3080                }
3081
3082                if(DlineageUtil.getTopStmt(stmt) == stmt){
3083                        modelManager.collectSqlHash(stmt);
3084                }
3085
3086                try {
3087                        if (stmt instanceof TUseDatabase) {
3088                                if (((TUseDatabase) stmt).getDatabaseName() != null) {
3089                                        ModelBindingManager.setGlobalDatabase(((TUseDatabase) stmt).getDatabaseName().toString());
3090                                }
3091                        } else if (stmt instanceof TUseSchema) {
3092                                if (((TUseSchema) stmt).getSchemaName() != null) {
3093                                        String schemaName = ((TUseSchema) stmt).getSchemaName().toString();
3094                                        List<String> splits = SQLUtil.parseNames(schemaName);
3095                                        if (splits.size() == 1) {
3096                                                ModelBindingManager.setGlobalSchema(schemaName);
3097                                        } else if (splits.size() > 1) {
3098                                                ModelBindingManager.setGlobalSchema(splits.get(splits.size() - 1));
3099                                                ModelBindingManager.setGlobalDatabase(splits.get(splits.size() - 2));
3100                                        }
3101                                }
3102                        } else if (stmt instanceof TPlsqlRecordTypeDefStmt) {
3103                                this.stmtStack.push(stmt);
3104                                this.analyzePlsqlRecordTypeDefStmt((TPlsqlRecordTypeDefStmt) stmt);
3105                                this.stmtStack.pop();
3106                        } else if (stmt instanceof TPlsqlCreateType_Placeholder) {
3107                                this.stmtStack.push(stmt);
3108                                TPlsqlCreateType_Placeholder placeholder = (TPlsqlCreateType_Placeholder) stmt;
3109                                if (placeholder.getObjectStatement() != null && pipelinedAnalyzer != null) {
3110                                        this.pipelinedAnalyzer.indexObjectType(placeholder.getObjectStatement());
3111                                }
3112                                if (placeholder.getNestedTableStatement() != null) {
3113                                        if (pipelinedAnalyzer != null) {
3114                                                this.pipelinedAnalyzer.indexCollectionType(placeholder.getNestedTableStatement());
3115                                        }
3116                                        this.analyzePlsqlTableTypeDefStmt(placeholder.getNestedTableStatement());
3117                                }
3118                                this.stmtStack.pop();
3119                        } else if (stmt instanceof TPlsqlCreateType) {
3120                                this.stmtStack.push(stmt);
3121                                if (pipelinedAnalyzer != null) {
3122                                        this.pipelinedAnalyzer.indexObjectType((TPlsqlCreateType) stmt);
3123                                }
3124                                this.stmtStack.pop();
3125                        } else if (stmt instanceof TPlsqlTableTypeDefStmt) {
3126                                this.stmtStack.push(stmt);
3127                                this.analyzePlsqlTableTypeDefStmt((TPlsqlTableTypeDefStmt) stmt);
3128                                if (pipelinedAnalyzer != null) {
3129                                        this.pipelinedAnalyzer.indexCollectionType((TPlsqlTableTypeDefStmt) stmt);
3130                                }
3131                                this.stmtStack.pop();
3132                        } else if (stmt instanceof TStoredProcedureSqlStatement) {
3133                                this.stmtStack.push(stmt);
3134                                this.analyzeStoredProcedureStmt((TStoredProcedureSqlStatement) stmt);
3135                                this.stmtStack.pop();
3136                        } else if (stmt instanceof TCreateTableSqlStatement) {
3137                                stmtStack.push(stmt);
3138                                analyzeCreateTableStmt((TCreateTableSqlStatement) stmt);
3139                                stmtStack.pop();
3140                        } else if (stmt instanceof TCreateStageStmt) {
3141                                stmtStack.push(stmt);
3142                                analyzeCreateStageStmt((TCreateStageStmt) stmt);
3143                                stmtStack.pop();
3144                        } else if (stmt instanceof TCreateExternalDataSourceStmt) {
3145                                stmtStack.push(stmt);
3146                                analyzeCreateExternalDataSourceStmt((TCreateExternalDataSourceStmt) stmt);
3147                                stmtStack.pop();
3148                        } else if (stmt instanceof TCreateStreamStmt) {
3149                                stmtStack.push(stmt);
3150                                analyzeCreateStreamStmt((TCreateStreamStmt) stmt);
3151                                stmtStack.pop();
3152                        } else if (stmt instanceof TSelectSqlStatement) {
3153                                analyzeSelectStmt((TSelectSqlStatement) stmt);
3154                        } else if (stmt instanceof TDropTableSqlStatement) {
3155                                stmtStack.push(stmt);
3156                                analyzeDropTableStmt((TDropTableSqlStatement) stmt);
3157                                stmtStack.pop();
3158                        } else if (stmt instanceof TTruncateStatement) {
3159                                stmtStack.push(stmt);
3160                                analyzeTruncateTableStmt((TTruncateStatement) stmt);
3161                                stmtStack.pop();
3162                        } else if (stmt instanceof TCreateMaterializedSqlStatement) {
3163                                stmtStack.push(stmt);
3164                                TCreateMaterializedSqlStatement view = (TCreateMaterializedSqlStatement) stmt;
3165                                analyzeCreateViewStmt(view, view.getSubquery(), view.getViewAliasClause(), view.getViewName());
3166                                stmtStack.pop();
3167                        } else if (stmt instanceof TCreateViewSqlStatement) {
3168                                stmtStack.push(stmt);
3169                                TCreateViewSqlStatement view = (TCreateViewSqlStatement) stmt;
3170                                analyzeCreateViewStmt(view, view.getSubquery(), view.getViewAliasClause(), view.getViewName());
3171                                stmtStack.pop();
3172                        } else if(stmt instanceof TDb2SqlVariableDeclaration){
3173                                stmtStack.push(stmt);
3174                                analyzeDb2Declare((TDb2SqlVariableDeclaration)stmt);
3175                                stmtStack.pop();
3176                        } else if (stmt instanceof TMssqlCreateType) {
3177                                stmtStack.push(stmt);
3178                                TMssqlCreateType createType = (TMssqlCreateType) stmt;
3179                                analyzeMssqlCreateType(createType);
3180                                stmtStack.pop();
3181                        } else if (stmt instanceof TMssqlDeclare) {
3182                                stmtStack.push(stmt);
3183                                TMssqlDeclare declare = (TMssqlDeclare) stmt;
3184                                analyzeMssqlDeclare(declare);
3185                                stmtStack.pop();
3186                        } else if (stmt instanceof TInsertSqlStatement) {
3187                                stmtStack.push(stmt);
3188                                TInsertSqlStatement insert = (TInsertSqlStatement)stmt;
3189                                analyzeInsertStmt(insert);
3190                                if(insert.getMultiInsertStatements()!=null) {
3191                                        for(int i=0;i<insert.getMultiInsertStatements().size();i++) {
3192                                                analyzeInsertStmt(insert.getMultiInsertStatements().get(i));
3193                                        }
3194                                }
3195                                stmtStack.pop();
3196                        } else if (stmt instanceof TRedshiftCopy) {
3197                                stmtStack.push(stmt);
3198                                analyzeRedshiftCopyStmt((TRedshiftCopy) stmt);
3199                                stmtStack.pop();
3200                        } else if (stmt instanceof TSnowflakeCopyIntoStmt) {
3201                                stmtStack.push(stmt);
3202                                analyzeCopyIntoStmt((TSnowflakeCopyIntoStmt) stmt);
3203                                stmtStack.pop();
3204                        } else if (stmt instanceof TUnloadStmt) {
3205                                stmtStack.push(stmt);
3206                                analyzeUnloadStmt((TUnloadStmt) stmt);
3207                                stmtStack.pop();
3208                        } else if (stmt instanceof TUpdateSqlStatement) {
3209                                stmtStack.push(stmt);
3210                                analyzeUpdateStmt((TUpdateSqlStatement) stmt);
3211                                stmtStack.pop();
3212                        } else if (stmt instanceof TMergeSqlStatement) {
3213                                stmtStack.push(stmt);
3214                                analyzeMergeStmt((TMergeSqlStatement) stmt);
3215                                stmtStack.pop();
3216                        } else if (stmt instanceof TDeleteSqlStatement) {
3217                                stmtStack.push(stmt);
3218                                analyzeDeleteStmt((TDeleteSqlStatement) stmt);
3219                                stmtStack.pop();
3220                        } else if (stmt instanceof TCursorDeclStmt) {
3221                                stmtStack.push(stmt);
3222                                analyzeCursorDeclStmt((TCursorDeclStmt) stmt);
3223                                stmtStack.pop();
3224                        } else if (stmt instanceof TFetchStmt) {
3225                                stmtStack.push(stmt);
3226                                analyzeFetchStmt((TFetchStmt) stmt);
3227                                stmtStack.pop();
3228                        } else if (stmt instanceof TMssqlFetch) {
3229                                stmtStack.push(stmt);
3230                                analyzeFetchStmt((TMssqlFetch) stmt);
3231                                stmtStack.pop();
3232                        } else if (stmt instanceof TForStmt) {
3233                                stmtStack.push(stmt);
3234                                analyzeForStmt((TForStmt) stmt);
3235                                stmtStack.pop();
3236                        } else if (stmt instanceof TOpenforStmt) {
3237                                stmtStack.push(stmt);
3238                                analyzeOpenForStmt((TOpenforStmt) stmt);
3239                                stmtStack.pop();
3240                        } else if (stmt instanceof TLoopStmt) {
3241                                stmtStack.push(stmt);
3242                                analyzeLoopStmt((TLoopStmt) stmt);
3243                                stmtStack.pop();
3244                        } else if (stmt instanceof TAssignStmt) {
3245                                stmtStack.push(stmt);
3246                                analyzeAssignStmt((TAssignStmt) stmt);
3247                                stmtStack.pop();
3248                        } else if (stmt instanceof TSetStmt) {
3249                                stmtStack.push(stmt);
3250                                analyzeSetStmt((TSetStmt) stmt);
3251                                stmtStack.pop();
3252                        } else if (stmt instanceof TMssqlSet) {
3253                                stmtStack.push(stmt);
3254                                analyzeMssqlSetStmt((TMssqlSet) stmt);
3255                                stmtStack.pop();
3256                        } else if (stmt instanceof TVarDeclStmt) {
3257                                stmtStack.push(stmt);
3258                                analyzeVarDeclStmt((TVarDeclStmt) stmt);
3259                                stmtStack.pop();
3260                        } else if (stmt instanceof TCreateDatabaseSqlStatement) {
3261                                stmtStack.push(stmt);
3262                                analyzeCloneDatabaseStmt((TCreateDatabaseSqlStatement) stmt);
3263                                stmtStack.pop();
3264                        } else if (stmt instanceof TCreateSchemaSqlStatement) {
3265                                stmtStack.push(stmt);
3266                                analyzeCloneSchemaStmt((TCreateSchemaSqlStatement) stmt);
3267                                stmtStack.pop();
3268                        } else if (stmt instanceof TAlterTableStatement) {
3269                                stmtStack.push(stmt);
3270                                analyzeAlterTableStmt((TAlterTableStatement) stmt);
3271                                stmtStack.pop();
3272                        } else if (stmt instanceof TAlterViewStatement) {
3273                                stmtStack.push(stmt);
3274                                analyzeAlterViewStmt((TAlterViewStatement) stmt);
3275                                stmtStack.pop();
3276                        } else if (stmt instanceof TRenameStmt) {
3277                                stmtStack.push(stmt);
3278                                analyzeRenameStmt((TRenameStmt) stmt);
3279                                stmtStack.pop();
3280                        } else if (stmt instanceof TCreateSynonymStmt) {
3281                                stmtStack.push(stmt);
3282                                analyzeCreateSynonymStmt((TCreateSynonymStmt) stmt);
3283                                stmtStack.pop();
3284                        } else if (stmt instanceof TLoadDataStmt) {
3285                                stmtStack.push(stmt);
3286                                analyzeLoadDataStmt((TLoadDataStmt) stmt);
3287                                stmtStack.pop();
3288                        } else if (stmt instanceof THiveLoad) {
3289                                stmtStack.push(stmt);
3290                                analyzeHiveLoadStmt((THiveLoad) stmt);
3291                                stmtStack.pop();
3292                        } else if (stmt instanceof TDb2ReturnStmt) {
3293                                stmtStack.push(stmt);
3294                                analyzeDb2ReturnStmt((TDb2ReturnStmt) stmt);
3295                                stmtStack.pop();
3296                        } else if (stmt instanceof TReturnStmt) {
3297                                stmtStack.push(stmt);
3298                                analyzeReturnStmt((TReturnStmt) stmt);
3299                                stmtStack.pop();
3300                        } else if (stmt instanceof TMssqlReturn) {
3301                                stmtStack.push(stmt);
3302                                analyzeMssqlReturnStmt((TMssqlReturn) stmt);
3303                                stmtStack.pop();
3304                        } else if (stmt instanceof TExecuteSqlStatement) {
3305                                String sqlText = ((TExecuteSqlStatement) stmt).getPreparedSqlText();
3306                                if(sqlText == null) {
3307                                        sqlText = ((TExecuteSqlStatement) stmt).getSqlText();
3308                                }
3309                                if (sqlText != null) {
3310                                        modelManager.collectDynamicSqlHash(stmt);
3311                                        TGSqlParser sqlparser = new TGSqlParser(option.getVendor());
3312                                        sqlparser.sqltext = SQLUtil.trimColumnStringQuote(sqlText);
3313                                        int result = sqlparser.parse();
3314                                        if (result != 0) {
3315                                                errors = sqlparser.getSyntaxErrors();
3316                                                if (errors != null && !errors.isEmpty()) {
3317                                                        for (int i = 0; i < errors.size(); i++) {
3318                                                                TSyntaxError error = errors.get(i);
3319                                                                ErrorInfo errorInfo = new ErrorInfo();
3320                                                                errorInfo.setErrorType(ErrorInfo.SYNTAX_ERROR);
3321                                                                errorInfo.setErrorMessage(getErrorMessage(error, ErrorInfo.SYNTAX_ERROR));
3322                                                                errorInfo.setStartPosition(new Pair3<Long, Long, String>(error.lineNo, error.columnNo,
3323                                                                                ModelBindingManager.getGlobalHash()));
3324                                                                String[] segments = error.tokentext.split("\n");
3325                                                                if (segments.length == 1) {
3326                                                                        errorInfo.setEndPosition(new Pair3<Long, Long, String>(error.lineNo,
3327                                                                                        error.columnNo + error.tokentext.length(),
3328                                                                                        ModelBindingManager.getGlobalHash()));
3329                                                                } else {
3330                                                                        errorInfo.setEndPosition(
3331                                                                                        new Pair3<Long, Long, String>(error.lineNo + segments.length - 1,
3332                                                                                                        (long) segments[segments.length - 1].length() + 1,
3333                                                                                                        ModelBindingManager.getGlobalHash()));
3334                                                                }
3335                                                                errorInfo.fillInfo(this);
3336                                                                errorInfos.add(errorInfo);
3337                                                        }
3338                                                }
3339                                        } else if (sqlparser.sqlstatements != null) {
3340                                                for (int i = 0; i < sqlparser.sqlstatements.size(); i++) {
3341                                                        analyzeCustomSqlStmt(sqlparser.sqlstatements.get(i));
3342                                                }
3343                                        }
3344                                }
3345                                else if (((TExecuteSqlStatement) stmt).getStmtString() != null) {
3346                                        modelManager.collectDynamicSqlHash(stmt);
3347                                }
3348                        } else if (stmt instanceof TMssqlExecute) {
3349                                TMssqlExecute executeStmt = (TMssqlExecute)stmt;
3350                                if (executeStmt.getSqlText() != null) {
3351                                        modelManager.collectDynamicSqlHash(stmt);
3352                                        TGSqlParser sqlparser = new TGSqlParser(option.getVendor());
3353                                        sqlparser.sqltext = ((TMssqlExecute) stmt).getSqlText();
3354                                        int result = sqlparser.parse();
3355                                        if (result != 0) {
3356                                                errors = sqlparser.getSyntaxErrors();
3357                                                if (errors != null && !errors.isEmpty()) {
3358                                                        for (int i = 0; i < errors.size(); i++) {
3359                                                                TSyntaxError error = errors.get(i);
3360                                                                ErrorInfo errorInfo = new ErrorInfo();
3361                                                                errorInfo.setErrorType(ErrorInfo.SYNTAX_ERROR);
3362                                                                errorInfo.setErrorMessage(getErrorMessage(error, ErrorInfo.SYNTAX_ERROR));
3363                                                                errorInfo.setStartPosition(new Pair3<Long, Long, String>(error.lineNo, error.columnNo,
3364                                                                                ModelBindingManager.getGlobalHash()));
3365                                                                String[] segments = error.tokentext.split("\n");
3366                                                                if (segments.length == 1) {
3367                                                                        errorInfo.setEndPosition(new Pair3<Long, Long, String>(error.lineNo,
3368                                                                                        error.columnNo + error.tokentext.length(),
3369                                                                                        ModelBindingManager.getGlobalHash()));
3370                                                                } else {
3371                                                                        errorInfo.setEndPosition(
3372                                                                                        new Pair3<Long, Long, String>(error.lineNo + segments.length - 1,
3373                                                                                                        (long) segments[segments.length - 1].length() + 1,
3374                                                                                                        ModelBindingManager.getGlobalHash()));
3375                                                                }
3376                                                                errorInfo.fillInfo(this);
3377                                                                errorInfos.add(errorInfo);
3378                                                        }
3379                                                }
3380                                        } else if (sqlparser.sqlstatements != null) {
3381                                                for (int i = 0; i < sqlparser.sqlstatements.size(); i++) {
3382                                                        analyzeCustomSqlStmt(sqlparser.sqlstatements.get(i));
3383                                                }
3384                                        }
3385                                } else if (executeStmt.getModuleName() != null) {
3386                                        stmtStack.push(stmt);
3387                                        analyzeMssqlExecute(executeStmt);
3388                                        stmtStack.pop();
3389                                }
3390                        } else if (stmt instanceof TExecImmeStmt) {
3391                                TExecImmeStmt execImmeStmt = (TExecImmeStmt) stmt;
3392                                modelManager.collectDynamicSqlHash(stmt);
3393                                synchronized (DataFlowAnalyzer.class) {
3394                                        TStatementList stmts = execImmeStmt.getDynamicStatements();
3395                                        if (stmts != null && stmts.size() > 0) {
3396                                                for (int i = 0; i < stmts.size(); i++) {
3397                                                        analyzeCustomSqlStmt(stmts.get(i));
3398                                                }
3399                                        }
3400
3401                                        // Only re-parse the raw dynamic SQL when getDynamicStatements()
3402                                        // produced nothing. getDynamicStatements() already parses and
3403                                        // analyzes the fragment with coordinates remapped back to the
3404                                        // original file position; re-parsing it here with a fresh parser
3405                                        // would analyze the same SQL a second time and emit duplicate
3406                                        // hints/lineage carrying fragment-relative (un-remapped)
3407                                        // coordinates. This fallback still covers the case where
3408                                        // getDynamicStatements() failed (e.g. dynamic SQL syntax error),
3409                                        // reporting the parse error below.
3410                                        String dynamicSql = (stmts == null || stmts.size() == 0) ? execImmeStmt.getDynamicSQL() : null;
3411                                        if (!SQLUtil.isEmpty(dynamicSql)) {
3412                                                TGSqlParser sqlparser = new TGSqlParser(option.getVendor());
3413                                                sqlparser.sqltext = dynamicSql;
3414                                                int result = sqlparser.parse();
3415                                                        // This fresh parse runs the dynamic SQL un-padded, so its
3416                                                        // coordinates are relative to the fragment (first char at
3417                                                        // line 1). Shift them back to the position the fragment
3418                                                        // occupies in the original file, mirroring the shift
3419                                                        // TExecImmeStmt.getDynamicStatements() applies; otherwise
3420                                                        // fallback parse errors for EXECUTE IMMEDIATE text located
3421                                                        // later in the file would report fragment-relative lines.
3422                                                        if (execImmeStmt.getDynamicStringExpr() != null
3423                                                                        && execImmeStmt.getDynamicStringExpr().getPlainTextLineNo() != -1) {
3424                                                                int deltaLine = (int) execImmeStmt.getDynamicStringExpr().getPlainTextLineNo() - 1;
3425                                                                int deltaColumn = (int) execImmeStmt.getDynamicStringExpr().getPlainTextColumnNo();
3426                                                                if ((deltaLine != 0 || deltaColumn != 0) && sqlparser.getSyntaxErrors() != null) {
3427                                                                        for (int ei = 0; ei < sqlparser.getSyntaxErrors().size(); ei++) {
3428                                                                                TSyntaxError dynErr = sqlparser.getSyntaxErrors().get(ei);
3429                                                                                if (dynErr == null) continue;
3430                                                                                boolean onFirstLine = (dynErr.lineNo == 1);
3431                                                                                dynErr.lineNo += deltaLine;
3432                                                                                if (onFirstLine) dynErr.columnNo += deltaColumn;
3433                                                                        }
3434                                                                }
3435                                                        }
3436                                                if (result != 0) {
3437                                                        errors = sqlparser.getSyntaxErrors();
3438                                                        if (errors != null && !errors.isEmpty()) {
3439                                                                for (int i = 0; i < errors.size(); i++) {
3440                                                                        TSyntaxError error = errors.get(i);
3441                                                                        ErrorInfo errorInfo = new ErrorInfo();
3442                                                                        errorInfo.setErrorType(ErrorInfo.SYNTAX_ERROR);
3443                                                                        errorInfo.setErrorMessage(getErrorMessage(error, ErrorInfo.SYNTAX_ERROR));
3444                                                                        errorInfo.setStartPosition(new Pair3<Long, Long, String>(error.lineNo, error.columnNo,
3445                                                                                        ModelBindingManager.getGlobalHash()));
3446                                                                        String[] segments = error.tokentext.split("\n");
3447                                                                        if (segments.length == 1) {
3448                                                                                errorInfo.setEndPosition(new Pair3<Long, Long, String>(error.lineNo,
3449                                                                                                error.columnNo + error.tokentext.length(),
3450                                                                                                ModelBindingManager.getGlobalHash()));
3451                                                                        } else {
3452                                                                                errorInfo.setEndPosition(
3453                                                                                                new Pair3<Long, Long, String>(error.lineNo + segments.length - 1,
3454                                                                                                                (long) segments[segments.length - 1].length() + 1,
3455                                                                                                                ModelBindingManager.getGlobalHash()));
3456                                                                        }
3457                                                                        errorInfo.fillInfo(this);
3458                                                                        errorInfos.add(errorInfo);
3459                                                                }
3460                                                        }
3461                                                } else if (sqlparser.sqlstatements != null) {
3462                                                        for (int i = 0; i < sqlparser.sqlstatements.size(); i++) {
3463                                                                analyzeCustomSqlStmt(sqlparser.sqlstatements.get(i));
3464                                                        }
3465                                                }
3466                                        }
3467                                }
3468                        } else if (stmt instanceof TCallStatement) {
3469                                TCallStatement callStmt = (TCallStatement)stmt;
3470                                stmtStack.push(stmt);
3471                                analyzeCallStmt(callStmt);
3472                                stmtStack.pop();
3473                        } else if (stmt instanceof TBasicStmt) {
3474                                TBasicStmt oracleBasicStmt = (TBasicStmt) stmt;
3475                                stmtStack.push(stmt);
3476                                analyzeOracleBasicStmt(oracleBasicStmt);
3477                                stmtStack.pop();
3478                        } else if (stmt instanceof TIfStmt) {
3479                                TIfStmt ifStmt = (TIfStmt) stmt;
3480                                stmtStack.push(stmt);
3481                                analyzeIfStmt(ifStmt);
3482                                stmtStack.pop();
3483                        } else if (stmt instanceof TElsifStmt) {
3484                                TElsifStmt elsIfStmt = (TElsifStmt) stmt;
3485                                stmtStack.push(stmt);
3486                                analyzeElsIfStmt(elsIfStmt);
3487                                stmtStack.pop();
3488                        } else if (stmt.getStatements() != null && stmt.getStatements().size() > 0) {
3489                                for (int i = 0; i < stmt.getStatements().size(); i++) {
3490                                        analyzeCustomSqlStmt(stmt.getStatements().get(i));
3491                                }
3492                        } else if (stmt instanceof TCreateIndexSqlStatement) {
3493                                stmtStack.push(stmt);
3494                                analyzeCreateIndexStageStmt((TCreateIndexSqlStatement) stmt);
3495                                stmtStack.pop();
3496                        } else if (stmt instanceof gudusoft.gsqlparser.stmt.mdx.TMdxSelect) {
3497                                stmtStack.push(stmt);
3498                                analyzeMdxSelectStmt((gudusoft.gsqlparser.stmt.mdx.TMdxSelect) stmt);
3499                                stmtStack.pop();
3500                        } else if (stmt instanceof TPowerQueryDocumentStmt) {
3501                                stmtStack.push(stmt);
3502                                analyzePowerQueryDocumentStmt((TPowerQueryDocumentStmt) stmt);
3503                                stmtStack.pop();
3504                        }
3505                } catch (Exception e) {
3506            StringBuffer errorMessage = new StringBuffer();
3507            errorMessage.append("analyze sql stmt failed, ");
3508            if (stmt.getStartToken() != null) {
3509                errorMessage.append("line: " + stmt.getStartToken().lineNo + ", column: " + stmt.getStartToken().columnNo).append(", ");
3510            }
3511            if (stmt.getGsqlparser() != null && !SQLUtil.isEmpty(stmt.getGsqlparser().sqlfilename)) {
3512                errorMessage.append("file: "+ stmt.getGsqlparser().sqlfilename).append(", ");
3513            }
3514            if (stmt.toString() != null) {
3515                errorMessage.append("sql:\n" + stmt.toString());
3516            }
3517            logger.error(errorMessage.toString(), e);
3518                        ErrorInfo errorInfo = new ErrorInfo();
3519                        errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
3520                        if (e.getMessage() == null) {
3521                                if (e.getStackTrace() != null && e.getStackTrace().length > 0) {
3522                                        errorInfo.setErrorMessage(e.getClass().getSimpleName() + ": " + e.getStackTrace()[0].toString());
3523                                } else {
3524                                        errorInfo.setErrorMessage(e.getClass().getSimpleName());
3525                                }
3526                        } else {
3527                                errorInfo.setErrorMessage(e.getClass().getSimpleName() + ": " + e.getMessage());
3528                        }
3529                        errorInfo.setStartPosition(new Pair3<Long, Long, String>(stmt.getStartToken().lineNo,
3530                                        stmt.getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
3531                        String[] segments = stmt.getEndToken().getAstext().split("\n", -1);
3532                        if (segments.length == 1) {
3533                                errorInfo.setEndPosition(new Pair3<Long, Long, String>(stmt.getEndToken().lineNo,
3534                                                stmt.getEndToken().columnNo + stmt.getEndToken().getAstext().length(),
3535                                                ModelBindingManager.getGlobalHash()));
3536                        } else {
3537                                errorInfo.setEndPosition(new Pair3<Long, Long, String>(stmt.getEndToken().lineNo + segments.length - 1,
3538                                                (long) segments[segments.length - 1].length() + 1, ModelBindingManager.getGlobalHash()));
3539                        }
3540                        errorInfo.fillInfo(this);
3541                        errorInfos.add(errorInfo);
3542                }
3543        }
3544
3545        private void analyzePlsqlRecordTypeDefStmt(TPlsqlRecordTypeDefStmt stmt) {
3546                TObjectName typeName = stmt.getTypeName();
3547                Variable variable = modelFactory.createVariable(typeName);
3548                variable.setSubType(SubType.record_type);
3549        
3550                if (stmt.getFieldDeclarations() != null) {
3551                        for (int i = 0; i < stmt.getFieldDeclarations().size(); i++) {
3552                                TParameterDeclaration param = stmt.getFieldDeclarations().getParameterDeclarationItem(i);
3553                                String dataTypeName = param.getDataType().getDataTypeName();
3554                                TObjectName columnName = param.getParameterName();      
3555                                TableColumn variableProperty = modelFactory.createTableColumn(variable, columnName, true);
3556                                if(dataTypeName.indexOf(".")!=-1) {
3557                                        String tableName = dataTypeName.substring(0, dataTypeName.lastIndexOf("."));
3558                                        Table table = modelFactory.createTableByName(tableName, true);
3559                                        if(table!=null) {
3560                                                TableColumn tableColumn = modelFactory.createInsertTableColumn(table, dataTypeName);
3561                                                if (tableColumn != null) {
3562                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
3563                                                        relation.setEffectType(EffectType.rowtype);
3564                                                        relation.setTarget(new TableColumnRelationshipElement(variableProperty));
3565                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
3566                                                }
3567                                        }
3568                                }
3569                        }
3570                        variable.setDetermined(true);
3571                }
3572                else {
3573                        TObjectName starColumn = new TObjectName();
3574                        starColumn.setString("*");
3575                }
3576        }
3577        
3578        private void analyzePlsqlTableTypeDefStmt(TPlsqlTableTypeDefStmt stmt) {
3579                TTypeName typeName = stmt.getElementDataType();
3580                if (typeName != null && typeName.toString().toUpperCase().indexOf("ROWTYPE") != -1) {
3581                        Variable cursorVariable = modelFactory.createVariable(stmt.getTypeName());
3582                        cursorVariable.setSubType(SubType.record_type);
3583
3584                        Table variableTable = modelFactory.createTableByName(typeName.getDataTypeName(), false);
3585                        if(!variableTable.isCreateTable()) {
3586                                TObjectName starColumn1 = new TObjectName();
3587                                starColumn1.setString("*");
3588                                TableColumn variableTableStarColumn = modelFactory.createTableColumn(variableTable, starColumn1, true);
3589                                variableTableStarColumn.setShowStar(false);
3590                                variableTableStarColumn.setExpandStar(true);
3591
3592                                TObjectName starColumn = new TObjectName();
3593                                starColumn.setString("*");
3594                                TableColumn variableProperty = modelFactory.createTableColumn(cursorVariable, starColumn, true);
3595                                variableProperty.setShowStar(false);
3596                                variableProperty.setExpandStar(true);
3597
3598                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
3599                                dataflowRelation.setEffectType(EffectType.rowtype);
3600                                dataflowRelation.addSource(new TableColumnRelationshipElement(variableTableStarColumn));
3601                                dataflowRelation.setTarget(new TableColumnRelationshipElement(variableProperty));
3602                        } else {
3603                                for (TableColumn sourceColumn : variableTable.getColumns()) {
3604                                        String columnName = sourceColumn.getName();
3605                                        TObjectName targetColumn = new TObjectName();
3606                                        targetColumn.setString(columnName);
3607                                        TableColumn variableProperty = modelFactory.createTableColumn(cursorVariable, targetColumn, true);
3608                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
3609                                        dataflowRelation.setEffectType(EffectType.rowtype);
3610                                        dataflowRelation.addSource(new TableColumnRelationshipElement(sourceColumn));
3611                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(variableProperty));
3612                                }
3613                        }
3614                } 
3615        }
3616
3617        private void analyzeMssqlCreateType(TMssqlCreateType createType) {
3618
3619        }
3620
3621        private void analyzeMssqlExecute(TMssqlExecute executeStmt) {
3622                if (executeStmt.getModuleName() != null) {
3623                        TObjectName module = executeStmt.getModuleName();
3624                        if(module.toString().toLowerCase().endsWith("sp_rename")) {
3625                                String oldTableName = SQLUtil.trimColumnStringQuote(executeStmt.getParameters().getExecParameter(0).toString());        
3626                                Table oldNameTableModel = modelFactory.createTableByName(oldTableName, true);
3627                                List<String> oldTableNames = SQLUtil.parseNames(oldNameTableModel.getName());
3628                                TObjectName oldStarColumn = new TObjectName();
3629                                oldStarColumn.setString("*");
3630                                TableColumn oldTableStarColumn = modelFactory.createTableColumn(oldNameTableModel, oldStarColumn, true);
3631                                
3632                                String newTableName = SQLUtil.trimColumnStringQuote(executeStmt.getParameters().getExecParameter(1).toString());
3633                                List<String> newTableNames = SQLUtil.parseNames(newTableName);
3634                                if (oldTableNames.size() > newTableNames.size()) {
3635                                        for (int i = oldTableNames.size() - newTableNames.size() - 1; i >= 0; i--) {
3636                                                newTableName = (oldTableNames.get(i) + ".") + newTableName;
3637                                        }
3638                                }
3639
3640                                Table newNameTableModel = modelFactory.createTableByName(newTableName, true);
3641                                TObjectName newStarColumn = new TObjectName();
3642                                newStarColumn.setString("*");
3643                                TableColumn newTableStarColumn = modelFactory.createTableColumn(newNameTableModel, newStarColumn, true);
3644                                
3645                                Process process = modelFactory.createProcess(executeStmt);
3646                                newNameTableModel.addProcess(process);
3647                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
3648                                relation.setEffectType(EffectType.rename_table);
3649                                relation.setTarget(new TableColumnRelationshipElement(newTableStarColumn));
3650                                relation.addSource(new TableColumnRelationshipElement(oldTableStarColumn));
3651                                relation.setProcess(process);
3652                                
3653                                if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) {
3654                                        oldTableStarColumn.setShowStar(false);
3655                                        relation.setShowStarRelation(false);
3656                                }
3657
3658                                if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) {
3659                                        newTableStarColumn.setShowStar(false);
3660                                        relation.setShowStarRelation(false);
3661                                }
3662                        }
3663                        else if(module.toString().toLowerCase().endsWith("sp_executesql")) {
3664                                String sql = SQLUtil.trimColumnStringQuote(executeStmt.getParameters().getExecParameter(0).toString()); 
3665                                if(sql.startsWith("N")) {
3666                                        sql =  SQLUtil.trimColumnStringQuote(sql.substring(1));
3667                                }
3668                                executeDynamicSql(sql);
3669                        }
3670                        else {
3671                                int argumentSize = executeStmt.getParameters() == null ? 0 : executeStmt.getParameters().size();
3672                                String procedureNameWithArgSize = module.toString() + "(" + argumentSize + ")";
3673                                if (argumentSize <= 0  || !DlineageUtil.supportFunctionOverride(option.getVendor())) {
3674                                        procedureNameWithArgSize = module.toString();
3675                                }
3676                                if (procedureDDLMap.containsKey(procedureNameWithArgSize)) {
3677                                        analyzeCustomSqlStmt(procedureDDLMap.get(procedureNameWithArgSize));
3678                                }
3679                                Procedure procedure = modelFactory.createProcedureByName(module, executeStmt.getParameters() == null ? 0 : executeStmt.getParameters().size());
3680                                String procedureParent = getProcedureParentName(executeStmt);
3681                                if (procedureParent != null) {
3682                                        Procedure caller = modelManager
3683                                                        .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
3684                                        if (caller != null) {
3685                                                CallRelationship callRelation = modelFactory.createCallRelation();
3686                                                callRelation.setCallObject(executeStmt);
3687                                                callRelation.setTarget(new ProcedureRelationshipElement(caller));
3688                                                callRelation.addSource(new ProcedureRelationshipElement(procedure));
3689                                                if(isBuiltInFunctionName(module) || isKeyword(module)){
3690                                                        callRelation.setBuiltIn(true);
3691                                                }
3692                                        }
3693                                }
3694
3695                                if (procedure.getArguments() != null) {
3696                                        for (int i = 0; i < procedure.getArguments().size(); i++) {
3697                                                Argument argument = procedure.getArguments().get(i);
3698                                                Variable variable = modelFactory.createVariable(procedure, argument.getName(), false);
3699                                                if (variable != null) {
3700                                                        if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
3701                                                                Transform transform = new Transform();
3702                                                                transform.setType(Transform.FUNCTION);
3703                                                                transform.setCode(module);
3704                                                                variable.getColumns().get(0).setTransform(transform);
3705                                                        }
3706                                                        Process process = modelFactory.createProcess(executeStmt);
3707                                                        variable.addProcess(process);
3708                                                        analyzeFunctionArgumentsDataFlowRelation(variable.getColumns().get(0), executeStmt, argument.getName(), i, process);
3709                                                }
3710                                        }
3711                                }
3712                        }
3713                }
3714        }
3715
3716        private void analyzeDropTableStmt(TDropTableSqlStatement stmt) {
3717                TTable dropTable = stmt.getTargetTable();
3718                if(dropTable == null) {
3719                        return;
3720                }
3721                Table tableModel = modelManager.getTableByName(DlineageUtil.getTableFullName(dropTable.getTableName().toString()));
3722                if(tableModel!=null) {
3723                        modelManager.dropTable(tableModel);
3724                }
3725                
3726                if(option.getAnalyzeMode() == AnalyzeMode.crud) {
3727                        tableModel = modelFactory.createTable(dropTable);
3728                        CrudRelationship crudRelationship = modelFactory.createCrudRelation();
3729                        crudRelationship.setTarget(new TableRelationshipElement(tableModel));
3730                        crudRelationship.setEffectType(EffectType.drop_table);
3731                }
3732        }
3733        
3734        private void analyzeTruncateTableStmt(TTruncateStatement stmt) {
3735                if(option.getAnalyzeMode() == AnalyzeMode.crud) {
3736                        TObjectName table = stmt.getTableName();
3737                        Table tableModel = modelFactory.createTableByName(table);
3738                        CrudRelationship crudRelationship = modelFactory.createCrudRelation();
3739                        crudRelationship.setTarget(new TableRelationshipElement(tableModel));
3740                        crudRelationship.setEffectType(EffectType.truncate_table);
3741                }
3742        }
3743
3744        private void analyzeCallStmt(TCallStatement callStmt) {
3745                if (callStmt.getRoutineExpr() != null && callStmt.getRoutineExpr().getFunctionCall() != null) {
3746
3747                        TFunctionCall functionCall = callStmt.getRoutineExpr().getFunctionCall();
3748                        Procedure callee = modelManager.getProcedureByName(
3749                                        DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
3750                        if (callee == null && procedureDDLMap.containsKey(DlineageUtil.getFunctionNameWithArgNum(functionCall))) {
3751                                analyzeCustomSqlStmt(procedureDDLMap.get(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
3752                                callee = modelManager.getProcedureByName(DlineageUtil
3753                                                .getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
3754                        }
3755                        if (callee != null) {
3756                                String procedureParent = getProcedureParentName(callStmt);
3757                                if (procedureParent != null) {
3758                                        Procedure caller = modelManager
3759                                                        .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
3760                                        if (caller != null) {
3761                                                CallRelationship callRelation = modelFactory.createCallRelation();
3762                                                callRelation.setCallObject(callStmt);
3763                                                callRelation.setTarget(new ProcedureRelationshipElement(caller));
3764                                                callRelation.addSource(new ProcedureRelationshipElement(callee));
3765                                                if (isBuiltInFunctionName(functionCall.getFunctionName())
3766                                                                || isKeyword(functionCall.getFunctionName())) {
3767                                                        callRelation.setBuiltIn(true);
3768                                                }
3769                                        }
3770                                }
3771                                if (callee.getArguments() != null) {
3772                                        for (int i = 0; i < callee.getArguments().size(); i++) {
3773                                                Argument argument = callee.getArguments().get(i);
3774                                                Variable variable = modelFactory.createVariable(callee, argument.getName(), false);
3775                                                if (variable != null) {
3776                                                        if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
3777                                                                Transform transform = new Transform();
3778                                                                transform.setType(Transform.FUNCTION);
3779                                                                transform.setCode(functionCall);
3780                                                                variable.getColumns().get(0).setTransform(transform);
3781                                                        }
3782                                                        Process process = modelFactory.createProcess(callStmt);
3783                                                        variable.addProcess(process);
3784                                                        analyzeFunctionArgumentsDataFlowRelation(variable.getColumns().get(0), functionCall, i,
3785                                                                        process);
3786                                                }
3787                                        }
3788                                }
3789                        } else {
3790                                Function function = (Function) createFunction(functionCall);
3791                                String procedureParent = getProcedureParentName(callStmt);
3792                                if (procedureParent != null) {
3793                                        Procedure caller = modelManager
3794                                                        .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
3795                                        if (caller != null) {
3796                                                CallRelationship callRelation = modelFactory.createCallRelation();
3797                                                callRelation.setCallObject(callStmt);
3798                                                callRelation.setTarget(new ProcedureRelationshipElement(caller));
3799                                                callRelation.addSource(new FunctionRelationshipElement(function));
3800                                                if (isBuiltInFunctionName(functionCall.getFunctionName())
3801                                                                || isKeyword(functionCall.getFunctionName())) {
3802                                                        callRelation.setBuiltIn(true);
3803                                                }
3804                                        }
3805                                }
3806                        }
3807                }
3808                else if (callStmt.getRoutineName() != null) {
3809                        TObjectName function = callStmt.getRoutineName();
3810                        String functionName = function.toString();
3811                        Procedure callee = modelManager.getProcedureByName(
3812                                        DlineageUtil.getIdentifierNormalTableName(functionName));
3813                        if (callee == null && procedureDDLMap.containsKey(functionName)) {
3814                                analyzeCustomSqlStmt(procedureDDLMap.get(functionName));
3815                                callee = modelManager.getProcedureByName(functionName);
3816                        }
3817                        if (callee != null) {
3818                                String procedureParent = getProcedureParentName(callStmt);
3819                                if (procedureParent != null) {
3820                                        Procedure caller = modelManager
3821                                                        .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
3822                                        if (caller != null) {
3823                                                CallRelationship callRelation = modelFactory.createCallRelation();
3824                                                callRelation.setCallObject(callStmt);
3825                                                callRelation.setTarget(new ProcedureRelationshipElement(caller));
3826                                                callRelation.addSource(new ProcedureRelationshipElement(callee));
3827                                                if (isBuiltInFunctionName(function)
3828                                                                || isKeyword(function)) {
3829                                                        callRelation.setBuiltIn(true);
3830                                                }
3831                                        }
3832                                }
3833                                if (callee.getArguments() != null) {
3834                                        for (int i = 0; i < callee.getArguments().size(); i++) {
3835                                                Argument argument = callee.getArguments().get(i);
3836                                                Variable variable = modelFactory.createVariable(callee, argument.getName(), false);
3837                                                if (variable != null) {
3838                                                        if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
3839                                                                Transform transform = new Transform();
3840                                                                transform.setType(Transform.FUNCTION);
3841                                                                transform.setCode(callStmt);
3842                                                                variable.getColumns().get(0).setTransform(transform);
3843                                                        }
3844                                                        Process process = modelFactory.createProcess(callStmt);
3845                                                        variable.addProcess(process);
3846                                                        analyzeFunctionArgumentsDataFlowRelation(variable.getColumns().get(0), callStmt, i,
3847                                                                        process);
3848                                                }
3849                                        }
3850                                }
3851                        }
3852                }
3853        }
3854
3855        private boolean analyzeCustomFunctionCall(TFunctionCall functionCall) {
3856                Procedure callee = modelManager.getProcedureByName(
3857                                DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
3858                if(callee == null && procedureDDLMap.containsKey(DlineageUtil.getFunctionNameWithArgNum(functionCall))) {
3859                        analyzeCustomSqlStmt(procedureDDLMap.get(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
3860                        callee = modelManager.getProcedureByName(
3861                                        DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
3862                }
3863                if (callee != null) {
3864                        String procedureParent = getProcedureParentName(stmtStack.peek());
3865                        if (procedureParent != null) {
3866                                Procedure caller = modelManager
3867                                                .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
3868                                if (caller != null) {
3869                                        CallRelationship callRelation = modelFactory.createCallRelation();
3870                                        callRelation.setCallObject(functionCall);
3871                                        callRelation.setTarget(new ProcedureRelationshipElement(caller));
3872                                        callRelation.addSource(new ProcedureRelationshipElement(callee));
3873                                        if(isBuiltInFunctionName(functionCall.getFunctionName()) || isKeyword(functionCall.getFunctionName())){
3874                                                callRelation.setBuiltIn(true);
3875                                        }
3876                                }
3877                        }
3878                        if (callee.getArguments() != null) {
3879                                for (int i = 0; i < callee.getArguments().size(); i++) {
3880                                        Argument argument = callee.getArguments().get(i);
3881                                        Variable variable = modelFactory.createVariable(callee, argument.getName(), false);
3882                                        if(variable!=null) {
3883                                                if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
3884                                                        Transform transform = new Transform();
3885                                                        transform.setType(Transform.FUNCTION);
3886                                                        transform.setCode(functionCall);
3887                                                        variable.getColumns().get(0).setTransform(transform);
3888                                                }
3889                                                Process process = modelFactory.createProcess(functionCall);
3890                                                variable.addProcess(process);
3891                                                analyzeFunctionArgumentsDataFlowRelation(variable.getColumns().get(0), functionCall, i, process);
3892                                        }
3893                                }
3894                        }
3895                        return true;
3896                }
3897                return false;
3898        }
3899
3900        private void analyzeOracleBasicStmt(TBasicStmt oracleBasicStmt) {
3901                if (oracleBasicStmt.getExpr() == null || oracleBasicStmt.getExpr().getFunctionCall() == null) {
3902                        return;
3903                }
3904
3905                TFunctionCall functionCall = oracleBasicStmt.getExpr().getFunctionCall();
3906                Procedure callee = modelManager.getProcedureByName(
3907                                DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
3908                if(callee == null && procedureDDLMap.containsKey(DlineageUtil.getFunctionNameWithArgNum(functionCall))) {
3909                        analyzeCustomSqlStmt(procedureDDLMap.get(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
3910                        callee = modelManager.getProcedureByName(
3911                                        DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
3912                }
3913                if (callee != null) {
3914                        String procedureParent = getProcedureParentName(oracleBasicStmt);
3915                        if (procedureParent != null) {
3916                                Procedure caller = modelManager
3917                                                .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
3918                                if (caller != null) {
3919                                        CallRelationship callRelation = modelFactory.createCallRelation();
3920                                        callRelation.setCallObject(functionCall);
3921                                        callRelation.setTarget(new ProcedureRelationshipElement(caller));
3922                                        callRelation.addSource(new ProcedureRelationshipElement(callee));
3923                                        if (functionChecker.isOraclePredefinedPackageFunction(functionCall.getFunctionName().toString())
3924                                                        || (isBuiltInFunctionName(functionCall.getFunctionName())
3925                                                        || isKeyword(functionCall.getFunctionName()))) {
3926                                                callRelation.setBuiltIn(true);
3927                                        }
3928                                }
3929                        }
3930                        if (callee.getArguments() != null) {
3931                                for (int i = 0; i < callee.getArguments().size(); i++) {
3932                                        Argument argument = callee.getArguments().get(i);
3933                                        Variable variable = modelFactory.createVariable(callee, argument.getName(), false);
3934                                        if(variable!=null) {
3935                                                if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
3936                                                        Transform transform = new Transform();
3937                                                        transform.setType(Transform.FUNCTION);
3938                                                        transform.setCode(functionCall);
3939                                                        variable.getColumns().get(0).setTransform(transform);
3940                                                }
3941                                                Process process = modelFactory.createProcess(functionCall);
3942                                                variable.addProcess(process);
3943                                                analyzeFunctionArgumentsDataFlowRelation(variable.getColumns().get(0), functionCall, i, process);
3944                                        }
3945                                }
3946                        }
3947                } else {
3948                        Function function = modelFactory.createFunction(functionCall);
3949                        String procedureParent = getProcedureParentName(oracleBasicStmt);
3950                        if (procedureParent != null) {
3951                                Procedure caller = modelManager
3952                                                .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
3953                                if (caller != null) {
3954                                        CallRelationship callRelation = modelFactory.createCallRelation();
3955                                        callRelation.setCallObject(functionCall);
3956                                        callRelation.setTarget(new ProcedureRelationshipElement(caller));
3957                                        callRelation.addSource(new FunctionRelationshipElement(function));
3958                                        if (functionChecker.isOraclePredefinedPackageFunction(functionCall.getFunctionName().toString())
3959                                                        || (isBuiltInFunctionName(functionCall.getFunctionName()) || isKeyword(functionCall.getFunctionName()))) {
3960                                                callRelation.setBuiltIn(true);
3961                                        }
3962                                }
3963                        }
3964                }
3965        }
3966
3967        private void analyzeIfStmt(TIfStmt ifStmt) {
3968                if (ifStmt.getCondition() != null) {
3969                        columnsInExpr visitor = new columnsInExpr();
3970                        ifStmt.getCondition().inOrderTraverse(visitor);
3971                        List<TParseTreeNode> functions = visitor.getFunctions();
3972
3973                        if (functions != null && !functions.isEmpty()) {
3974                                for (int i = 0; i < functions.size(); i++) {
3975                                        if (!(functions.get(i) instanceof TFunctionCall))
3976                                                continue;
3977                                        TFunctionCall functionCall = (TFunctionCall) functions.get(i);
3978                                        Procedure callee = modelManager.getProcedureByName(DlineageUtil
3979                                                        .getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
3980                                        if(callee == null && procedureDDLMap.containsKey(DlineageUtil.getFunctionNameWithArgNum(functionCall))) {
3981                                                analyzeCustomSqlStmt(procedureDDLMap.get(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
3982                                                callee = modelManager.getProcedureByName(
3983                                                                DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
3984                                        }
3985                                        if (callee != null) {
3986                                                String procedureParent = getProcedureParentName(ifStmt);
3987                                                if (procedureParent != null) {
3988                                                        Procedure caller = modelManager
3989                                                                        .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
3990                                                        if (caller != null) {
3991                                                                CallRelationship callRelation = modelFactory.createCallRelation();
3992                                                                callRelation.setCallObject(functionCall);
3993                                                                callRelation.setTarget(new ProcedureRelationshipElement(caller));
3994                                                                callRelation.addSource(new ProcedureRelationshipElement(callee));
3995                                                                if (isBuiltInFunctionName(functionCall.getFunctionName()) || isKeyword(functionCall.getFunctionName())) {
3996                                                                        callRelation.setBuiltIn(true);
3997                                                                }
3998                                                        }
3999                                                }
4000                                                if (callee.getArguments() != null) {
4001                                                        for (int j = 0; j < callee.getArguments().size(); j++) {
4002                                                                Argument argument = callee.getArguments().get(j);
4003                                                                Variable variable = modelFactory.createVariable(callee, argument.getName(), false);
4004                                                                if(variable!=null) {
4005                                                                        if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
4006                                                                                Transform transform = new Transform();
4007                                                                                transform.setType(Transform.FUNCTION);
4008                                                                                transform.setCode(functionCall);
4009                                                                                variable.getColumns().get(0).setTransform(transform);
4010                                                                        }
4011                                                                        Process process = modelFactory.createProcess(functionCall);
4012                                                                        variable.addProcess(process);
4013                                                                        analyzeFunctionArgumentsDataFlowRelation(variable.getColumns().get(0), functionCall, j, process);
4014                                                                }
4015                                                        }
4016                                                }
4017                                        } else {
4018                                                Function function = modelFactory.createFunction(functionCall);
4019                                                String procedureParent = getProcedureParentName(ifStmt);
4020                                                if (procedureParent != null) {
4021                                                        Procedure caller = modelManager
4022                                                                        .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
4023                                                        if (caller != null) {
4024                                                                CallRelationship callRelation = modelFactory.createCallRelation();
4025                                                                callRelation.setCallObject(functionCall);
4026                                                                callRelation.setTarget(new ProcedureRelationshipElement(caller));
4027                                                                callRelation.addSource(new FunctionRelationshipElement(function));
4028                                                                if (isBuiltInFunctionName(functionCall.getFunctionName()) || isKeyword(functionCall.getFunctionName())) {
4029                                                                        callRelation.setBuiltIn(true);
4030                                                                }
4031                                                        }
4032                                                }
4033                                        }
4034                                }
4035                        }
4036                }
4037
4038                if (ifStmt.getThenStatements() != null) {
4039                        for (int i = 0; i < ifStmt.getThenStatements().size(); ++i) {
4040                                analyzeCustomSqlStmt(ifStmt.getThenStatements().get(i));
4041                                ResultSet returnResult = modelFactory.createResultSet(ifStmt.getThenStatements().get(i), false);
4042                                if(returnResult!=null){
4043                                        for(ResultColumn resultColumn: returnResult.getColumns()){
4044                                                analyzeFilterCondition(resultColumn, ifStmt.getCondition(), null, null, EffectType.function);
4045                                        }
4046                                }
4047                        }
4048                }
4049
4050                if (ifStmt.getElseifStatements() != null) {
4051                        for (int i = 0; i < ifStmt.getElseifStatements().size(); ++i) {
4052                                analyzeCustomSqlStmt(ifStmt.getElseifStatements().get(i));
4053                        }
4054                }
4055
4056                if (ifStmt.getElseStatements() != null) {
4057                        for (int i = 0; i < ifStmt.getElseStatements().size(); ++i) {
4058                                analyzeCustomSqlStmt(ifStmt.getElseStatements().get(i));
4059                        }
4060                }
4061        }
4062
4063        private void analyzeElsIfStmt(TElsifStmt elsIfStmt) {
4064                if (elsIfStmt.getCondition() != null) {
4065                        columnsInExpr visitor = new columnsInExpr();
4066                        elsIfStmt.getCondition().inOrderTraverse(visitor);
4067                        List<TParseTreeNode> functions = visitor.getFunctions();
4068
4069                        if (functions != null && !functions.isEmpty()) {
4070                                for (int i = 0; i < functions.size(); i++) {
4071                                        if (!(functions.get(i) instanceof TFunctionCall))
4072                                                continue;
4073                                        TFunctionCall functionCall = (TFunctionCall) functions.get(i);
4074                                        Procedure callee = modelManager.getProcedureByName(DlineageUtil
4075                                                        .getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
4076                                        if(callee == null && procedureDDLMap.containsKey(DlineageUtil.getFunctionNameWithArgNum(functionCall))) {
4077                                                analyzeCustomSqlStmt(procedureDDLMap.get(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
4078                                                callee = modelManager.getProcedureByName(
4079                                                                DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
4080                                        }
4081                                        if (callee != null) {
4082                                                String procedureParent = getProcedureParentName(elsIfStmt);
4083                                                if (procedureParent != null) {
4084                                                        Procedure caller = modelManager
4085                                                                        .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
4086                                                        if (caller != null) {
4087                                                                CallRelationship callRelation = modelFactory.createCallRelation();
4088                                                                callRelation.setCallObject(functionCall);
4089                                                                callRelation.setTarget(new ProcedureRelationshipElement(caller));
4090                                                                callRelation.addSource(new ProcedureRelationshipElement(callee));
4091                                                                if(isBuiltInFunctionName(functionCall.getFunctionName()) || isKeyword(functionCall.getFunctionName())){
4092                                                                        callRelation.setBuiltIn(true);
4093                                                                }
4094                                                        }
4095                                                }
4096                                                if (callee.getArguments() != null) {
4097                                                        for (int j = 0; j < callee.getArguments().size(); j++) {
4098                                                                Argument argument = callee.getArguments().get(j);
4099                                                                Variable variable = modelFactory.createVariable(callee, argument.getName(), false);
4100                                                                if(variable!=null) {
4101                                                                        if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
4102                                                                                Transform transform = new Transform();
4103                                                                                transform.setType(Transform.FUNCTION);
4104                                                                                transform.setCode(functionCall);
4105                                                                                variable.getColumns().get(0).setTransform(transform);
4106                                                                        }
4107                                                                        Process process = modelFactory.createProcess(functionCall);
4108                                                                        variable.addProcess(process);
4109                                                                        analyzeFunctionArgumentsDataFlowRelation(variable.getColumns().get(0), functionCall, j, process);
4110                                                                }
4111                                                        }
4112                                                }
4113                                        } else {
4114                                                Function function = modelFactory.createFunction(functionCall);
4115                                                String procedureParent = getProcedureParentName(elsIfStmt);
4116                                                if (procedureParent != null) {
4117                                                        Procedure caller = modelManager
4118                                                                        .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
4119                                                        if (caller != null) {
4120                                                                CallRelationship callRelation = modelFactory.createCallRelation();
4121                                                                callRelation.setCallObject(functionCall);
4122                                                                callRelation.setTarget(new ProcedureRelationshipElement(caller));
4123                                                                callRelation.addSource(new FunctionRelationshipElement(function));
4124                                                                if(isBuiltInFunctionName(functionCall.getFunctionName()) || isKeyword(functionCall.getFunctionName())){
4125                                                                        callRelation.setBuiltIn(true);
4126                                                                }
4127                                                        }
4128                                                }
4129                                        }
4130                                }
4131                        }
4132                }
4133
4134                if (elsIfStmt.getThenStatements() != null) {
4135                        for (int i = 0; i < elsIfStmt.getThenStatements().size(); ++i) {
4136                                ResultSet returnResult = modelFactory.createResultSet(elsIfStmt.getThenStatements().get(i), false);
4137                                if (returnResult != null) {
4138                                        for (ResultColumn resultColumn : returnResult.getColumns()) {
4139                                                analyzeFilterCondition(resultColumn, elsIfStmt.getCondition(), null, null, EffectType.function);
4140                                        }
4141                                }
4142                                analyzeCustomSqlStmt(elsIfStmt.getThenStatements().get(i));
4143                        }
4144                }
4145        }
4146
4147        private void analyzeCloneTableStmt(TCreateTableSqlStatement stmt) {
4148                if (stmt.getCloneSourceTable() != null) {
4149                        Table sourceTable = modelFactory.createTableByName(stmt.getCloneSourceTable());
4150                        Table cloneTable = modelFactory.createTableByName(stmt.getTableName());
4151                        Process process = modelFactory.createProcess(stmt);
4152                        cloneTable.addProcess(process);
4153
4154                        if (sourceTable.isDetermined()) {
4155                                for (int k = 0; k < sourceTable.getColumns().size(); k++) {
4156                                        TableColumn sourceColumn = sourceTable.getColumns().get(k);
4157                                        TObjectName objectName = new TObjectName();
4158                                        objectName.setString(sourceColumn.getName());
4159                                        TableColumn tableColumn = modelFactory.createTableColumn(cloneTable, objectName, true);
4160                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
4161                                        dataflowRelation.setEffectType(EffectType.clone_table);
4162                                        dataflowRelation
4163                                                        .addSource(new TableColumnRelationshipElement(sourceColumn));
4164                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(tableColumn));
4165                                        dataflowRelation.setProcess(process);
4166                                }
4167                                cloneTable.setDetermined(true);
4168                                cloneTable.setFromDDL(true);
4169                        } else {
4170                                TObjectName sourceName = new TObjectName();
4171                                sourceName.setString("*");
4172                                TableColumn sourceTableColumn = modelFactory.createTableColumn(sourceTable, sourceName, false);
4173                                TObjectName targetName = new TObjectName();
4174                                targetName.setString("*");
4175                                TableColumn targetTableColumn = modelFactory.createTableColumn(cloneTable, targetName, false);
4176                                if(sourceTableColumn!=null && targetTableColumn!=null) {
4177                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
4178                                        dataflowRelation.setEffectType(EffectType.clone_table);
4179                                        dataflowRelation
4180                                                        .addSource(new TableColumnRelationshipElement(sourceTableColumn));
4181                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(targetTableColumn));
4182                                        dataflowRelation.setProcess(process);
4183                                }
4184                        }
4185                }
4186        }
4187
4188        private void analyzeCloneDatabaseStmt(TCreateDatabaseSqlStatement stmt) {
4189                if (stmt.getCloneSourceDb() != null) {
4190                        Database sourceDatabase = modelFactory.createDatabase(stmt.getCloneSourceDb());
4191                        Database cloneDatabase = modelFactory.createDatabase(stmt.getDatabaseName());
4192                        Process process = modelFactory.createProcess(stmt);
4193                        cloneDatabase.addProcess(process);
4194                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
4195                        relation.setEffectType(EffectType.clone_database);
4196                        relation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(cloneDatabase.getRelationRows()));
4197                        relation.addSource(
4198                                        new RelationRowsRelationshipElement<TableRelationRows>(sourceDatabase.getRelationRows()));
4199                        relation.setProcess(process);
4200                }
4201        }
4202
4203        private void analyzeCloneSchemaStmt(TCreateSchemaSqlStatement stmt) {
4204                if (stmt.getCloneSourceSchema() != null) {
4205                        Schema sourceSchema = modelFactory.createSchema(stmt.getCloneSourceSchema());
4206                        Schema cloneSchema = modelFactory.createSchema(stmt.getSchemaName());
4207                        Process process = modelFactory.createProcess(stmt);
4208                        cloneSchema.addProcess(process);
4209                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
4210                        relation.setEffectType(EffectType.clone_schema);
4211                        relation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(cloneSchema.getRelationRows()));
4212                        relation.addSource(new RelationRowsRelationshipElement<TableRelationRows>(sourceSchema.getRelationRows()));
4213                        relation.setProcess(process);
4214                }
4215        }
4216
4217        private void executeDynamicSql(String sql) {
4218                TGSqlParser sqlparser = new TGSqlParser(option.getVendor());
4219                sqlparser.sqltext = sql;
4220                int result = sqlparser.parse();
4221                if (result == 0) {
4222                        for (int i = 0; i < sqlparser.sqlstatements.size(); i++) {
4223                                analyzeCustomSqlStmt(sqlparser.sqlstatements.get(i));
4224                        }
4225                }
4226        }
4227
4228        private void extractSnowflakeSQLFromProcedure(TCreateProcedureStmt procedure) {
4229                Map<String, String> argMap = new LinkedHashMap<String, String>();
4230                if (procedure.getParameterDeclarations() != null) {
4231                        for (int i = 0; i < procedure.getParameterDeclarations().size(); i++) {
4232                                TParameterDeclaration def = procedure.getParameterDeclarations().getParameterDeclarationItem(i);
4233                                argMap.put(def.getParameterName().toString(), def.getDataType().getDataTypeName());
4234                        }
4235                }
4236                StringBuilder buffer = new StringBuilder();
4237                buffer.append("(function(");
4238                String[] args = argMap.keySet().toArray(new String[0]);
4239                for (int i = 0; i < args.length; i++) {
4240                        buffer.append(args[i].toUpperCase());
4241                        if (i < args.length - 1) {
4242                                buffer.append(",");
4243                        }
4244                }
4245                buffer.append("){\n");
4246
4247                int start = -1;
4248                int end = -1;
4249                boolean dollar = false;
4250                boolean quote = false;
4251                if (procedure.getRoutineBody().indexOf("$$") != -1) {
4252                        start = procedure.getRoutineBody().indexOf("$$") + 2;
4253                        end = procedure.getRoutineBody().lastIndexOf("$$") - 1;
4254                        dollar = true;
4255                } else if (procedure.getRoutineBody().indexOf("'") != -1) {
4256                        start = procedure.getRoutineBody().indexOf("'") + 1;
4257                        end = procedure.getRoutineBody().lastIndexOf("'");
4258                        quote = true;
4259                }
4260                String body = procedure.getRoutineBody().substring(start, end);
4261                if (dollar && body.indexOf("`") != -1) {
4262                        Pattern pattern = Pattern.compile("`.+?`", Pattern.CASE_INSENSITIVE | Pattern.DOTALL);
4263                        Matcher matcher = pattern.matcher(body);
4264                        StringBuffer replaceBuffer = new StringBuffer();
4265                        while (matcher.find()) {
4266                                String condition = matcher.group().replace("\r\n", "\n").replace("'", "\\\\'")
4267                                                .replace("\n", "\\\\n'\n+'").replace("`", "'").replace("$", "RDS_CHAR_DOLLAR");
4268                                matcher.appendReplacement(replaceBuffer, condition);
4269                        }
4270                        matcher.appendTail(replaceBuffer);
4271                        body = replaceBuffer.toString().replace("RDS_CHAR_DOLLAR", "$");
4272                }
4273                if (quote && body.indexOf("'") != -1) {
4274                        body = body.replace("''", "'");
4275                }
4276                buffer.append(body);
4277                buffer.append("})(");
4278                for (int i = 0; i < args.length; i++) {
4279                        String type = argMap.get(args[i]);
4280                        if (type.equalsIgnoreCase("VARCHAR")) {
4281                                buffer.append("'pseudo'");
4282                        } else if (type.equalsIgnoreCase("STRING")) {
4283                                buffer.append("'pseudo'");
4284                        } else if (type.equalsIgnoreCase("CHAR")) {
4285                                buffer.append("'pseudo'");
4286                        } else if (type.equalsIgnoreCase("CHARACTER")) {
4287                                buffer.append("'pseudo'");
4288                        } else if (type.equalsIgnoreCase("TEXT")) {
4289                                buffer.append("'pseudo'");
4290                        } else if (type.equalsIgnoreCase("BINARY")) {
4291                                buffer.append("'pseudo'");
4292                        } else if (type.equalsIgnoreCase("VARBINARY")) {
4293                                buffer.append("'pseudo'");
4294                        } else if (type.equalsIgnoreCase("BOOLEAN")) {
4295                                buffer.append(true);
4296                        } else if (type.equalsIgnoreCase("FLOAT")) {
4297                                buffer.append("1.0");
4298                        } else if (type.equalsIgnoreCase("FLOAT4")) {
4299                                buffer.append("1.0");
4300                        } else if (type.equalsIgnoreCase("FLOAT8")) {
4301                                buffer.append("1.0");
4302                        } else if (type.equalsIgnoreCase("DOUBLE")) {
4303                                buffer.append("1.0");
4304                        } else if (type.equalsIgnoreCase("DOUBLE PRECISION")) {
4305                                buffer.append("1.0");
4306                        } else if (type.equalsIgnoreCase("REAL")) {
4307                                buffer.append("1.0");
4308                        } else if (type.equalsIgnoreCase("NUMBER")) {
4309                                buffer.append("1.0");
4310                        } else if (type.equalsIgnoreCase("DECIMAL")) {
4311                                buffer.append("1.0");
4312                        } else if (type.equalsIgnoreCase("NUMERIC")) {
4313                                buffer.append("1.0");
4314                        } else if (type.equalsIgnoreCase("INT")) {
4315                                buffer.append("1");
4316                        } else if (type.equalsIgnoreCase("INTEGER")) {
4317                                buffer.append("1");
4318                        } else if (type.equalsIgnoreCase("BIGINT")) {
4319                                buffer.append("1");
4320                        } else if (type.equalsIgnoreCase("SMALLINT")) {
4321                                buffer.append("1");
4322                        } else if (type.equalsIgnoreCase("DATE")) {
4323                                buffer.append("new Date()");
4324                        } else if (type.equalsIgnoreCase("DATETIME")) {
4325                                buffer.append("new Date()");
4326                        } else if (type.equalsIgnoreCase("TIME")) {
4327                                buffer.append("new Date()");
4328                        } else if (type.equalsIgnoreCase("TIMESTAMP")) {
4329                                buffer.append("new Date()");
4330                        } else if (type.equalsIgnoreCase("TIMESTAMP_LTZ")) {
4331                                buffer.append("new Date()");
4332                        } else if (type.equalsIgnoreCase("TIMESTAMP_NTZ")) {
4333                                buffer.append("new Date()");
4334                        } else if (type.equalsIgnoreCase("TIMESTAMP_TZ")) {
4335                                buffer.append("new Date()");
4336                        } else if (type.equalsIgnoreCase("VARIANT")) {
4337                                buffer.append("{}");
4338                        } else if (type.equalsIgnoreCase("OBJECT")) {
4339                                buffer.append("{}");
4340                        } else if (type.equalsIgnoreCase("ARRAY")) {
4341                                buffer.append("[]");
4342                        } else if (type.equalsIgnoreCase("GEOGRAPHY")) {
4343                                buffer.append("{}");
4344                        }
4345                        if (i < args.length - 1) {
4346                                buffer.append(",");
4347                        }
4348                }
4349                buffer.append(");");
4350
4351                try {
4352                        ScriptEngineManager scriptEngineManager = new ScriptEngineManager();
4353                        ScriptEngine nashorn = scriptEngineManager.getEngineByName("nashorn");
4354                        nashorn.put("analyzer", this);
4355                        nashorn.eval(new InputStreamReader(
4356                                        getClass().getResourceAsStream("/gudusoft/gsqlparser/parser/snowflake/snowflake.js")));
4357                        nashorn.eval(new StringReader(buffer.toString()));
4358                } catch (ScriptException e) {
4359                        TGSqlParser sqlparser = new TGSqlParser(option.getVendor());
4360                        sqlparser.sqltext = body;
4361                        int result = sqlparser.parse();
4362                        if (result == 0) {
4363                                for (int i = 0; i < sqlparser.sqlstatements.size(); i++) {
4364                                        analyzeCustomSqlStmt(sqlparser.sqlstatements.get(i));
4365                                }
4366                                return;
4367                        }
4368                        ErrorInfo errorInfo = new ErrorInfo();
4369                        errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
4370                        errorInfo.setErrorMessage("Invoke script error: " + e.getMessage());
4371                        errorInfo.setStartPosition(new Pair3<Long, Long, String>(procedure.getStartToken().lineNo,
4372                                        procedure.getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
4373                        errorInfo.setEndPosition(new Pair3<Long, Long, String>(procedure.getEndToken().lineNo,
4374                                        procedure.getEndToken().columnNo + procedure.getEndToken().getAstext().length(),
4375                                        ModelBindingManager.getGlobalHash()));
4376                        errorInfos.add(errorInfo);
4377                }
4378        }
4379
4380        private void analyzeHiveLoadStmt(THiveLoad stmt) {
4381                if (stmt.getPath() != null && stmt.getTable() != null) {
4382                        Table uriFile = modelFactory.createTableByName(stmt.getPath(), true);
4383                        uriFile.setPath(true);
4384                        uriFile.setCreateTable(true);
4385                        TObjectName fileUri = new TObjectName();
4386                        fileUri.setString("uri=" + stmt.getPath());
4387                        TableColumn fileUriColumn = modelFactory.createFileUri(uriFile, fileUri);
4388
4389                        Table tableModel = modelFactory.createTable(stmt.getTable());
4390                        Process process = modelFactory.createProcess(stmt);
4391                        tableModel.addProcess(process);
4392
4393                        TPartitionExtensionClause p = stmt.getTable().getPartitionExtensionClause();
4394                        if (p.getKeyValues() != null && p.getKeyValues().size() > 0) {
4395                                for (int i = 0; i < p.getKeyValues().size(); i++) {
4396                                        TExpression expression = p.getKeyValues().getExpression(i);
4397                                        if (expression.getLeftOperand().getExpressionType() == EExpressionType.simple_object_name_t) {
4398                                                modelFactory.createTableColumn(tableModel, expression.getLeftOperand().getObjectOperand(),
4399                                                                true);
4400                                        }
4401                                }
4402                        }
4403
4404                        for (int j = 0; j < tableModel.getColumns().size(); j++) {
4405                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
4406                                relation.addSource(new TableColumnRelationshipElement(fileUriColumn));
4407                                relation.setTarget(new TableColumnRelationshipElement(tableModel.getColumns().get(j)));
4408                                relation.setProcess(process);
4409                        }
4410                }
4411        }
4412        
4413        private void analyzeLoadDataStmt(TLoadDataStmt stmt) {
4414                
4415        }
4416
4417        private void analyzeUnloadStmt(TUnloadStmt unloadStmt) {
4418                if (unloadStmt.getSelectSqlStatement() != null && unloadStmt.getS3() != null) {
4419
4420                        Table uriFile = modelFactory.createTableByName(unloadStmt.getS3(), true);
4421                        uriFile.setPath(true);
4422                        uriFile.setCreateTable(true);
4423                        TObjectName fileUri = new TObjectName();
4424                        fileUri.setString("uri=" + unloadStmt.getS3());
4425                        TableColumn fileUriColumn = modelFactory.createFileUri(uriFile, fileUri);
4426
4427                        Process process = modelFactory.createProcess(unloadStmt);
4428                        uriFile.addProcess(process);
4429
4430                        TCustomSqlStatement stmt = unloadStmt.getSelectSqlStatement();
4431                        analyzeCustomSqlStmt(stmt);
4432                        if (stmt instanceof TSelectSqlStatement) {
4433                                TSelectSqlStatement select = (TSelectSqlStatement) stmt;
4434                                ResultSet resultSetModel = (ResultSet) modelManager.getModel(select);
4435                                if (resultSetModel != null) {
4436                                        for (int j = 0; j < resultSetModel.getColumns().size(); j++) {
4437                                                ResultColumn resultColumn = resultSetModel.getColumns().get(j);
4438                                                if (resultColumn.hasStarLinkColumn()
4439                                                                && resultColumn.getStarLinkColumnNames().size() > 0) {
4440                                                        for (int k = 0; k < resultColumn.getStarLinkColumnNames().size(); k++) {
4441                                                                ResultColumn expandStarColumn = modelFactory.createResultColumn(resultSetModel,
4442                                                                                resultColumn.getStarLinkColumnName(k), false);
4443                                                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
4444                                                                dataflowRelation.setEffectType(EffectType.unload);
4445                                                                dataflowRelation
4446                                                                                .addSource(new ResultColumnRelationshipElement(expandStarColumn));
4447                                                                dataflowRelation.setTarget(new TableColumnRelationshipElement(fileUriColumn));
4448                                                                dataflowRelation.setProcess(process);
4449                                                        }
4450                                                }
4451                                                if (!resultColumn.hasStarLinkColumn() || resultColumn.isShowStar()) {
4452                                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
4453                                                        dataflowRelation.setEffectType(EffectType.unload);
4454                                                        dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn));
4455                                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(fileUriColumn));
4456                                                        dataflowRelation.setProcess(process);
4457                                                }
4458                                        }
4459                                }
4460                        }
4461                }
4462
4463        }
4464
4465        private void analyzeCopyIntoStmt(TSnowflakeCopyIntoStmt stmt) {
4466                if (stmt.getTableName() != null) {
4467                        if (stmt.getStageLocation() != null) {
4468                                Table intoTable = modelManager
4469                                                .getTableByName(DlineageUtil.getTableFullName(stmt.getTableName().toString()));
4470                                if (intoTable == null) {
4471                                        intoTable = modelFactory.createTableByName(stmt.getTableName(), false);
4472                                        TObjectName starColumn = new TObjectName();
4473                                        starColumn.setString("*");
4474                                        TableColumn column = modelFactory.createTableColumn(intoTable, starColumn, false);
4475                                        if (column != null) {
4476                                                column.setExpandStar(false);
4477                                                column.setPseduo(true);
4478                                        }
4479                                }
4480                                Process process = modelFactory.createProcess(stmt);
4481                                intoTable.addProcess(process);
4482
4483                                TObjectName stageName = stmt.getStageLocation().getStageName();
4484                                if (stageName == null || stmt.getStageLocation().getTableName() != null) {
4485                                        stageName = stmt.getStageLocation().getTableName();
4486                                }
4487                                if (stageName != null) {
4488                                        String stageFullName = DlineageUtil.getTableFullName(stageName.toString());
4489                                        Table stage = modelManager.getTableByName(stageFullName);
4490                                        if (stage == null) {
4491                                                stage = modelFactory.createStage(stageName);
4492                                                stage.setCreateTable(true);
4493                                                stage.setStage(true);
4494                                                String stagePath = stmt.getStageLocation().getPath() == null ? null
4495                                                                : stmt.getStageLocation().getPath().toString();
4496                                                if (stagePath != null) {
4497                                                        stage.setLocation(stagePath);
4498                                                        TObjectName location = new TObjectName();
4499                                                        location.setString(stagePath);
4500                                                        modelFactory.createStageLocation(stage, location);
4501                                                } else {
4502                                                        stage.setLocation("unknownPath");
4503                                                        TObjectName location = new TObjectName();
4504                                                        location.setString("unknownPath");
4505                                                        modelFactory.createStageLocation(stage, location);
4506                                                }
4507                                        }
4508
4509                                        if (stage != null && intoTable != null) {
4510                                                if (intoTable != null && !intoTable.getColumns().isEmpty() && !stage.getColumns().isEmpty()) {
4511                                                        for (int i = 0; i < intoTable.getColumns().size(); i++) {
4512                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
4513                                                                relation.addSource(new TableColumnRelationshipElement(stage.getColumns().get(0)));
4514                                                                relation.setTarget(new TableColumnRelationshipElement(intoTable.getColumns().get(i)));
4515                                                                relation.setProcess(process);
4516                                                        }
4517                                                }
4518                                        }
4519                                } else if (stmt.getStageLocation().getExternalLocation() != null) {
4520                                        Table pathModel = modelFactory.createTableByName(stmt.getStageLocation().getExternalLocation(),
4521                                                        true);
4522                                        pathModel.setPath(true);
4523                                        pathModel.setCreateTable(true);
4524                                        TableColumn fileUriColumn = modelFactory.createFileUri(pathModel,
4525                                                        stmt.getStageLocation().getExternalLocation());
4526                                        if (intoTable != null) {
4527                                                if (intoTable != null && !intoTable.getColumns().isEmpty()) {
4528                                                        for (int i = 0; i < intoTable.getColumns().size(); i++) {
4529                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
4530                                                                relation.addSource(new TableColumnRelationshipElement(fileUriColumn));
4531                                                                relation.setTarget(new TableColumnRelationshipElement(intoTable.getColumns().get(i)));
4532                                                                relation.setProcess(process);
4533                                                        }
4534                                                }
4535                                        }
4536                                }
4537                        }
4538                        else if (stmt.getSubQuery() != null) {
4539                                analyzeSelectStmt(stmt.getSubQuery());
4540                                
4541                                Table intoTable = modelManager
4542                                                .getTableByName(DlineageUtil.getTableFullName(stmt.getTableName().toString()));
4543                                if (intoTable == null) {
4544                                        intoTable = modelFactory.createTableByName(stmt.getTableName(), false);
4545                                        TObjectName starColumn = new TObjectName();
4546                                        starColumn.setString("*");
4547                                        TableColumn column = modelFactory.createTableColumn(intoTable, starColumn, false);
4548                                        if (column != null) {
4549                                                column.setExpandStar(true);
4550                                                column.setPseduo(true);
4551                                        }
4552
4553                                        Process process = modelFactory.createProcess(stmt);
4554                                        intoTable.addProcess(process);
4555
4556                                        ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt.getSubQuery());
4557                                        if (resultSetModel != null && column != null) {
4558                                                for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
4559                                                        ResultColumn resultColumn = resultSetModel.getColumns().get(i);
4560                                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
4561                                                        dataflowRelation.setEffectType(EffectType.copy);
4562                                                        dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn));
4563                                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(column));
4564                                                        dataflowRelation.setProcess(process);
4565                                                }
4566                                        }
4567                                }
4568                        }
4569                }
4570        }
4571
4572        private void analyzeRedshiftCopyStmt(TRedshiftCopy stmt) {
4573                if (stmt.getTableName() != null && stmt.getFromSource() != null) {
4574
4575                        Table intoTable = modelManager
4576                                        .getTableByName(DlineageUtil.getTableFullName(stmt.getTableName().toString()));
4577                        if (intoTable == null) {
4578                                intoTable = modelFactory.createTableByName(stmt.getTableName(), false);
4579                                if (stmt.getColumnList() == null || stmt.getColumnList().size() == 0) {
4580                                        TObjectName starColumn = new TObjectName();
4581                                        starColumn.setString("*");
4582                                        TableColumn column = modelFactory.createTableColumn(intoTable, starColumn, false);
4583                                        if (column != null) {
4584                                                column.setExpandStar(false);
4585                                                column.setPseduo(true);
4586                                        }
4587                                } else {
4588                                        for (TObjectName columnName : stmt.getColumnList()) {
4589                                                modelFactory.createTableColumn(intoTable, columnName, true);
4590                                        }
4591                                }
4592                        }
4593                        Process process = modelFactory.createProcess(stmt);
4594                        intoTable.addProcess(process);
4595
4596                        if (stmt.getFromSource() != null) {
4597                                Table pathModel = modelFactory.createTableByName(stmt.getFromSource(), true);
4598                                pathModel.setPath(true);
4599                                pathModel.setCreateTable(true);
4600                                TObjectName fileUri = new TObjectName();
4601                                fileUri.setString(stmt.getFromSource());
4602                                TableColumn fileUriColumn = modelFactory.createFileUri(pathModel, fileUri);
4603                                if (intoTable != null) {
4604                                        if (intoTable != null && !intoTable.getColumns().isEmpty()) {
4605                                                for (int i = 0; i < intoTable.getColumns().size(); i++) {
4606                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
4607                                                        relation.addSource(new TableColumnRelationshipElement(fileUriColumn));
4608                                                        relation.setTarget(new TableColumnRelationshipElement(intoTable.getColumns().get(i)));
4609                                                        relation.setProcess(process);
4610                                                }
4611                                        }
4612                                }
4613                        }
4614                }
4615        }
4616
4617        private void analyzeCreateIndexExpressionOperand(TExpression expr, Table tableModel){
4618                if(expr == null) return;
4619                Deque<TExpression> stack = new ArrayDeque<>();
4620                stack.push(expr);
4621                while (!stack.isEmpty()) {
4622                        TExpression current = stack.pop();
4623                        if (current == null) continue;
4624                        TObjectName columnObj = current.getObjectOperand();
4625                        if (columnObj != null) {
4626                                TableColumn tableConstraint = modelFactory.createTableColumn(tableModel, columnObj, true);
4627                                tableConstraint.setIndexKey(true);
4628                        } else {
4629                                if (current.getRightOperand() != null) {
4630                                        stack.push(current.getRightOperand());
4631                                }
4632                                if (current.getLeftOperand() != null) {
4633                                        stack.push(current.getLeftOperand());
4634                                }
4635                        }
4636                }
4637        }
4638        private void analyzeCreateIndexStageStmt(TCreateIndexSqlStatement stmt) {
4639        if(stmt.getTableName() == null){
4640            return;
4641        }
4642                Table tableModel = modelFactory.createTableByName(stmt.getTableName());
4643                TOrderByItemList columns = stmt.getColumnNameList();
4644        if(columns!=null) {
4645            for (int i = 0; i < columns.size(); i++) {
4646                TExpression expr = columns.getOrderByItem(i).getSortKey();
4647                analyzeCreateIndexExpressionOperand(expr, tableModel);
4648            }
4649        }
4650        }
4651
4652        private void analyzeCreateSynonymStmt(TCreateSynonymStmt stmt) {
4653                TObjectName sourceTableName = stmt.getForName();
4654                if(sourceTableName == null) {
4655                        return;
4656                }
4657                TCustomSqlStatement createView = viewDDLMap
4658                                .get(DlineageUtil.getTableFullName(sourceTableName.toString()));
4659                if (createView != null) {
4660                        analyzeCustomSqlStmt(createView);
4661                }
4662                Table sourceTableModel = modelFactory.createTableByName(sourceTableName);
4663                Process process = modelFactory.createProcess(stmt);
4664                sourceTableModel.addProcess(process);
4665                if(stmt.getSynonymName()!=null) {
4666                        Table synonymTableModel = modelFactory.createTableByName(stmt.getSynonymName());
4667                        synonymTableModel.setSubType(SubType.synonym);
4668                        if(sourceTableModel.isCreateTable()) {
4669                                synonymTableModel.setCreateTable(true);
4670                                for(TableColumn sourceTableColumn: sourceTableModel.getColumns()) {
4671                                        TObjectName columnName = new TObjectName();
4672                                        columnName.setString(sourceTableColumn.getName());
4673                                        TableColumn synonymTableColumn = new TableColumn(synonymTableModel, columnName);
4674                                        synonymTableModel.addColumn(synonymTableColumn);
4675                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
4676                                        relation.setEffectType(EffectType.create_synonym);
4677                                        relation.setTarget(new TableColumnRelationshipElement(synonymTableColumn));
4678                                        relation.addSource(new TableColumnRelationshipElement(sourceTableColumn));
4679                                        relation.setProcess(process);
4680                                }
4681                        }
4682                        else {
4683                                TObjectName synonymStarColumn = new TObjectName();
4684                                synonymStarColumn.setString("*");
4685                                TableColumn synonymTableStarColumn = modelFactory.createTableColumn(synonymTableModel,
4686                                                synonymStarColumn, true);
4687                                TObjectName sourceStarColumn = new TObjectName();
4688                                sourceStarColumn.setString("*");
4689                                TableColumn sourceTableStarColumn = modelFactory.createTableColumn(sourceTableModel, sourceStarColumn, true);
4690                                
4691                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
4692                                relation.setEffectType(EffectType.create_synonym);
4693                                relation.setTarget(new TableColumnRelationshipElement(synonymTableStarColumn));
4694                                relation.addSource(new TableColumnRelationshipElement(sourceTableStarColumn));
4695                                relation.setProcess(process);
4696                        }
4697                }
4698        }
4699        
4700        private void analyzeRenameStmt(TRenameStmt stmt) {
4701                TObjectName oldTableName = stmt.getOldName();
4702                TObjectName newTableName = stmt.getNewName();
4703                
4704                Table oldNameTableModel = modelFactory.createTableByName(oldTableName);
4705                TObjectName oldStarColumn = new TObjectName();
4706                oldStarColumn.setString("*");
4707                TableColumn oldTableStarColumn = modelFactory.createTableColumn(oldNameTableModel, oldStarColumn, true);
4708                
4709                Table newNameTableModel = modelFactory.createTableByName(newTableName);
4710                TObjectName newStarColumn = new TObjectName();
4711                newStarColumn.setString("*");
4712                TableColumn newTableStarColumn = modelFactory.createTableColumn(newNameTableModel, newStarColumn, true);
4713                
4714                Process process = modelFactory.createProcess(stmt);
4715                newNameTableModel.addProcess(process);
4716                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
4717                relation.setEffectType( EffectType.rename_table);
4718                relation.setTarget(new TableColumnRelationshipElement(newTableStarColumn));
4719                relation.addSource(new TableColumnRelationshipElement(oldTableStarColumn));
4720                relation.setProcess(process);
4721                
4722                if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) {
4723                        oldTableStarColumn.setShowStar(false);
4724                        relation.setShowStarRelation(false);
4725                }
4726
4727                if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) {
4728                        newTableStarColumn.setShowStar(false);
4729                        relation.setShowStarRelation(false);
4730                }
4731        }
4732        
4733        private void analyzeAlterTableStmt(TAlterTableStatement stmt) {
4734                TTable oldNameTable = stmt.getTargetTable();
4735                if (oldNameTable == null) {
4736                        return;
4737                }
4738
4739                Table oldNameTableModel = modelFactory.createTable(oldNameTable);
4740                TObjectName oldStarColumn = new TObjectName();
4741                oldStarColumn.setString("*");
4742                TableColumn oldTableStarColumn = modelFactory.createTableColumn(oldNameTableModel, oldStarColumn, true);
4743
4744                for (int i = 0; stmt.getAlterTableOptionList() != null && i < stmt.getAlterTableOptionList().size(); i++) {
4745                        TAlterTableOption option = stmt.getAlterTableOptionList().getAlterTableOption(i);
4746                        if (option.getOptionType() == EAlterTableOptionType.RenameTable
4747                                        || option.getOptionType() == EAlterTableOptionType.swapWith) {
4748                                TObjectName newTableName = option.getNewTableName();
4749                                Stack<TParseTreeNode> list = newTableName.getStartToken().getNodesStartFromThisToken();
4750                                boolean containsTable = false;
4751                                for (int j = 0; j < list.size(); j++) {
4752                                        if (list.get(j) instanceof TTable) {
4753                                                TTable newTableTable = (TTable) list.get(j);
4754                                                Table newNameTableModel = modelFactory.createTable(newTableTable);
4755                                                newNameTableModel.setStarStmt("rename_table");
4756
4757                                                TObjectName newStarColumn = new TObjectName();
4758                                                newStarColumn.setString("*");
4759                                                TableColumn newTableStarColumn = modelFactory.createTableColumn(newNameTableModel,
4760                                                                newStarColumn, true);
4761
4762                                                Process process = modelFactory.createProcess(stmt);
4763                                                newNameTableModel.addProcess(process);
4764                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
4765                                                relation.setEffectType(
4766                                                                option.getOptionType() == EAlterTableOptionType.RenameTable ? EffectType.rename_table
4767                                                                                : EffectType.swap_table);                                               
4768                                                if (option.getOptionType() == EAlterTableOptionType.RenameTable) {
4769                                                        relation.setTarget(new TableColumnRelationshipElement(newTableStarColumn));
4770                                                        relation.addSource(new TableColumnRelationshipElement(oldTableStarColumn));
4771                                                } else if (option.getOptionType() == EAlterTableOptionType.swapWith) {
4772                                                        relation.setTarget(new TableColumnRelationshipElement(oldTableStarColumn));
4773                                                        relation.addSource(new TableColumnRelationshipElement(newTableStarColumn));
4774                                                }
4775                                                relation.setProcess(process);
4776                                                containsTable = true;
4777
4778                                                if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) {
4779                                                        oldTableStarColumn.setShowStar(false);
4780                                                        relation.setShowStarRelation(false);
4781                                                }
4782
4783                                                if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) {
4784                                                        newTableStarColumn.setShowStar(false);
4785                                                        relation.setShowStarRelation(false);
4786                                                }
4787                                        }
4788                                }
4789                                if (!containsTable) {
4790                                        Table newNameTableModel = modelFactory.createTableByName(newTableName);
4791                                        newNameTableModel.setStarStmt("rename_table");
4792                                        TObjectName newStarColumn = new TObjectName();
4793                                        newStarColumn.setString("*");
4794                                        TableColumn newTableStarColumn = modelFactory.createTableColumn(newNameTableModel, newStarColumn,
4795                                                        true);
4796
4797                                        Process process = modelFactory.createProcess(stmt);
4798                                        newNameTableModel.addProcess(process);
4799                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
4800                                        relation.setEffectType(
4801                                                        option.getOptionType() == EAlterTableOptionType.RenameTable ? EffectType.rename_table
4802                                                                        : EffectType.swap_table);
4803                                        if (option.getOptionType() == EAlterTableOptionType.RenameTable) {
4804                                                relation.setTarget(new TableColumnRelationshipElement(newTableStarColumn));
4805                                                relation.addSource(new TableColumnRelationshipElement(oldTableStarColumn));
4806                                        } else if (option.getOptionType() == EAlterTableOptionType.swapWith) {
4807                                                relation.setTarget(new TableColumnRelationshipElement(oldTableStarColumn));
4808                                                relation.addSource(new TableColumnRelationshipElement(newTableStarColumn));
4809                                        }
4810                                        relation.setProcess(process);
4811                                        if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) {
4812                                                oldTableStarColumn.setShowStar(false);
4813                                                relation.setShowStarRelation(false);
4814                                        }
4815
4816                                        if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) {
4817                                                newTableStarColumn.setShowStar(false);
4818                                                relation.setShowStarRelation(false);
4819                                        }
4820
4821                                }
4822                        }
4823                        else if (option.getOptionType() == EAlterTableOptionType.appendFrom) {
4824                                TObjectName newTableName = option.getSourceTableName();
4825                                Stack<TParseTreeNode> list = newTableName.getStartToken().getNodesStartFromThisToken();
4826                                boolean containsTable = false;
4827                                for (int j = 0; j < list.size(); j++) {
4828                                        if (list.get(j) instanceof TTable) {
4829                                                TTable newTableTable = (TTable) list.get(j);
4830                                                Table newNameTableModel = modelFactory.createTable(newTableTable);
4831                                                newNameTableModel.setStarStmt("append_from");
4832
4833                                                TObjectName newStarColumn = new TObjectName();
4834                                                newStarColumn.setString("*");
4835                                                TableColumn newTableStarColumn = modelFactory.createTableColumn(newNameTableModel,
4836                                                                newStarColumn, true);
4837
4838                                                Process process = modelFactory.createProcess(stmt);
4839                                                newNameTableModel.addProcess(process);
4840                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
4841                                                relation.setEffectType(EffectType.append_from);
4842                                                relation.setTarget(new TableColumnRelationshipElement(oldTableStarColumn));
4843                                                relation.addSource(new TableColumnRelationshipElement(newTableStarColumn));
4844                                                relation.setProcess(process);
4845                                                containsTable = true;
4846
4847                                                if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) {
4848                                                        oldTableStarColumn.setShowStar(false);
4849                                                        relation.setShowStarRelation(false);
4850                                                }
4851
4852                                                if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) {
4853                                                        newTableStarColumn.setShowStar(false);
4854                                                        relation.setShowStarRelation(false);
4855                                                }
4856                                        }
4857                                }
4858                                if (!containsTable) {
4859                                        Table newNameTableModel = modelFactory.createTableByName(newTableName);
4860                                        newNameTableModel.setStarStmt("append_from");
4861                                        TObjectName newStarColumn = new TObjectName();
4862                                        newStarColumn.setString("*");
4863                                        TableColumn newTableStarColumn = modelFactory.createTableColumn(newNameTableModel, newStarColumn,
4864                                                        true);
4865
4866                                        Process process = modelFactory.createProcess(stmt);
4867                                        newNameTableModel.addProcess(process);
4868                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
4869                                        relation.setEffectType(EffectType.append_from);
4870                                        relation.setTarget(new TableColumnRelationshipElement(oldTableStarColumn));
4871                                        relation.addSource(new TableColumnRelationshipElement(newTableStarColumn));
4872                                        relation.setProcess(process);
4873                                        if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) {
4874                                                oldTableStarColumn.setShowStar(false);
4875                                                relation.setShowStarRelation(false);
4876                                        }
4877
4878                                        if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) {
4879                                                newTableStarColumn.setShowStar(false);
4880                                                relation.setShowStarRelation(false);
4881                                        }
4882
4883                                }
4884                        }
4885                        else if (option.getOptionType() == EAlterTableOptionType.exchangePartition) {
4886                                TObjectName newTableName = option.getNewTableName();
4887                                Stack<TParseTreeNode> list = newTableName.getStartToken().getNodesStartFromThisToken();
4888                                boolean containsTable = false;
4889                                for (int j = 0; j < list.size(); j++) {
4890                                        if (list.get(j) instanceof TTable) {
4891                                                TTable newTableTable = (TTable) list.get(j);
4892                                                Table newNameTableModel = modelFactory.createTable(newTableTable);
4893                                                newNameTableModel.setStarStmt("exchange_partition");
4894
4895                                                TObjectName newStarColumn = new TObjectName();
4896                                                newStarColumn.setString("*");
4897                                                TableColumn newTableStarColumn = modelFactory.createTableColumn(newNameTableModel,
4898                                                                newStarColumn, true);
4899
4900                                                Process process = modelFactory.createProcess(stmt);
4901                                                newNameTableModel.addProcess(process);
4902                                                {
4903                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
4904                                                        relation.setEffectType(EffectType.exchange_partition);
4905                                                        relation.setTarget(new TableColumnRelationshipElement(oldTableStarColumn));
4906                                                        relation.addSource(new TableColumnRelationshipElement(newTableStarColumn));
4907                                                        relation.setProcess(process);
4908                                                        if (option.getPartitionName() != null) {
4909                                                                relation.setPartition(option.getPartitionName().toString());
4910                                                        }
4911                                                        if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) {
4912                                                                oldTableStarColumn.setShowStar(false);
4913                                                                relation.setShowStarRelation(false);
4914                                                        }
4915
4916                                                        if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) {
4917                                                                newTableStarColumn.setShowStar(false);
4918                                                                relation.setShowStarRelation(false);
4919                                                        }
4920                                                }
4921                                                {
4922                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
4923                                                        relation.setEffectType(EffectType.exchange_partition);
4924                                                        relation.setTarget(new TableColumnRelationshipElement(newTableStarColumn));
4925                                                        relation.addSource(new TableColumnRelationshipElement(oldTableStarColumn));
4926                                                        relation.setProcess(process);
4927                                                        if (option.getPartitionName() != null) {
4928                                                                relation.setPartition(option.getPartitionName().toString());
4929                                                        }
4930                                                        if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) {
4931                                                                oldTableStarColumn.setShowStar(false);
4932                                                                relation.setShowStarRelation(false);
4933                                                        }
4934
4935                                                        if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) {
4936                                                                newTableStarColumn.setShowStar(false);
4937                                                                relation.setShowStarRelation(false);
4938                                                        }
4939                                                }
4940                                                containsTable = true;   
4941                                        }
4942                                }
4943                                if (!containsTable) {
4944                                        Table newNameTableModel = modelFactory.createTableByName(newTableName);
4945                                        newNameTableModel.setStarStmt("exchange_partition");
4946                                        TObjectName newStarColumn = new TObjectName();
4947                                        newStarColumn.setString("*");
4948                                        TableColumn newTableStarColumn = modelFactory.createTableColumn(newNameTableModel, newStarColumn,
4949                                                        true);
4950
4951                                        Process process = modelFactory.createProcess(stmt);
4952                                        newNameTableModel.addProcess(process);
4953                                        {
4954                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
4955                                                relation.setEffectType(EffectType.exchange_partition);
4956                                                relation.setTarget(new TableColumnRelationshipElement(oldTableStarColumn));
4957                                                relation.addSource(new TableColumnRelationshipElement(newTableStarColumn));
4958                                                if (option.getPartitionName() != null) {
4959                                                        relation.setPartition(option.getPartitionName().toString());
4960                                                }
4961                                                relation.setProcess(process);
4962                                                if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) {
4963                                                        oldTableStarColumn.setShowStar(false);
4964                                                        relation.setShowStarRelation(false);
4965                                                }
4966
4967                                                if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) {
4968                                                        newTableStarColumn.setShowStar(false);
4969                                                        relation.setShowStarRelation(false);
4970                                                }
4971                                        }
4972                                        {
4973                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
4974                                                relation.setEffectType(EffectType.exchange_partition);
4975                                                relation.setTarget(new TableColumnRelationshipElement(newTableStarColumn));
4976                                                relation.addSource(new TableColumnRelationshipElement(oldTableStarColumn));
4977                                                if (option.getPartitionName() != null) {
4978                                                        relation.setPartition(option.getPartitionName().toString());
4979                                                }
4980                                                relation.setProcess(process);
4981                                                if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) {
4982                                                        oldTableStarColumn.setShowStar(false);
4983                                                        relation.setShowStarRelation(false);
4984                                                }
4985
4986                                                if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) {
4987                                                        newTableStarColumn.setShowStar(false);
4988                                                        relation.setShowStarRelation(false);
4989                                                }
4990                                        }
4991                                }
4992                        }
4993                        else if(option.getOptionType() == EAlterTableOptionType.setLocation) {
4994                                TObjectName location = option.getTableLocation();
4995                                Process process = modelFactory.createProcess(stmt);
4996                                process.setType("Set Table Location");
4997                                oldNameTableModel.addProcess(process);
4998                                Table uriFile = modelFactory.createTableByName(location, true);
4999                                uriFile.setPath(true);
5000                                for (int j = 0; j < oldNameTableModel.getColumns().size(); j++) {
5001                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
5002                                        TObjectName fileUri = new TObjectName();
5003                                        fileUri.setString("*");
5004                                        TableColumn fileUriColumn = modelFactory.createFileUri(uriFile, fileUri);
5005                                        relation.addSource(new TableColumnRelationshipElement(fileUriColumn));
5006                                        relation.setTarget(new TableColumnRelationshipElement(oldNameTableModel.getColumns().get(j)));
5007                                        relation.setProcess(process);
5008                                }
5009                        }
5010                        else if (option.getOptionType() == EAlterTableOptionType.AddColumn
5011                                        || option.getOptionType() == EAlterTableOptionType.addColumnIfNotExists) {
5012                                if (option.getColumnDefinitionList() != null) {
5013                                        for (TColumnDefinition column : option.getColumnDefinitionList()) {
5014                                                if (column != null && column.getColumnName() != null) {
5015                                                        TableColumn tableColumn = modelFactory.createTableColumn(oldNameTableModel, column.getColumnName(), true);
5016                                                        if(this.option.getAnalyzeMode() == AnalyzeMode.crud) {
5017                                                                CrudRelationship crudRelationship = modelFactory.createCrudRelation();
5018                                                                crudRelationship.setTarget(new TableColumnRelationshipElement(tableColumn));
5019                                                                crudRelationship.setEffectType(EffectType.add_table_column);
5020                                                        }
5021                                                }
5022                                        }
5023                                }
5024                        }
5025                        else if (option.getOptionType() == EAlterTableOptionType.DropColumn && this.option.getAnalyzeMode() == AnalyzeMode.crud) {
5026                                if (option.getColumnNameList() != null) {
5027                                        for (TObjectName column : option.getColumnNameList()) {
5028                                                TableColumn tableColumn = modelFactory.createTableColumn(oldNameTableModel, column, true);
5029                                                CrudRelationship crudRelationship = modelFactory.createCrudRelation();
5030                                                crudRelationship.setTarget(new TableColumnRelationshipElement(tableColumn));
5031                                                crudRelationship.setEffectType(EffectType.drop_table_column);
5032                                        }
5033                                }
5034                        }
5035                        else if(option.getOptionType() == EAlterTableOptionType.AddConstraint || option.getOptionType() == EAlterTableOptionType.AddConstraintFK
5036                                        || option.getOptionType() == EAlterTableOptionType.AddConstraintPK || option.getOptionType() == EAlterTableOptionType.AddConstraintUnique
5037                                        || option.getOptionType() == EAlterTableOptionType.AddConstraintIndex){
5038                                if (option.getTableConstraint() != null) {
5039                                        TConstraint alertTableConstraint = option.getTableConstraint();
5040                                        TPTNodeList<TColumnWithSortOrder> keyNames = alertTableConstraint.getColumnList();
5041                                        for (int k = 0; k < keyNames.size(); k++) {
5042                                                TObjectName keyName = keyNames.getElement(k).getColumnName();
5043                                                TObjectName referencedTableName = alertTableConstraint.getReferencedObject();
5044                                                Table tableModel = modelFactory.createTableByName(stmt.getTableName());
5045                                                TableColumn tableConstraint = modelFactory.createTableColumn(tableModel, keyName, true);
5046                                                if(alertTableConstraint.getConstraint_type() == EConstraintType.primary_key){
5047                                                        tableConstraint.setPrimaryKey(true);
5048                                                }
5049                                                else if(alertTableConstraint.getConstraint_type() == EConstraintType.table_index){
5050                                                        tableConstraint.setIndexKey(true);
5051                                                }
5052                                                else if(alertTableConstraint.getConstraint_type() == EConstraintType.unique){
5053                                                        tableConstraint.setUnqiueKey(true);
5054                                                }
5055                                                else if (alertTableConstraint.getConstraint_type() == EConstraintType.foreign_key) {
5056                                                        tableConstraint.setForeignKey(true);
5057                                                        Table referencedTable = modelManager.getTableByName(DlineageUtil.getTableFullName(referencedTableName.toString()));
5058                                                        if (referencedTable == null) {
5059                                                                referencedTable = modelFactory.createTableByName(referencedTableName);
5060                                                        }
5061                                                        TObjectNameList referencedTableColumns = alertTableConstraint.getReferencedColumnList();
5062                                                        if (referencedTableColumns != null) {
5063                                                                for (int j = 0; j < referencedTableColumns.size(); j++) {
5064                                                                        TableColumn tableColumn = modelFactory.createTableColumn(referencedTable,
5065                                                                                        referencedTableColumns.getObjectName(j), false);
5066                                                                        if (tableColumn != null) {
5067                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
5068                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
5069                                                                                relation.setTarget(new TableColumnRelationshipElement(tableConstraint));
5070                                                                                relation.setEffectType(EffectType.foreign_key);
5071                                                                                Process process = modelFactory.createProcess(stmt);
5072                                                                                relation.setProcess(process);
5073                                                                                if(this.option.isShowERDiagram()){
5074                                                                                        ERRelationship erRelation = modelFactory.createERRelation();
5075                                                                                        erRelation.addSource(new TableColumnRelationshipElement(tableColumn));
5076                                                                                        erRelation.setTarget(new TableColumnRelationshipElement(tableConstraint));
5077                                                                                }
5078                                                                        }
5079                                                                }
5080                                                        }
5081                                                        else{
5082                                                                TableColumn tableColumn = modelFactory.createTableColumn(referencedTable, keyName, false);
5083                                                                if (tableColumn != null) {
5084                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
5085                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
5086                                                                        relation.setTarget(new TableColumnRelationshipElement(tableConstraint));
5087                                                                        relation.setEffectType(EffectType.foreign_key);
5088                                                                        if(this.option.isShowERDiagram()){
5089                                                                                ERRelationship erRelation = modelFactory.createERRelation();
5090                                                                                erRelation.addSource(new TableColumnRelationshipElement(tableColumn));
5091                                                                                erRelation.setTarget(new TableColumnRelationshipElement(tableConstraint));
5092                                                                        }
5093                                                                }
5094                                                        }
5095                                                }
5096                                        }
5097                                }
5098                                else if (option.getConstraintList() != null) {
5099                                        for(int iCons=0; iCons<option.getConstraintList().size(); iCons++){
5100                                                TConstraint alertTableConstraint = option.getConstraintList().getConstraint(iCons);
5101                                                TPTNodeList<TColumnWithSortOrder> keyNames = alertTableConstraint.getColumnList();
5102                                                if(keyNames != null){
5103                                                        for (int k = 0; k < keyNames.size(); k++) {
5104                                                                TObjectName keyName = keyNames.getElement(k).getColumnName();
5105                                                                TObjectName referencedTableName = alertTableConstraint.getReferencedObject();
5106                                                                Table tableModel = modelFactory.createTableByName(stmt.getTableName());
5107                                                                TableColumn tableConstraint = modelFactory.createTableColumn(tableModel, keyName, true);
5108                                                                if(alertTableConstraint.getConstraint_type() == EConstraintType.primary_key){
5109                                                                        tableConstraint.setPrimaryKey(true);
5110                                                                }
5111                                                                else if(alertTableConstraint.getConstraint_type() == EConstraintType.table_index){
5112                                                                        tableConstraint.setIndexKey(true);
5113                                                                }
5114                                                                else if(alertTableConstraint.getConstraint_type() == EConstraintType.unique){
5115                                                                        tableConstraint.setUnqiueKey(true);
5116                                                                }
5117                                                                else if (alertTableConstraint.getConstraint_type() == EConstraintType.foreign_key) {
5118                                                                        tableConstraint.setForeignKey(true);
5119                                                                        Table referencedTable = modelManager.getTableByName(DlineageUtil.getTableFullName(referencedTableName.toString()));
5120                                                                        if (referencedTable == null) {
5121                                                                                referencedTable = modelFactory.createTableByName(referencedTableName);
5122                                                                        }
5123                                                                        TObjectNameList referencedTableColumns = alertTableConstraint.getReferencedColumnList();
5124                                                                        if (referencedTableColumns != null) {
5125                                                                                for (int j = 0; j < referencedTableColumns.size(); j++) {
5126                                                                                        TableColumn tableColumn = modelFactory.createTableColumn(referencedTable,
5127                                                                                                        referencedTableColumns.getObjectName(j), false);
5128                                                                                        if (tableColumn != null) {
5129                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
5130                                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
5131                                                                                                relation.setTarget(new TableColumnRelationshipElement(tableConstraint));
5132                                                                                                relation.setEffectType(EffectType.foreign_key);
5133                                                                                                Process process = modelFactory.createProcess(stmt);
5134                                                                                                relation.setProcess(process);
5135                                                                                                if(this.option.isShowERDiagram()){
5136                                                                                                        ERRelationship erRelation = modelFactory.createERRelation();
5137                                                                                                        erRelation.addSource(new TableColumnRelationshipElement(tableColumn));
5138                                                                                                        erRelation.setTarget(new TableColumnRelationshipElement(tableConstraint));
5139                                                                                                }
5140                                                                                        }
5141                                                                                }
5142                                                                        }
5143                                                                        else{
5144                                                                                TableColumn tableColumn = modelFactory.createTableColumn(referencedTable, keyName, false);
5145                                                                                if (tableColumn != null) {
5146                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
5147                                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
5148                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableConstraint));
5149                                                                                        relation.setEffectType(EffectType.foreign_key);
5150                                                                                        Process process = modelFactory.createProcess(stmt);
5151                                                                                        relation.setProcess(process);
5152                                                                                        if(this.option.isShowERDiagram()){
5153                                                                                                ERRelationship erRelation = modelFactory.createERRelation();
5154                                                                                                erRelation.addSource(new TableColumnRelationshipElement(tableColumn));
5155                                                                                                erRelation.setTarget(new TableColumnRelationshipElement(tableConstraint));
5156                                                                                        }
5157                                                                                }
5158                                                                        }
5159                                                                }
5160                                                        }
5161                                                }
5162                                        }
5163                                }
5164                                else if (option.getIndexCols() != null){
5165                                        TPTNodeList<TColumnWithSortOrder> keyNames = option.getIndexCols();
5166                                        for (int k = 0; k < keyNames.size(); k++) {
5167                                                TObjectName keyName = keyNames.getElement(k).getColumnName();
5168                                                Table tableModel = modelFactory.createTableByName(stmt.getTableName());
5169                                                TableColumn tableConstraint = modelFactory.createTableColumn(tableModel, keyName, true);
5170                                                if(option.getOptionType() == EAlterTableOptionType.AddConstraintPK){
5171                                                        tableConstraint.setPrimaryKey(true);
5172                                                }
5173                                                else if(option.getOptionType() == EAlterTableOptionType.AddConstraintIndex){
5174                                                        tableConstraint.setIndexKey(true);
5175                                                }
5176                                                else if(option.getOptionType() == EAlterTableOptionType.AddConstraintUnique){
5177                                                        tableConstraint.setUnqiueKey(true);
5178                                                }
5179                                                else if (option.getOptionType() == EAlterTableOptionType.AddConstraintFK) {
5180                                                        TObjectName referencedTableName = option.getReferencedObjectName();
5181                                                        tableConstraint.setForeignKey(true);
5182                                                        Table referencedTable = modelManager.getTableByName(DlineageUtil.getTableFullName(referencedTableName.toString()));
5183                                                        if (referencedTable == null) {
5184                                                                referencedTable = modelFactory.createTableByName(referencedTableName);
5185                                                        }
5186                                                        TObjectNameList referencedTableColumns = option.getReferencedColumnList();
5187                                                        if (referencedTableColumns != null) {
5188                                                                for (int j = 0; j < referencedTableColumns.size(); j++) {
5189                                                                        TableColumn tableColumn = modelFactory.createTableColumn(referencedTable,
5190                                                                                        referencedTableColumns.getObjectName(j), false);
5191                                                                        if (tableColumn != null) {
5192                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
5193                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
5194                                                                                relation.setTarget(new TableColumnRelationshipElement(tableConstraint));
5195                                                                                relation.setEffectType(EffectType.foreign_key);
5196                                                                                Process process = modelFactory.createProcess(stmt);
5197                                                                                relation.setProcess(process);
5198                                                                                if(this.option.isShowERDiagram()){
5199                                                                                        ERRelationship erRelation = modelFactory.createERRelation();
5200                                                                                        erRelation.addSource(new TableColumnRelationshipElement(tableColumn));
5201                                                                                        erRelation.setTarget(new TableColumnRelationshipElement(tableConstraint));
5202                                                                                }
5203                                                                        }
5204                                                                }
5205                                                        }
5206                                                        else{
5207                                                                TableColumn tableColumn = modelFactory.createTableColumn(referencedTable, keyName, false);
5208                                                                if (tableColumn != null) {
5209                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
5210                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
5211                                                                        relation.setTarget(new TableColumnRelationshipElement(tableConstraint));
5212                                                                        relation.setEffectType(EffectType.foreign_key);
5213                                                                        Process process = modelFactory.createProcess(stmt);
5214                                                                        relation.setProcess(process);
5215                                                                        if(this.option.isShowERDiagram()){
5216                                                                                ERRelationship erRelation = modelFactory.createERRelation();
5217                                                                                erRelation.addSource(new TableColumnRelationshipElement(tableColumn));
5218                                                                                erRelation.setTarget(new TableColumnRelationshipElement(tableConstraint));
5219                                                                        }
5220                                                                }
5221                                                        }
5222                                                }
5223                                        }
5224                                }
5225                        }
5226                }
5227        }
5228
5229        private void analyzeAlterViewStmt(TAlterViewStatement stmt) {
5230                if (stmt.getAlterViewOption() == EAlterViewOption.asSelect) {
5231                        analyzeCreateViewStmt(stmt, stmt.getSelectSqlStatement(), null, stmt.getViewName());
5232                } else {
5233                        throw new UnsupportedOperationException("Can't handle this alter view statement case, alter option = "+ stmt.getAlterViewOption().name());
5234                }
5235        }
5236        
5237        private void analyzeDeleteStmt(TDeleteSqlStatement stmt) {
5238                TTable table = stmt.getTargetTable();
5239                if (table == null)
5240                        return;
5241                
5242                if (table.getCTE() != null) {
5243                        table = table.getCTE().getSubquery().getTables().getTable(0);
5244                } else if (table.getLinkTable() != null && table.getLinkTable().getSubquery() != null) {
5245                        table = table.getLinkTable().getSubquery().getTables().getTable(0);
5246                } else if (table.getSubquery() != null) {
5247                        table = table.getSubquery().getTables().getTable(0);
5248                }
5249                
5250                Table tableModel = modelFactory.createTable(table);
5251                if (getTableLinkedColumns(table) != null && getTableLinkedColumns(table).size() > 0) {
5252                        for (int j = 0; j < getTableLinkedColumns(table).size(); j++) {
5253                                TObjectName object = getTableLinkedColumns(table).getObjectName(j);
5254
5255                                if (object.getDbObjectType() == EDbObjectType.variable) {
5256                                        continue;
5257                                }
5258
5259                                if (object.getColumnNameOnly().startsWith("@")
5260                                                && (option.getVendor() == EDbVendor.dbvmssql || option.getVendor() == EDbVendor.dbvazuresql)) {
5261                                        continue;
5262                                }
5263
5264                                if (object.getColumnNameOnly().startsWith(":")
5265                                                && (option.getVendor() == EDbVendor.dbvhana || option.getVendor() == EDbVendor.dbvteradata)) {
5266                                        continue;
5267                                }
5268
5269                                if (!isBuiltInFunctionName(object)) {
5270                                        if (object.getSourceTable() == null || object.getSourceTable() == table) {
5271                                                modelFactory.createTableColumn(tableModel, object, false);
5272                                        }
5273                                }
5274                        }
5275                }
5276
5277                if(option.getAnalyzeMode() == AnalyzeMode.crud) {
5278                        CrudRelationship crudRelationship = modelFactory.createCrudRelation();
5279                        crudRelationship.setTarget(new TableRelationshipElement(tableModel));
5280                        crudRelationship.setEffectType(EffectType.delete);
5281                }
5282                
5283                if (stmt.getWhereClause() != null && stmt.getWhereClause().getCondition() != null) {
5284                        analyzeFilterCondition(null, stmt.getWhereClause().getCondition(), null, JoinClauseType.where,
5285                                        EffectType.delete);
5286                }
5287        }
5288
5289        private TObjectName getProcedureName(TStoredProcedureSqlStatement stmt) {
5290                if (stmt instanceof TTeradataCreateProcedure) {
5291                        return ((TTeradataCreateProcedure) stmt).getProcedureName();
5292                }
5293                return stmt.getStoredProcedureName();
5294        }
5295
5296        private void analyzePlsqlCreatePackage(TPlsqlCreatePackage stmt) {
5297                TObjectName procedureName = getProcedureName(stmt);
5298                OraclePackage oraclePackage;
5299                if (procedureName != null) {
5300                        if (this.modelManager.getOraclePackageByName(
5301                                        DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getProcedureNameWithArgs(stmt))) == null) {
5302                                oraclePackage = this.modelFactory.createOraclePackage(stmt);
5303
5304                                if (stmt.getParameterDeclarations() != null) {
5305                                        TParameterDeclarationList parameters = stmt.getParameterDeclarations();
5306
5307                                        for (int i = 0; i < parameters.size(); ++i) {
5308                                                TParameterDeclaration parameter = parameters.getParameterDeclarationItem(i);
5309                                                if (parameter.getParameterName() != null) {
5310                                                        this.modelFactory.createProcedureArgument(oraclePackage, parameter, i + 1);
5311                                                } else if (parameter.getDataType() != null) {
5312                                                        this.modelFactory.createProcedureArgument(oraclePackage, parameter, i + 1);
5313                                                }
5314                                        }
5315                                }
5316
5317                                ModelBindingManager.setGlobalOraclePackage(oraclePackage);
5318                        } else {
5319                                ModelBindingManager.setGlobalOraclePackage(this.modelManager.getOraclePackageByName(
5320                                                DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getProcedureNameWithArgs(stmt))));
5321                        }
5322                        
5323                        try {
5324                                if (stmt.getDeclareStatements() != null) {
5325                                        for (int i = 0; i < stmt.getDeclareStatements().size(); ++i) {
5326                                                analyzeCustomSqlStmt(stmt.getDeclareStatements().get(i));
5327                                        }
5328                                }
5329                        } finally {
5330                                ModelBindingManager.removeGlobalOraclePackage();
5331                        }
5332                }
5333        }
5334
5335        private void analyzeStoredProcedureStmt(TStoredProcedureSqlStatement stmt) {
5336
5337                if (stmt instanceof TPlsqlCreatePackage) {
5338                        analyzePlsqlCreatePackage((TPlsqlCreatePackage) stmt);
5339                        return;
5340                }
5341                
5342                ModelBindingManager.setGlobalProcedure(stmt);
5343
5344                try {
5345                        Procedure procedure = null;
5346        
5347                        TObjectName procedureName = getProcedureName(stmt);
5348                        if (procedureName != null) {
5349                                procedure = this.modelFactory.createProcedure(stmt);
5350                                if (procedure != null) {
5351                                        modelManager.bindModel(stmt, procedure);
5352                                }
5353                                if (ModelBindingManager.getGlobalOraclePackage() != null) {
5354                                        ModelBindingManager.getGlobalOraclePackage().addProcedure(procedure);
5355                                        procedure.setParentPackage(ModelBindingManager.getGlobalOraclePackage());
5356                                }
5357                                if (stmt.getParameterDeclarations() != null) {
5358                                        TParameterDeclarationList parameters = stmt.getParameterDeclarations();
5359        
5360                                        for (int i = 0; i < parameters.size(); ++i) {
5361                                                TParameterDeclaration parameter = parameters.getParameterDeclarationItem(i);
5362                                                Argument argument = null;
5363                                                TObjectName argumentName = null;
5364                                                if (parameter.getParameterName() != null) {
5365                                                        argument = this.modelFactory.createProcedureArgument(procedure, parameter, i + 1);
5366                                                        argumentName = parameter.getParameterName();
5367                                                } else if (parameter.getDataType() != null) {
5368                                                        argument = this.modelFactory.createProcedureArgument(procedure, parameter, i + 1);
5369                                                }
5370                                                
5371                                                if (argument != null) {
5372                                                        if (argumentName == null) {
5373                                                                argumentName = new TObjectName();
5374                                                                argumentName.setString(argument.getName());
5375                                                        }
5376                                                        Variable variable = modelFactory.createVariable(argument.getName());
5377                                                        if (argument.getMode() == EParameterMode.in || argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
5378                                                                variable.setSubType(SubType.of(argument.getMode().name()));
5379                                                        } else {
5380                                                                variable.setSubType(SubType.argument);
5381                                                        }
5382                                                        if(isSimpleDataType(argument.getDataType())){
5383                                                                modelFactory.createTableColumn(variable, argumentName, true);
5384                                                        }
5385                                                        else {
5386                                                                TObjectName variableProperties = new TObjectName();
5387                                                                variableProperties.setString("*");
5388                                                                modelFactory.createTableColumn(variable, variableProperties, true);
5389                                                        }
5390                                                }
5391                                        }
5392                                }
5393                        }
5394        
5395                        if (stmt instanceof TCreateTriggerStmt) {
5396                                TCreateTriggerStmt trigger = (TCreateTriggerStmt) stmt;
5397        
5398                                if (trigger.getFunctionCall() != null) {
5399                                        modelFactory.createProcedureFromFunctionCall(trigger.getFunctionCall());
5400                                }
5401        
5402                                if (trigger.getTables() != null) {
5403                                        for (int i = 0; i < trigger.getTables().size(); i++) {
5404                                                Table tableModel = this.modelFactory.createTriggerOnTable(trigger.getTables().getTable(i));
5405                                        }
5406                                }
5407                        }
5408        
5409                        if (stmt instanceof TPlsqlCreateTrigger
5410                                        && ((TPlsqlCreateTrigger) stmt).getTriggeringClause().getEventClause() instanceof TDmlEventClause) {
5411                                TPlsqlCreateTrigger trigger = (TPlsqlCreateTrigger) stmt;
5412                                TDmlEventClause clause = (TDmlEventClause) ((TPlsqlCreateTrigger) stmt).getTriggeringClause()
5413                                                .getEventClause();
5414                                Table sourceTable = modelFactory.createTableByName(clause.getTableName());
5415        
5416                                for (TTriggerEventItem item : clause.getEventItems()) {
5417                                        if (item instanceof TDmlEventItem) {
5418                                                if (((TDmlEventItem) item).getColumnList() != null) {
5419                                                        for (TObjectName column : ((TDmlEventItem) item).getColumnList()) {
5420                                                                modelFactory.createTableColumn(sourceTable, column, true);
5421                                                        }
5422                                                }
5423                                        }
5424                                }
5425        
5426                                for (TCustomSqlStatement subStmt : ((TPlsqlCreateTrigger) stmt).getStatements()) {
5427                                        if (!(subStmt instanceof TCommonBlock))
5428                                                continue;
5429                                        for (TCustomSqlStatement blockSubStmt : ((TCommonBlock) subStmt).getStatements()) {
5430                                                if (!(blockSubStmt instanceof TBasicStmt))
5431                                                        continue;
5432                                                TBasicStmt basicStmt = (TBasicStmt) blockSubStmt;
5433                                                TExpression expression = basicStmt.getExpr();
5434                                                if (expression != null && expression.getExpressionType() == EExpressionType.function_t) {
5435                                                        Procedure targetProcedure = modelManager.getProcedureByName(DlineageUtil
5436                                                                        .getTableFullName(expression.getFunctionCall().getFunctionName().toString()));
5437                                                        if (targetProcedure == null) {
5438                                                                targetProcedure = modelManager
5439                                                                                .getProcedureByName(DlineageUtil.getTableFullName(procedure.getSchema() + "."
5440                                                                                                + expression.getFunctionCall().getFunctionName().toString()));
5441                                                        }
5442                                                        if (targetProcedure != null && expression.getFunctionCall().getArgs() != null && expression
5443                                                                        .getFunctionCall().getArgs().size() == targetProcedure.getArguments().size()) {
5444                                                                for (int j = 0; j < expression.getFunctionCall().getArgs().size(); j++) {
5445                                                                        TExpression columnExpr = expression.getFunctionCall().getArgs().getExpression(j);
5446                                                                        if (columnExpr.getExpressionType() == EExpressionType.simple_object_name_t) {
5447                                                                                TObjectName columnObject = columnExpr.getObjectOperand();
5448                                                                                if (columnObject.toString().indexOf(":") != -1) {
5449                                                                                        TableColumn tableColumn = modelFactory.createTableColumn(sourceTable,
5450                                                                                                        columnObject, true);
5451                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
5452                                                                                        relation.setEffectType(EffectType.trigger);
5453                                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
5454                                                                                        relation.setTarget(
5455                                                                                                        new ArgumentRelationshipElement(targetProcedure.getArguments().get(j)));
5456                                                                                        Process process = modelFactory.createProcess(stmt);
5457                                                                                        relation.setProcess(process);
5458                                                                                }
5459                                                                        }
5460                                                                }
5461                                                        }
5462                                                }
5463                                        }
5464                                }
5465                        }
5466        
5467                        if (stmt instanceof TMssqlCreateFunction) {
5468                                TMssqlCreateFunction createFunction = (TMssqlCreateFunction) stmt;
5469                                if (createFunction.getReturnTableVaraible() != null && createFunction.getReturnTableDefinitions() != null) {
5470                                        Variable tableModel = this.modelFactory.createVariable(createFunction.getReturnTableVaraible());
5471                                        tableModel.setVariable(true);
5472                                        tableModel.setCreateTable(true);
5473                                        String procedureParent = createFunction.getFunctionName().toString();
5474                                        if (procedureParent != null) {
5475                                                tableModel.setParent(procedureParent);
5476                                        }
5477                                        modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent), tableModel);
5478        
5479                                        if (createFunction.getReturnTableDefinitions() != null) {
5480                                                for (int j = 0; j < createFunction.getReturnTableDefinitions().size(); j++) {
5481                                                        TTableElement tableElement = createFunction.getReturnTableDefinitions().getTableElement(j);
5482                                                        TColumnDefinition column = tableElement.getColumnDefinition();
5483                                                        if (column != null && column.getColumnName() != null) {
5484                                                                modelFactory.createTableColumn(tableModel, column.getColumnName(), true);
5485                                                        }
5486                                                }
5487                                        }
5488                                }
5489        
5490                                if (createFunction.getReturnStmt() != null && createFunction.getReturnStmt().getSubquery() != null) {
5491                                        String procedureParent = createFunction.getFunctionName().toString();
5492                                        analyzeSelectStmt(createFunction.getReturnStmt().getSubquery());
5493                                        ResultSet resultSetModel = (ResultSet) modelManager
5494                                                        .getModel(createFunction.getReturnStmt().getSubquery());
5495                                        modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent),
5496                                                        resultSetModel);
5497                                }
5498                        } else if (stmt instanceof TCreateFunctionStmt) {
5499                                TCreateFunctionStmt createFunction = (TCreateFunctionStmt) stmt;
5500                                if (createFunction.getReturnDataType() != null
5501                                                && createFunction.getReturnDataType().getColumnDefList() != null) {
5502                                        Table tableModel = this.modelFactory.createTableByName(createFunction.getFunctionName(), true);
5503                                        tableModel.setCreateTable(true);
5504                                        String procedureParent = createFunction.getFunctionName().toString();
5505                                        if (procedureParent != null) {
5506                                                tableModel.setParent(procedureParent);
5507                                        }
5508        
5509                                        modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent), tableModel);
5510                                        for (int j = 0; j < createFunction.getReturnDataType().getColumnDefList().size(); j++) {
5511                                                TColumnDefinition column = createFunction.getReturnDataType().getColumnDefList().getColumn(j);
5512                                                if (column != null && column.getColumnName() != null) {
5513                                                        modelFactory.createTableColumn(tableModel, column.getColumnName(), true);
5514                                                }
5515                                        }
5516        
5517                                        if (createFunction.getSqlQuery() != null) {
5518                                                analyzeSelectStmt(createFunction.getSqlQuery());
5519                                                ResultSet resultSetModel = (ResultSet) modelManager.getModel(createFunction.getSqlQuery());
5520                                                if (resultSetModel != null) {
5521                                                        for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
5522                                                                ResultColumn resultColumn = resultSetModel.getColumns().get(i);
5523                                                                for (int j = 0; j < tableModel.getColumns().size(); j++) {
5524                                                                        TableColumn tableColumn = tableModel.getColumns().get(j);
5525                                                                        if (DlineageUtil.compareColumnIdentifier(getColumnName(resultColumn.getName()),
5526                                                                                        DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
5527                                                                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
5528                                                                                dataflowRelation.setEffectType(EffectType.select);
5529                                                                                dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn));
5530                                                                                dataflowRelation.setTarget(new TableColumnRelationshipElement(tableColumn));
5531                                                                        }
5532                                                                }
5533                                                        }
5534                                                }
5535                                        }
5536                                }
5537                                
5538                                if (createFunction.getReturnStmt() != null && createFunction.getReturnStmt().getSubquery() != null) {
5539                                        String procedureParent = createFunction.getFunctionName().toString();
5540                                        analyzeSelectStmt(createFunction.getReturnStmt().getSubquery());
5541                                        ResultSet resultSetModel = (ResultSet) modelManager
5542                                                        .getModel(createFunction.getReturnStmt().getSubquery());
5543                                        modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent),
5544                                                        resultSetModel);
5545                                }
5546                                
5547                                if (createFunction.getReturnStmt() != null && createFunction.getReturnStmt().getReturnExpr() != null) {
5548                                        TExpression returnExpression =  createFunction.getReturnStmt().getReturnExpr();
5549                                        ResultSet returnResult = modelFactory.createResultSet(createFunction.getReturnStmt(), false);
5550                                        ResultColumn resultColumn = modelFactory.createResultColumn(returnResult, returnExpression);
5551                                        
5552                                        TExpression expression = createFunction.getReturnStmt().getReturnExpr();
5553                                        analyzeResultColumnExpressionRelation(resultColumn, expression);
5554        
5555                                        String procedureParent = createFunction.getFunctionName().toString();
5556                                        modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent),
5557                                                        returnResult);
5558                                }
5559                        }
5560        
5561                        if (stmt instanceof TCreateProcedureStmt) {
5562                                TCreateProcedureStmt createProcedure = (TCreateProcedureStmt) stmt;
5563                                if (EDbVendor.dbvsnowflake == option.getVendor() && createProcedure.getRoutineBodyInConstant() != null) {
5564                                        extractSnowflakeSQLFromProcedure(createProcedure);
5565                                }
5566                        }
5567                                
5568                        if (stmt.getStatements().size() > 0) {
5569                                for (int i = 0; i < stmt.getStatements().size(); ++i) {
5570                                        this.analyzeCustomSqlStmt(stmt.getStatements().get(i));
5571                                }
5572                        }
5573
5574                        if (stmt.getBodyStatements().size() > 0) {
5575                                for (int i = 0; i < stmt.getBodyStatements().size(); ++i) {
5576                                        this.analyzeCustomSqlStmt(stmt.getBodyStatements().get(i));
5577                                }
5578                        }
5579                        
5580                        // Detect pipelined functions and build signatures
5581                        if (stmt instanceof TPlsqlCreateFunction && pipelinedAnalyzer != null) {
5582                                try {
5583                                        pipelinedAnalyzer.analyzePipelinedFunction((TPlsqlCreateFunction) stmt);
5584                                } catch (Exception e) {
5585                                        // Don't let pipelined analysis failure break main flow
5586                                }
5587                        }
5588
5589                        if (procedure != null && !getLastSelectStmt(stmt).isEmpty()) {
5590                                for (TSelectSqlStatement select : getLastSelectStmt(stmt)) {
5591                                        ResultSet resultSet = (ResultSet) modelManager.getModel(select);
5592                                        modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedure.getName()),
5593                                                        resultSet);
5594                                }
5595//                              List<Argument> outArgs = new ArrayList<Argument>();
5596//                              for (int i = 0; i < procedure.getArguments().size(); i++) {
5597//                                      Argument argument = procedure.getArguments().get(i);
5598//                                      if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
5599//                                              outArgs.add(argument);
5600//                                      }
5601//                              }
5602//
5603//                              if (resultSet != null && resultSet.getColumns().size() == outArgs.size()) {
5604//                                      for (int i = 0; i < outArgs.size(); i++) {
5605//                                              Argument argument = outArgs.get(i);
5606//                                              Variable variable = modelFactory.createVariable(argument.getName(), false);
5607//                                              if (variable != null) {
5608//                                                      DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
5609//                                                      dataflowRelation.setEffectType(EffectType.output);
5610//                                                      dataflowRelation
5611//                                                                      .addSource(new ResultColumnRelationshipElement(resultSet.getColumns().get(i)));
5612//                                                      dataflowRelation
5613//                                                                      .setTarget(new TableColumnRelationshipElement(variable.getColumns().get(0)));
5614//                                              }
5615//                                      }
5616//
5617//                              }
5618                        }
5619
5620                        if (stmt instanceof TCreateFunctionStmt && ((TCreateFunctionStmt)stmt).getSqlExpression()!=null) {
5621                                TCreateFunctionStmt createFunction = (TCreateFunctionStmt) stmt;
5622                                TExpression returnExpression = createFunction.getSqlExpression();
5623
5624                                ResultSet returnResult = modelFactory.createResultSet(stmt, false);
5625                                ResultColumn resultColumn = modelFactory.createResultColumn(returnResult, returnExpression);
5626
5627                                columnsInExpr visitor = new columnsInExpr();
5628                                returnExpression.inOrderTraverse(visitor);
5629
5630                                List<TObjectName> objectNames = visitor.getObjectNames();
5631                                List<TParseTreeNode> functions = visitor.getFunctions();
5632                                List<TParseTreeNode> constants = visitor.getConstants();
5633                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
5634
5635                                if (functions != null && !functions.isEmpty()) {
5636                                        analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function);
5637                                }
5638                                if (subquerys != null && !subquerys.isEmpty()) {
5639                                        analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.select);
5640                                }
5641                                if (objectNames != null && !objectNames.isEmpty()) {
5642                                        analyzeDataFlowRelation(resultColumn, objectNames, EffectType.select, functions);
5643                                }
5644                                if (constants != null && !constants.isEmpty()) {
5645                                        analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.select, functions);
5646                                }
5647
5648                                String procedureParent = SQLUtil.trimColumnStringQuote(getProcedureParentName(stmt));
5649                                modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent),
5650                                                returnResult);
5651                        }
5652                } finally {
5653                        ModelBindingManager.removeGlobalProcedure();
5654                }
5655
5656        }
5657
5658        private boolean isSimpleDataType(TTypeName dataType) {
5659                if (dataType.getDataType() == EDataType.variant_t) {
5660                        return false;
5661                }
5662                if (dataType.getDataType() == EDataType.cursor_t) {
5663                        return false;
5664                }
5665                if (dataType.getDataType() == EDataType.generic_t) {
5666                        return false;
5667                }
5668                if (dataType.getDataType() == EDataType.unknown_t) {
5669                        return false;
5670                }
5671                if (dataType.getDataType() == EDataType.sql_variant_t) {
5672                        return false;
5673                }
5674                if (dataType.getDataType() == EDataType.table_t) {
5675                        return false;
5676                }
5677                if (dataType.getDataType() == EDataType.raw_t) {
5678                        return false;
5679                }
5680                if (dataType.getDataType() == EDataType.resultset_t) {
5681                        return false;
5682                }
5683                if (dataType.getDataType() == EDataType.row_t) {
5684                        return false;
5685                }
5686                if (dataType.getDataType() == EDataType.map_t) {
5687                        return false;
5688                }
5689                if (dataType.getDataType() == EDataType.anyType_t) {
5690                        return false;
5691                }
5692                if (dataType.getDataType() == EDataType.struct_t) {
5693                        return false;
5694                }
5695                if (dataType.getDataType() == EDataType.structType_t) {
5696                        return false;
5697                }
5698                if (dataType.getDataType() == EDataType.mapType_t) {
5699                        return false;
5700                }
5701                return true;
5702        }
5703
5704        protected void analyzeResultColumnExpressionRelation(Object resultColumn, TExpression expression) {
5705                columnsInExpr visitor = new columnsInExpr();
5706                expression.inOrderTraverse(visitor);
5707                List<TObjectName> objectNames = visitor.getObjectNames();
5708                List<TParseTreeNode> functions = visitor.getFunctions();
5709                List<TParseTreeNode> constants = visitor.getConstants();
5710                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
5711
5712                if (functions != null && !functions.isEmpty()) {
5713                        analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function);
5714                }
5715                if (subquerys != null && !subquerys.isEmpty()) {
5716                        analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.select);
5717                }
5718                if (objectNames != null && !objectNames.isEmpty()) {
5719                        analyzeDataFlowRelation(resultColumn, objectNames, EffectType.select, functions);
5720                }
5721                if (constants != null && !constants.isEmpty()) {
5722                        analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.select, functions);
5723                }
5724        }
5725
5726        private void analyzeDb2ReturnStmt(TDb2ReturnStmt stmt) {
5727                if (stmt.getReturnExpr() != null) {
5728                        TExpression returnExpression = stmt.getReturnExpr();
5729                        ResultSet returnResult = modelFactory.createResultSet(stmt, true);
5730                        ResultColumn resultColumn = modelFactory.createResultColumn(returnResult, returnExpression);
5731                        
5732                        columnsInExpr visitor = new columnsInExpr();
5733                        stmt.getReturnExpr().inOrderTraverse(visitor);
5734                        
5735                        List<TObjectName> objectNames = visitor.getObjectNames();
5736                        List<TParseTreeNode> functions = visitor.getFunctions();
5737                        List<TParseTreeNode> constants = visitor.getConstants();
5738                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
5739
5740                        if (functions != null && !functions.isEmpty()) {
5741                                analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function);
5742                        }
5743                        if (subquerys != null && !subquerys.isEmpty()) {
5744                                analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.select);
5745                        }
5746                        if (objectNames != null && !objectNames.isEmpty()) {
5747                                analyzeDataFlowRelation(resultColumn, objectNames, EffectType.select, functions);
5748                        }
5749                        if (constants != null && !constants.isEmpty()) {
5750                                analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.select, functions);
5751                        }
5752
5753                        String procedureParent = SQLUtil.trimColumnStringQuote(getProcedureParentName(stmt));
5754                        if (procedureParent != null) {
5755                                modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent),
5756                                                returnResult);
5757                        }
5758                }
5759        }
5760
5761        private void analyzeReturnStmt(TReturnStmt stmt) {
5762                if (stmt.getResultColumnList() != null) {
5763                        ResultSet returnResult = modelFactory.createResultSet(stmt, true);
5764                        for (TResultColumn column : stmt.getResultColumnList()) {
5765                                ResultColumn resultColumn = modelFactory.createResultColumn(returnResult, column);
5766
5767                                columnsInExpr visitor = new columnsInExpr();
5768                                column.getExpr().inOrderTraverse(visitor);
5769
5770                                List<TObjectName> objectNames = visitor.getObjectNames();
5771                                List<TParseTreeNode> functions = visitor.getFunctions();
5772                                List<TParseTreeNode> constants = visitor.getConstants();
5773                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
5774
5775                                if (functions != null && !functions.isEmpty()) {
5776                                        analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function);
5777                                }
5778                                if (subquerys != null && !subquerys.isEmpty()) {
5779                                        analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.select);
5780                                }
5781                                if (objectNames != null && !objectNames.isEmpty()) {
5782                                        analyzeDataFlowRelation(resultColumn, objectNames, EffectType.select, functions);
5783                                }
5784                                if (constants != null && !constants.isEmpty()) {
5785                                        analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.select, functions);
5786                                }
5787
5788                        }
5789
5790                        String procedureParent = SQLUtil.trimColumnStringQuote(getProcedureParentName(stmt));
5791                        modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent),
5792                                        returnResult);
5793                }
5794                else if(stmt.getExpression()!=null){
5795                        TExpression returnExpression = stmt.getExpression();
5796                        ResultSet returnResult = modelFactory.createResultSet(stmt, true);
5797                        
5798                        columnsInExpr visitor = null;
5799                        List<TSelectSqlStatement> subquerys = null;
5800                        
5801                        if (returnExpression.getFunctionCall() != null
5802                                        && returnExpression.getFunctionCall().getFunctionName().toString().equalsIgnoreCase("table")
5803                                        && returnExpression.getFunctionCall().getArgs() != null
5804                                        && returnExpression.getFunctionCall().getArgs().size()>0) {
5805                                visitor = new columnsInExpr();
5806                                returnExpression.getFunctionCall().getArgs().getExpression(0).inOrderTraverse(visitor);
5807                                subquerys = visitor.getSubquerys();
5808                                if (subquerys != null && !subquerys.isEmpty()) {
5809                                        analyzeSelectStmt(subquerys.get(0));
5810                                        ResultSet resultSet = (ResultSet) modelManager.getModel(subquerys.get(0));
5811                                        if (resultSet != null && resultSet.getColumns() != null) {
5812                                                for (ResultColumn column : resultSet.getColumns()) {
5813                                                        TObjectName columnName = new TObjectName();
5814                                                        columnName.setString(column.getName());
5815                                                        ResultColumn resultColumn = modelFactory.createResultColumn(returnResult, columnName);
5816                                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
5817                                                        dataflowRelation.setEffectType(EffectType.select);
5818                                                        dataflowRelation.addSource(new ResultColumnRelationshipElement(column));
5819                                                        dataflowRelation.setTarget(new ResultColumnRelationshipElement(resultColumn));
5820                                                }
5821                                        }
5822                                        returnResult.setDetermined(resultSet.isDetermined());
5823                                        String procedureParent = SQLUtil.trimColumnStringQuote(getProcedureParentName(stmt));
5824                                        modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent),
5825                                                        returnResult);
5826                                        return;
5827                                }
5828                        }
5829                        
5830                        ResultColumn resultColumn = null;
5831                        if (returnExpression.getExpressionType() == EExpressionType.simple_object_name_t) {
5832                                TObjectName columnName = new TObjectName();
5833                                columnName.setString("*");
5834                                resultColumn = modelFactory.createResultColumn(returnResult, columnName);
5835                        }
5836                        else {
5837                                resultColumn = modelFactory.createResultColumn(returnResult, returnExpression);
5838                        }
5839
5840                        visitor = new columnsInExpr();
5841                        stmt.getExpression().inOrderTraverse(visitor);
5842
5843                        List<TObjectName> objectNames = visitor.getObjectNames();
5844                        List<TParseTreeNode> functions = visitor.getFunctions();
5845                        List<TParseTreeNode> constants = visitor.getConstants();
5846                        subquerys = visitor.getSubquerys();
5847
5848                        if (functions != null && !functions.isEmpty()) {
5849                                analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function);
5850                        }
5851                        if (subquerys != null && !subquerys.isEmpty()) {
5852                                analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.select);
5853                        }
5854                        if (objectNames != null && !objectNames.isEmpty()) {
5855                                DataFlowRelationship relation = analyzeDataFlowRelation(resultColumn, objectNames, EffectType.select, functions);
5856                                //如果variable对应的不是一个复杂结构，则resultColumn不要设置为*
5857                                if (relation != null && relation.getTarget().getElement() == resultColumn && relation.getSources().size() == 1) {
5858                                        Object column = relation.getSources().iterator().next().getElement();
5859                                        boolean star = true;
5860                                        if (column instanceof TableColumn && ((TableColumn) column).getName().indexOf("*") == -1) {
5861                                                star = false;
5862                                        }
5863                                        if (column instanceof ResultColumn && ((ResultColumn) column).getName().indexOf("*") == -1) {
5864                                                star = false;
5865                                        }
5866                                        if (!star && returnExpression.getExpressionType() == EExpressionType.simple_object_name_t) {
5867                                                resultColumn = modelFactory.createResultColumn(returnResult, returnExpression.getObjectOperand());
5868                                                returnResult.getColumns().clear();
5869                                                returnResult.addColumn(resultColumn);
5870                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
5871                                        }
5872                                }
5873                        }
5874                        if (constants != null && !constants.isEmpty()) {
5875                                analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.select, functions);
5876                        }
5877
5878                        String procedureParent = SQLUtil.trimColumnStringQuote(getProcedureParentName(stmt));
5879                        if (procedureParent != null) {
5880                                modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent),
5881                                                returnResult);
5882                        }
5883                }
5884        }
5885
5886        private void analyzeMssqlReturnStmt(TMssqlReturn stmt) {
5887                if (stmt.getResultColumnList() != null) {
5888                        ResultSet returnResult = modelFactory.createResultSet(stmt, true);
5889                        for (TResultColumn column : stmt.getResultColumnList()) {
5890                                ResultColumn resultColumn = modelFactory.createResultColumn(returnResult, column);
5891
5892                                columnsInExpr visitor = new columnsInExpr();
5893                                column.getExpr().inOrderTraverse(visitor);
5894
5895                                List<TObjectName> objectNames = visitor.getObjectNames();
5896                                List<TParseTreeNode> functions = visitor.getFunctions();
5897                                List<TParseTreeNode> constants = visitor.getConstants();
5898                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
5899
5900                                if (functions != null && !functions.isEmpty()) {
5901                                        analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function);
5902                                }
5903                                if (subquerys != null && !subquerys.isEmpty()) {
5904                                        analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.select);
5905                                }
5906                                if (objectNames != null && !objectNames.isEmpty()) {
5907                                        analyzeDataFlowRelation(resultColumn, objectNames, EffectType.select, functions);
5908                                }
5909                                if (constants != null && !constants.isEmpty()) {
5910                                        analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.select, functions);
5911                                }
5912
5913                        }
5914
5915                        String procedureParent = SQLUtil.trimColumnStringQuote(getProcedureParentName(stmt));
5916                        modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent),
5917                                        returnResult);
5918                }
5919                else if(stmt.getReturnExpr()!=null){
5920                        TExpression returnExpression = stmt.getReturnExpr();
5921                        ResultSet returnResult = modelFactory.createResultSet(stmt, true);
5922                        
5923                        if (returnExpression.getSubQuery() == null) {
5924                                ResultColumn resultColumn = modelFactory.createResultColumn(returnResult, returnExpression);
5925                                columnsInExpr visitor = new columnsInExpr();
5926                                stmt.getReturnExpr().inOrderTraverse(visitor);
5927
5928                                List<TObjectName> objectNames = visitor.getObjectNames();
5929                                List<TParseTreeNode> functions = visitor.getFunctions();
5930                                List<TParseTreeNode> constants = visitor.getConstants();
5931                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
5932
5933                                if (functions != null && !functions.isEmpty()) {
5934                                        analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function);
5935                                }
5936                                if (subquerys != null && !subquerys.isEmpty()) {
5937                                        analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.select);
5938                                }
5939                                if (objectNames != null && !objectNames.isEmpty()) {
5940                                        analyzeDataFlowRelation(resultColumn, objectNames, EffectType.select, functions);
5941                                }
5942                                if (constants != null && !constants.isEmpty()) {
5943                                        analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.select, functions);
5944                                }
5945                        }
5946                        else {
5947                                analyzeSelectStmt(returnExpression.getSubQuery());
5948                                ResultSet subResultSet = (ResultSet)modelManager.getModel(returnExpression.getSubQuery());
5949                                if (subResultSet != null) {
5950                                        for (ResultColumn subResultColumn : subResultSet.getColumns()) {
5951                                                TObjectName objectName = new TObjectName();
5952                                                objectName.setString(subResultColumn.getName());
5953                                                ResultColumn resultColumn = modelFactory.createResultColumn(returnResult, objectName);
5954                                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
5955                                                dataflowRelation.setEffectType(EffectType.select);
5956                                                dataflowRelation.addSource(new ResultColumnRelationshipElement(subResultColumn));
5957                                                dataflowRelation.setTarget(new ResultColumnRelationshipElement(resultColumn));
5958
5959                                        }
5960                                        
5961                                        if(subResultSet.getRelationRows().hasRelation()) {
5962                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
5963                                                impactRelation.setEffectType(EffectType.select);
5964                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
5965                                                                subResultSet.getRelationRows()));
5966                                                impactRelation.setTarget(
5967                                                                new RelationRowsRelationshipElement<ResultSetRelationRows>(returnResult.getRelationRows()));
5968                                        }
5969                                }
5970                        }
5971                        
5972                        String procedureParent = SQLUtil.trimColumnStringQuote(getProcedureParentName(stmt));
5973                        if (procedureParent != null) {
5974                                modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent),
5975                                                returnResult);
5976                        }
5977                }
5978        }
5979        
5980        private void analyzeFetchStmt(TFetchStmt stmt) {
5981                if (stmt.getVariableNames() != null) {
5982                        for (int i = 0; i < stmt.getVariableNames().size(); i++) {
5983                                TExpression variableExpression = stmt.getVariableNames().getExpression(i);
5984                                if (variableExpression.getExpressionType() == EExpressionType.simple_object_name_t) {
5985                                        TObjectName columnObject = variableExpression.getObjectOperand();
5986                                        if (columnObject.getDbObjectType() == EDbObjectType.variable) {
5987                                                continue;
5988                                        }
5989
5990                                        if (columnObject.getColumnNameOnly().startsWith("@") && (option.getVendor() == EDbVendor.dbvmssql
5991                                                        || option.getVendor() == EDbVendor.dbvazuresql)) {
5992                                                continue;
5993                                        }
5994
5995                                        if (columnObject.getColumnNameOnly().startsWith(":") && (option.getVendor() == EDbVendor.dbvhana
5996                                                        || option.getVendor() == EDbVendor.dbvteradata)) {
5997                                                continue;
5998                                        }
5999
6000                                        Variable cursorVariable = modelFactory.createVariable(columnObject);
6001                                        cursorVariable.setSubType(SubType.record);
6002                                        if (cursorVariable.isDetermined()) {
6003                                                if (stmt.getCursorName() != null) {
6004                                                        String procedureName = DlineageUtil.getProcedureParentName(stmt);
6005                                                        String variableString = stmt.getCursorName().toString();
6006                                                        if (variableString.startsWith(":")) {
6007                                                                variableString = variableString.substring(variableString.indexOf(":") + 1);
6008                                                        }
6009                                                        if (!SQLUtil.isEmpty(procedureName)) {
6010                                                                variableString = procedureName + "."
6011                                                                                + SQLUtil.getIdentifierNormalTableName(variableString);
6012                                                        }
6013                                                        Table cursor = modelManager.getTableByName(DlineageUtil.getTableFullName(variableString));
6014                                                        if (cursor != null) {
6015                                                                for (TableColumn variableProperty : cursorVariable.getColumns()) {
6016                                                                        boolean flag = false;
6017                                                                        for (TableColumn cursorColumn : cursor.getColumns()) {
6018                                                                                if (getColumnName(cursorColumn.getName())
6019                                                                                                .equalsIgnoreCase(variableProperty.getName())) {
6020                                                                                        DataFlowRelationship dataflowRelation = modelFactory
6021                                                                                                        .createDataFlowRelation();
6022                                                                                        dataflowRelation.setEffectType(EffectType.cursor);
6023                                                                                        dataflowRelation
6024                                                                                                        .addSource(new TableColumnRelationshipElement(cursorColumn));
6025                                                                                        dataflowRelation
6026                                                                                                        .setTarget(new TableColumnRelationshipElement(variableProperty));
6027                                                                                        flag = true;
6028                                                                                        break;
6029                                                                                }
6030                                                                        }
6031
6032                                                                        if (!flag) {
6033                                                                                if (cursor.getColumns().size() == stmt.getVariableNames().size()) {
6034                                                                                        DataFlowRelationship dataflowRelation = modelFactory
6035                                                                                                        .createDataFlowRelation();
6036                                                                                        dataflowRelation.setEffectType(EffectType.cursor);
6037                                                                                        dataflowRelation
6038                                                                                                        .setTarget(new TableColumnRelationshipElement(variableProperty));
6039                                                                                        dataflowRelation.addSource(
6040                                                                                                        new TableColumnRelationshipElement(cursor.getColumns().get(i)));
6041                                                                                }
6042                                                                                else {
6043                                                                                        for (int j = 0; j < cursor.getColumns().size(); j++) {
6044                                                                                                DataFlowRelationship dataflowRelation = modelFactory
6045                                                                                                                .createDataFlowRelation();
6046                                                                                                dataflowRelation.setEffectType(EffectType.cursor);
6047                                                                                                if (stmt.getVariableNames().size() == 1) {
6048                                                                                                        dataflowRelation.addSource(new TableColumnRelationshipElement(
6049                                                                                                                        cursor.getColumns().get(j)));
6050                                                                                                } else {
6051                                                                                                        dataflowRelation.addSource(new TableColumnRelationshipElement(
6052                                                                                                                        cursor.getColumns().get(j), i));
6053                                                                                                }
6054                                                                                                dataflowRelation.setTarget(
6055                                                                                                                new TableColumnRelationshipElement(variableProperty));
6056                                                                                        }
6057                                                                                }
6058                                                                        }
6059                                                                }
6060                                                        }
6061                                                }
6062                                                
6063                                        } else {
6064                                                TableColumn variableProperty = null;
6065                                                if (stmt.getVariableNames().size() == 1) {
6066                                                        if (cursorVariable.getColumns() == null || cursorVariable.getColumns().isEmpty()) {
6067                                                                TObjectName starColumn = new TObjectName();
6068                                                                starColumn.setString("*");
6069                                                                variableProperty = modelFactory.createTableColumn(cursorVariable, starColumn, true);
6070                                                        } else {
6071                                                                variableProperty = cursorVariable.getColumns().get(0);
6072                                                        }
6073                                                } else {
6074                                                        variableProperty = modelFactory.createTableColumn(cursorVariable, columnObject, true);
6075                                                }
6076
6077                                                if (stmt.getCursorName() != null) {
6078                                                        String procedureName = DlineageUtil.getProcedureParentName(stmt);
6079                                                        String variableString = stmt.getCursorName().toString();
6080                                                        if (variableString.startsWith(":")) {
6081                                                                variableString = variableString.substring(variableString.indexOf(":") + 1);
6082                                                        }
6083                                                        if (!SQLUtil.isEmpty(procedureName)) {
6084                                                                variableString = procedureName + "."
6085                                                                                + SQLUtil.getIdentifierNormalTableName(variableString);
6086                                                        }
6087                                                        Table cursor = modelManager.getTableByName(DlineageUtil.getTableFullName(variableString));
6088                                                        if (cursor != null) {
6089                                                                for (int j = 0; j < cursor.getColumns().size(); j++) {
6090                                                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
6091                                                                        dataflowRelation.setEffectType(EffectType.cursor);
6092                                                                        if (stmt.getVariableNames().size() == 1) {
6093                                                                                dataflowRelation.addSource(
6094                                                                                                new TableColumnRelationshipElement(cursor.getColumns().get(j)));
6095                                                                        } else {
6096                                                                                dataflowRelation.addSource(
6097                                                                                                new TableColumnRelationshipElement(cursor.getColumns().get(j), i));
6098                                                                        }
6099                                                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(variableProperty));
6100                                                                }
6101                                                        }
6102                                                }
6103                                        }
6104                                }
6105                        }
6106                }
6107        }
6108
6109        private void analyzeFetchStmt(TMssqlFetch stmt) {
6110                if (stmt.getVariableNames() != null) {
6111                        for (int i = 0; i < stmt.getVariableNames().size(); i++) {
6112                                TObjectName columnObject = stmt.getVariableNames().getObjectName(i);
6113                                Variable cursorVariable = modelFactory.createVariable(columnObject);
6114                                cursorVariable.setCreateTable(true);
6115                                cursorVariable.setSubType(SubType.record);
6116                                TableColumn variableProperty = null;
6117                                if (stmt.getVariableNames().size() == 1) {
6118                                        if (cursorVariable.getColumns() == null || cursorVariable.getColumns().isEmpty()) {
6119                                                TObjectName starColumn = new TObjectName();
6120                                                starColumn.setString("*");
6121                                                variableProperty = modelFactory.createTableColumn(cursorVariable, starColumn, true);
6122                                        } else {
6123                                                variableProperty = cursorVariable.getColumns().get(0);
6124                                        }
6125                                } else {
6126                                        variableProperty = modelFactory.createTableColumn(cursorVariable, columnObject, true);
6127                                }
6128
6129                                if (stmt.getCursorName() != null) {
6130                                        String procedureName = DlineageUtil.getProcedureParentName(stmt);
6131                                        String variableString = stmt.getCursorName().toString();
6132                                        if (variableString.startsWith(":")) {
6133                                                variableString = variableString.substring(variableString.indexOf(":") + 1);
6134                                        }
6135                                        if (!SQLUtil.isEmpty(procedureName)) {
6136                                                variableString = procedureName + "." + SQLUtil.getIdentifierNormalTableName(variableString);
6137                                        }
6138                                        Table cursor = modelManager
6139                                                        .getTableByName(DlineageUtil.getTableFullName(variableString));
6140                                        if (cursor != null) {
6141                                                for (int j = 0; j < cursor.getColumns().size(); j++) {
6142                                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
6143                                                        dataflowRelation.setEffectType(EffectType.cursor);
6144                                                        if (stmt.getVariableNames().size() == 1) {
6145                                                                dataflowRelation
6146                                                                                .addSource(new TableColumnRelationshipElement(cursor.getColumns().get(j)));
6147                                                        } else {
6148                                                                dataflowRelation
6149                                                                                .addSource(new TableColumnRelationshipElement(cursor.getColumns().get(j), i));
6150                                                        }
6151                                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(variableProperty));
6152                                                }
6153                                        }
6154                                }
6155                        }
6156
6157                }
6158        }
6159
6160        private void analyzeLoopStmt(TLoopStmt stmt) {
6161
6162                if (stmt.getCursorName() != null && stmt.getIndexName() != null) {
6163                        modelManager.bindCursorIndex(stmt.getIndexName(), stmt.getCursorName());
6164                }
6165
6166                if (stmt.getRecordName() != null && stmt.getSubquery() != null) {
6167                        Variable cursorTempTable = modelFactory.createCursor(stmt);
6168                        cursorTempTable.setVariable(true);
6169                        cursorTempTable.setSubType(SubType.cursor);
6170                        modelManager.bindCursorModel(stmt, cursorTempTable);
6171                        analyzeSelectStmt(stmt.getSubquery());
6172
6173                        TableColumn cursorColumn = null;
6174                        if (cursorTempTable.getColumns() == null || cursorTempTable.getColumns().isEmpty()) {
6175                                TObjectName starColumn = new TObjectName();
6176                                starColumn.setString("*");
6177                                cursorColumn = modelFactory.createTableColumn(cursorTempTable, starColumn, true);
6178                        } else {
6179                                cursorColumn = cursorTempTable.getColumns().get(0);
6180                        }
6181
6182
6183                        ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt.getSubquery());
6184                        if (resultSetModel != null) {
6185                                for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
6186                                        ResultColumn resultColumn = resultSetModel.getColumns().get(i);
6187                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
6188                                        dataflowRelation.setEffectType(EffectType.cursor);
6189                                        dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn));
6190                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(cursorColumn));
6191                                }
6192                        }
6193                }
6194
6195                for (int i = 0; i < stmt.getStatements().size(); i++) {
6196                        analyzeCustomSqlStmt(stmt.getStatements().get(i));
6197                }
6198        }
6199
6200        private void analyzeForStmt(TForStmt stmt) {
6201                if (stmt.getSubquery() == null) {
6202                        return;
6203                }
6204
6205                Variable cursorTempTable = modelFactory.createCursor(stmt);
6206                cursorTempTable.setVariable(true);
6207                cursorTempTable.setSubType(SubType.cursor);
6208                cursorTempTable.setCreateTable(true);
6209                modelManager.bindCursorModel(stmt, cursorTempTable);
6210                analyzeSelectStmt(stmt.getSubquery());
6211
6212                TableColumn cursorColumn = null;
6213                if (cursorTempTable.getColumns() == null || cursorTempTable.getColumns().isEmpty()) {
6214                        TObjectName starColumn = new TObjectName();
6215                        starColumn.setString("*");
6216                        cursorColumn = modelFactory.createTableColumn(cursorTempTable, starColumn, true);
6217                } else {
6218                        cursorColumn = cursorTempTable.getColumns().get(0);
6219                }
6220
6221
6222                ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt.getSubquery());
6223                if (resultSetModel != null) {
6224                        for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
6225                                ResultColumn resultColumn = resultSetModel.getColumns().get(i);
6226                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
6227                                dataflowRelation.setEffectType(EffectType.cursor);
6228                                dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn));
6229                                dataflowRelation.setTarget(new TableColumnRelationshipElement(cursorColumn));
6230                        }
6231                }
6232
6233                if (stmt.getStatements() != null && stmt.getStatements().size() > 0) {
6234                        for (int i = 0; i < stmt.getStatements().size(); i++) {
6235                                analyzeCustomSqlStmt(stmt.getStatements().get(i));
6236                        }
6237                }
6238        }
6239
6240        private void analyzeVarDeclStmt(TVarDeclStmt stmt) {
6241                TTypeName typeName = stmt.getDataType();
6242                if (typeName != null && typeName.toString().toUpperCase().indexOf("ROWTYPE") != -1) {
6243                        Variable cursorVariable = modelFactory.createVariable(stmt.getElementName());
6244                        cursorVariable.setSubType(SubType.record_type);
6245
6246                        Table variableTable = modelFactory.createTableByName(typeName.getDataTypeName(), false);
6247                        if(!variableTable.isCreateTable()) {
6248                                TObjectName starColumn1 = new TObjectName();
6249                                starColumn1.setString("*");
6250                                TableColumn variableTableStarColumn = modelFactory.createTableColumn(variableTable, starColumn1, true);
6251                                variableTableStarColumn.setShowStar(false);
6252                                variableTableStarColumn.setExpandStar(true);
6253
6254                                TObjectName starColumn = new TObjectName();
6255                                starColumn.setString("*");
6256                                TableColumn variableProperty = modelFactory.createTableColumn(cursorVariable, starColumn, true);
6257                                variableProperty.setShowStar(false);
6258                                variableProperty.setExpandStar(true);
6259
6260                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
6261                                dataflowRelation.setEffectType(EffectType.rowtype);
6262                                dataflowRelation.addSource(new TableColumnRelationshipElement(variableTableStarColumn));
6263                                dataflowRelation.setTarget(new TableColumnRelationshipElement(variableProperty));
6264                        } else {
6265                                for (TableColumn sourceColumn : variableTable.getColumns()) {
6266                                        String columnName = sourceColumn.getName();
6267                                        TObjectName targetColumn = new TObjectName();
6268                                        targetColumn.setString(columnName);
6269                                        TableColumn variableProperty = modelFactory.createTableColumn(cursorVariable, targetColumn, true);
6270                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
6271                                        dataflowRelation.setEffectType(EffectType.rowtype);
6272                                        dataflowRelation.addSource(new TableColumnRelationshipElement(sourceColumn));
6273                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(variableProperty));
6274                                }
6275                        }
6276                } else if (stmt.getElementName() != null) {
6277                        Variable variable = modelFactory.createVariable(stmt.getElementName());
6278                        variable.setCreateTable(true);
6279                        variable.setSubType(SubType.record);
6280                        TableColumn tableColumn = null;
6281                        if (stmt.getDataType() != null && (modelFactory.createVariable(stmt.getDataType().getDataTypeName(), false)!=null || isCursorType(stmt.getDataType().getDataTypeName()))) {
6282                                Variable cursorVariable = modelFactory.createVariable(stmt.getDataType().getDataTypeName(), false);
6283                                if (cursorVariable != null) {
6284                                        if (cursorVariable.getSubType() == SubType.record_type) {
6285                                                variable.setSubType(SubType.record_type);
6286                                        }
6287                                        if(cursorVariable.isDetermined()) {
6288                                                for (int k = 0; k < cursorVariable.getColumns().size(); k++) {
6289                                                        TableColumn sourceColumn = cursorVariable.getColumns().get(k);
6290                                                        TObjectName objectName = new TObjectName();
6291                                                        objectName.setString(sourceColumn.getName());
6292                                                        tableColumn = modelFactory.createTableColumn(variable, objectName, true);
6293                                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
6294                                                        dataflowRelation.setEffectType(EffectType.cursor);
6295                                                        dataflowRelation
6296                                                                        .addSource(new TableColumnRelationshipElement(sourceColumn));
6297                                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(tableColumn));
6298                                                }
6299                                                variable.setDetermined(true);
6300                                                return;
6301                                        }
6302                                        else {
6303                                                TObjectName objectName = new TObjectName();
6304                                                objectName.setString("*");
6305                                                tableColumn = modelFactory.createTableColumn(variable, objectName, true);
6306                                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
6307                                                dataflowRelation.setEffectType(EffectType.cursor);
6308                                                dataflowRelation
6309                                                                .addSource(new TableColumnRelationshipElement(cursorVariable.getColumns().get(0)));
6310                                                dataflowRelation.setTarget(new TableColumnRelationshipElement(tableColumn));
6311                                        }
6312                                }
6313                                else {
6314                                        TObjectName objectName = new TObjectName();
6315                                        objectName.setString("*");
6316                                        tableColumn = modelFactory.createTableColumn(variable, objectName, true);                                       
6317                                }
6318                        } else {
6319                                tableColumn = modelFactory.createTableColumn(variable, stmt.getElementName(), true);
6320                                tableColumn.setVariant(true);
6321                        }
6322
6323                        if (stmt.getDefaultValue() != null) {
6324                                columnsInExpr visitor = new columnsInExpr();
6325                                stmt.getDefaultValue().inOrderTraverse(visitor);
6326                                List<TObjectName> objectNames = visitor.getObjectNames();
6327                                List<TParseTreeNode> functions = visitor.getFunctions();
6328                                List<TParseTreeNode> constants = visitor.getConstants();
6329                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
6330
6331                                if (functions != null && !functions.isEmpty()) {
6332                                        analyzeFunctionDataFlowRelation(tableColumn, functions, EffectType.function);
6333                                }
6334                                if (subquerys != null && !subquerys.isEmpty()) {
6335                                        analyzeSubqueryDataFlowRelation(tableColumn, subquerys, EffectType.select);
6336                                }
6337                                if (objectNames != null && !objectNames.isEmpty()) {
6338                                        analyzeDataFlowRelation(tableColumn, objectNames, EffectType.select, functions);
6339                                }
6340                                if (constants != null && !constants.isEmpty()) {
6341                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select, functions);
6342                                }
6343                        }
6344                        
6345                        
6346                }
6347        }
6348
6349        private boolean isCursorType(String dataTypeName) {
6350                if (dataTypeName != null && dataTypeName.toLowerCase().contains("cursor")) {
6351                        return true;
6352                }
6353                return false;
6354        }
6355
6356        private void analyzeSetStmt(TSetStmt stmt) {
6357                TExpression right = stmt.getVariableValue();
6358                TObjectName columnObject = stmt.getVariableName();
6359                if (columnObject != null) {
6360                        TableColumn tableColumn = null;
6361                        Variable tableModel;
6362                        if (columnObject.toString().indexOf(".") != -1) {
6363                                List<String> splits = SQLUtil.parseNames(columnObject.toString());
6364                                tableModel = modelFactory.createVariable(splits.get(splits.size() - 2));
6365                        } else {
6366                                tableModel = modelFactory.createVariable(columnObject);
6367                        }
6368                        tableModel.setCreateTable(true);
6369                        tableModel.setSubType(SubType.record);
6370
6371                        if (tableModel.getColumns() == null || tableModel.getColumns().isEmpty()) {
6372                                tableColumn = modelFactory.createTableColumn(tableModel, columnObject, true);
6373                        } else {
6374                                tableColumn = tableModel.getColumns().get(0);
6375                        }
6376
6377                        if (tableColumn != null && right!=null) {
6378                                columnsInExpr visitor = new columnsInExpr();
6379                                right.inOrderTraverse(visitor);
6380                                List<TObjectName> objectNames = visitor.getObjectNames();
6381                                List<TParseTreeNode> functions = visitor.getFunctions();
6382                                List<TParseTreeNode> constants = visitor.getConstants();
6383                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
6384
6385                                if (functions != null && !functions.isEmpty()) {
6386                                        analyzeFunctionDataFlowRelation(tableColumn, functions, EffectType.function);
6387                                }
6388                                if (subquerys != null && !subquerys.isEmpty()) {
6389                                        analyzeSubqueryDataFlowRelation(tableColumn, subquerys, EffectType.select);
6390                                }
6391                                if (objectNames != null && !objectNames.isEmpty()) {
6392                                        analyzeDataFlowRelation(tableColumn, objectNames, EffectType.select, functions);
6393                                }
6394                                if (constants != null && !constants.isEmpty()) {
6395                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select, functions);
6396                                }
6397                                
6398                                if(columnObject.toString().equalsIgnoreCase("search_path") && !constants.isEmpty()) {
6399                                        ModelBindingManager.setGlobalSchema(constants.get(0).toString());
6400                                }
6401                        }
6402                }
6403
6404                if(stmt.getAssignments()!=null){
6405                        for (int i = 0; i < stmt.getAssignments().size(); i++) {
6406                                TSetAssignment assignStmt = stmt.getAssignments().getElement(i);
6407                                analyzeSetAssignmentStmt(assignStmt);
6408                        }
6409                }
6410        }
6411
6412        private void analyzeSetAssignmentStmt(TSetAssignment stmt) {
6413                TExpression right = stmt.getParameterValue();
6414                TObjectName columnObject = stmt.getParameterName();
6415                if (columnObject != null) {
6416                        TableColumn tableColumn = null;
6417                        Variable tableModel;
6418                        if (columnObject.toString().indexOf(".") != -1) {
6419                                List<String> splits = SQLUtil.parseNames(columnObject.toString());
6420                                tableModel = modelFactory.createVariable(splits.get(splits.size() - 2));
6421                        } else {
6422                                tableModel = modelFactory.createVariable(columnObject);
6423                        }
6424                        tableModel.setCreateTable(true);
6425                        tableModel.setSubType(SubType.record);
6426                        if (tableModel.getColumns() == null || tableModel.getColumns().isEmpty()) {
6427                                tableColumn = modelFactory.createTableColumn(tableModel, columnObject, true);
6428                        } else {
6429                                tableColumn = tableModel.getColumns().get(0);
6430                        }
6431
6432                        if (tableColumn != null && right!=null) {
6433                                columnsInExpr visitor = new columnsInExpr();
6434                                right.inOrderTraverse(visitor);
6435                                List<TObjectName> objectNames = visitor.getObjectNames();
6436                                List<TParseTreeNode> functions = visitor.getFunctions();
6437                                List<TParseTreeNode> constants = visitor.getConstants();
6438                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
6439
6440                                if (functions != null && !functions.isEmpty()) {
6441                                        analyzeFunctionDataFlowRelation(tableColumn, functions, EffectType.function);
6442                                }
6443                                if (subquerys != null && !subquerys.isEmpty()) {
6444                                        analyzeSubqueryDataFlowRelation(tableColumn, subquerys, EffectType.select);
6445                                }
6446                                if (objectNames != null && !objectNames.isEmpty()) {
6447                                        analyzeDataFlowRelation(tableColumn, objectNames, EffectType.select, functions);
6448                                }
6449                                if (constants != null && !constants.isEmpty()) {
6450                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select, functions);
6451                                }
6452
6453                                if(columnObject.toString().equalsIgnoreCase("search_path") && !constants.isEmpty()) {
6454                                        ModelBindingManager.setGlobalSchema(constants.get(0).toString());
6455                                }
6456                        }
6457                }
6458        }
6459
6460        private void analyzeMssqlSetStmt(TMssqlSet stmt) {
6461                TExpression right = stmt.getVarExpr();
6462                TObjectName columnObject = stmt.getVarName();
6463                if (columnObject != null) {
6464                        TableColumn tableColumn = null;
6465                        Variable tableModel;
6466                        if (columnObject.toString().indexOf(".") != -1) {
6467                                List<String> splits = SQLUtil.parseNames(columnObject.toString());
6468                                tableModel = modelFactory.createVariable(splits.get(splits.size() - 2));
6469                        } else {
6470                                tableModel = modelFactory.createVariable(columnObject);
6471                        }
6472                        tableModel.setCreateTable(true);
6473                        tableModel.setSubType(SubType.record);
6474                        if (tableModel.getColumns() == null || tableModel.getColumns().isEmpty()) {
6475                                tableColumn = modelFactory.createTableColumn(tableModel, columnObject, true);
6476                        }
6477                        else {
6478                                tableColumn = tableModel.getColumns().get(0);
6479                        }
6480
6481                        if (tableColumn != null && right!=null) {
6482                                columnsInExpr visitor = new columnsInExpr();
6483                                right.inOrderTraverse(visitor);
6484                                List<TObjectName> objectNames = visitor.getObjectNames();
6485                                List<TParseTreeNode> functions = visitor.getFunctions();
6486                                List<TParseTreeNode> constants = visitor.getConstants();
6487                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
6488
6489                                if (functions != null && !functions.isEmpty()) {
6490                                        analyzeFunctionDataFlowRelation(tableColumn, functions, EffectType.function);
6491                                }
6492                                if (subquerys != null && !subquerys.isEmpty()) {
6493                                        analyzeSubqueryDataFlowRelation(tableColumn, subquerys, EffectType.select);
6494                                }
6495                                if (objectNames != null && !objectNames.isEmpty()) {
6496                                        analyzeDataFlowRelation(tableColumn, objectNames, EffectType.select, functions);
6497                                }
6498                                if (constants != null && !constants.isEmpty()) {
6499                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select, functions);
6500                                }
6501
6502                                if(columnObject.toString().equalsIgnoreCase("search_path") && !constants.isEmpty()) {
6503                                        ModelBindingManager.setGlobalSchema(constants.get(0).toString());
6504                                }
6505                        }
6506                        else if(tableColumn!=null && stmt.getSubquery()!=null){
6507                                analyzeCustomSqlStmt(stmt.getSubquery());
6508                                analyzeSubqueryDataFlowRelation(tableColumn, Arrays.asList(stmt.getSubquery()), EffectType.select);
6509                        }
6510                }
6511        }
6512
6513        private void analyzeAssignStmt(TAssignStmt stmt) {
6514                TExpression left = stmt.getLeft();
6515                TExpression right = stmt.getExpression();
6516                TObjectName columnObject = null;
6517                if (left == null) {
6518                        columnObject = stmt.getVariableName();
6519                } else if (left.getExpressionType() == EExpressionType.simple_object_name_t) {
6520                        columnObject = left.getObjectOperand();
6521                }
6522                if (columnObject != null) {
6523                        TableColumn tableColumn = null;
6524                        if (columnObject.getDbObjectType() == EDbObjectType.variable || stmt.getVariableName() != null) {
6525                                Variable tableModel;
6526                                if (columnObject.toString().indexOf(".") != -1) {
6527                                        List<String> splits = SQLUtil.parseNames(columnObject.toString());
6528                                        tableModel = modelFactory.createVariable(splits.get(splits.size() - 2));
6529                                } else {
6530                                        tableModel = modelFactory.createVariable(columnObject);
6531                                }
6532                                tableModel.setCreateTable(true);
6533                                tableModel.setSubType(SubType.record);
6534                                if (tableModel.getColumns() == null || tableModel.getColumns().isEmpty()) {
6535                                        tableColumn = modelFactory.createTableColumn(tableModel, columnObject, true);
6536                                } else {
6537                                        tableColumn = tableModel.getColumns().get(0);
6538                                }
6539                        } else {
6540                                List<String> splits = SQLUtil.parseNames(columnObject.toString());
6541                                if (splits.size() > 1) {
6542                                        Table tableModel = modelManager
6543                                                        .getTableByName(DlineageUtil.getTableFullName(splits.get(splits.size() - 2)));
6544                                        if (tableModel == null) {
6545                                                String procedureName = DlineageUtil.getProcedureParentName(stmt);
6546                                                String variableString = splits.get(splits.size() - 2).toString();
6547                                                if (variableString.startsWith(":")) {
6548                                                        variableString = variableString.substring(variableString.indexOf(":") + 1);
6549                                                }
6550                                                if (!SQLUtil.isEmpty(procedureName)) {
6551                                                        variableString = procedureName + "." + SQLUtil.getIdentifierNormalTableName(variableString);
6552                                                }
6553                                                tableModel = modelManager.getTableByName(DlineageUtil.getTableFullName(variableString));
6554                                        }
6555                                        if (tableModel != null) {
6556                                                tableColumn = modelFactory.createTableColumn(tableModel, columnObject, true);
6557                                        }
6558                                }
6559                        }
6560
6561                        if (tableColumn != null && right != null) {
6562                                Transform transform = new Transform();
6563                                transform.setType(Transform.EXPRESSION);
6564                                transform.setCode(right);
6565                                tableColumn.setTransform(transform);;
6566                                columnsInExpr visitor = new columnsInExpr();
6567                                right.inOrderTraverse(visitor);
6568                                List<TObjectName> objectNames = visitor.getObjectNames();
6569                                List<TParseTreeNode> functions = visitor.getFunctions();
6570                                List<TParseTreeNode> constants = visitor.getConstants();
6571                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
6572
6573                                if (functions != null && !functions.isEmpty()) {
6574                                        analyzeFunctionDataFlowRelation(tableColumn, functions, EffectType.function);
6575                                }
6576                                if (subquerys != null && !subquerys.isEmpty()) {
6577                                        analyzeSubqueryDataFlowRelation(tableColumn, subquerys, EffectType.select);
6578                                }
6579                                if (objectNames != null && !objectNames.isEmpty()) {
6580                                        analyzeDataFlowRelation(tableColumn, objectNames, EffectType.select, functions);
6581                                }
6582                                if (constants != null && !constants.isEmpty()) {
6583                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select, functions);
6584                                }
6585                        }
6586                }
6587        }
6588
6589        private void analyzeOpenForStmt(TOpenforStmt stmt) {
6590                if (stmt.getSubquery() == null) {
6591                        return;
6592                }
6593
6594                Variable cursorTempTable = modelFactory.createCursor(stmt);
6595                cursorTempTable.setVariable(true);
6596                cursorTempTable.setSubType(SubType.cursor);
6597                modelManager.bindCursorModel(stmt, cursorTempTable);
6598                analyzeSelectStmt(stmt.getSubquery());
6599
6600                TableColumn cursorColumn = null;
6601                if (cursorTempTable.getColumns() == null || cursorTempTable.getColumns().isEmpty()) {
6602                        TObjectName starColumn = new TObjectName();
6603                        starColumn.setString("*");
6604                        cursorColumn = modelFactory.createTableColumn(cursorTempTable, starColumn, true);
6605                } else {
6606                        cursorColumn = cursorTempTable.getColumns().get(0);
6607                }
6608
6609                ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt.getSubquery());
6610                if (resultSetModel != null) {
6611                        for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
6612                                ResultColumn resultColumn = resultSetModel.getColumns().get(i);
6613                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
6614                                dataflowRelation.setEffectType(EffectType.cursor);
6615                                dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn));
6616                                dataflowRelation.setTarget(new TableColumnRelationshipElement(cursorColumn));
6617                        }
6618                }
6619        }
6620
6621        private void analyzeCursorDeclStmt(TCursorDeclStmt stmt) {
6622                if (stmt.getSubquery() == null) {
6623                        return;
6624                }
6625
6626                Variable cursorTempTable = modelFactory.createCursor(stmt);
6627                cursorTempTable.setVariable(true);
6628                cursorTempTable.setSubType(SubType.cursor);
6629                modelManager.bindCursorModel(stmt, cursorTempTable);
6630                analyzeSelectStmt(stmt.getSubquery());
6631                
6632                ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt.getSubquery());
6633                if(resultSetModel!=null && resultSetModel.isDetermined()) {
6634                        for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
6635                                ResultColumn resultColumn = resultSetModel.getColumns().get(i);
6636                                TObjectName columnName = new TObjectName();
6637                                columnName.setString(resultColumn.getName());
6638                                TableColumn cursorColumn = modelFactory.createTableColumn(cursorTempTable, columnName, true);           
6639                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
6640                                dataflowRelation.setEffectType(EffectType.cursor);
6641                                dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn));
6642                                dataflowRelation.setTarget(new TableColumnRelationshipElement(cursorColumn));
6643                        }
6644                }
6645                else {
6646                        TableColumn cursorColumn = null;
6647                        if (cursorTempTable.getColumns() == null || cursorTempTable.getColumns().isEmpty()) {
6648                                TObjectName starColumn = new TObjectName();
6649                                starColumn.setString("*");
6650                                cursorColumn = modelFactory.createTableColumn(cursorTempTable, starColumn, true);
6651                        } else {
6652                                cursorColumn = cursorTempTable.getColumns().get(0);
6653                        }
6654                        if (resultSetModel != null) {
6655                                for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
6656                                        ResultColumn resultColumn = resultSetModel.getColumns().get(i);
6657                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
6658                                        dataflowRelation.setEffectType(EffectType.cursor);
6659                                        dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn));
6660                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(cursorColumn));
6661                                }
6662                        }
6663                }
6664                
6665        }
6666
6667        private void analyzeDb2Declare(TDb2SqlVariableDeclaration stmt) {
6668                TDeclareVariableList variables = stmt.getVariables();
6669                if (variables == null) {
6670                        return;
6671                }
6672                for (int i = 0; i < variables.size(); i++) {
6673                        TDeclareVariable variable = variables.getDeclareVariable(i);
6674                        if (variable.getTableTypeDefinitions() != null && variable.getTableTypeDefinitions().size() > 0) {
6675
6676
6677                                TObjectName tableName = variable.getVariableName();
6678                                TTableElementList columns = variable.getTableTypeDefinitions();
6679
6680                                Table tableModel = modelFactory.createTableByName(tableName, true);
6681                                tableModel.setCreateTable(true);
6682                                String procedureParent = getProcedureParentName(stmt);
6683                                if (procedureParent != null) {
6684                                        tableModel.setParent(procedureParent);
6685                                }
6686                                modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent), tableModel);
6687
6688                                for (int j = 0; j < columns.size(); j++) {
6689                                        TTableElement tableElement = columns.getTableElement(j);
6690                                        TColumnDefinition column = tableElement.getColumnDefinition();
6691                                        if (column != null && column.getColumnName() != null) {
6692                                                modelFactory.createTableColumn(tableModel, column.getColumnName(), true);
6693                                        }
6694                                }
6695                        } else if (variable.getVariableName() != null) {
6696                                Variable cursorVariable = modelFactory.createVariable(variable.getVariableName());
6697                                cursorVariable.setCreateTable(true);
6698                                cursorVariable.setSubType(SubType.record);
6699                                if(variable.getDatatype()!=null && isSimpleDataType(variable.getDatatype())){
6700                                        TableColumn variableProperty = modelFactory.createTableColumn(cursorVariable,
6701                                                        variable.getVariableName(), true);
6702                                }
6703                                else {
6704                                        TObjectName variableProperties = new TObjectName();
6705                                        variableProperties.setString("*");
6706                                        TableColumn variableProperty = modelFactory.createTableColumn(cursorVariable,
6707                                                        variableProperties, true);
6708                                }
6709                        }
6710                }
6711        }
6712
6713        private void analyzeMssqlDeclare(TMssqlDeclare stmt) {
6714                if (stmt.getDeclareType() == EDeclareType.variable) {
6715                        TDeclareVariableList variables = stmt.getVariables();
6716                        if (variables == null) {
6717                                return;
6718                        }
6719                        for (int i = 0; i < variables.size(); i++) {
6720                                TDeclareVariable variable = variables.getDeclareVariable(i);
6721                                if (variable.getTableTypeDefinitions() != null && variable.getTableTypeDefinitions().size() > 0) {
6722
6723
6724                                        TObjectName tableName = variable.getVariableName();
6725                                        TTableElementList columns = variable.getTableTypeDefinitions();
6726
6727                                        Variable tableModel = modelFactory.createVariable(tableName);
6728                                        tableModel.setVariable(true);
6729                    tableModel.setCreateTable(true);
6730                                        String procedureParent = getProcedureParentName(stmt);
6731                                        if (procedureParent != null) {
6732                                                tableModel.setParent(procedureParent);
6733                                        }
6734                                        modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent), tableModel);
6735
6736                                        for (int j = 0; j < columns.size(); j++) {
6737                                                TTableElement tableElement = columns.getTableElement(j);
6738                                                TColumnDefinition column = tableElement.getColumnDefinition();
6739                                                if (column != null && column.getColumnName() != null) {
6740                                                        modelFactory.createTableColumn(tableModel, column.getColumnName(), true);
6741                                                }
6742                                        }
6743                                } else if (variable.getVariableName() != null) {
6744                                        Variable cursorVariable = modelFactory.createVariable(variable.getVariableName());
6745                                        cursorVariable.setCreateTable(true);
6746                                        cursorVariable.setSubType(SubType.record);
6747                                        TableColumn variableProperty = null;
6748                                        if (variable.getDatatype() != null && isSimpleDataType(variable.getDatatype())) {
6749                                                variableProperty = modelFactory.createTableColumn(cursorVariable, variable.getVariableName(),
6750                                                                true);
6751                                        } else {
6752                                                TObjectName variableProperties = new TObjectName();
6753                                                variableProperties.setString("*");
6754                                                variableProperty = modelFactory.createTableColumn(cursorVariable, variableProperties, true);
6755                                        }
6756                                        
6757                                        if (variable.getDefaultValue() != null && variable.getDefaultValue().getSubQuery() != null) {
6758                                                analyzeSelectStmt(variable.getDefaultValue().getSubQuery());
6759                                                ResultSet resultSetModel = (ResultSet) modelManager
6760                                                                .getModel(variable.getDefaultValue().getSubQuery());
6761                                                if (variableProperty != null && resultSetModel != null) {
6762                                                        for (ResultColumn column : resultSetModel.getColumns()) {
6763                                                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
6764                                                                dataflowRelation.setEffectType(EffectType.select);
6765                                                                dataflowRelation.addSource(new ResultColumnRelationshipElement(column));
6766                                                                dataflowRelation.setTarget(new TableColumnRelationshipElement(variableProperty));
6767                                                        }
6768                                                }
6769                                        }
6770                                }
6771                        }
6772                } else if (stmt.getDeclareType() == EDeclareType.cursor) {
6773                        Variable cursorTempTable = modelFactory.createCursor(stmt);
6774                        cursorTempTable.setVariable(true);
6775                        cursorTempTable.setSubType(SubType.cursor);
6776                        modelManager.bindCursorModel(stmt, cursorTempTable);
6777                        analyzeSelectStmt(stmt.getSubquery());
6778                        ResultSet resultSetModel = (ResultSet)modelManager.getModel(stmt.getSubquery());
6779                        if (resultSetModel != null && resultSetModel.isDetermined()) {
6780                                for(ResultColumn resultColumn: resultSetModel.getColumns()){
6781                                        TObjectName starColumn = new TObjectName();
6782                                        starColumn.setString(resultColumn.getName());
6783                                        TableColumn cursorColumn = modelFactory.createTableColumn(cursorTempTable, starColumn, true);
6784                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
6785                                        dataflowRelation.setEffectType(EffectType.cursor);
6786                                        dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn));
6787                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(cursorColumn));
6788                                }
6789                        }
6790                        else {
6791                                TableColumn cursorColumn = null;
6792                                if (cursorTempTable.getColumns() == null || cursorTempTable.getColumns().isEmpty()) {
6793                                        TObjectName starColumn = new TObjectName();
6794                                        starColumn.setString("*");
6795                                        cursorColumn = modelFactory.createTableColumn(cursorTempTable, starColumn, true);
6796                                        cursorColumn.setShowStar(false);
6797                                        cursorColumn.setExpandStar(true);
6798                                } else {
6799                                        cursorColumn = cursorTempTable.getColumns().get(0);
6800                                }
6801
6802                                if (resultSetModel != null) {
6803                                        for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
6804                                                ResultColumn resultColumn = resultSetModel.getColumns().get(i);
6805                                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
6806                                                dataflowRelation.setEffectType(EffectType.cursor);
6807                                                dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn));
6808                                                dataflowRelation.setTarget(new TableColumnRelationshipElement(cursorColumn));
6809                                        }
6810                                }
6811                        }
6812                }
6813        }
6814
6815
6816        private boolean analyzeMssqlJsonDeclare(TMssqlDeclare stmt, String jsonName, Table jsonTable) {
6817                TDeclareVariableList variables = stmt.getVariables();
6818                if (variables == null) {
6819                        return false;
6820                }
6821                for (int i = 0; i < variables.size(); i++) {
6822                        TDeclareVariable variable = variables.getDeclareVariable(i);
6823                        TObjectName variableName = variable.getVariableName();
6824                        if (DlineageUtil.getIdentifierNormalTableName(variableName.toString())
6825                                        .equals(DlineageUtil.getIdentifierNormalTableName(jsonName))) {
6826                                if (variable.getDefaultValue() != null) {
6827                                        Table variableTable = modelFactory.createJsonVariable(variableName);
6828                                        variableTable.setVariable(true);
6829                                        variableTable.setSubType(SubType.scalar);
6830                                        variableTable.setCreateTable(true);
6831                                        TableColumn property = modelFactory.createVariableProperty(variableTable, variable);
6832
6833                                        for (int j = 0; j < jsonTable.getColumns().size(); j++) {
6834                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
6835                                                relation.setEffectType(EffectType.select);
6836                                                relation.setTarget(new TableColumnRelationshipElement(jsonTable.getColumns().get(j)));
6837                                                relation.addSource(new TableColumnRelationshipElement(property));
6838                                        }
6839                                }
6840                                return true;
6841                        }
6842                }
6843                return false;
6844        }
6845
6846        private void analyzeCreateTableStmt(TCreateTableSqlStatement stmt) {
6847                if (stmt.getCloneSourceTable() != null) {
6848                        analyzeCloneTableStmt(stmt);
6849                        return;
6850                }
6851
6852                TTable table = stmt.getTargetTable();
6853
6854                boolean hasDefinition = false;
6855
6856                if (stmt.getColumnList() != null && stmt.getColumnList().size() > 0) {
6857                        hasDefinition = true;
6858                }
6859
6860                if (table != null) {
6861                        Table tableModel = modelFactory.createTableFromCreateDDL(table, hasDefinition || (stmt.getSubQuery() == null && hasDefinition)
6862                                        || (stmt.getSubQuery()!=null && stmt.getSubQuery().getSetOperatorType() == ESetOperatorType.none && stmt.getSubQuery().getResultColumnList().toString().indexOf("*") == -1));
6863                        if (stmt.isExternal()) {
6864                                tableModel.setExternal(true);
6865                        }
6866
6867                        if (stmt.getSubQuery() != null) {
6868                                Process process = modelFactory.createProcess(stmt);
6869                                tableModel.addProcess(process);
6870                        }
6871
6872                        String procedureParent = getProcedureParentName(stmt);
6873                        if (procedureParent != null) {
6874                                tableModel.setParent(procedureParent);
6875                        }
6876
6877                        if (stmt.isUsingTemplate()) {
6878                                // Snowflake CREATE TABLE ... USING TEMPLATE <query>: the query only
6879                                // infers the column definitions (e.g. ARRAY_AGG(OBJECT_CONSTRUCT(*))
6880                                // over INFER_SCHEMA); it does NOT populate the table. Analyze it so its
6881                                // own sources (the stage / INFER_SCHEMA) still resolve, but do not
6882                                // project its result columns onto the target table or emit CTAS-style
6883                                // data-flow edges that would misrepresent it as data population.
6884                                if (stmt.getSubQuery() != null) {
6885                                        analyzeSelectStmt(stmt.getSubQuery());
6886                                }
6887                                return;
6888                        }
6889
6890                        if (hasDefinition) {
6891                                if(stmt.getSubQuery()!=null) {
6892                                        TSelectSqlStatement subquery = stmt.getSubQuery();
6893                                        analyzeSelectStmt(subquery);
6894                                        Process process = modelFactory.createProcess(stmt);
6895                                        ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt.getSubQuery());
6896                                        if(resultSetModel.isDetermined()){
6897                                                tableModel.setFromDDL(true);
6898                                        }
6899                                        if (resultSetModel != null) {
6900                                                int resultSetSize = resultSetModel.getColumns().size();
6901                                                int stmtColumnSize = stmt.getColumnList().size();
6902                                                int j = 0;
6903                                                int tableColumnSize = stmtColumnSize;
6904                                                if (resultSetModel.isDetermined() && resultSetSize > tableColumnSize) {
6905                                                        tableColumnSize = resultSetSize;
6906                                                }
6907                                                for (int i = 0; i < tableColumnSize && j < resultSetSize; i++) {
6908                                                        ResultColumn resultColumn = resultSetModel.getColumns().get(j);
6909                                                        if (i < stmtColumnSize) {
6910                                                                TObjectName alias = stmt.getColumnList().getColumn(i).getColumnName();
6911                                                                
6912                                                                if (!resultSetModel.getColumns().get(j).getName().contains("*")) {
6913                                                                        j++;
6914                                                                } else {
6915                                                                        if (resultSetSize - j == stmt.getColumnList().size() - i) {
6916                                                                                j++;
6917                                                                        }
6918                                                                }
6919
6920                                                                if (alias != null) {
6921                                                                        TableColumn viewColumn = modelFactory.createTableColumn(tableModel, alias, true);
6922                                                                        if (resultColumn != null) {
6923                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
6924                                                                                relation.setEffectType(EffectType.create_table);
6925                                                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
6926                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
6927                                                                                relation.setProcess(process);
6928                                                                        }
6929                                                                } else if (resultColumn.getColumnObject() instanceof TObjectName) {
6930                                                                        TableColumn viewColumn = modelFactory.createTableColumn(tableModel,
6931                                                                                        (TObjectName) resultColumn.getColumnObject(), true);
6932                                                                        if (resultColumn != null) {
6933                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
6934                                                                                relation.setEffectType(EffectType.create_table);
6935                                                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
6936                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
6937                                                                                relation.setProcess(process);
6938                                                                        }
6939                                                                } else if (resultColumn.getColumnObject() instanceof TResultColumn) {
6940                                                                        TableColumn viewColumn = modelFactory.createTableColumn(tableModel,
6941                                                                                        ((TResultColumn) resultColumn.getColumnObject()).getFieldAttr(), true);
6942                                                                        ResultColumn column = (ResultColumn) modelManager
6943                                                                                        .getModel(resultColumn.getColumnObject());
6944                                                                        if (column != null && !column.getStarLinkColumns().isEmpty()) {
6945                                                                                viewColumn.bindStarLinkColumns(column.getStarLinkColumns());
6946                                                                        }
6947                                                                        if (resultColumn != null) {
6948                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
6949                                                                                relation.setEffectType(EffectType.create_table);
6950                                                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
6951                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
6952                                                                                relation.setProcess(process);
6953                                                                        }
6954                                                                }
6955                                                        }
6956                                                        else if(resultSetModel.isDetermined()){
6957                                                                TObjectName tableName = new TObjectName();
6958                                                                tableName.setString(resultColumn.getName());
6959                                                                TableColumn viewColumn = modelFactory.createTableColumn(tableModel, tableName, true);
6960                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
6961                                                                relation.setEffectType(EffectType.create_table);
6962                                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
6963                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
6964                                                                relation.setProcess(process);
6965                                                                j++;
6966                                                        }
6967                                                }
6968                                                if (resultSetModel != null && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
6969                                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
6970                                                        impactRelation.setEffectType(EffectType.create_table);
6971                                                        impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
6972                                                                        resultSetModel.getRelationRows()));
6973                                                        impactRelation.setTarget(
6974                                                                        new RelationRowsRelationshipElement<TableRelationRows>(tableModel.getRelationRows()));
6975                                                }
6976                                        }
6977
6978                                        if (subquery.getResultColumnList() == null && subquery.getValueClause() != null
6979                                                        && subquery.getValueClause().getValueRows().size() == stmt.getColumnList().size()) {
6980                                                for (int i = 0; i < stmt.getColumnList().size(); i++) {
6981                                                        TObjectName alias = stmt.getColumnList().getColumn(i).getColumnName();
6982
6983                                                        if (alias != null) {
6984                                                                TableColumn viewColumn = modelFactory.createTableColumn(tableModel, alias, true);
6985
6986                                                                TExpression expression = subquery.getValueClause().getValueRows().getValueRowItem(i).getExpr();
6987
6988                                                                columnsInExpr visitor = new columnsInExpr();
6989                                                                expression.inOrderTraverse(visitor);
6990                                                                List<TObjectName> objectNames = visitor.getObjectNames();
6991                                                                List<TParseTreeNode> functions = visitor.getFunctions();
6992
6993                                                                if (functions != null && !functions.isEmpty()) {
6994                                                                        analyzeFunctionDataFlowRelation(viewColumn, functions, EffectType.select);
6995
6996                                                                }
6997
6998                                                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
6999                                                                if (subquerys != null && !subquerys.isEmpty()) {
7000                                                                        analyzeSubqueryDataFlowRelation(viewColumn, subquerys, EffectType.select);
7001                                                                }
7002
7003                                                                analyzeDataFlowRelation(viewColumn, objectNames, EffectType.select, functions);
7004                                                                List<TParseTreeNode> constants = visitor.getConstants();
7005                                                                analyzeConstantDataFlowRelation(viewColumn, constants, EffectType.select, functions);
7006                                                        }
7007                                                }
7008                                        }
7009                                        
7010                                        return;
7011                                }
7012                                else {
7013                                        for (int i = 0; i < stmt.getColumnList().size(); i++) {
7014                                                TColumnDefinition column = stmt.getColumnList().getColumn(i);
7015                                                if (column.getDatatype() != null && column.getDatatype().getTypeOfList() != null
7016                                                                && column.getDatatype().getTypeOfList().getColumnDefList() != null) {
7017                                                        for (int j = 0; j < column.getDatatype().getTypeOfList().getColumnDefList().size(); j++) {
7018                                                                TObjectName columnName = new TObjectName();
7019                                                                if (column.getDatatype().getDataType() == EDataType.array_t) {
7020//                                                                      columnName.setString(column.getColumnName().getColumnNameOnly() + ".array."
7021//                                                                                      + column.getDatatype().getTypeOfList().getColumnDefList().getColumn(j)
7022//                                                                                                      .getColumnName().getColumnNameOnly());
7023                                                                        columnName.setString(column.getColumnName().getColumnNameOnly() + "."
7024                                                                                        + column.getDatatype().getTypeOfList().getColumnDefList().getColumn(j)
7025                                                                                        .getColumnName().getColumnNameOnly());
7026                                                                } else {
7027                                                                        columnName.setString(column.getColumnName().getColumnNameOnly() + "."
7028                                                                                        + column.getDatatype().getTypeOfList().getColumnDefList().getColumn(j)
7029                                                                                                        .getColumnName().getColumnNameOnly());
7030                                                                }
7031                                                                TableColumn tableColumn = modelFactory.createTableColumn(tableModel, columnName,
7032                                                                                hasDefinition);
7033                                                                tableColumn.setStruct(true);
7034                                                                tableColumn.setColumnIndex(i);
7035                                                                appendTableColumnToSQLEnv(tableModel, tableColumn);
7036                                                                
7037                                                                if(option.getAnalyzeMode() == AnalyzeMode.crud) {
7038                                                                        CrudRelationship crudRelationship = modelFactory.createCrudRelation();
7039                                                                        crudRelationship.setTarget(new TableColumnRelationshipElement(tableColumn));
7040                                                                        crudRelationship.setEffectType(EffectType.create_table);
7041                                                                }
7042                                                        }
7043                                                        continue;
7044                                                }
7045                                                if (column.getDatatype() != null && column.getDatatype().getColumnDefList() != null) {
7046                                                        Stack<TColumnDefinition> columnPaths = new Stack<TColumnDefinition>();
7047                                                        flattenStructColumns(hasDefinition, tableModel, column, columnPaths, i);
7048                                                        continue;
7049                                                }
7050                                                TableColumn tableColumn = modelFactory.createTableColumn(tableModel, column.getColumnName(),
7051                                                                hasDefinition);
7052                                                if (tableColumn == null) {
7053                                                        continue;
7054                                                }
7055                                                
7056                                                if(option.getAnalyzeMode() == AnalyzeMode.crud) {
7057                                                        CrudRelationship crudRelationship = modelFactory.createCrudRelation();
7058                                                        crudRelationship.setTarget(new TableColumnRelationshipElement(tableColumn));
7059                                                        crudRelationship.setEffectType(EffectType.create_table);
7060                                                }
7061                                                
7062                                                if (column.getDatatype() != null && column.getDatatype().getDataType() == EDataType.variant_t) {
7063                                                        if (tableColumn != null) {
7064                                                                tableColumn.setVariant(true);
7065                                                        }
7066                                                }
7067                                                if (column.getDatatype() != null) {
7068                                                        String dType = column.getDatatype().getDataTypeName();
7069                                                        if ((!SQLUtil.isEmpty(dType)) && (dType.indexOf("_") > 0)) {
7070                                                                dType = dType.split("_")[0];
7071                                                        }
7072                                                        tableColumn.setDataType(dType);
7073                                                }
7074                                                appendTableColumnToSQLEnv(tableModel, tableColumn);
7075                                        }
7076                                }
7077                        }
7078
7079                        if (stmt.getExternalTableOption("DATA_SOURCE") != null) {
7080                                String dataSourceName = stmt.getExternalTableOption("DATA_SOURCE");
7081                                Table dataSource = modelManager.getTableByName(DlineageUtil.getTableFullName(dataSourceName));
7082                                if (dataSource != null) {
7083                                        TableColumn dataSourceColumn = dataSource.getColumns().get(0);
7084                                        for (int i = 0; i < tableModel.getColumns().size(); i++) {
7085                                                TableColumn column = tableModel.getColumns().get(i);
7086                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7087                                                relation.setTarget(new TableColumnRelationshipElement(column));
7088                                                relation.addSource(new TableColumnRelationshipElement(dataSourceColumn));
7089
7090                                                appendTableColumnToSQLEnv(tableModel, column);
7091                                        }
7092                                }
7093                        }
7094
7095                        if (stmt.getSubQuery() != null) {
7096
7097                                analyzeSelectStmt(stmt.getSubQuery());
7098
7099                                ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt.getSubQuery());
7100                                if (resultSetModel != null && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
7101                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
7102                                        impactRelation.setEffectType(EffectType.create_table);
7103                                        impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
7104                                                        resultSetModel.getRelationRows()));
7105                                        impactRelation.setTarget(
7106                                                        new RelationRowsRelationshipElement<TableRelationRows>(tableModel.getRelationRows()));
7107                                }
7108                        }
7109
7110                        if (stmt.getSubQuery() != null && !stmt.getSubQuery().isCombinedQuery()) {
7111                                SelectResultSet resultSetModel = (SelectResultSet) modelManager
7112                                                .getModel(stmt.getSubQuery().getResultColumnList());
7113                                if(resultSetModel.isDetermined()){
7114                                        tableModel.setDetermined(true);
7115                                        tableModel.setFromDDL(true);
7116                                }
7117                                for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
7118                                        ResultColumn resultColumn = resultSetModel.getColumns().get(i);
7119                                        if (resultSetModel.isDetermined() && resultColumn.getName().endsWith("*")) {
7120                                                continue;
7121                                        }
7122
7123                                        if (resultColumn.getColumnObject() instanceof TResultColumn) {
7124                                                TResultColumn columnObject = (TResultColumn) resultColumn.getColumnObject();
7125
7126                                                TAliasClause alias = columnObject.getAliasClause();
7127                                                if (alias != null && alias.getAliasName() != null) {
7128                                                        TableColumn tableColumn = null;
7129                                                        if (!hasDefinition) {
7130                                                                tableColumn = modelFactory.createTableColumn(tableModel, alias.getAliasName(),
7131                                                                                !hasDefinition);
7132                                                        } else {
7133                                                                tableColumn = tableModel.getColumns().get(i);
7134                                                        }
7135
7136                                                        if (!tableColumn.getName().endsWith("*") && tableModel.isDetermined()) {
7137                                                                appendTableColumnToSQLEnv(tableModel, tableColumn);
7138                                                        }
7139
7140                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7141                                                        relation.setEffectType(EffectType.create_table);
7142                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
7143                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
7144                                                        Process process = modelFactory.createProcess(stmt);
7145                                                        relation.setProcess(process);
7146                                                } else if (columnObject.getFieldAttr() != null || (columnObject.getExpr()!=null && columnObject.getExpr().getExpressionType() == EExpressionType.typecast_t)) {
7147                                                        TableColumn tableColumn = null;
7148                                                        TObjectName columnObj = columnObject.getFieldAttr();
7149                                                        if(columnObj == null) {
7150                                                                columnObj = columnObject.getExpr().getLeftOperand().getObjectOperand();
7151                                                        }
7152                                                        if ((columnObj == null || columnObj.toString().endsWith("*")) && !resultColumn.getName().endsWith("*")) {
7153                                                                columnObj = new TObjectName();
7154                                                                columnObj.setString(resultColumn.getName());
7155                                                        }
7156
7157                                                        if(columnObj == null){
7158                                                                logger.info("Can't handle column " + resultColumn.getName());
7159                                                                continue;
7160                                                        }
7161
7162                                                        if (!hasDefinition) {
7163                                                                tableColumn = modelFactory.createTableColumn(tableModel, columnObj,
7164                                                                                !hasDefinition);
7165                                                                if (tableColumn == null) {
7166                                                                        if (tableModel.getColumns().isEmpty()) {
7167                                                                                logger.info("Add table " + tableModel.getName() + " column " + columnObj.toString() + " failed");
7168                                                                        }
7169                                                                        else if (resultColumn.getName().endsWith("*")) {
7170                                                                                for (int j = 0; j < tableModel.getColumns().size(); j++) {
7171                                                                                        tableColumn = tableModel.getColumns().get(j);
7172                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7173                                                                                        relation.setEffectType(EffectType.create_table);
7174                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
7175                                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
7176                                                                                        if (tableColumn.getName().endsWith("*")
7177                                                                                                        && resultColumn.getName().endsWith("*")) {
7178                                                                                                tableModel.setStarStmt("create_table");
7179                                                                                        }
7180                                                                                        Process process = modelFactory.createProcess(stmt);
7181                                                                                        relation.setProcess(process);
7182                                                                                }
7183                                                                        }
7184                                                                        continue;
7185                                                                }
7186                                                                if (!tableColumn.getName().endsWith("*") && tableModel.isDetermined()) {
7187                                                                        appendTableColumnToSQLEnv(tableModel, tableColumn);
7188                                                                }
7189
7190                                                                Object model = modelManager
7191                                                                                .getModel(resultColumn.getColumnObject());
7192                                                                if (model instanceof ResultColumn) {
7193                                                                        ResultColumn column = (ResultColumn) model;
7194                                                                        if (tableColumn.getName().endsWith("*") && column != null
7195                                                                                        && !column.getStarLinkColumns().isEmpty()) {
7196                                                                                tableColumn.bindStarLinkColumns(column.getStarLinkColumns());
7197                                                                        }
7198
7199                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7200                                                                        relation.setEffectType(EffectType.create_table);
7201                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
7202                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
7203                                                                        if (tableColumn.getName().endsWith("*") && resultColumn.getName().endsWith("*")) {
7204                                                                                tableModel.setStarStmt("create_table");
7205                                                                        }
7206                                                                        Process process = modelFactory.createProcess(stmt);
7207                                                                        relation.setProcess(process);
7208                                                                }
7209                                                                else if(model instanceof LinkedHashMap) {
7210                                                                        String columnName = getColumnNameOnly(resultColumn.getName());
7211                                                                        LinkedHashMap<String, ResultColumn> resultColumns =  (LinkedHashMap<String, ResultColumn>)model;
7212                                                                        if (columnObj.toString().endsWith("*")) {
7213                                                                                for (String key : resultColumns.keySet()) {
7214                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel, resultColumns.get(key).getName());
7215                                                                                        DataFlowRelationship relation = modelFactory
7216                                                                                                        .createDataFlowRelation();
7217                                                                                        relation.setEffectType(EffectType.create_table);
7218                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
7219                                                                                        relation.addSource(
7220                                                                                                        new ResultColumnRelationshipElement(resultColumns.get(key)));
7221                                                                                        Process process = modelFactory.createProcess(stmt);
7222                                                                                        relation.setProcess(process);
7223                                                                                }
7224                                                                        } else if (resultColumns.containsKey(columnName)) {
7225                                                                                ResultColumn column = resultColumns.get(columnName);
7226                                                                                tableColumn = modelFactory.createInsertTableColumn(tableModel, column.getName());
7227                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7228                                                                                relation.setEffectType(EffectType.create_table);
7229                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
7230                                                                                relation.addSource(new ResultColumnRelationshipElement(column));
7231                                                                                Process process = modelFactory.createProcess(stmt);
7232                                                                                relation.setProcess(process);
7233                                                                        }
7234                                                                }
7235                                                        } else {
7236                                                                if (resultColumn.getName().endsWith("*")) {
7237                                                                        for (int j = 0; j < tableModel.getColumns().size(); j++) {
7238                                                                                tableColumn = tableModel.getColumns().get(j);
7239                                                                                ResultColumn column = (ResultColumn) modelManager
7240                                                                                                .getModel(resultColumn.getColumnObject());
7241                                                                                if (tableColumn.getName().endsWith("*") && column != null
7242                                                                                                && !column.getStarLinkColumns().isEmpty()) {
7243                                                                                        tableColumn.bindStarLinkColumns(column.getStarLinkColumns());
7244                                                                                }
7245
7246                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7247                                                                                relation.setEffectType(EffectType.create_table);
7248                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
7249                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
7250                                                                                if (tableColumn.getName().endsWith("*")
7251                                                                                                && resultColumn.getName().endsWith("*")) {
7252                                                                                        tableModel.setStarStmt("create_table");
7253                                                                                        ;
7254                                                                                }
7255                                                                                Process process = modelFactory.createProcess(stmt);
7256                                                                                relation.setProcess(process);
7257                                                                        }
7258                                                                } else {
7259                                                                        tableColumn = tableModel.getColumns().get(i);
7260                                                                        Object model = modelManager
7261                                                                                        .getModel(resultColumn.getColumnObject());
7262                                                                        String columnName = getColumnNameOnly(resultColumn.getName());
7263                                                                        if(model instanceof LinkedHashMap) {
7264                                                                                LinkedHashMap<String, ResultColumn> resultColumns =  (LinkedHashMap<String, ResultColumn>)model;
7265                                                                                if (resultColumns.size() == tableModel.getColumns().size()) {
7266                                                                                        int j = 0;
7267                                                                                        for (String key : resultColumns.keySet()) {
7268                                                                                                if (j == i) {
7269                                                                                                        ResultColumn column = resultColumns.get(key);
7270                                                                                                        DataFlowRelationship relation = modelFactory
7271                                                                                                                        .createDataFlowRelation();
7272                                                                                                        relation.setEffectType(EffectType.create_table);
7273                                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
7274                                                                                                        relation.addSource(new ResultColumnRelationshipElement(column));
7275                                                                                                        Process process = modelFactory.createProcess(stmt);
7276                                                                                                        relation.setProcess(process);
7277                                                                                                }
7278                                                                                                j++;
7279                                                                                        }
7280                                                                                } else if (resultColumns.containsKey(columnName)) {
7281                                                                                        ResultColumn column = resultColumns.get(columnName);
7282                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel, column.getName());
7283                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7284                                                                                        relation.setEffectType(EffectType.create_table);
7285                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
7286                                                                                        relation.addSource(new ResultColumnRelationshipElement(column));
7287                                                                                        Process process = modelFactory.createProcess(stmt);
7288                                                                                        relation.setProcess(process);
7289                                                                                } else {
7290                                                                                        throw new UnsupportedOperationException("Can't handle this star case.");
7291                                                                                }
7292                                                                        }
7293                                                                        else if (model instanceof ResultColumn) {
7294                                                                                ResultColumn column = (ResultColumn) modelManager
7295                                                                                                .getModel(resultColumn.getColumnObject());
7296                                                                                if (tableColumn.getName().endsWith("*") && column != null
7297                                                                                                && !column.getStarLinkColumns().isEmpty()) {
7298                                                                                        tableColumn.bindStarLinkColumns(column.getStarLinkColumns());
7299                                                                                }
7300
7301                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7302                                                                                relation.setEffectType(EffectType.create_table);
7303                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
7304                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
7305                                                                                if (tableColumn.getName().endsWith("*")
7306                                                                                                && resultColumn.getName().endsWith("*")) {
7307                                                                                        tableModel.setStarStmt("create_table");
7308                                                                                }
7309                                                                                Process process = modelFactory.createProcess(stmt);
7310                                                                                relation.setProcess(process);
7311                                                                        }
7312                                                                }
7313                                                        }
7314                                                } else {
7315                                                        TableColumn tableColumn = null;
7316                                                        if (!hasDefinition) {
7317                                                                TObjectName columnName = new TObjectName();
7318                                                                columnName.setString(resultColumn.getColumnObject().toString());
7319                                                                tableColumn = modelFactory.createTableColumn(tableModel, columnName, !hasDefinition);
7320                                                                if(tableColumn == null){
7321                                                                        continue;
7322                                                                }
7323                                                        } else {
7324                                                                tableColumn = tableModel.getColumns().get(i);
7325                                                        }
7326                                                        ResultColumn column = (ResultColumn) modelManager.getModel(resultColumn.getColumnObject());
7327                                                        if (column != null && !column.getStarLinkColumns().isEmpty()) {
7328                                                                tableColumn.bindStarLinkColumns(column.getStarLinkColumns());
7329                                                        }
7330
7331                                                        if (!tableColumn.getName().endsWith("*") && tableModel.isDetermined()) {
7332                                                                appendTableColumnToSQLEnv(tableModel, tableColumn);
7333                                                        }
7334
7335                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7336                                                        relation.setEffectType(EffectType.create_table);
7337                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
7338                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
7339                                                        Process process = modelFactory.createProcess(stmt);
7340                                                        relation.setProcess(process);
7341                                                }
7342                                        } else if (resultColumn.getColumnObject() instanceof TObjectName) {
7343                                                TableColumn tableColumn = null;
7344                                                if (!hasDefinition) {
7345                                                        tableColumn = modelFactory.createTableColumn(tableModel,
7346                                                                        (TObjectName) resultColumn.getColumnObject(), !hasDefinition);
7347                                                } else {
7348                                                        tableColumn = tableModel.getColumns().get(i);
7349                                                }
7350
7351                                                if (!tableColumn.getName().endsWith("*") && tableModel.isDetermined()) {
7352                                                        appendTableColumnToSQLEnv(tableModel, tableColumn);
7353                                                }
7354
7355                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7356                                                relation.setEffectType(EffectType.create_table);
7357                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
7358                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
7359                                                Process process = modelFactory.createProcess(stmt);
7360                                                relation.setProcess(process);
7361                                        }
7362                                }
7363                        } else if (stmt.getSubQuery() != null) {
7364                                SelectSetResultSet resultSetModel = (SelectSetResultSet) modelManager.getModel(stmt.getSubQuery());
7365                                if(resultSetModel.isDetermined()){
7366                                        tableModel.setDetermined(true);
7367                                        tableModel.setFromDDL(true);
7368                                }
7369                                for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
7370                                        ResultColumn resultColumn = resultSetModel.getColumns().get(i);
7371                                        if (resultColumn.getColumnObject() instanceof TResultColumn) {
7372                                                TResultColumn columnObject = (TResultColumn) resultColumn.getColumnObject();
7373
7374                                                TAliasClause alias = columnObject.getAliasClause();
7375                                                if (alias != null && alias.getAliasName() != null) {
7376                                                        TableColumn tableColumn = null;
7377                                                        if (!hasDefinition) {
7378                                                                tableColumn = modelFactory.createTableColumn(tableModel, alias.getAliasName(),
7379                                                                                !hasDefinition);
7380                                                                if (tableColumn == null) {
7381                                                                        continue;
7382                                                                }
7383                                                        } else {
7384                                                                tableColumn = tableModel.getColumns().get(i);
7385                                                        }
7386
7387                                                        if (!tableColumn.getName().endsWith("*") && tableModel.isDetermined()) {
7388                                                                appendTableColumnToSQLEnv(tableModel, tableColumn);
7389                                                        }
7390
7391                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7392                                                        relation.setEffectType(EffectType.create_table);
7393                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
7394                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
7395                                                        Process process = modelFactory.createProcess(stmt);
7396                                                        relation.setProcess(process);
7397                                                } else if (columnObject.getFieldAttr() != null || (columnObject.getExpr()!=null && columnObject.getExpr().getExpressionType() == EExpressionType.typecast_t)) {
7398                                                        TableColumn tableColumn = null;
7399                                                        TObjectName columnObj = columnObject.getFieldAttr();
7400                                                        if(columnObj == null) {
7401                                                                columnObj = columnObject.getExpr().getLeftOperand().getObjectOperand();
7402                                                        }
7403                                                        if (!hasDefinition) {
7404                                                                tableColumn = modelFactory.createTableColumn(tableModel, columnObj,
7405                                                                                !hasDefinition);
7406                                                                if (tableColumn == null) {
7407                                                                        continue;
7408                                                                }
7409                                                        } else {
7410                                                                tableColumn = tableModel.getColumns().get(i);
7411                                                        }
7412                                                        ResultColumn column = (ResultColumn) modelManager.getModel(resultColumn.getColumnObject());
7413                                                        if (column != null && !column.getStarLinkColumns().isEmpty()) {
7414                                                                tableColumn.bindStarLinkColumns(column.getStarLinkColumns());
7415                                                        }
7416
7417                                                        if (!tableColumn.getName().endsWith("*") && tableModel.isDetermined()) {
7418                                                                appendTableColumnToSQLEnv(tableModel, tableColumn);
7419                                                        }
7420
7421                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7422                                                        relation.setEffectType(EffectType.create_table);
7423                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
7424                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
7425                                                        Process process = modelFactory.createProcess(stmt);
7426                                                        relation.setProcess(process);
7427                                                } else {
7428                                                        ErrorInfo errorInfo = new ErrorInfo();
7429                                                        errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
7430                                                        errorInfo.setErrorMessage("Can't handle the table column " + columnObject.toString());
7431                                                        errorInfo.setStartPosition(new Pair3<Long, Long, String>(
7432                                                                        columnObject.getStartToken().lineNo, columnObject.getStartToken().columnNo,
7433                                                                        ModelBindingManager.getGlobalHash()));
7434                                                        errorInfo.setEndPosition(new Pair3<Long, Long, String>(columnObject.getEndToken().lineNo,
7435                                                                        columnObject.getEndToken().columnNo + columnObject.getEndToken().getAstext().length(),
7436                                                                        ModelBindingManager.getGlobalHash()));
7437                                                        errorInfos.add(errorInfo);
7438                                                        continue;
7439                                                }
7440                                        } else if (resultColumn.getColumnObject() instanceof TObjectName) {
7441                                                TableColumn tableColumn = null;
7442                                                if (!hasDefinition) {
7443                                                        tableColumn = modelFactory.createTableColumn(tableModel,
7444                                                                        (TObjectName) resultColumn.getColumnObject(), !hasDefinition);
7445                                                } else {
7446                                                        tableColumn = tableModel.getColumns().get(i);
7447                                                }
7448
7449                                                if (!tableColumn.getName().endsWith("*") && tableModel.isDetermined()) {
7450                                                        appendTableColumnToSQLEnv(tableModel, tableColumn);
7451                                                }
7452
7453                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7454                                                relation.setEffectType(EffectType.create_table);
7455                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
7456                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
7457                                                Process process = modelFactory.createProcess(stmt);
7458                                                relation.setProcess(process);
7459                                        }
7460                                }
7461                        } else if (stmt.getLikeTableName() != null) {
7462                                Table likeTableModel = modelFactory.createTableByName(stmt.getLikeTableName());
7463
7464                                if(likeTableModel.isCreateTable()){
7465                                        for(TableColumn column: likeTableModel.getColumns()){
7466                                                TObjectName tableColumn = new TObjectName();
7467                                                tableColumn.setString(column.getName());
7468                                                TableColumn createTableColumn = modelFactory.createTableColumn(tableModel, tableColumn, true);
7469                                                createTableColumn.setPrimaryKey(column.getPrimaryKey());
7470                                                createTableColumn.setForeignKey(column.getForeignKey());
7471                                                createTableColumn.setIndexKey(column.getIndexKey());
7472                                                createTableColumn.setUnqiueKey(column.getUnqiueKey());
7473                                                createTableColumn.setDataType(column.getDataType());
7474                                        }
7475                                        tableModel.setCreateTable(true);
7476                                }
7477
7478//                              Process process = modelFactory.createProcess(stmt);
7479//                              process.setType("Like Table");
7480//                              tableModel.addProcess(process);
7481//
7482//
7483//                              DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7484//                              relation.setEffectType(EffectType.like_table);
7485//                              relation.setTarget(
7486//                                              new RelationRowsRelationshipElement<TableRelationRows>(tableModel.getRelationRows()));
7487//                              relation.addSource(
7488//                                              new RelationRowsRelationshipElement<TableRelationRows>(likeTableModel.getRelationRows()));
7489//                              relation.setProcess(process);
7490                        }
7491
7492                        if (stmt.getStageLocation() != null && stmt.getStageLocation().getStageName() != null) {
7493                                tableModel
7494                                                .setLocation(DlineageUtil.getTableFullName(stmt.getStageLocation().getStageName().toString()));
7495                                Process process = modelFactory.createProcess(stmt);
7496                                process.setType("Create External Table");
7497                                tableModel.addProcess(process);
7498                                Table stage = modelManager.getTableByName(DlineageUtil.getTableFullName(tableModel.getLocation()));
7499                                if (stage == null) {
7500                                        stage = modelManager.getTableByName(
7501                                                        DlineageUtil.getTableFullName(stmt.getStageLocation().toString().replaceFirst("@", "")));
7502                                }
7503                                if (stage == null) {
7504                                        stage = modelFactory.createTableByName(stmt.getStageLocation().toString().replaceFirst("@", ""),
7505                                                        false);
7506                                        stage.setCreateTable(false);
7507                                        stage.setStage(true);
7508                                        if (stmt.getStageLocation().getPath() != null) {
7509                                                stage.setLocation(stmt.getStageLocation().getPath().toString());
7510                                                TObjectName location = new TObjectName();
7511                                                location.setString(stmt.getStageLocation().getPath().toString());
7512                                                modelFactory.createStageLocation(stage, location);
7513                                        } else if (stmt.getRegex_pattern() != null) {
7514                                                stage.setLocation(stmt.getRegex_pattern());
7515                                                TObjectName location = new TObjectName();
7516                                                location.setString(stmt.getRegex_pattern());
7517                                                modelFactory.createStageLocation(stage, location);
7518                                        } else {
7519                                                stage.setLocation("unknownPath");
7520                                                TObjectName location = new TObjectName();
7521                                                location.setString("unknownPath");
7522                                                modelFactory.createStageLocation(stage, location);
7523                                        }
7524                                }
7525                                if (stage != null && !stage.getColumns().isEmpty()) {
7526                                        if (!tableModel.getColumns().isEmpty()) {
7527                                                for (int i = 0; i < tableModel.getColumns().size(); i++) {
7528                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7529                                                        relation.addSource(new TableColumnRelationshipElement(stage.getColumns().get(0)));
7530                                                        relation.setTarget(new TableColumnRelationshipElement(tableModel.getColumns().get(i)));
7531                                                        relation.setProcess(process);
7532                                                }
7533                                        } else {
7534                                                TObjectName starColumn = new TObjectName();
7535                                                starColumn.setString("*");
7536                                                TableColumn tableColumn = modelFactory.createTableColumn(tableModel, starColumn, true);
7537                                                tableColumn.setExpandStar(false);
7538                                                tableColumn.setShowStar(true);
7539                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7540                                                relation.addSource(new TableColumnRelationshipElement(stage.getColumns().get(0)));
7541                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
7542                                                relation.setProcess(process);
7543                                        }
7544                                }
7545                        } else if (stmt.getTableOptions() != null && !stmt.getTableOptions().isEmpty()) {
7546                                for (int i = 0; i < stmt.getTableOptions().size(); i++) {
7547                                        TCreateTableOption createTableOption = stmt.getTableOptions().get(i);
7548                                        if (createTableOption.getCreateTableOptionType() != ECreateTableOption.etoBigQueryExternal)
7549                                                continue;
7550                                        List<String> uris = createTableOption.getUris();
7551                                        if (uris == null || uris.isEmpty())
7552                                                continue;
7553                                        Process process = modelFactory.createProcess(stmt);
7554                                        process.setType("Create External Table");
7555                                        tableModel.addProcess(process);
7556                                        if (tableModel.getColumns().isEmpty()) {
7557                                                TObjectName starColumn = new TObjectName();
7558                                                starColumn.setString("*");
7559                                                TableColumn tableColumn = modelFactory.createTableColumn(tableModel, starColumn, true);
7560                                                tableColumn.setExpandStar(false);
7561                                                tableColumn.setShowStar(true);
7562                                        }
7563                                        for (String uri : uris) {
7564                                                Table uriFile = modelFactory.createTableByName(uri, true);
7565                                                uriFile.setPath(true);
7566                                                uriFile.setFileFormat(createTableOption.getFormat());
7567                                                for (int j = 0; j < tableModel.getColumns().size(); j++) {
7568                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7569                                                        if (stmt.getColumnList() != null) {
7570                                                                TObjectName fileUri = new TObjectName();
7571                                                                fileUri.setString(tableModel.getColumns().get(j).getColumnObject().toString());
7572                                                                TableColumn fileUriColumn = modelFactory.createFileUri(uriFile, fileUri);
7573                                                                relation.addSource(new TableColumnRelationshipElement(fileUriColumn));
7574                                                        } else {
7575                                                                TObjectName fileUri = new TObjectName();
7576                                                                fileUri.setString("*");
7577                                                                TableColumn fileUriColumn = modelFactory.createFileUri(uriFile, fileUri);
7578                                                                relation.addSource(new TableColumnRelationshipElement(fileUriColumn));
7579                                                        }
7580                                                        relation.setTarget(new TableColumnRelationshipElement(tableModel.getColumns().get(j)));
7581                                                        relation.setProcess(process);
7582                                                }
7583                                                if (tableModel.getColumns().isEmpty()) {
7584                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7585                                                        TObjectName fileUri = new TObjectName();
7586                                                        fileUri.setString("*");
7587                                                        TableColumn fileUriColumn = modelFactory.createFileUri(uriFile, fileUri);
7588                                                        relation.addSource(new TableColumnRelationshipElement(fileUriColumn));
7589                                                        relation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(
7590                                                                        tableModel.getRelationRows()));
7591                                                        relation.setProcess(process);
7592                                                }
7593                                        }
7594                                }
7595                        } else if (stmt.getSubQuery() == null && stmt.getTableLocation() != null) {
7596                                Process process = modelFactory.createProcess(stmt);
7597                                process.setType("Create External Table");
7598                                tableModel.addProcess(process);
7599                                Table uriFile = modelFactory.createTableByName(stmt.getTableLocation(), true);
7600                                uriFile.setPath(true);
7601                                for (int j = 0; j < tableModel.getColumns().size(); j++) {
7602                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7603                                        if (stmt.getColumnList() != null) {
7604                                                TObjectName fileUri = new TObjectName();
7605                                                fileUri.setString(tableModel.getColumns().get(j).getColumnObject().toString());
7606                                                TableColumn fileUriColumn = modelFactory.createFileUri(uriFile, fileUri);
7607                                                relation.addSource(new TableColumnRelationshipElement(fileUriColumn));
7608                                        } else {
7609                                                TObjectName fileUri = new TObjectName();
7610                                                fileUri.setString("*");
7611                                                TableColumn fileUriColumn = modelFactory.createFileUri(uriFile, fileUri);
7612                                                relation.addSource(new TableColumnRelationshipElement(fileUriColumn));
7613                                        }
7614                                        relation.setTarget(new TableColumnRelationshipElement(tableModel.getColumns().get(j)));
7615                                        relation.setProcess(process);
7616                                }
7617                        }
7618
7619                        if (stmt.getTableConstraints() != null && stmt.getTableConstraints().size() > 0) {
7620                                for (int i = 0; i < stmt.getTableConstraints().size(); i++) {
7621                                        TConstraint createTableConstraint = stmt.getTableConstraints().getConstraint(i);
7622                                        TPTNodeList<TColumnWithSortOrder> keyNames = createTableConstraint.getColumnList();
7623                                        if (keyNames != null) {
7624                                                for (int k = 0; k < keyNames.size(); k++) {
7625                                                        TObjectName keyName = keyNames.getElement(k).getColumnName();
7626                                                        // Skip functional indexes (expression-based indexes) where columnName is null
7627                                                        if (keyName == null) {
7628                                                                continue;
7629                                                        }
7630                                                        TObjectName referencedTableName = createTableConstraint.getReferencedObject();
7631                                                        TObjectNameList referencedTableColumns = createTableConstraint.getReferencedColumnList();
7632
7633                                                        TableColumn tableConstraint = modelFactory.createTableColumn(tableModel, keyName, true);
7634                                                        if (createTableConstraint.getConstraint_type() == EConstraintType.primary_key) {
7635                                                                tableConstraint.setPrimaryKey(true);
7636                                                        } else if (createTableConstraint.getConstraint_type() == EConstraintType.table_index) {
7637                                                                tableConstraint.setIndexKey(true);
7638                                                        } else if (createTableConstraint.getConstraint_type() == EConstraintType.unique) {
7639                                                                tableConstraint.setUnqiueKey(true);
7640                                                        } else if (createTableConstraint.getConstraint_type() == EConstraintType.foreign_key) {
7641                                                                tableConstraint.setForeignKey(true);
7642                                                                Table referencedTable = modelManager
7643                                                                                .getTableByName(DlineageUtil.getTableFullName(referencedTableName.toString()));
7644                                                                if (referencedTable == null) {
7645                                                                        referencedTable = modelFactory.createTableByName(referencedTableName);
7646                                                                }
7647
7648                                                                if (referencedTableColumns != null) {
7649                                                                        for (int j = 0; j < referencedTableColumns.size(); j++) {
7650                                                                                TableColumn tableColumn = modelFactory.createTableColumn(referencedTable,
7651                                                                                                referencedTableColumns.getObjectName(j), false);
7652                                                                                if (tableColumn != null) {
7653                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7654                                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
7655                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableConstraint));
7656                                                                                        relation.setEffectType(EffectType.foreign_key);
7657                                                                                        Process process = modelFactory.createProcess(stmt);
7658                                                                                        relation.setProcess(process);
7659                                                                                        if (this.option.isShowERDiagram()) {
7660                                                                                                ERRelationship erRelation = modelFactory.createERRelation();
7661                                                                                                erRelation.addSource(new TableColumnRelationshipElement(tableColumn));
7662                                                                                                erRelation
7663                                                                                                                .setTarget(new TableColumnRelationshipElement(tableConstraint));
7664                                                                                        }
7665                                                                                }
7666                                                                        }
7667                                                                }
7668                                                        }
7669                                                }
7670                                        }
7671                                }
7672                        }
7673
7674                        if (stmt.getHiveTablePartition() != null && stmt.getHiveTablePartition().getColumnDefList() != null) {
7675                                for (int i = 0; i < stmt.getHiveTablePartition().getColumnDefList().size(); i++) {
7676                                        TColumnDefinition column = stmt.getHiveTablePartition().getColumnDefList().getColumn(i);
7677                                        modelFactory.createTableColumn(tableModel, column.getColumnName(), true);
7678                                        appendTableColumnToSQLEnv(tableModel, column.getColumnName());
7679                                }
7680                        }
7681
7682                } else {
7683                        ErrorInfo errorInfo = new ErrorInfo();
7684                        errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
7685                        errorInfo.setErrorMessage("Can't get target table. CreateTableSqlStatement is " + stmt.toString());
7686                        errorInfo.setStartPosition(new Pair3<Long, Long, String>(stmt.getStartToken().lineNo,
7687                                        stmt.getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
7688                        errorInfo.setEndPosition(new Pair3<Long, Long, String>(stmt.getEndToken().lineNo,
7689                                        stmt.getEndToken().columnNo + stmt.getEndToken().getAstext().length(),
7690                                        ModelBindingManager.getGlobalHash()));
7691                        errorInfos.add(errorInfo);
7692                }
7693        }
7694
7695        protected void flattenStructColumns(boolean hasDefinition, Table tableModel, TColumnDefinition column,
7696                        Stack<TColumnDefinition> columnPaths, int index) {
7697                columnPaths.push(column);
7698                for (int j = 0; j < column.getDatatype().getColumnDefList().size(); j++) {
7699                        TColumnDefinition columnDefinition = column.getDatatype().getColumnDefList().getColumn(j);
7700                        if (columnDefinition.getDatatype().getColumnDefList() != null) {
7701                                flattenStructColumns(hasDefinition, tableModel, columnDefinition, columnPaths, index);
7702                        } else {
7703                                TObjectName columnName = new TObjectName();
7704                                columnName.setString(getColumnName(columnPaths, columnDefinition));
7705                                TableColumn tableColumn = modelFactory.createTableColumn(tableModel, columnName, hasDefinition);
7706                                tableColumn.setColumnIndex(index);
7707                                tableColumn.setStruct(true);
7708                                
7709                                if(option.getAnalyzeMode() == AnalyzeMode.crud) {
7710                                        CrudRelationship crudRelationship = modelFactory.createCrudRelation();
7711                                        crudRelationship.setTarget(new TableColumnRelationshipElement(tableColumn));
7712                                        crudRelationship.setEffectType(EffectType.create_table);
7713                                }
7714                                
7715                                appendTableColumnToSQLEnv(tableModel, tableColumn);
7716                        }
7717                }
7718                columnPaths.pop();
7719        }
7720
7721        private String getColumnName(Stack<TColumnDefinition> columnPaths, TColumnDefinition column) {
7722                StringBuilder buffer = new StringBuilder();
7723                Iterator<TColumnDefinition> iter = columnPaths.iterator();
7724                while(iter.hasNext()) {
7725                        buffer.append(iter.next().getColumnName().getColumnNameOnly()).append(".");
7726                }
7727                buffer.append(column.getColumnName().getColumnNameOnly());
7728                return buffer.toString();
7729        }
7730
7731        private void appendTableColumnToSQLEnv(Table tableModel, TableColumn tableColumn) {
7732                //tableModel如果非determined，请不要添加到sqlenv里
7733                if (sqlenv != null && tableColumn!=null) {
7734                        TSQLSchema schema = sqlenv.getSQLSchema(DlineageUtil.getTableSchema(tableModel), true);
7735                        if (schema != null) {
7736                                TSQLTable tempTable = schema.createTable(DlineageUtil.getSimpleTableName(tableModel.getName()));
7737                                if (tableColumn.hasStarLinkColumn()) {
7738                                        for (String column : tableColumn.getStarLinkColumnNames()) {
7739                                                tempTable.addColumn(DlineageUtil.getColumnNameOnly(column));
7740                                        }
7741                                } else {
7742                                        tempTable.addColumn(DlineageUtil.getColumnNameOnly(tableColumn.getName()));
7743                                }
7744                        }
7745                }
7746        }
7747
7748        private void appendTableColumnToSQLEnv(Table tableModel, TObjectName tableColumn) {
7749                if (sqlenv != null) {
7750                        TSQLSchema schema = sqlenv.getSQLSchema(DlineageUtil.getTableSchema(tableModel), true);
7751                        if (schema != null) {
7752                                TSQLTable tempTable = schema.createTable(DlineageUtil.getSimpleTableName(tableModel.getName()));
7753                                tempTable.addColumn(DlineageUtil.getColumnNameOnly(tableColumn.getColumnNameOnly()));
7754                        }
7755                }
7756        }
7757
7758        private void analyzeCreateStageStmt(TCreateStageStmt stmt) {
7759                TObjectName stageName = stmt.getStageName();
7760
7761                if (stageName != null) {
7762
7763                        Table tableModel = modelFactory.createStage(stageName);
7764                        tableModel.setCreateTable(true);
7765                        tableModel.setStage(true);
7766                        tableModel.setLocation(stmt.getExternalStageURL());
7767
7768                        Process process = modelFactory.createProcess(stmt);
7769                        tableModel.addProcess(process);
7770
7771                        TableColumn locationColumn = null;
7772                        if (stmt.getExternalStageURL() != null) {
7773                                TObjectName location = new TObjectName();
7774                                location.setString(stmt.getExternalStageURL());
7775                                locationColumn = modelFactory.createStageLocation(tableModel, location);
7776
7777                                Table pathModel = modelFactory.createTableByName(stmt.getExternalStageURL(), true);
7778                                pathModel.setPath(true);
7779                                pathModel.setCreateTable(true);
7780                                TObjectName fileUri = new TObjectName();
7781                                fileUri.setString(stmt.getExternalStageURL());
7782                                TableColumn fileUriColumn = modelFactory.createFileUri(pathModel, fileUri);
7783
7784                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7785                                relation.addSource(new TableColumnRelationshipElement(fileUriColumn));
7786                                relation.setTarget(new TableColumnRelationshipElement(locationColumn));
7787                                relation.setProcess(process);
7788
7789                        } else {
7790                                for (TCustomSqlStatement temp : stmt.getGsqlparser().getSqlstatements()) {
7791                                        if (temp instanceof TPutStmt) {
7792                                                TPutStmt put = (TPutStmt) temp;
7793                                                TStageLocation stageLocation = put.getStageLocation();
7794                                                if (stageLocation != null && stageLocation.getStageName() != null) {
7795                                                        Table stage = modelManager.getTableByName(
7796                                                                        DlineageUtil.getTableFullName(stageLocation.getStageName().toString()));
7797                                                        if (stage == tableModel) {
7798                                                                TObjectName location = new TObjectName();
7799                                                                if (!SQLUtil.isEmpty(put.getFileName())) {
7800                                                                        location.setString(put.getFileName());
7801                                                                        locationColumn = modelFactory.createStageLocation(tableModel, location);
7802                                                                }
7803                                                                break;
7804                                                        }
7805                                                }
7806                                        }
7807                                }
7808                        }
7809
7810                        String fileFormat = stmt.getFileFormatName();
7811                        if (fileFormat != null) {
7812                                for (TCustomSqlStatement temp : stmt.getGsqlparser().getSqlstatements()) {
7813                                        if (temp instanceof TCreateFileFormatStmt) {
7814                                                TCreateFileFormatStmt fileFormatStmt = (TCreateFileFormatStmt) temp;
7815                                                if (fileFormatStmt.getFileFormatName() != null
7816                                                                && fileFormatStmt.getFileFormatName().toString().equalsIgnoreCase(fileFormat)) {
7817                                                        tableModel.setFileType(fileFormatStmt.getTypeName());
7818                                                        break;
7819                                                }
7820                                        }
7821                                }
7822                        }
7823
7824                        String procedureParent = getProcedureParentName(stmt);
7825                        if (procedureParent != null) {
7826                                tableModel.setParent(procedureParent);
7827                        }
7828
7829                        if (locationColumn != null) {
7830                                List<Table> tables = modelManager.getTablesByName();
7831                                if (tables != null) {
7832                                        for (Table referTable : tables) {
7833                                                if (referTable.getLocation() != null && referTable.getLocation()
7834                                                                .equals(DlineageUtil.getIdentifierNormalTableName(tableModel.getName()))) {
7835                                                        for (int i = 0; i < referTable.getColumns().size(); i++) {
7836                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7837                                                                relation.addSource(new TableColumnRelationshipElement(locationColumn));
7838                                                                relation.setTarget(new TableColumnRelationshipElement(referTable.getColumns().get(i)));
7839                                                        }
7840                                                }
7841                                        }
7842                                }
7843                        }
7844                } else {
7845                        ErrorInfo errorInfo = new ErrorInfo();
7846                        errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
7847                        errorInfo.setErrorMessage("Can't get target table. CreateStageStmt is " + stmt.toString());
7848                        errorInfo.setStartPosition(new Pair3<Long, Long, String>(stmt.getStartToken().lineNo,
7849                                        stmt.getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
7850                        errorInfo.setEndPosition(new Pair3<Long, Long, String>(stmt.getEndToken().lineNo,
7851                                        stmt.getEndToken().columnNo + stmt.getEndToken().getAstext().length(),
7852                                        ModelBindingManager.getGlobalHash()));
7853                        errorInfo.fillInfo(this);
7854                        errorInfos.add(errorInfo);
7855                }
7856        }
7857
7858        private void analyzeCreateExternalDataSourceStmt(TCreateExternalDataSourceStmt stmt) {
7859                TObjectName dataSourceName = stmt.getDataSourceName();
7860                String locationUrl = stmt.getOption("LOCATION");
7861                if (dataSourceName != null && locationUrl != null) {
7862                        Table tableModel = modelFactory.createDataSource(dataSourceName);
7863                        tableModel.setCreateTable(true);
7864                        tableModel.setDataSource(true);
7865                        tableModel.setLocation(locationUrl);
7866
7867                        Process process = modelFactory.createProcess(stmt);
7868                        tableModel.addProcess(process);
7869
7870                        TObjectName location = new TObjectName();
7871                        location.setString(locationUrl);
7872                        modelFactory.createTableColumn(tableModel, location, true);
7873
7874                        String procedureParent = getProcedureParentName(stmt);
7875                        if (procedureParent != null) {
7876                                tableModel.setParent(procedureParent);
7877                        }
7878                } else {
7879                        ErrorInfo errorInfo = new ErrorInfo();
7880                        errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
7881                        errorInfo.setErrorMessage("Can't get target table. CreateExternalDataSourceStmt is " + stmt.toString());
7882                        errorInfo.setStartPosition(new Pair3<Long, Long, String>(stmt.getStartToken().lineNo,
7883                                        stmt.getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
7884                        errorInfo.setEndPosition(new Pair3<Long, Long, String>(stmt.getEndToken().lineNo,
7885                                        stmt.getEndToken().columnNo + stmt.getEndToken().getAstext().length(),
7886                                        ModelBindingManager.getGlobalHash()));
7887                        errorInfo.fillInfo(this);
7888                        errorInfos.add(errorInfo);
7889                }
7890        }
7891
7892        private void analyzeCreateStreamStmt(TCreateStreamStmt stmt) {
7893                TObjectName streamName = stmt.getStreamName();
7894
7895                if (streamName != null) {
7896                        Table tableModel = modelFactory.createTableByName(stmt.getTableName());
7897                        tableModel.setCreateTable(true);
7898
7899                        Table streamModel = modelFactory.createStream(streamName);
7900                        streamModel.setCreateTable(true);
7901                        streamModel.setStream(true);
7902
7903                        Process process = modelFactory.createProcess(stmt);
7904                        tableModel.addProcess(process);
7905
7906                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7907                        relation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(streamModel.getRelationRows()));
7908                        relation.addSource(new RelationRowsRelationshipElement<TableRelationRows>(tableModel.getRelationRows()));
7909                        relation.setProcess(process);
7910                }
7911        }
7912
7913        private String getProcedureParentName(TCustomSqlStatement stmt) {
7914                if (stmt instanceof TStoredProcedureSqlStatement) {
7915                        if (((TStoredProcedureSqlStatement) stmt).getStoredProcedureName() != null) {
7916                                return ((TStoredProcedureSqlStatement) stmt).getStoredProcedureName().toString();
7917                        }
7918                }
7919
7920                stmt = stmt.getParentStmt();
7921                if (stmt == null)
7922                        return null;
7923
7924        if (stmt instanceof TCommonBlock) {
7925                        if(((TCommonBlock) stmt).getBlockBody().getParentObjectName() instanceof TStoredProcedureSqlStatement) {
7926                                stmt = (TStoredProcedureSqlStatement)((TCommonBlock) stmt).getBlockBody().getParentObjectName();
7927                        }
7928                }
7929
7930                if (stmt instanceof TStoredProcedureSqlStatement) {
7931                        if (((TStoredProcedureSqlStatement) stmt).getStoredProcedureName() != null) {
7932                                return ((TStoredProcedureSqlStatement) stmt).getStoredProcedureName().toString();
7933                        }
7934                }
7935                if (stmt instanceof TTeradataCreateProcedure) {
7936                        if (((TTeradataCreateProcedure) stmt).getProcedureName() != null) {
7937                                return ((TTeradataCreateProcedure) stmt).getProcedureName().toString();
7938                        }
7939                }
7940
7941                return getProcedureParentName(stmt);
7942        }
7943
7944        private TStoredProcedureSqlStatement getProcedureParent(TCustomSqlStatement stmt) {
7945                if (stmt instanceof TStoredProcedureSqlStatement) {
7946                        return (TStoredProcedureSqlStatement) stmt;
7947                }
7948
7949                stmt = stmt.getParentStmt();
7950                if (stmt == null)
7951                        return null;
7952
7953                if (stmt instanceof TCommonBlock) {
7954                        if (((TCommonBlock) stmt).getBlockBody().getParentObjectName() instanceof TStoredProcedureSqlStatement) {
7955                                stmt = (TStoredProcedureSqlStatement) ((TCommonBlock) stmt).getBlockBody().getParentObjectName();
7956                        }
7957                }
7958
7959                if (stmt instanceof TStoredProcedureSqlStatement) {
7960                        return ((TStoredProcedureSqlStatement) stmt);
7961                }
7962                if (stmt instanceof TTeradataCreateProcedure) {
7963                        return ((TTeradataCreateProcedure) stmt);
7964                }
7965
7966                return getProcedureParent(stmt);
7967        }
7968
7969        private void analyzeMergeStmt(TMergeSqlStatement stmt) {
7970                Object tableModel;
7971                Process process;
7972                if (stmt.getUsingTable() != null) {
7973                        TTable table = stmt.getTargetTable();
7974                        if(table.getSubquery()!=null) {
7975                                tableModel = modelFactory.createQueryTable(table);
7976                                analyzeSelectStmt(table.getSubquery());
7977                                process = modelFactory.createProcess(stmt);
7978                        }
7979                        else {
7980                                tableModel = modelFactory.createTable(table);
7981                                process = modelFactory.createProcess(stmt);
7982                                ((Table)tableModel).addProcess(process);
7983                        }
7984                        
7985                        for(TTable item: stmt.tables) {
7986                                if(item.getSubquery()!=null) {
7987                                        continue;
7988                                }
7989                                Table tableItemModel = modelFactory.createTable(item);                          
7990                                if (tableItemModel.getColumns() == null || tableItemModel.getColumns().isEmpty()) {
7991                                        tableItemModel.addColumnsFromSQLEnv();
7992                                }
7993                        }
7994
7995                        if (stmt.getUsingTable().getSubquery() != null) {
7996                                QueryTable queryTable = modelFactory.createQueryTable(stmt.getUsingTable());
7997                                analyzeSelectStmt(stmt.getUsingTable().getSubquery());
7998
7999                                ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt.getUsingTable().getSubquery());
8000                                
8001                                if (queryTable != null && resultSetModel != null && queryTable != resultSetModel) {
8002                                        if (queryTable.getColumns().size() == resultSetModel.getColumns().size()) {
8003                                                for (int i = 0; i < queryTable.getColumns().size(); i++) {
8004                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8005                                                        relation.setEffectType(EffectType.select);
8006                                                        relation.setTarget(new ResultColumnRelationshipElement(queryTable.getColumns().get(i)));
8007                                                        relation.addSource(new ResultColumnRelationshipElement(resultSetModel.getColumns().get(i)));
8008                                                        relation.setProcess(process);
8009                                                }
8010                                        }
8011                                }
8012                                
8013                                if (resultSetModel != null && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
8014                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
8015                                        impactRelation.setEffectType(EffectType.merge);
8016                                        impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
8017                                                        resultSetModel.getRelationRows()));
8018                                        if (tableModel instanceof Table) {
8019                                                impactRelation.setTarget(
8020                                                                new RelationRowsRelationshipElement<TableRelationRows>(((Table)tableModel).getRelationRows()));
8021                                        }
8022                                        else {
8023                                                impactRelation.setTarget(
8024                                                                new RelationRowsRelationshipElement<ResultSetRelationRows>(((ResultSet)tableModel).getRelationRows()));
8025                                        }
8026                                }
8027                        } else {
8028                                if (stmt.getUsingTable().getAliasClause() != null && stmt.getUsingTable().getAliasClause().getColumns() != null && stmt.getUsingTable().getValueClause().getRows()!=null) {
8029                                        Table usingTable = modelFactory.createTableFromCreateDDL(stmt.getUsingTable(), false, stmt.getUsingTable().getAliasName() + stmt.getUsingTable().getTableName());
8030                                        usingTable.setCreateTable(true);
8031                                        usingTable.setSubType(SubType.function);
8032                                        for (int z=0;z<stmt.getUsingTable().getAliasClause().getColumns().size();z++) {
8033                                                TObjectName columnName = stmt.getUsingTable().getAliasClause().getColumns().getObjectName(z);
8034                                                TableColumn tableColumn = modelFactory.createTableColumn(usingTable, columnName, true);
8035                                                TResultColumn resultColumn = stmt.getUsingTable().getValueClause().getRows().get(0).getResultColumn(z);
8036                                                modelManager.bindModel(resultColumn, tableColumn);
8037                                                analyzeResultColumnExpressionRelation(tableColumn, resultColumn.getExpr());                                             }
8038                                }
8039                                else {
8040                                        modelFactory.createTable(stmt.getUsingTable());
8041                                }
8042                        }
8043                
8044
8045                        if (stmt.getWhenClauses() != null && stmt.getWhenClauses().size() > 0) {
8046                                for (int i = 0; i < stmt.getWhenClauses().size(); i++) {
8047                                        TMergeWhenClause clause = stmt.getWhenClauses().getElement(i);
8048                                        if (clause.getCondition() != null) {
8049                                                analyzeFilterCondition(null, clause.getCondition(), null, null, EffectType.merge_when);
8050                                        }
8051                                        if (clause.getUpdateClause() != null) {
8052                                                TResultColumnList columns = clause.getUpdateClause().getUpdateColumnList();
8053                                                if (columns == null || columns.size() == 0)
8054                                                        continue;
8055
8056                                                ResultSet resultSet = modelFactory.createResultSet(clause.getUpdateClause(), false);
8057                                                createPseudoImpactRelation(stmt, resultSet, EffectType.merge_update);
8058
8059                                                for (int j = 0; j < columns.size(); j++) {
8060                                                        TResultColumn resultColumn = columns.getResultColumn(j);
8061                                                        if (resultColumn.getExpr().getLeftOperand()
8062                                                                        .getExpressionType() == EExpressionType.simple_object_name_t) {
8063                                                                TObjectName columnObject = resultColumn.getExpr().getLeftOperand().getObjectOperand();
8064
8065                                                                if (columnObject.getDbObjectType() == EDbObjectType.variable) {
8066                                                                        continue;
8067                                                                }
8068
8069                                                                if (columnObject.getColumnNameOnly().startsWith("@")
8070                                                                                && (option.getVendor() == EDbVendor.dbvmssql
8071                                                                                                || option.getVendor() == EDbVendor.dbvazuresql)) {
8072                                                                        continue;
8073                                                                }
8074
8075                                                                if (columnObject.getColumnNameOnly().startsWith(":")
8076                                                                                && (option.getVendor() == EDbVendor.dbvhana
8077                                                                                                || option.getVendor() == EDbVendor.dbvteradata)) {
8078                                                                        continue;
8079                                                                }
8080
8081                                                                ResultColumn updateColumn = modelFactory.createMergeResultColumn(resultSet,
8082                                                                                columnObject);
8083
8084                                                                TExpression valueExpression = resultColumn.getExpr().getRightOperand();
8085                                                                if (valueExpression == null)
8086                                                                        continue;
8087
8088                                                                columnsInExpr visitor = new columnsInExpr();
8089                                                                valueExpression.inOrderTraverse(visitor);
8090                                                                List<TObjectName> objectNames = visitor.getObjectNames();
8091                                                                List<TParseTreeNode> functions = visitor.getFunctions();
8092
8093                                                                if (functions != null && !functions.isEmpty()) {
8094                                                                        analyzeFunctionDataFlowRelation(updateColumn, functions, EffectType.merge_update);
8095                                                                }
8096
8097                                                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
8098                                                                if (subquerys != null && !subquerys.isEmpty()) {
8099                                                                        analyzeSubqueryDataFlowRelation(updateColumn, subquerys, EffectType.merge_update);
8100                                                                }
8101
8102                                                                analyzeDataFlowRelation(updateColumn, objectNames, EffectType.merge_update, functions);
8103
8104                                                                List<TParseTreeNode> constants = visitor.getConstants();
8105                                                                analyzeConstantDataFlowRelation(updateColumn, constants, EffectType.merge_update,
8106                                                                                functions);
8107
8108                                                                if (tableModel instanceof Table) {
8109                                                                        TableColumn tableColumn = modelFactory.createTableColumn((Table)tableModel, columnObject,
8110                                                                                        false);
8111
8112                                                                        if (tableColumn != null) {
8113                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8114                                                                                relation.setEffectType(EffectType.merge_update);
8115                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
8116                                                                                relation.addSource(new ResultColumnRelationshipElement(updateColumn));
8117                                                                                relation.setProcess(process);
8118                                                                        }
8119                                                                }
8120                                                                else {
8121                                                                        TTable targetTable = stmt.getTargetTable().getSubquery().getTables().getTable(0);
8122                                                                        if (targetTable != null && modelManager.getModel(targetTable) instanceof Table) {
8123                                                                                Table targetTableModel = (Table) modelManager.getModel(targetTable);
8124                                                                                TableColumn tableColumn = modelFactory
8125                                                                                                .createTableColumn((Table) targetTableModel, columnObject, false);
8126                                                                                if (tableColumn != null) {
8127                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8128                                                                                        relation.setEffectType(EffectType.merge_update);
8129                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
8130                                                                                        relation.addSource(new ResultColumnRelationshipElement(updateColumn));
8131                                                                                        relation.setProcess(process);
8132                                                                                }
8133                                                                        }
8134                                                                }
8135                                                        }
8136                                                }
8137                                        }
8138                                        if (clause.getInsertClause() != null && tableModel instanceof Table) {
8139                                                TExpression insertValue = clause.getInsertClause().getInsertValue();
8140                                                if (insertValue != null
8141                                                                && insertValue.getExpressionType() == EExpressionType.objectConstruct_t) {
8142                                                        ResultSet resultSet = modelFactory.createResultSet(clause.getInsertClause(), false);
8143
8144                                                        createPseudoImpactRelation(stmt, resultSet, EffectType.merge_insert);
8145
8146                                                        TObjectConstruct objectConstruct = insertValue.getObjectConstruct();
8147                                                        for (int z = 0; z < objectConstruct.getPairs().size(); z++) {
8148                                                                TPair pair = objectConstruct.getPairs().getElement(z);
8149
8150                                                                if (pair.getKeyName().getExpressionType() == EExpressionType.simple_constant_t) {
8151                                                                        TObjectName columnObject = new TObjectName();
8152                                                                        TConstant constant = pair.getKeyName().getConstantOperand();
8153                                                                        TSourceToken newSt = new TSourceToken(
8154                                                                                        constant.getValueToken().getTextWithoutQuoted());
8155                                                                        columnObject.setPartToken(newSt);
8156                                                                        columnObject.setSourceTable(stmt.getTargetTable());
8157                                                                        columnObject.setStartToken(constant.getStartToken());
8158                                                                        columnObject.setEndToken(constant.getEndToken());
8159
8160                                                                        ResultColumn insertColumn = modelFactory.createMergeResultColumn(resultSet,
8161                                                                                        columnObject);
8162
8163                                                                        TExpression valueExpression = pair.getKeyValue();
8164                                                                        if (valueExpression == null)
8165                                                                                continue;
8166
8167                                                                        columnsInExpr visitor = new columnsInExpr();
8168                                                                        valueExpression.inOrderTraverse(visitor);
8169                                                                        List<TObjectName> objectNames = visitor.getObjectNames();
8170                                                                        List<TParseTreeNode> functions = visitor.getFunctions();
8171
8172                                                                        if (functions != null && !functions.isEmpty()) {
8173                                                                                analyzeFunctionDataFlowRelation(insertColumn, functions,
8174                                                                                                EffectType.merge_insert);
8175                                                                        }
8176
8177                                                                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
8178                                                                        if (subquerys != null && !subquerys.isEmpty()) {
8179                                                                                analyzeSubqueryDataFlowRelation(insertColumn, subquerys,
8180                                                                                                EffectType.merge_insert);
8181                                                                        }
8182
8183                                                                        analyzeDataFlowRelation(insertColumn, objectNames, EffectType.merge_insert,
8184                                                                                        functions);
8185
8186                                                                        List<TParseTreeNode> constants = visitor.getConstants();
8187                                                                        analyzeConstantDataFlowRelation(insertColumn, constants, EffectType.merge_insert,
8188                                                                                        functions);
8189
8190                                                                        TableColumn tableColumn = modelFactory.createTableColumn((Table)tableModel,
8191                                                                                        columnObject, false);
8192
8193                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8194                                                                        relation.setEffectType(EffectType.merge_insert);
8195                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
8196                                                                        relation.addSource(new ResultColumnRelationshipElement(insertColumn));
8197                                                                        relation.setProcess(process);
8198                                                                }
8199                                                        }
8200                                                } else {
8201                                                        TObjectNameList columns = clause.getInsertClause().getColumnList();
8202                                                        TResultColumnList values = clause.getInsertClause().getValuelist();
8203                                                        if (values == null || values.size() == 0) {
8204                                                                if (clause.getInsertClause().toString().toLowerCase().indexOf("row") != -1) {
8205                                                                        if (stmt.getUsingTable().getSubquery() != null) {
8206                                                                                ResultSet sourceResultSet = modelFactory.createQueryTable(stmt.getUsingTable());
8207                                                                                TObjectName targetStarColumn = new TObjectName();
8208                                                                                targetStarColumn.setString("*");
8209                                                                                TableColumn targetTableColumn = modelFactory.createTableColumn((Table)tableModel,
8210                                                                                                targetStarColumn, true);
8211                                                                                if (sourceResultSet.getColumns() == null
8212                                                                                                || sourceResultSet.getColumns().isEmpty()) {
8213                                                                                        TObjectName sourceStarColumn = new TObjectName();
8214                                                                                        sourceStarColumn.setString("*");
8215                                                                                        modelFactory.createResultColumn(sourceResultSet, sourceStarColumn);
8216                                                                                }
8217                                                                                for (ResultColumn sourceColumn : sourceResultSet.getColumns()) {
8218                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8219                                                                                        relation.setEffectType(EffectType.merge_insert);
8220                                                                                        relation.setTarget(new TableColumnRelationshipElement(targetTableColumn));
8221                                                                                        relation.addSource(new ResultColumnRelationshipElement(sourceColumn));
8222                                                                                        relation.setProcess(process);
8223                                                                                }
8224                                                                        } else {
8225                                                                                Table sourceTable = modelFactory.createTable(stmt.getUsingTable());
8226                                                                                TObjectName sourceStarColumn = new TObjectName();
8227                                                                                sourceStarColumn.setString("*");
8228                                                                                TableColumn sourceTableColumn = modelFactory.createTableColumn(sourceTable,
8229                                                                                                sourceStarColumn, true);
8230                                                                                TObjectName targetStarColumn = new TObjectName();
8231                                                                                targetStarColumn.setString("*");
8232                                                                                TableColumn targetTableColumn = modelFactory.createTableColumn(((Table)tableModel),
8233                                                                                                targetStarColumn, true);
8234                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8235                                                                                relation.setEffectType(EffectType.merge_insert);
8236                                                                                relation.setTarget(new TableColumnRelationshipElement(targetTableColumn));
8237                                                                                relation.addSource(new TableColumnRelationshipElement(sourceTableColumn));
8238                                                                                relation.setProcess(process);
8239                                                                        }
8240
8241                                                                }
8242                                                                continue;
8243                                                        }
8244
8245                                                        List<TObjectName> tableColumns = new ArrayList<TObjectName>();
8246                                                        if (columns == null || columns.size() == 0) {
8247//                                                              if (!((Table)tableModel).getColumns().isEmpty()) {
8248//                                                                      for (int j = 0; j < ((Table)tableModel).getColumns().size(); j++) {
8249//                                                                              if (((Table)tableModel).getColumns().get(j).getColumnObject() == null) {
8250//                                                                                      continue;
8251//                                                                              }
8252//                                                                              tableColumns.add(((Table)tableModel).getColumns().get(j).getColumnObject());
8253//                                                                      }
8254//                                                              } else {
8255                                                                        for (int j = 0; j < values.size(); j++) {
8256                                                                                TResultColumn column = values.getResultColumn(j);
8257                                                                                if (column.getAliasClause() != null) {
8258                                                                                        tableColumns.add(column.getAliasClause().getAliasName());
8259                                                                                } else if (column.getFieldAttr() != null) {
8260                                                                                        tableColumns.add(column.getFieldAttr());
8261                                                                                } else {
8262                                                                                        TObjectName columnName = new TObjectName();
8263                                                                                        columnName.setString(column.toString());
8264                                                                                        tableColumns.add(columnName);
8265                                                                                }
8266//                                                                      }
8267                                                                }
8268                                                        } else {
8269                                                                for (int j = 0; j < columns.size(); j++) {
8270                                                                        tableColumns.add(columns.getObjectName(j));
8271                                                                }
8272                                                        }
8273
8274                                                        ResultSet resultSet = modelFactory.createResultSet(clause.getInsertClause(), false);
8275
8276                                                        createPseudoImpactRelation(stmt, resultSet, EffectType.merge_insert);
8277
8278                                                        for (int j = 0; j < tableColumns.size() && j < values.size(); j++) {
8279                                                                TObjectName columnObject = tableColumns.get(j);
8280
8281                                                                ResultColumn insertColumn = modelFactory.createMergeResultColumn(resultSet,
8282                                                                                columnObject);
8283
8284                                                                TExpression valueExpression = values.getResultColumn(j).getExpr();
8285                                                                if (valueExpression == null)
8286                                                                        continue;
8287
8288                                                                columnsInExpr visitor = new columnsInExpr();
8289                                                                valueExpression.inOrderTraverse(visitor);
8290                                                                List<TObjectName> objectNames = visitor.getObjectNames();
8291                                                                List<TParseTreeNode> functions = visitor.getFunctions();
8292
8293                                                                if (functions != null && !functions.isEmpty()) {
8294                                                                        analyzeFunctionDataFlowRelation(insertColumn, functions, EffectType.merge_insert);
8295                                                                }
8296
8297                                                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
8298                                                                if (subquerys != null && !subquerys.isEmpty()) {
8299                                                                        analyzeSubqueryDataFlowRelation(insertColumn, subquerys, EffectType.merge_insert);
8300                                                                }
8301
8302                                                                analyzeDataFlowRelation(insertColumn, objectNames, EffectType.merge_insert, functions);
8303
8304                                                                List<TParseTreeNode> constants = visitor.getConstants();
8305                                                                analyzeConstantDataFlowRelation(insertColumn, constants, EffectType.merge_insert,
8306                                                                                functions);
8307
8308                                                                TableColumn tableColumn = modelFactory.createTableColumn(((Table)tableModel), columnObject,
8309                                                                                false);
8310                                                                if(tableColumn == null) {
8311                                                                        if (((Table) tableModel).isCreateTable()) {
8312                                                                                tableColumn = ((Table) tableModel).getColumns().get(j);
8313                                                                        }
8314                                                                        else {
8315                                                                                continue;
8316                                                                        }
8317                                                                }
8318
8319                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8320                                                                relation.setEffectType(EffectType.merge_insert);
8321                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
8322                                                                relation.addSource(new ResultColumnRelationshipElement(insertColumn));
8323                                                                relation.setProcess(process);
8324                                                        }
8325                                                }
8326                                        }
8327                                }
8328
8329                        }
8330
8331                        if (stmt.getCondition() != null) {
8332                                analyzeFilterCondition(null, stmt.getCondition(), null, JoinClauseType.on, EffectType.merge);
8333                        }
8334                }
8335        }
8336
8337        private List<TableColumn> bindInsertTableColumn(Table tableModel, TInsertIntoValue value, List<TObjectName> keyMap,
8338                        List<TResultColumn> valueMap) {
8339                List<TableColumn> tableColumns = new ArrayList<TableColumn>();
8340                if (value.getColumnList() != null) {
8341                        for (int z = 0; z < value.getColumnList().size(); z++) {
8342                                TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
8343                                                value.getColumnList().getObjectName(z));
8344                                tableColumns.add(tableColumn);
8345                                keyMap.add(tableColumn.getColumnObject());
8346                        }
8347                }
8348
8349                if (value.getTargetList() != null) {
8350                        for (int z = 0; z < value.getTargetList().size(); z++) {
8351                                TMultiTarget target = value.getTargetList().getMultiTarget(z);
8352                                TResultColumnList columns = target.getColumnList();
8353                                for (int i = 0; i < columns.size(); i++) {
8354                                        if (value.getColumnList() == null) {
8355                                                TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
8356                                                                columns.getResultColumn(i).getFieldAttr());
8357                                                tableColumns.add(tableColumn);
8358                                        }
8359                                        valueMap.add(columns.getResultColumn(i));
8360                                }
8361                        }
8362                }
8363
8364                return tableColumns;
8365        }
8366
8367        private TableColumn matchColumn(List<TableColumn> tableColumns, TableColumn targetColumn) {
8368                String columnName = targetColumn.getName();
8369                if (tableColumns == null) {
8370                        return null;
8371                }
8372                for (int i = 0; i < tableColumns.size(); i++) {
8373                        TableColumn column = tableColumns.get(i);
8374                        if (column.getColumnObject() == null) {
8375                                continue;
8376                        }
8377                        if(column.isStruct() && targetColumn.isStruct()) {
8378                                List<String> names = SQLUtil.parseNames(column.getName());
8379                                List<String> targetNames = SQLUtil
8380                                                .parseNames(targetColumn.getName());
8381                                if (!getColumnName(targetNames.get(0))
8382                                                .equals(getColumnName(names.get(0)))) {
8383                                        continue;
8384                                }
8385                        }
8386                        if (getColumnName(column.getColumnObject().toString()).equals(getColumnName(columnName))) {
8387                                return column;
8388                        }
8389                }
8390                return null;
8391        }
8392        
8393        private TableColumn matchColumn(List<TableColumn> tableColumns, TObjectName columnName) {
8394                if (tableColumns == null) {
8395                        return null;
8396                }
8397                for (int i = 0; i < tableColumns.size(); i++) {
8398                        TableColumn column = tableColumns.get(i);
8399                        if (column.getColumnObject() == null) {
8400                                continue;
8401                        }
8402                        if (DlineageUtil.getColumnName(column.getColumnObject()).equalsIgnoreCase(DlineageUtil.getColumnName(columnName)))
8403                                return column;
8404                }
8405                return null;
8406        }
8407
8408        private ResultColumn matchResultColumn(List<ResultColumn> resultColumns, ResultColumn resultColumn) {
8409                if (resultColumns == null) {
8410                        return null;
8411                }
8412
8413                TObjectName columnName = getObjectName(resultColumn);
8414                if (columnName == null) {
8415                        return null;
8416                }
8417
8418                for (int i = 0; i < resultColumns.size(); i++) {
8419                        ResultColumn column = resultColumns.get(i);
8420                        if (column.getAlias() != null
8421                                        && getColumnName(column.getAlias()).equalsIgnoreCase(getColumnName(columnName)))
8422                                return column;
8423                        if (column.getName() != null && getColumnName(column.getName()).equalsIgnoreCase(getColumnName(columnName)))
8424                                return column;
8425                        if (column.getName() != null && column.getName().endsWith("*")) {
8426                                if ("*".equals(column.getColumnObject().toString())) {
8427                                        return column;
8428                                } else {
8429                                        TObjectName columnObjectName = getObjectName(column);
8430                                        if (columnObjectName.getTableString() != null
8431                                                        && columnObjectName.getTableString().equals(getResultSetAlias(resultColumn))) {
8432                                                return column;
8433                                        }
8434                                }
8435                        }
8436                }
8437                return null;
8438        }
8439
8440        private String getResultSetAlias(ResultColumn resultColumn) {
8441                ResultSet resultSet = resultColumn.getResultSet();
8442                if (resultSet instanceof QueryTable) {
8443                        return ((QueryTable) resultSet).getAlias();
8444                }
8445                return null;
8446        }
8447
8448        private ResultColumn matchResultColumn(List<ResultColumn> resultColumns, TObjectName columnName) {
8449                if (resultColumns == null) {
8450                        return null;
8451                }
8452                for (int i = 0; i < resultColumns.size(); i++) {
8453                        ResultColumn column = resultColumns.get(i);
8454                        if (column.getAlias() != null
8455                                        && getColumnName(column.getAlias()).equalsIgnoreCase(getColumnName(columnName)))
8456                                return column;
8457                        if (column.getName() != null && getColumnName(column.getName()).equalsIgnoreCase(getColumnName(columnName)))
8458                                return column;
8459                        if (column.getName() != null && column.getName().endsWith("*")) {
8460                                if ("*".equals(column.getColumnObject().toString())) {
8461                                        return column;
8462                                } else {
8463                                        TObjectName columnObjectName = getObjectName(column);
8464                                        if (columnObjectName.getTableString() != null
8465                                                        && columnObjectName.getTableString().equals(columnName.getTableString())) {
8466                                                return column;
8467                                        }
8468                                }
8469                        }
8470                }
8471                return null;
8472        }
8473
8474        private void analyzeInsertStmt(TInsertSqlStatement stmt) {
8475                Map<Table, List<TObjectName>> insertTableKeyMap = new LinkedHashMap<Table, List<TObjectName>>();
8476                Map<Table, List<TResultColumn>> insertTableValueMap = new LinkedHashMap<Table, List<TResultColumn>>();
8477                Map<String, List<TableColumn>> tableColumnMap = new LinkedHashMap<String, List<TableColumn>>();
8478                List<Table> inserTables = new ArrayList<Table>();
8479                List<TExpression> expressions = new ArrayList<TExpression>();
8480                boolean hasInsertColumns = false;
8481                
8482                EffectType effectType = EffectType.insert;
8483                if(stmt.getInsertToken()!=null && stmt.getInsertToken().toString().toLowerCase().startsWith("replace")) {
8484                        effectType = EffectType.replace;
8485                }
8486                
8487                if (stmt.getInsertConditions() != null && stmt.getInsertConditions().size() > 0) {
8488                        for (int i = 0; i < stmt.getInsertConditions().size(); i++) {
8489                                TInsertCondition condition = stmt.getInsertConditions().getElement(i);
8490                                if (condition.getCondition() != null) {
8491                                        expressions.add(condition.getCondition());
8492                                }
8493                                for (int j = 0; j < condition.getInsertIntoValues().size(); j++) {
8494                                        TInsertIntoValue value = condition.getInsertIntoValues().getElement(j);
8495                                        TTable table = value.getTable();
8496                                        Table tableModel = modelFactory.createTable(table);
8497
8498                                        inserTables.add(tableModel);
8499                                        List<TObjectName> keyMap = new ArrayList<TObjectName>();
8500                                        List<TResultColumn> valueMap = new ArrayList<TResultColumn>();
8501                                        insertTableKeyMap.put(tableModel, keyMap);
8502                                        insertTableValueMap.put(tableModel, valueMap);
8503
8504                                        List<TableColumn> tableColumns = bindInsertTableColumn(tableModel, value, keyMap, valueMap);
8505                                        if (tableColumnMap.get(DlineageUtil.getIdentifierNormalTableName(table.getFullName())) == null
8506                                                        && !tableColumns.isEmpty()) {
8507                                                tableColumnMap.put(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName()),
8508                                                                tableColumns);
8509                                        }
8510
8511                                        // if (stmt.getSubQuery() != null)
8512                                        {
8513                                                Process process = modelFactory.createProcess(stmt);
8514                                                tableModel.addProcess(process);
8515                                        }
8516                                }
8517                        }
8518                        hasInsertColumns = true;
8519                } else if (stmt.getInsertIntoValues() != null && stmt.getInsertIntoValues().size() > 0) {
8520                        for (int i = 0; i < stmt.getInsertIntoValues().size(); i++) {
8521                                TInsertIntoValue value = stmt.getInsertIntoValues().getElement(i);
8522                                TTable table = value.getTable();
8523                                Table tableModel = modelFactory.createTable(table);
8524
8525                                inserTables.add(tableModel);
8526                                List<TObjectName> keyMap = new ArrayList<TObjectName>();
8527                                List<TResultColumn> valueMap = new ArrayList<TResultColumn>();
8528                                insertTableKeyMap.put(tableModel, keyMap);
8529                                insertTableValueMap.put(tableModel, valueMap);
8530
8531                                List<TableColumn> tableColumns = bindInsertTableColumn(tableModel, value, keyMap, valueMap);
8532                                if (tableColumnMap.get(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName())) == null && !tableColumns.isEmpty()) {
8533                                        tableColumnMap.put(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName()), tableColumns);
8534                                }
8535
8536                                // if (stmt.getSubQuery() != null)
8537                                {
8538                                        Process process = modelFactory.createProcess(stmt);
8539                                        tableModel.addProcess(process);
8540                                }
8541                        }
8542                        hasInsertColumns = true;
8543                } else if (stmt.getColumnList() != null && stmt.getColumnList().size() > 0) {
8544                        TTable table = stmt.getTargetTable();
8545                        Table tableModel = modelFactory.createTable(table);
8546
8547                        inserTables.add(tableModel);
8548                        List<TObjectName> keyMap = new ArrayList<TObjectName>();
8549                        insertTableKeyMap.put(tableModel, keyMap);
8550                        List<TableColumn> tableColumns = new ArrayList<TableColumn>();
8551                        for (int i = 0; i < stmt.getColumnList().size(); i++) {
8552                                TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
8553                                                stmt.getColumnList().getObjectName(i));
8554                                tableColumns.add(tableColumn);
8555                        }
8556                        if (tableColumnMap.get(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName())) == null && !tableColumns.isEmpty()) {
8557                                tableColumnMap.put(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName()), tableColumns);
8558                        }
8559
8560                        // if (stmt.getSubQuery() != null)
8561                        {
8562                                Process process = modelFactory.createProcess(stmt);
8563                                tableModel.addProcess(process);
8564                        }
8565                        hasInsertColumns = true;
8566                } else if (stmt.getOutputClause() != null && stmt.getOutputClause().getSelectItemList().size() > 0) {
8567                        TTable table = stmt.getTargetTable();
8568                        Table tableModel = modelFactory.createTable(table);
8569
8570                        inserTables.add(tableModel);
8571                        List<TObjectName> keyMap = new ArrayList<TObjectName>();
8572                        insertTableKeyMap.put(tableModel, keyMap);
8573                        List<TableColumn> tableColumns = new ArrayList<TableColumn>();
8574                        for (int i = 0; i < stmt.getOutputClause().getSelectItemList().size(); i++) {
8575                                TObjectName columnName = stmt.getOutputClause().getSelectItemList().getResultColumn(i).getFieldAttr();
8576                                if (columnName.getPseudoTableType() != EPseudoTableType.none) {
8577                                        // Phase 1 already swapped tokens; getColumnNameOnly() returns actual column name
8578                                        String column = columnName.getColumnNameOnly();
8579                                        columnName = new TObjectName();
8580                                        columnName.setString(column);
8581                                } else {
8582                                        String column = columnName.toString().toLowerCase();
8583                                        if ((column.startsWith("inserted.") || column.startsWith("deleted."))
8584                                                        && columnName.getPropertyToken() != null) {
8585                                                column = columnName.getPropertyToken().getAstext();
8586                                                columnName = new TObjectName();
8587                                                columnName.setString(column);
8588                                        }
8589                                }
8590                                TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel, columnName);
8591                                tableColumns.add(tableColumn);
8592                        }
8593                        if (tableColumnMap.get(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName())) == null && !tableColumns.isEmpty()) {
8594                                tableColumnMap.put(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName()), tableColumns);
8595                        }
8596
8597                        // if (stmt.getSubQuery() != null)
8598                        {
8599                                Process process = modelFactory.createProcess(stmt);
8600                                tableModel.addProcess(process);
8601                        }
8602                        hasInsertColumns = true;
8603                } else {
8604                        TTable table = stmt.getTargetTable();
8605                        Table tableModel;
8606                        if (table != null) {
8607                                tableModel = modelFactory.createTable(table);
8608                                // if (stmt.getSubQuery() != null)
8609                                {
8610                                        Process process = modelFactory.createProcess(stmt);
8611                                        tableModel.addProcess(process);
8612                                }
8613                                if (tableModel.getColumns() == null || tableModel.getColumns().isEmpty()) {
8614                                        tableModel.addColumnsFromSQLEnv();
8615                                }
8616                        } else if (stmt.getDirectoryName() != null) {
8617                                tableModel = modelFactory.createTableByName(stmt.getDirectoryName(), true);
8618                                tableModel.setPath(true);
8619                                tableModel.setCreateTable(true);
8620                                TObjectName fileUri = new TObjectName();
8621                                fileUri.setString("uri=" + stmt.getDirectoryName());
8622                                TableColumn tableColumn = modelFactory.createFileUri(tableModel, fileUri);
8623                                // if (stmt.getSubQuery() != null)
8624                                {
8625                                        Process process = modelFactory.createProcess(stmt);
8626                                        tableModel.addProcess(process);
8627                                }
8628                        } else {
8629                                ErrorInfo errorInfo = new ErrorInfo();
8630                                errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
8631                                errorInfo.setErrorMessage("Can't get target table. InsertSqlStatement is " + stmt.toString());
8632                                errorInfo.setStartPosition(new Pair3<Long, Long, String>(stmt.getStartToken().lineNo,
8633                                                stmt.getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
8634                                errorInfo.setEndPosition(new Pair3<Long, Long, String>(stmt.getEndToken().lineNo,
8635                                                stmt.getEndToken().columnNo + stmt.getEndToken().getAstext().length(),
8636                                                ModelBindingManager.getGlobalHash()));
8637                                errorInfo.fillInfo(this);
8638                                errorInfos.add(errorInfo);
8639                                return;
8640                        }
8641                        inserTables.add(tableModel);
8642                        if (table != null
8643                                        && tableColumnMap.get(DlineageUtil.getIdentifierNormalTableName(table.getFullName())) == null) {
8644                                if (tableModel.getColumns() != null && !tableModel.getColumns().isEmpty()) {
8645                                        tableColumnMap.put(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName()),
8646                                                        tableModel.getColumns());
8647                                } else {
8648                                        tableColumnMap.put(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName()), null);
8649                                }
8650                        }
8651                }
8652
8653                if (stmt.getSubQuery() != null) {
8654                        analyzeSelectStmt(stmt.getSubQuery());
8655                }
8656
8657                Iterator<Table> tableIter = inserTables.iterator();
8658                while (tableIter.hasNext()) {
8659                        Table tableModel = tableIter.next();
8660                        List<TableColumn> tableColumns = tableColumnMap.get(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName()));
8661                        List<TObjectName> keyMap = insertTableKeyMap.get(tableModel);
8662                        List<TResultColumn> valueMap = insertTableValueMap.get(tableModel);
8663                        boolean initColumn = (hasInsertColumns && tableColumns != null && !containStarColumn(tableColumns));
8664
8665                        if (stmt.getSubQuery() != null) {
8666
8667                                List<TSelectSqlStatement> subquerys = new ArrayList<TSelectSqlStatement>();
8668                                if (stmt.getSubQuery().getResultColumnList() != null || stmt.getSubQuery().getTransformClause() != null) {
8669                                        subquerys.add(stmt.getSubQuery());
8670                                } else if (stmt.getSubQuery().getValueClause() != null
8671                                                && stmt.getSubQuery().getValueClause().getRows() != null) {
8672                                        for (TResultColumnList resultColumnList : stmt.getSubQuery().getValueClause().getRows()) {
8673                                                for(TResultColumn resultColumn: resultColumnList) {
8674                                                        if(resultColumn.getExpr()!=null && resultColumn.getExpr().getSubQuery()!=null) {
8675                                                                analyzeSelectStmt(resultColumn.getExpr().getSubQuery());
8676                                                                subquerys.add(resultColumn.getExpr().getSubQuery());
8677                                                        }
8678                                                }
8679                                        }
8680                                }
8681
8682                                for(TSelectSqlStatement subquery: subquerys) {
8683                                        if ((tableModel.isCreateTable() && tableModel.getColumns() != null)
8684                                                        || (subquery.getSetOperatorType() == ESetOperatorType.none
8685                                                                        && stmt.getColumnList() != null && stmt.getColumnList().size() > 0)) {
8686
8687                                                ResultSet resultSetModel = null;
8688
8689                                                if (subquery != null) {
8690                                                        resultSetModel = (ResultSet) modelManager.getModel(subquery);
8691                                                }
8692
8693                                                TResultColumnList resultset = subquery.getResultColumnList();
8694                                                if (resultSetModel == null && resultset != null) {
8695                                                        resultSetModel = (ResultSet) modelManager.getModel(resultset);
8696                                                }
8697
8698                                                if (resultSetModel == null) {
8699                                                        ErrorInfo errorInfo = new ErrorInfo();
8700                                                        errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
8701                                                        errorInfo.setErrorMessage("Can't get resultset model");
8702                                                        errorInfo.setStartPosition(new Pair3<Long, Long, String>(resultset.getStartToken().lineNo,
8703                                                                        resultset.getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
8704                                                        errorInfo.setEndPosition(new Pair3<Long, Long, String>(resultset.getEndToken().lineNo,
8705                                                                        resultset.getEndToken().columnNo + resultset.getEndToken().getAstext().length(),
8706                                                                        ModelBindingManager.getGlobalHash()));
8707                                                        errorInfos.add(errorInfo);
8708                                                }
8709
8710                                                if (!resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
8711                                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
8712                                                        impactRelation.setEffectType(effectType);
8713                                                        impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
8714                                                                        resultSetModel.getRelationRows()));
8715                                                        impactRelation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(
8716                                                                        tableModel.getRelationRows()));
8717                                                }
8718
8719                                                int resultSetSize = resultSetModel.getColumns().size();
8720                                                int j = 0;
8721                                                int starIndex = 0;
8722                                                TObjectNameList items = stmt.getColumnList();
8723                                                List<String> itemNames = new ArrayList<String>();
8724                                                int starColumnCount = 0;
8725                                                for (ResultColumn item : resultSetModel.getColumns()) {
8726                                                        if (item.getName().endsWith("*")) {
8727                                                                starColumnCount += 1;
8728                                                        }
8729                                                }
8730                                                if (items != null) {
8731                                                        for (int i = 0; i < items.size() && j < resultSetSize; i++) {
8732                                                                TObjectName column = items.getObjectName(i);
8733
8734                                                                if (column.getDbObjectType() == EDbObjectType.variable) {
8735                                                                        continue;
8736                                                                }
8737
8738                                                                if (column.getColumnNameOnly().startsWith("@")
8739                                                                                && (option.getVendor() == EDbVendor.dbvmssql
8740                                                                                                || option.getVendor() == EDbVendor.dbvazuresql)) {
8741                                                                        continue;
8742                                                                }
8743
8744                                                                if (column.getColumnNameOnly().startsWith(":")
8745                                                                                && (option.getVendor() == EDbVendor.dbvhana
8746                                                                                                || option.getVendor() == EDbVendor.dbvteradata)) {
8747                                                                        continue;
8748                                                                }
8749
8750                                                                ResultColumn resultColumn = resultSetModel.getColumns().get(j);
8751                                                                if (!resultSetModel.getColumns().get(j).getName().contains("*")) {
8752                                                                        j++;
8753                                                                } else {
8754                                                                        starIndex++;
8755                                                                        if (resultSetSize - j == items.size() - i) {
8756                                                                                j++;
8757
8758                                                                        }
8759                                                                }
8760                                                                if (column != null) {
8761                                                                        TableColumn tableColumn;
8762                                                                        // if (!initColumn) {
8763                                                                        tableColumn = matchColumn(tableModel.getColumns(), column);
8764                                                                        if (tableColumn == null) {
8765                                                                                if (tableModel.isCreateTable() && !containStarColumn(tableModel.getColumns())) {
8766                                                                                        if (tableModel.getColumns().size() <= i) {
8767                                                                                                continue;
8768                                                                                        }
8769                                                                                        tableColumn = tableModel.getColumns().get(i);
8770                                                                                } else {
8771                                                                                        tableColumn = modelFactory.createTableColumn(tableModel, column, false);
8772                                                                                        if(tableColumn == null) {
8773                                                                                                continue;
8774                                                                                        }
8775                                                                                }
8776                                                                        }
8777//                                                      } else {
8778//                                                              tableColumn = matchColumn(tableColumns, column);
8779//                                                              if (tableColumn == null) {
8780//                                                                      continue;
8781//                                                              }
8782//                                                      }
8783                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8784                                                                        relation.setEffectType(effectType);
8785                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
8786                                                                        if (resultColumn.hasStarLinkColumn()
8787                                                                                        && resultColumn.getStarLinkColumnNames().size() > starIndex - 1 && starColumnCount<=1) {
8788                                                                                boolean find = false;
8789                                                                                while (resultColumn.getStarLinkColumnNames().size() > starIndex - 1) {
8790                                                                                        TObjectName name = resultColumn.getStarLinkColumnName(starIndex - 1);
8791                                                                                        if (itemNames.contains(name.toString())) {
8792                                                                                                starIndex++;
8793                                                                                                continue;
8794                                                                                        }
8795                                                                                        ResultColumn expandStarColumn = modelFactory
8796                                                                                                        .createResultColumn(resultSetModel, name, false);
8797                                                                                        relation.addSource(new ResultColumnRelationshipElement(expandStarColumn));
8798                                                                                        itemNames.add(resultColumn.getName());
8799                                                                                        find = true;
8800                                                                                        break;
8801                                                                                }
8802                                                                                if (!find) {
8803                                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
8804                                                                                        itemNames.add(resultColumn.getName());
8805                                                                                }
8806                                                                        } else {
8807                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
8808                                                                                itemNames.add(resultColumn.getName());
8809                                                                        }
8810                                                                        Process process = modelFactory.createProcess(stmt);
8811                                                                        relation.setProcess(process);
8812                                                                }
8813                                                        }
8814                                                } else {
8815                                                        List<TableColumn> columns = tableModel.getColumns();
8816                                                        if (columns.size() == 1 && tableModel.isPath()) {
8817                                                                for(int i=0;i<resultSetSize;i++) {
8818                                                                        ResultColumn resultColumn = resultSetModel.getColumns().get(i);
8819                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8820                                                                        relation.setEffectType(effectType);
8821                                                                        relation.setTarget(new TableColumnRelationshipElement(tableModel.getColumns().get(0)));
8822                                                                        if (resultColumn.hasStarLinkColumn()
8823                                                                                        && resultColumn.getStarLinkColumnNames().size() > starIndex - 1) {
8824                                                                                ResultColumn expandStarColumn = modelFactory.createResultColumn(
8825                                                                                                resultSetModel, resultColumn.getStarLinkColumnName(starIndex - 1),
8826                                                                                                false);
8827                                                                                relation.addSource(new ResultColumnRelationshipElement(expandStarColumn));
8828                                                                        } else {
8829                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
8830                                                                        }
8831                                                                        Process process = modelFactory.createProcess(stmt);
8832                                                                        relation.setProcess(process);
8833                                                                }
8834                                                        } else {
8835                                                                boolean fromStruct = false;
8836                                                                for (int i = 0; i < columns.size() && j < resultSetSize; i++) {
8837                                                                        String column = columns.get(i).getName();
8838                                                                        ResultColumn resultColumn = resultSetModel.getColumns().get(j);
8839                                                                        if (!resultColumn.getName().contains("*")) {
8840                                                                                if (resultColumn.getName().equals(resultColumn.getRefColumnName())
8841                                                                                                && resultColumn.getColumnObject().toString().endsWith("*")
8842                                                                                                && resultSetSize == 1) {
8843                                                                                        starIndex++;
8844                                                                                        if (resultSetSize - j == columns.size() - i) {
8845                                                                                                j++;
8846
8847                                                                                        }
8848                                                                                }
8849                                                                                else {
8850                                                                                        j++;
8851                                                                                }
8852                                                                        } else {
8853                                                                                starIndex++;
8854                                                                                if (resultSetSize - j == columns.size() - i) {
8855                                                                                        j++;
8856
8857                                                                                }
8858                                                                        }
8859                                                                        if (column != null) {
8860                                                                                TableColumn tableColumn;
8861                                                                                // if (!initColumn) {
8862                                                                                tableColumn = matchColumn(tableModel.getColumns(), columns.get(i));
8863                                                                                if (tableColumn == null) {
8864                                                                                        if (tableModel.isCreateTable()
8865                                                                                                        && !containStarColumn(tableModel.getColumns())) {
8866                                                                                                if (tableModel.getColumns().size() <= i) {
8867                                                                                                        continue;
8868                                                                                                }
8869                                                                                                tableColumn = tableModel.getColumns().get(i);
8870                                                                                        } else {
8871                                                                                                TObjectName columnName = new TObjectName();
8872                                                                                                columnName.setString(column);
8873                                                                                                tableColumn = modelFactory.createTableColumn(tableModel, columnName,
8874                                                                                                                false);
8875                                                                                        }
8876                                                                                }
8877                                                                                else if (!resultColumn.isStruct() && tableColumn.isStruct() && columns.size() != resultSetSize) {
8878                                                                                        j--;
8879                                                                                        fromStruct = true;
8880                                                                                }
8881                                                                                if(fromStruct && !tableColumn.isStruct()) {
8882                                                                                        fromStruct = false;
8883                                                                                        resultColumn = resultSetModel.getColumns().get(j);
8884                                                                                        j++;
8885                                                                                }
8886                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8887                                                                                relation.setEffectType(effectType);
8888                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
8889                                                                                if (resultColumn.hasStarLinkColumn()
8890                                                                                                && resultColumn.getStarLinkColumnNames().size() > starIndex - 1) {
8891                                                                                        ResultColumn expandStarColumn = modelFactory.createResultColumn(
8892                                                                                                        resultSetModel, resultColumn.getStarLinkColumnName(starIndex - 1),
8893                                                                                                        false);
8894                                                                                        relation.addSource(new ResultColumnRelationshipElement(expandStarColumn));
8895                                                                                } else {
8896                                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn, starIndex - 1));
8897                                                                                }
8898                                                                                Process process = modelFactory.createProcess(stmt);
8899                                                                                relation.setProcess(process);
8900                                                                        }
8901                                                                }
8902                                                        }
8903                                                }
8904                                        } else if (!subquery.isCombinedQuery()) {
8905                                                SelectResultSet resultSetModel = (SelectResultSet) modelManager
8906                                                                .getModel(subquery.getResultColumnList() != null ? subquery.getResultColumnList()
8907                                                                                : subquery.getTransformClause());
8908
8909                                                if (resultSetModel != null && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
8910                                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
8911                                                        impactRelation.setEffectType(effectType);
8912                                                        impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
8913                                                                        resultSetModel.getRelationRows()));
8914                                                        impactRelation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(
8915                                                                        tableModel.getRelationRows()));
8916                                                }
8917
8918                                                List<ResultColumn> columnsSnapshot = new ArrayList<ResultColumn>();
8919                                                columnsSnapshot.addAll(resultSetModel.getColumns());
8920                                                if(resultSetModel.isDetermined() && stmt.getColumnList() == null) {
8921                                                        tableModel.setDetermined(true);
8922                                                }
8923                                                for (int i = 0; i < columnsSnapshot.size(); i++) {
8924                                                        ResultColumn resultColumn = columnsSnapshot.get(i);
8925                                                        if (resultColumn.getColumnObject() instanceof TObjectName) {
8926                                                                TableColumn tableColumn;
8927                                                                if (!initColumn) {
8928                                                                        if (tableModel.isCreateTable() && !containStarColumn(tableModel.getColumns())) {
8929                                                                                if (tableModel.getColumns().size() <= i) {
8930                                                                                        continue;
8931                                                                                }
8932                                                                                tableColumn = tableModel.getColumns().get(i);
8933                                                                        } else {
8934                                                                                tableColumn = modelFactory.createInsertTableColumn(tableModel,
8935                                                                                                (TObjectName) resultColumn.getColumnObject());
8936                                                                        }
8937                                                                        if (containStarColumn(tableColumns)) {
8938                                                                                getStarColumn(tableColumns)
8939                                                                                                .bindStarLinkColumn((TObjectName) resultColumn.getColumnObject());
8940                                                                        }
8941                                                                } else {
8942                                                                        TObjectName matchedColumnName = (TObjectName) resultColumn.getColumnObject();
8943                                                                        tableColumn = matchColumn(tableColumns, matchedColumnName);
8944                                                                        if (tableColumn == null) {
8945                                                                                if (!isEmptyCollection(valueMap)) {
8946                                                                                        int index = indexOfColumn(valueMap, matchedColumnName);
8947                                                                                        if (index != -1) {
8948                                                                                                if (!isEmptyCollection(keyMap) && index < keyMap.size()) {
8949                                                                                                        tableColumn = matchColumn(tableColumns, keyMap.get(index));
8950                                                                                                } else if (isEmptyCollection(keyMap) && index < tableColumns.size()) {
8951                                                                                                        tableColumn = tableColumns.get(index);
8952                                                                                                } else {
8953                                                                                                        continue;
8954                                                                                                }
8955                                                                                        } else {
8956                                                                                                continue;
8957                                                                                        }
8958                                                                                } else if (!isEmptyCollection(keyMap) && i < keyMap.size()) {
8959                                                                                        tableColumn = matchColumn(tableColumns, keyMap.get(i));
8960                                                                                } else if (isEmptyCollection(keyMap) && isEmptyCollection(valueMap)
8961                                                                                                && i < tableColumns.size()) {
8962                                                                                        tableColumn = tableColumns.get(i);
8963                                                                                } else {
8964                                                                                        continue;
8965                                                                                }
8966                                                                        }
8967                                                                }
8968
8969                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8970                                                                relation.setEffectType(effectType);
8971                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
8972                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
8973                                                                Process process = modelFactory.createProcess(stmt);
8974                                                                relation.setProcess(process);
8975                                                        } else {
8976                                                                TAliasClause alias = ((TResultColumn) resultColumn.getColumnObject()).getAliasClause();
8977                                                                if (alias != null && alias.getAliasName() != null) {
8978                                                                        TableColumn tableColumn;
8979                                                                        if (!initColumn) {
8980                                                                                if (tableModel.isCreateTable() && !containStarColumn(tableModel.getColumns())) {
8981                                                                                        if (tableModel.getColumns().size() <= i) {
8982                                                                                                if (tableModel.isPath()) {
8983                                                                                                        tableColumn = tableModel.getColumns().get(0);
8984                                                                                                } else {
8985                                                                                                        continue;
8986                                                                                                }
8987                                                                                        } else {
8988                                                                                                tableColumn = tableModel.getColumns().get(i);
8989                                                                                        }
8990                                                                                } else {
8991                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
8992                                                                                                        alias.getAliasName());
8993                                                                                        if (containStarColumn(resultSetModel)) {
8994                                                                                                tableColumn.notBindStarLinkColumn(true);
8995                                                                                        }
8996                                                                                }
8997                                                                                if (containStarColumn(tableColumns)) {
8998                                                                                        getStarColumn(tableColumns).bindStarLinkColumn(alias.getAliasName());
8999                                                                                }
9000                                                                        } else {
9001                                                                                TObjectName matchedColumnName = alias.getAliasName();
9002                                                                                tableColumn = matchColumn(tableColumns, matchedColumnName);
9003                                                                                if (tableColumn == null) {
9004                                                                                        if (!isEmptyCollection(valueMap)) {
9005                                                                                                int index = indexOfColumn(valueMap, matchedColumnName);
9006                                                                                                if (index != -1) {
9007                                                                                                        if (!isEmptyCollection(keyMap) && index < keyMap.size()) {
9008                                                                                                                tableColumn = matchColumn(tableColumns, keyMap.get(index));
9009                                                                                                        } else if (isEmptyCollection(keyMap)
9010                                                                                                                        && index < tableColumns.size()) {
9011                                                                                                                tableColumn = tableColumns.get(index);
9012                                                                                                        } else {
9013                                                                                                                continue;
9014                                                                                                        }
9015                                                                                                } else {
9016                                                                                                        continue;
9017                                                                                                }
9018                                                                                        } else if (!isEmptyCollection(keyMap) && i < keyMap.size()) {
9019                                                                                                tableColumn = matchColumn(tableColumns, keyMap.get(i));
9020                                                                                        } else {
9021                                                                                                tableColumn = modelFactory.createInsertTableColumn(tableModel,
9022                                                                                                                alias.getAliasName());
9023                                                                                        }
9024                                                                                }
9025
9026                                                                        }
9027
9028                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9029                                                                        relation.setEffectType(effectType);
9030                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9031                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9032                                                                        Process process = modelFactory.createProcess(stmt);
9033                                                                        relation.setProcess(process);
9034
9035                                                                } else if (((TResultColumn) resultColumn.getColumnObject()).getFieldAttr() != null) {
9036                                                                        TObjectName fieldAttr = ((TResultColumn) resultColumn.getColumnObject())
9037                                                                                        .getFieldAttr();
9038
9039                                                                        Object model = modelManager.getModel( resultColumn.getColumnObject());
9040                                                                        
9041                                                                        TableColumn tableColumn;
9042                                                                        if (!initColumn) {
9043                                                                                if (tableModel.isCreateTable() && !containStarColumn(tableModel.getColumns())) {
9044                                                                                        if (fieldAttr.toString().endsWith("*")) {
9045                                                                                                int starIndex = 0;
9046                                                                                                for (TableColumn column : tableModel.getColumns()) {
9047                                                                                                        starIndex++;
9048                                                                                                        DataFlowRelationship relation = modelFactory
9049                                                                                                                        .createDataFlowRelation();
9050                                                                                                        relation.setEffectType(effectType);
9051                                                                                                        relation.setTarget(new TableColumnRelationshipElement(column));
9052                                                                                                        if (resultColumn.getStarLinkColumnList().size() == tableModel
9053                                                                                                                        .getColumns().size()) {
9054                                                                                                                ResultColumn expandStarColumn = modelFactory.createResultColumn(
9055                                                                                                                                resultSetModel,
9056                                                                                                                                resultColumn.getStarLinkColumnList().get(starIndex - 1),
9057                                                                                                                                false);
9058                                                                                                                relation.addSource(
9059                                                                                                                                new ResultColumnRelationshipElement(expandStarColumn));
9060                                                                                                        } else {
9061                                                                                                                relation.addSource(
9062                                                                                                                                new ResultColumnRelationshipElement(resultColumn));
9063                                                                                                        }
9064                                                                                                        Process process = modelFactory.createProcess(stmt);
9065                                                                                                        relation.setProcess(process);
9066                                                                                                }
9067                                                                                                continue;
9068                                                                                        }
9069                                                                                        if (tableModel.getColumns().size() <= i) {
9070                                                                                                continue;
9071                                                                                        }
9072                                                                                        tableColumn = tableModel.getColumns().get(i);
9073                                                                                } else {
9074                                                                                        if(model instanceof LinkedHashMap) {
9075                                                                                                LinkedHashMap<String, ResultColumn> resultColumns = (LinkedHashMap<String, ResultColumn>)model;
9076                                                                                                for(String key: resultColumns.keySet()) {
9077                                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel, resultColumns.get(key).getName());
9078                                                                                                        DataFlowRelationship relation = modelFactory
9079                                                                                                                        .createDataFlowRelation();
9080                                                                                                        relation.setEffectType(effectType);
9081                                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9082                                                                                                        relation.addSource(
9083                                                                                                                        new ResultColumnRelationshipElement(resultColumns.get(key)));
9084                                                                                                        Process process = modelFactory.createProcess(stmt);
9085                                                                                                        relation.setProcess(process);
9086                                                                                                }
9087                                                                                                continue;
9088                                                                                        }
9089                                                                                        else {
9090                                                                                                tableColumn = modelFactory.createInsertTableColumn(tableModel, fieldAttr);
9091                                                                                        }
9092                                                                                }
9093                                                                        } else if (tableModel.isDetermined() && i < tableModel.getColumns().size()) {
9094                                                                                if (!isEmptyCollection(valueMap)) {
9095                                                                                        TObjectName matchedColumnName = fieldAttr;
9096                                                                                        int index = indexOfColumn(valueMap, matchedColumnName);
9097                                                                                        if (index != -1) {
9098                                                                                                if (!isEmptyCollection(keyMap) && index < keyMap.size()) {
9099                                                                                                        tableColumn = matchColumn(tableColumns, keyMap.get(index));
9100                                                                                                } else if (isEmptyCollection(keyMap)
9101                                                                                                                && index < tableColumns.size()) {
9102                                                                                                        tableColumn = tableColumns.get(index);
9103                                                                                                } else {
9104                                                                                                        continue;
9105                                                                                                }
9106                                                                                        } else {
9107                                                                                                continue;
9108                                                                                        }
9109                                                                                } else if (!isEmptyCollection(keyMap) && i < keyMap.size()) {
9110                                                                                        tableColumn = matchColumn(tableColumns, keyMap.get(i));
9111                                                                                } else {
9112                                                                                        tableColumn = tableModel.getColumns().get(i);
9113                                                                                }
9114                                                                        } else {
9115                                                                                TObjectName matchedColumnName = fieldAttr;
9116                                                                                tableColumn = matchColumn(tableColumns, matchedColumnName);
9117                                                                                if (tableColumn == null) {
9118                                                                                        if (!isEmptyCollection(valueMap)) {
9119                                                                                                int index = indexOfColumn(valueMap, matchedColumnName);
9120                                                                                                if (index != -1) {
9121                                                                                                        if (!isEmptyCollection(keyMap) && index < keyMap.size()) {
9122                                                                                                                tableColumn = matchColumn(tableColumns, keyMap.get(index));
9123                                                                                                        } else if (isEmptyCollection(keyMap)
9124                                                                                                                        && index < tableColumns.size()) {
9125                                                                                                                tableColumn = tableColumns.get(index);
9126                                                                                                        } else {
9127                                                                                                                continue;
9128                                                                                                        }
9129                                                                                                } else {
9130                                                                                                        continue;
9131                                                                                                }
9132                                                                                        } else if (!isEmptyCollection(keyMap) && i < keyMap.size()) {
9133                                                                                                tableColumn = matchColumn(tableColumns, keyMap.get(i));
9134                                                                                        } else {
9135                                                                                                tableColumn = modelFactory.createInsertTableColumn(tableModel,
9136                                                                                                                fieldAttr);
9137                                                                                        }
9138                                                                                }
9139                                                                        }
9140
9141                                                                        if (!"*".equals(getColumnName(tableColumn.getColumnObject()))
9142                                                                                        && "*".equals(getColumnName(fieldAttr))) {
9143                                                                                TObjectName columnObject = fieldAttr;
9144                                                                                TTable sourceTable = columnObject.getSourceTable();
9145                                                                                if (columnObject.getTableToken() != null && sourceTable != null) {
9146                                                                                        TObjectName[] columns = modelManager.getTableColumns(sourceTable);
9147                                                                                        for (int j = 0; j < columns.length; j++) {
9148                                                                                                TObjectName columnName = columns[j];
9149                                                                                                if (columnName == null || "*".equals(getColumnName(columnName))) {
9150                                                                                                        continue;
9151                                                                                                }
9152                                                                                                resultColumn.bindStarLinkColumn(columnName);
9153                                                                                        }
9154                                                                                } else {
9155                                                                                        TTableList tables = stmt.getTables();
9156                                                                                        for (int k = 0; k < tables.size(); k++) {
9157                                                                                                TTable tableElement = tables.getTable(k);
9158                                                                                                TObjectName[] columns = modelManager.getTableColumns(tableElement);
9159                                                                                                for (int j = 0; j < columns.length; j++) {
9160                                                                                                        TObjectName columnName = columns[j];
9161                                                                                                        if (columnName == null || "*".equals(getColumnName(columnName))) {
9162                                                                                                                continue;
9163                                                                                                        }
9164                                                                                                        resultColumn.bindStarLinkColumn(columnName);
9165                                                                                                }
9166                                                                                        }
9167                                                                                }
9168                                                                        }
9169
9170                                                                        if ("*".equals(getColumnName(tableColumn.getColumnObject())) && resultColumn != null
9171                                                                                        && !resultColumn.getStarLinkColumns().isEmpty()) {
9172                                                                                tableColumn.bindStarLinkColumns(resultColumn.getStarLinkColumns());
9173                                                                        }
9174
9175                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9176                                                                        relation.setEffectType(effectType);
9177                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9178                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9179
9180                                                                        if (tableColumn.getName().endsWith("*") && resultColumn.getName().endsWith("*")) {
9181                                                                                tableColumn.getTable().setStarStmt("insert");
9182                                                                        }
9183
9184                                                                        Process process = modelFactory.createProcess(stmt);
9185                                                                        relation.setProcess(process);
9186                                                                } else if (((TResultColumn) resultColumn.getColumnObject()).getExpr()
9187                                                                                .getExpressionType() == EExpressionType.simple_constant_t) {
9188                                                                        if (!initColumn) {
9189                                                                                TableColumn tableColumn;
9190                                                                                if (tableModel.isCreateTable() && !containStarColumn(tableModel.getColumns())) {
9191                                                                                        if (tableModel.getColumns().size() <= i) {
9192                                                                                                continue;
9193                                                                                        }
9194                                                                                        tableColumn = tableModel.getColumns().get(i);
9195                                                                                } else {
9196                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
9197                                                                                                        ((TResultColumn) resultColumn.getColumnObject()).getExpr()
9198                                                                                                                        .getConstantOperand(),
9199                                                                                                        i);
9200                                                                                }
9201
9202                                                                                if (DlineageUtil.isTempTable(tableModel, option.getVendor()) && sqlenv != null
9203                                                                                                && tableModel.getDatabase() != null && tableModel.getSchema() != null) {
9204                                                                                        TSQLSchema schema = sqlenv.getSQLSchema(
9205                                                                                                        tableModel.getDatabase() + "." + tableModel.getSchema(), true);
9206                                                                                        if (schema != null) {
9207                                                                                                TSQLTable tempTable = schema.createTable(
9208                                                                                                                DlineageUtil.getSimpleTableName(tableModel.getName()));
9209                                                                                                tempTable.addColumn(tableColumn.getName());
9210                                                                                        }
9211                                                                                }
9212                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9213                                                                                relation.setEffectType(effectType);
9214                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9215                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9216                                                                                Process process = modelFactory.createProcess(stmt);
9217                                                                                relation.setProcess(process);
9218                                                                        } else {
9219                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9220                                                                                relation.setEffectType(effectType);
9221                                                                                relation.setTarget(
9222                                                                                                new TableColumnRelationshipElement(tableModel.getColumns().get(i)));
9223                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9224                                                                                Process process = modelFactory.createProcess(stmt);
9225                                                                                relation.setProcess(process);
9226                                                                        }
9227                                                                } else {
9228                                                                        if (!initColumn) {
9229                                                                                TableColumn tableColumn;
9230                                                                                if (tableModel.isCreateTable() && !containStarColumn(tableModel.getColumns())) {
9231                                                                                        if (tableModel.getColumns().size() <= i) {
9232                                                                                                continue;
9233                                                                                        }
9234                                                                                        tableColumn = tableModel.getColumns().get(i);
9235                                                                                } else {
9236                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
9237                                                                                                        ((TResultColumn) resultColumn.getColumnObject()).getExpr(), i);
9238                                                                                }
9239                                                                                if (DlineageUtil.isTempTable(tableModel, option.getVendor()) && sqlenv != null
9240                                                                                                && tableModel.getDatabase() != null && tableModel.getSchema() != null) {
9241                                                                                        TSQLSchema schema = sqlenv.getSQLSchema(
9242                                                                                                        tableModel.getDatabase() + "." + tableModel.getSchema(), true);
9243                                                                                        if (schema != null) {
9244                                                                                                TSQLTable tempTable = schema.createTable(
9245                                                                                                                DlineageUtil.getSimpleTableName(tableModel.getName()));
9246                                                                                                tempTable.addColumn(tableColumn.getName());
9247                                                                                        }
9248                                                                                }
9249                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9250                                                                                relation.setEffectType(effectType);
9251                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9252                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9253                                                                                Process process = modelFactory.createProcess(stmt);
9254                                                                                relation.setProcess(process);
9255                                                                        } else {
9256                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9257                                                                                relation.setEffectType(effectType);
9258                                                                                relation.setTarget(
9259                                                                                                new TableColumnRelationshipElement(tableModel.getColumns().get(i)));
9260                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9261                                                                                Process process = modelFactory.createProcess(stmt);
9262                                                                                relation.setProcess(process);
9263                                                                        }
9264                                                                }
9265                                                        }
9266                                                }
9267                                        } else if (stmt.getSubQuery() != null) {
9268                                                ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt.getSubQuery());
9269                                                if (resultSetModel != null) {
9270
9271                                                        if (!resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
9272                                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
9273                                                                impactRelation.setEffectType(effectType);
9274                                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
9275                                                                                resultSetModel.getRelationRows()));
9276                                                                impactRelation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(
9277                                                                                tableModel.getRelationRows()));
9278                                                        }
9279
9280                                                        if(stmt.getColumnList()!=null && stmt.getColumnList().size()>0) {
9281                                                                for(int i=0;i<stmt.getColumnList().size();i++) {
9282                                                                        TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
9283                                                                                        stmt.getColumnList().getObjectName(i));
9284                                                                }
9285                                                                
9286                                                                List<TableColumn> columns = tableModel.getColumns();
9287                                                                int resultSetSize = resultSetModel.getColumns().size();
9288                                                                int starIndex = 0; 
9289                                                                int j = 0;
9290                                                                boolean fromStruct = false;
9291                                                                for (int i = 0; i < columns.size() && j < resultSetSize; i++) {
9292                                                                        String column = columns.get(i).getName();
9293                                                                        ResultColumn resultColumn = resultSetModel.getColumns().get(j);
9294                                                                        if (!resultColumn.getName().contains("*")) {
9295                                                                                if (resultColumn.getName().equals(resultColumn.getRefColumnName())
9296                                                                                                && resultColumn.getColumnObject().toString().endsWith("*")
9297                                                                                                && resultSetSize == 1) {
9298                                                                                        starIndex++;
9299                                                                                        if (resultSetSize - j == columns.size() - i) {
9300                                                                                                j++;
9301
9302                                                                                        }
9303                                                                                }
9304                                                                                else {
9305                                                                                        j++;
9306                                                                                }
9307                                                                        } else {
9308                                                                                starIndex++;
9309                                                                                if (resultSetSize - j == columns.size() - i) {
9310                                                                                        j++;
9311
9312                                                                                }
9313                                                                        }
9314                                                                        if (column != null) {
9315                                                                                TableColumn tableColumn;
9316                                                                                // if (!initColumn) {
9317                                                                                tableColumn = matchColumn(tableModel.getColumns(), columns.get(i));
9318                                                                                if (tableColumn == null) {
9319                                                                                        if (tableModel.isCreateTable()
9320                                                                                                        && !containStarColumn(tableModel.getColumns())) {
9321                                                                                                if (tableModel.getColumns().size() <= i) {
9322                                                                                                        continue;
9323                                                                                                }
9324                                                                                                tableColumn = tableModel.getColumns().get(i);
9325                                                                                        } else {
9326                                                                                                TObjectName columnName = new TObjectName();
9327                                                                                                columnName.setString(column);
9328                                                                                                tableColumn = modelFactory.createTableColumn(tableModel, columnName,
9329                                                                                                                false);
9330                                                                                        }
9331                                                                                }
9332                                                                                else if (!resultColumn.isStruct() && tableColumn.isStruct() && columns.size() != resultSetSize) {
9333                                                                                        j--;
9334                                                                                        fromStruct = true;
9335                                                                                }
9336                                                                                if(fromStruct && !tableColumn.isStruct()) {
9337                                                                                        fromStruct = false;
9338                                                                                        resultColumn = resultSetModel.getColumns().get(j);
9339                                                                                        j++;
9340                                                                                }
9341                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9342                                                                                relation.setEffectType(effectType);
9343                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9344                                                                                if (resultColumn.hasStarLinkColumn()
9345                                                                                                && resultColumn.getStarLinkColumnNames().size() > starIndex - 1) {
9346                                                                                        ResultColumn expandStarColumn = modelFactory.createResultColumn(
9347                                                                                                        resultSetModel, resultColumn.getStarLinkColumnName(starIndex - 1),
9348                                                                                                        false);
9349                                                                                        relation.addSource(new ResultColumnRelationshipElement(expandStarColumn));
9350                                                                                } else {
9351                                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn, starIndex - 1));
9352                                                                                }
9353                                                                                Process process = modelFactory.createProcess(stmt);
9354                                                                                relation.setProcess(process);
9355                                                                        }
9356                                                                }
9357                                                        }
9358                                                        else {
9359                                                                for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
9360                                                                        ResultColumn resultColumn = resultSetModel.getColumns().get(i);
9361                                                                        TAliasClause alias = null;
9362                                                                        if(resultColumn.getColumnObject() instanceof TResultColumn) {
9363                                                                                alias = ((TResultColumn) resultColumn.getColumnObject()).getAliasClause();
9364                                                                        }
9365                                                                        if (stmt.getColumnList() != null) {
9366                                                                                if (i < stmt.getColumnList().size()) {
9367                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9368                                                                                        relation.setEffectType(effectType);
9369                                                                                        TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
9370                                                                                                        stmt.getColumnList().getObjectName(i));
9371                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9372                                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9373                                                                                        Process process = modelFactory.createProcess(stmt);
9374                                                                                        relation.setProcess(process);
9375                                                                                }
9376                                                                        } else {
9377                                                                                if (alias != null && alias.getAliasName() != null) {
9378                                                                                        TableColumn tableColumn;
9379                                                                                        if (!initColumn) {
9380                                                                                                if (tableModel.isCreateTable()
9381                                                                                                                && !containStarColumn(tableModel.getColumns())) {
9382                                                                                                        if (tableModel.getColumns().size() <= i) {
9383                                                                                                                continue;
9384                                                                                                        }
9385                                                                                                        tableColumn = tableModel.getColumns().get(i);
9386                                                                                                } else {
9387                                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
9388                                                                                                                        alias.getAliasName());
9389                                                                                                }
9390                                                                                        } else {
9391                                                                                                tableColumn = matchColumn(tableColumns, alias.getAliasName());
9392                                                                                                if (tableColumn == null) {
9393                                                                                                        continue;
9394                                                                                                }
9395                                                                                        }
9396        
9397                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9398                                                                                        relation.setEffectType(effectType);
9399                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9400                                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9401                                                                                        Process process = modelFactory.createProcess(stmt);
9402                                                                                        relation.setProcess(process);
9403                                                                                } else if (resultColumn.getColumnObject() instanceof TObjectName 
9404                                                                                                || ( resultColumn.getColumnObject() instanceof TResultColumn && ((TResultColumn) resultColumn.getColumnObject())
9405                                                                                                .getFieldAttr() != null)) {
9406                                                                                        TObjectName fieldAttr = null;
9407                                                                                        if (resultColumn.getColumnObject() instanceof TObjectName) {
9408                                                                                                fieldAttr = (TObjectName)resultColumn.getColumnObject();
9409                                                                                        }
9410                                                                                        else if (resultColumn.getColumnObject() instanceof TResultColumn) {
9411                                                                                                fieldAttr = ((TResultColumn) resultColumn.getColumnObject())
9412                                                                                                                .getFieldAttr();
9413                                                                                        }
9414                                
9415                                                                                        TableColumn tableColumn;
9416                                                                                        if (!initColumn) {
9417                                                                                                if (tableModel.isCreateTable()
9418                                                                                                                && !containStarColumn(tableModel.getColumns())) {
9419                                                                                                        if (tableModel.getColumns().size() <= i) {
9420                                                                                                                continue;
9421                                                                                                        }
9422                                                                                                        tableColumn = tableModel.getColumns().get(i);
9423                                                                                                } else {
9424                                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
9425                                                                                                                        fieldAttr);
9426                                                                                                }
9427                                                                                        } else {
9428                                                                                                tableColumn = matchColumn(tableColumns, fieldAttr);
9429                                                                                                if (tableColumn == null) {
9430                                                                                                        continue;
9431                                                                                                }
9432                                                                                        }
9433        
9434                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9435                                                                                        relation.setEffectType(effectType);
9436                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9437                                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9438                                                                                        Process process = modelFactory.createProcess(stmt);
9439                                                                                        relation.setProcess(process);
9440                                                                                } else if (((TResultColumn) resultColumn.getColumnObject()).getExpr()
9441                                                                                                .getExpressionType() == EExpressionType.simple_constant_t) {
9442                                                                                        if (!initColumn) {
9443                                                                                                TableColumn tableColumn;
9444                                                                                                if (tableModel.isCreateTable()
9445                                                                                                                && !containStarColumn(tableModel.getColumns())) {
9446                                                                                                        if (tableModel.getColumns().size() <= i) {
9447                                                                                                                continue;
9448                                                                                                        }
9449                                                                                                        tableColumn = tableModel.getColumns().get(i);
9450                                                                                                } else {
9451                                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
9452                                                                                                                        ((TResultColumn) resultColumn.getColumnObject()).getExpr()
9453                                                                                                                                        .getConstantOperand(),
9454                                                                                                                        i);
9455                                                                                                }
9456                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9457                                                                                                relation.setEffectType(effectType);
9458                                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9459                                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9460                                                                                                Process process = modelFactory.createProcess(stmt);
9461                                                                                                relation.setProcess(process);
9462                                                                                        }
9463                                                                                } else {
9464                                                                                        if (!initColumn) {
9465                                                                                                TableColumn tableColumn;
9466                                                                                                if (tableModel.isCreateTable()
9467                                                                                                                && !containStarColumn(tableModel.getColumns())) {
9468                                                                                                        if (tableModel.getColumns().size() <= i) {
9469                                                                                                                continue;
9470                                                                                                        }
9471                                                                                                        tableColumn = tableModel.getColumns().get(i);
9472                                                                                                } else {
9473                                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
9474                                                                                                                        ((TResultColumn) resultColumn.getColumnObject()).getExpr(), i);
9475                                                                                                }
9476                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9477                                                                                                relation.setEffectType(effectType);
9478                                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9479                                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9480                                                                                                Process process = modelFactory.createProcess(stmt);
9481                                                                                                relation.setProcess(process);
9482                                                                                        }
9483                                                                                }
9484                                                                        }
9485                                                                }
9486                                                        }
9487                                                }
9488                                        }
9489                                }
9490                        } else if (stmt.getColumnList() != null && stmt.getColumnList().size() > 0) {
9491                                TObjectNameList items = stmt.getColumnList();
9492                                TMultiTargetList values = stmt.getValues();
9493                                if (values != null) {
9494                                        for (int k = 0; values != null && k < values.size(); k++) {
9495                                                int j = 0;
9496                                                for (int i = 0; i < items.size(); i++) {
9497                                                        TObjectName column = items.getObjectName(i);
9498                                                        TableColumn tableColumn;
9499                                                        if (!initColumn) {
9500                                                                if (tableModel.isCreateTable() && !containStarColumn(tableModel.getColumns())) {
9501                                                                        if (tableModel.getColumns().size() <= i) {
9502                                                                                continue;
9503                                                                        }
9504                                                                        tableColumn = tableModel.getColumns().get(i);
9505                                                                } else {
9506                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel, column);
9507                                                                }
9508                                                        } else {
9509                                                                tableColumn = matchColumn(tableColumns, column);
9510                                                                if (tableColumn == null) {
9511                                                                        continue;
9512                                                                }
9513                                                        }
9514                                                        TResultColumn columnObject = values.getMultiTarget(k).getColumnList().getResultColumn(j);
9515                                                        if (columnObject == null) {
9516                                                                continue;
9517                                                        }
9518                                                        TExpression valueExpr = columnObject.getExpr();
9519                                                        columnsInExpr visitor = new columnsInExpr();
9520                                                        valueExpr.inOrderTraverse(visitor);
9521                                                        List<TObjectName> objectNames = visitor.getObjectNames();
9522                                                        List<TParseTreeNode> constants = visitor.getConstants();
9523                                                        List<TParseTreeNode> functions = visitor.getFunctions();
9524                                                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
9525
9526                                                        Process process = modelFactory.createProcess(stmt);
9527
9528
9529
9530                                                        if (functions != null && !functions.isEmpty()) {
9531                                                                analyzeFunctionDataFlowRelation(tableColumn, functions, effectType, process);
9532                                                        }
9533
9534                                                        if (subquerys != null && !subquerys.isEmpty()) {
9535                                                                analyzeSubqueryDataFlowRelation(tableColumn, subquerys, effectType, process);
9536                                                        }
9537                                                        if (objectNames != null && !objectNames.isEmpty()) {
9538                                                                analyzeDataFlowRelation(tableColumn, objectNames, null, effectType, functions,
9539                                                                                process, i);
9540                                                        }
9541                                                        //insert into values generate too many constant relations, ignore constant relations.
9542                                                        if (constants != null && !constants.isEmpty()) {
9543                                                                if (!option.isIgnoreInsertIntoValues() || stmt.getParentStmt() != null) {
9544                                                                        analyzeConstantDataFlowRelation(tableColumn, constants, effectType,
9545                                                                                        functions, process);
9546                                                                }
9547                                                        }
9548                                                        j++;
9549                                                }
9550                                        }
9551                                } else if (stmt.getExecuteStmt() != null && stmt.getExecuteStmt().getModuleName() != null) {
9552                                        analyzeCustomSqlStmt(stmt.getExecuteStmt());
9553                                        Procedure procedure = modelManager.getProcedureByName(DlineageUtil
9554                                                        .getIdentifierNormalTableName(stmt.getExecuteStmt().getModuleName().toString()));
9555                                        if (procedure!=null && procedure.getProcedureObject() instanceof TStoredProcedureSqlStatement) {
9556                                                TStoredProcedureSqlStatement procedureStmt = (TStoredProcedureSqlStatement) procedure
9557                                                                .getProcedureObject();
9558                                                List<TSelectSqlStatement> stmtItems = getLastSelectStmt(procedureStmt);
9559                                                if (stmtItems != null) {
9560                                                        for(TSelectSqlStatement stmtItem: stmtItems) {
9561                                                                ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmtItem);
9562                                                                if (resultSetModel != null) {
9563                                                                        for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
9564                                                                                ResultColumn resultColumn = resultSetModel.getColumns().get(i);
9565                                                                                Transform transform = new Transform();
9566                                                                                transform.setType(Transform.FUNCTION);
9567                                                                                transform.setCode(stmt.getExecuteStmt().getModuleName());
9568                                                                                resultColumn.setTransform(transform);
9569                                                                                
9570                                                                                if (stmt.getColumnList() != null) {
9571                                                                                        if (i < stmt.getColumnList().size()) {
9572                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9573                                                                                                relation.setEffectType(effectType);
9574                                                                                                TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
9575                                                                                                                stmt.getColumnList().getObjectName(i));                                                                                         
9576                                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9577                                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9578                                                                                                Process process = modelFactory.createProcess(stmt);
9579                                                                                                relation.setProcess(process);                                   
9580                                                                                        }
9581                                                                                } else {
9582                                                                                        if (resultColumn.getColumnObject() instanceof TObjectName) {
9583                                                                                                TObjectName fieldAttr = ((TObjectName) resultColumn.getColumnObject());
9584                                                                                                TableColumn tableColumn;
9585                                                                                                if (!initColumn) {
9586                                                                                                        if (tableModel.isCreateTable()
9587                                                                                                                        && !containStarColumn(tableModel.getColumns())) {
9588                                                                                                                if (tableModel.getColumns().size() <= i) {
9589                                                                                                                        continue;
9590                                                                                                                }
9591                                                                                                                tableColumn = tableModel.getColumns().get(i);
9592                                                                                                        } else {
9593                                                                                                                tableColumn = modelFactory.createInsertTableColumn(tableModel,
9594                                                                                                                                fieldAttr);
9595                                                                                                        }
9596                                                                                                } else {
9597                                                                                                        tableColumn = matchColumn(tableColumns, fieldAttr);
9598                                                                                                        if (tableColumn == null) {
9599                                                                                                                continue;
9600                                                                                                        }
9601                                                                                                }
9602        
9603                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9604                                                                                                relation.setEffectType(effectType);
9605                                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9606                                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9607                                                                                                Process process = modelFactory.createProcess(stmt);
9608                                                                                                relation.setProcess(process);
9609
9610                                                                                                
9611                                                                                        } else {
9612                                                                                                TAliasClause alias = ((TResultColumn) resultColumn.getColumnObject())
9613                                                                                                                .getAliasClause();
9614                                                                                                if (alias != null && alias.getAliasName() != null) {
9615                                                                                                        TableColumn tableColumn;
9616                                                                                                        if (!initColumn) {
9617                                                                                                                if (tableModel.isCreateTable()
9618                                                                                                                                && !containStarColumn(tableModel.getColumns())) {
9619                                                                                                                        if (tableModel.getColumns().size() <= i) {
9620                                                                                                                                continue;
9621                                                                                                                        }
9622                                                                                                                        tableColumn = tableModel.getColumns().get(i);
9623                                                                                                                } else {
9624                                                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
9625                                                                                                                                        alias.getAliasName());
9626                                                                                                                }
9627                                                                                                        } else {
9628                                                                                                                tableColumn = matchColumn(tableColumns, alias.getAliasName());
9629                                                                                                                if (tableColumn == null) {
9630                                                                                                                        continue;
9631                                                                                                                }
9632                                                                                                        }
9633                                                                                                        
9634                                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9635                                                                                                        relation.setEffectType(effectType);
9636                                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9637                                                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9638                                                                                                        Process process = modelFactory.createProcess(stmt);
9639                                                                                                        relation.setProcess(process);
9640                                                                                                } else if (((TResultColumn) resultColumn.getColumnObject())
9641                                                                                                                .getFieldAttr() != null) {
9642                                                                                                        TObjectName fieldAttr = ((TResultColumn) resultColumn.getColumnObject())
9643                                                                                                                        .getFieldAttr();
9644                                                                                                        TableColumn tableColumn;
9645                                                                                                        if (!initColumn) {
9646                                                                                                                if (tableModel.isCreateTable()
9647                                                                                                                                && !containStarColumn(tableModel.getColumns())) {
9648                                                                                                                        if (tableModel.getColumns().size() <= i) {
9649                                                                                                                                continue;
9650                                                                                                                        }
9651                                                                                                                        tableColumn = tableModel.getColumns().get(i);
9652                                                                                                                } else {
9653                                                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
9654                                                                                                                                        fieldAttr);
9655                                                                                                                }
9656                                                                                                        } else {
9657                                                                                                                tableColumn = matchColumn(tableColumns, fieldAttr);
9658                                                                                                                if (tableColumn == null) {
9659                                                                                                                        continue;
9660                                                                                                                }
9661                                                                                                        }
9662                                                                                                        
9663                                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9664                                                                                                        relation.setEffectType(effectType);
9665                                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9666                                                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9667                                                                                                        Process process = modelFactory.createProcess(stmt);
9668                                                                                                        relation.setProcess(process);                                                                                                   
9669                                                                                                } else if (((TResultColumn) resultColumn.getColumnObject()).getExpr()
9670                                                                                                                .getExpressionType() == EExpressionType.simple_constant_t) {
9671                                                                                                        if (!initColumn) {
9672                                                                                                                TableColumn tableColumn;
9673                                                                                                                if (tableModel.isCreateTable()
9674                                                                                                                                && !containStarColumn(tableModel.getColumns())) {
9675                                                                                                                        if (tableModel.getColumns().size() <= i) {
9676                                                                                                                                continue;
9677                                                                                                                        }
9678                                                                                                                        tableColumn = tableModel.getColumns().get(i);
9679                                                                                                                } else {
9680                                                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
9681                                                                                                                                        ((TResultColumn) resultColumn.getColumnObject()).getExpr()
9682                                                                                                                                                        .getConstantOperand(),
9683                                                                                                                                        i);
9684                                                                                                                }
9685                                                                                                                
9686                                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9687                                                                                                                relation.setEffectType(effectType);
9688                                                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9689                                                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9690                                                                                                                Process process = modelFactory.createProcess(stmt);
9691                                                                                                                relation.setProcess(process);
9692                                                                                                        }
9693                                                                                                } else {
9694                                                                                                        if (!initColumn) {
9695                                                                                                                TableColumn tableColumn;
9696                                                                                                                if (tableModel.isCreateTable()
9697                                                                                                                                && !containStarColumn(tableModel.getColumns())) {
9698                                                                                                                        if (tableModel.getColumns().size() <= i) {
9699                                                                                                                                continue;
9700                                                                                                                        }
9701                                                                                                                        tableColumn = tableModel.getColumns().get(i);
9702                                                                                                                } else {
9703                                                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
9704                                                                                                                                        ((TResultColumn) resultColumn.getColumnObject()).getExpr(),
9705                                                                                                                                        i);
9706                                                                                                                }
9707                                                                                                                
9708                                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9709                                                                                                                relation.setEffectType(effectType);
9710                                                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9711                                                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9712                                                                                                                Process process = modelFactory.createProcess(stmt);
9713                                                                                                                relation.setProcess(process);
9714                                                                                                        }
9715                                                                                                }
9716                                                                                        }
9717                                                                                }
9718                                                                        }
9719                                                                }
9720                                                        }
9721                                                }
9722                                        }
9723                                        else if (procedure!=null && procedure.getProcedureObject() instanceof TObjectName) {
9724                                                TObjectName functionName = new TObjectName();
9725                                                functionName.setString(procedure.getName());
9726                                                Function function = (Function)createFunction(functionName);
9727                                                if (stmt.getColumnList() != null) {
9728                                                        for (int i = 0; i < stmt.getColumnList().size(); i++) {
9729                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9730                                                                relation.setEffectType(effectType);
9731                                                                TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
9732                                                                                stmt.getColumnList().getObjectName(i));
9733                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9734                                                                relation.addSource(new ResultColumnRelationshipElement(function.getColumns().get(0)));
9735                                                                Process process = modelFactory.createProcess(stmt);
9736                                                                relation.setProcess(process);
9737                                                        }
9738                                                }
9739                                        }
9740                                }
9741                        } else if (stmt.getValues() != null && stmt.getValues().size() > 0 && tableModel.isCreateTable() && !tableModel.getColumns().isEmpty()) {
9742                                for (int k = 0; stmt.getValues() != null && k < stmt.getValues().size(); k++) {
9743                                        TResultColumnList columns = stmt.getValues().getMultiTarget(k).getColumnList();
9744                                        boolean allConstant = true;
9745                                        Process process = modelFactory.createProcess(stmt);
9746                                        for (int x = 0; x < columns.size(); x++) {
9747                                                TableColumn tableColumn = tableModel.getColumns().get(x);
9748                                                TResultColumn columnObject = columns.getResultColumn(x);
9749                                                if (columnObject == null) {
9750                                                        continue;
9751                                                }
9752                                                TExpression valueExpr = columnObject.getExpr();
9753                                                columnsInExpr visitor = new columnsInExpr();
9754                                                valueExpr.inOrderTraverse(visitor);
9755                                                List<TObjectName> objectNames = visitor.getObjectNames();
9756                                                List<TParseTreeNode> constants = visitor.getConstants();
9757                                                List<TParseTreeNode> functions = visitor.getFunctions();
9758                                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
9759
9760                                                if (functions != null && !functions.isEmpty()) {
9761                                                        analyzeFunctionDataFlowRelation(tableColumn, functions, effectType, process);
9762                                                        allConstant = false;
9763                                                }
9764
9765                                                if (subquerys != null && !subquerys.isEmpty()) {
9766                                                        analyzeSubqueryDataFlowRelation(tableColumn, subquerys, effectType, process);
9767                                                        allConstant = false;
9768                                                }
9769                                                if (objectNames != null && !objectNames.isEmpty()) {
9770                                                        analyzeDataFlowRelation(tableColumn, objectNames, null, effectType, functions,
9771                                                                        process);
9772                                                        allConstant = false;
9773                                                }
9774                                                //insert into values generate too many constant relations, ignore constant relations.
9775                                                if (constants != null && !constants.isEmpty() && stmt.getParentStmt() != null) {
9776                                                        analyzeConstantDataFlowRelation(tableColumn, constants, effectType, functions,
9777                                                                        process);
9778                                                        allConstant = false;
9779                                                }
9780                                        }
9781                                        
9782                                        if(allConstant) {
9783                                                modelManager.unbindProcessModel(stmt);
9784                                                tableModel.removeProcess(process);
9785                                        }
9786                                }
9787                        } else if (stmt.getRecordName() != null) {
9788                                String procedureName = DlineageUtil.getProcedureParentName(stmt);
9789                                String variableString = stmt.getRecordName().toString();
9790                                if (variableString.startsWith(":")) {
9791                                        variableString = variableString.substring(variableString.indexOf(":") + 1);
9792                                }
9793                                if (!SQLUtil.isEmpty(procedureName)) {
9794                                        variableString = procedureName + "." + SQLUtil.getIdentifierNormalTableName(variableString);
9795                                }
9796                                
9797                                Table recordTable = modelManager
9798                                                .getTableByName(DlineageUtil.getTableFullName(variableString));
9799                                if (recordTable != null) {
9800                                        for (int i = 0; i < recordTable.getColumns().size(); i++) {
9801                                                TableColumn sourceTableColumn = recordTable.getColumns().get(i);
9802                                                TableColumn targetTableColumn = modelFactory.createTableColumn(tableModel,
9803                                                                sourceTableColumn.getColumnObject(), false);
9804                                                if (targetTableColumn != null) {
9805                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9806                                                        relation.setEffectType(effectType);
9807                                                        relation.setTarget(new TableColumnRelationshipElement(targetTableColumn));
9808                                                        relation.addSource(new TableColumnRelationshipElement(sourceTableColumn));
9809                                                        Process process = modelFactory.createProcess(stmt);
9810                                                        relation.setProcess(process);
9811                                                } else if (sourceTableColumn.getName().endsWith("*") && tableModel.isCreateTable()) {
9812                                                        for (TableColumn column : tableModel.getColumns()) {
9813                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9814                                                                relation.setEffectType(effectType);
9815                                                                relation.setTarget(new TableColumnRelationshipElement(column));
9816                                                                relation.addSource(new TableColumnRelationshipElement(sourceTableColumn));
9817                                                                Process process = modelFactory.createProcess(stmt);
9818                                                                relation.setProcess(process);
9819                                                        }
9820                                                }
9821                                        }
9822                                }
9823                        } else if (stmt.getInsertSource() == EInsertSource.values_function && stmt.getFunctionCall() != null) {
9824                                Table cursor = modelManager.getTableByName(
9825                                                DlineageUtil.getTableFullName(stmt.getFunctionCall().getFunctionName().toString()));
9826                                if (cursor != null) {
9827                                        TObjectName starColumn = new TObjectName();
9828                                        starColumn.setString("*");
9829                                        TableColumn insertColumn = modelFactory.createTableColumn(tableModel, starColumn, true);
9830                                        insertColumn.setShowStar(false);
9831                                        insertColumn.setExpandStar(true);
9832                                        for (int j = 0; j < cursor.getColumns().size(); j++) {
9833                                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
9834                                                dataflowRelation.setEffectType(effectType);
9835                                                dataflowRelation.addSource(new TableColumnRelationshipElement(cursor.getColumns().get(j)));
9836                                                dataflowRelation.setTarget(new TableColumnRelationshipElement(insertColumn));
9837                                                Process process = modelFactory.createProcess(stmt);
9838                                                dataflowRelation.setProcess(process);
9839                                        }
9840                                }
9841
9842                        } else if (stmt.getInsertSource() == EInsertSource.values && stmt.getValues() != null) {
9843                                TObjectName starColumn = new TObjectName();
9844                                starColumn.setString("*");
9845                                TableColumn insertColumn = modelFactory.createTableColumn(tableModel, starColumn, true);
9846                                insertColumn.setShowStar(false);
9847                                insertColumn.setExpandStar(true);
9848                                for (int k = 0; stmt.getValues() != null && k < stmt.getValues().size(); k++) {
9849                                        TResultColumnList columns = stmt.getValues().getMultiTarget(k).getColumnList();
9850                                        for (int x = 0; x < columns.size(); x++) {
9851                                                TResultColumn columnObject = columns.getResultColumn(x);
9852                                                if (columnObject == null) {
9853                                                        continue;
9854                                                }
9855                                                TExpression valueExpr = columnObject.getExpr();
9856                                                columnsInExpr visitor = new columnsInExpr();
9857                                                valueExpr.inOrderTraverse(visitor);
9858                                                List<TObjectName> objectNames = visitor.getObjectNames();
9859                                                List<TParseTreeNode> constants = visitor.getConstants();
9860                                                List<TParseTreeNode> functions = visitor.getFunctions();
9861                                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
9862
9863                                                Process process = modelFactory.createProcess(stmt);
9864                                                if (functions != null && !functions.isEmpty()) {
9865                                                        analyzeFunctionDataFlowRelation(insertColumn, functions, effectType, process);
9866                                                }
9867
9868                                                if (subquerys != null && !subquerys.isEmpty()) {
9869                                                        analyzeSubqueryDataFlowRelation(insertColumn, subquerys, effectType, process);
9870                                                }
9871                                                if (objectNames != null && !objectNames.isEmpty()) {
9872                                                        analyzeDataFlowRelation(insertColumn, objectNames, null, effectType, functions,
9873                                                                        process);
9874                                                }
9875                                                //insert into values generate too many constant relations, ignore constant relations.
9876                                                if (constants != null && !constants.isEmpty() && stmt.getParentStmt() != null) {
9877                                                        analyzeConstantDataFlowRelation(insertColumn, constants, effectType, functions,
9878                                                                        process);
9879                                                }
9880                                        }
9881                                }
9882                        }else if (stmt.getExecuteStmt() != null && stmt.getExecuteStmt().getModuleName() != null) {
9883                                analyzeCustomSqlStmt(stmt.getExecuteStmt());
9884                                Procedure procedure = modelManager.getProcedureByName(DlineageUtil
9885                                                .getIdentifierNormalTableName(stmt.getExecuteStmt().getModuleName().toString()));
9886                                if (procedure!=null && procedure.getProcedureObject() instanceof TStoredProcedureSqlStatement) {
9887                                        TStoredProcedureSqlStatement procedureStmt = (TStoredProcedureSqlStatement) procedure
9888                                                        .getProcedureObject();
9889                                        List<TSelectSqlStatement> stmtItems = getLastSelectStmt(procedureStmt);
9890                                        if (stmtItems != null) {
9891                                                for(TSelectSqlStatement stmtItem: stmtItems) {
9892                                                        ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmtItem);
9893                                                        if (resultSetModel != null) {
9894                                                                for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
9895                                                                        ResultColumn resultColumn = resultSetModel.getColumns().get(i);
9896                                                                        
9897                                                                        Transform transform = new Transform();
9898                                                                        transform.setType(Transform.FUNCTION);
9899                                                                        transform.setCode(stmt.getExecuteStmt().getModuleName());
9900                                                                        resultColumn.setTransform(transform);
9901                                                                        
9902                                                                        TAliasClause alias = null;
9903
9904                                                                        if (resultColumn.getColumnObject() instanceof TResultColumn) {
9905                                                                                alias = ((TResultColumn) resultColumn.getColumnObject())
9906                                                                                                .getAliasClause();
9907                                                                        }
9908                                                                        
9909                                                                        if (alias != null && alias.getAliasName() != null) {
9910                                                                                TableColumn tableColumn;
9911                                                                                if (!initColumn) {
9912                                                                                        if (tableModel.isCreateTable()
9913                                                                                                        && !containStarColumn(tableModel.getColumns())) {
9914                                                                                                if(resultColumn.getName().endsWith("*")) {
9915                                                                                                        for(TableColumn tableColumnItem: tableModel.getColumns()) {
9916                                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9917                                                                                                                relation.setEffectType(effectType);
9918                                                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumnItem));
9919                                                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9920                                                                                                                Process process = modelFactory.createProcess(stmt);
9921                                                                                                                relation.setProcess(process);
9922                                                                                                        }
9923                                                                                                        continue;
9924                                                                                                }
9925                                                                                                else {
9926                                                                                                        if (tableModel.getColumns().size() <= i) {
9927                                                                                                                continue;
9928                                                                                                        }
9929                                                                                                        tableColumn = tableModel.getColumns().get(i);
9930                                                                                                }
9931                                                                                        } else {
9932                                                                                                tableColumn = modelFactory.createInsertTableColumn(tableModel,
9933                                                                                                                alias.getAliasName());
9934                                                                                        }
9935                                                                                } else {
9936                                                                                        tableColumn = matchColumn(tableColumns, alias.getAliasName());
9937                                                                                        if (tableColumn == null) {
9938                                                                                                continue;
9939                                                                                        }
9940                                                                                }
9941                                        
9942                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9943                                                                                relation.setEffectType(effectType);
9944                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9945                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9946                                                                                Process process = modelFactory.createProcess(stmt);
9947                                                                                relation.setProcess(process);
9948                                                                        } else if (resultColumn.getColumnObject() instanceof TObjectName
9949                                                                                        || (resultColumn.getColumnObject() instanceof TResultColumn
9950                                                                                                        && ((TResultColumn) resultColumn.getColumnObject())
9951                                                                                                                        .getFieldAttr() != null)) {
9952                                                                                TObjectName fieldAttr = null;
9953                                                                                if (resultColumn.getColumnObject() instanceof TObjectName) {
9954                                                                                        fieldAttr = (TObjectName) resultColumn.getColumnObject();
9955                                                                                } else {
9956                                                                                        fieldAttr = ((TResultColumn) resultColumn.getColumnObject()).getFieldAttr();
9957                                                                                }
9958                                                                                TableColumn tableColumn;
9959                                                                                if (!initColumn) {
9960                                                                                        if (tableModel.isCreateTable()
9961                                                                                                        && !containStarColumn(tableModel.getColumns())) {
9962                                                                                                if(resultColumn.getName().endsWith("*")) {
9963                                                                                                        for(TableColumn tableColumnItem: tableModel.getColumns()) {
9964                                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9965                                                                                                                relation.setEffectType(effectType);
9966                                                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumnItem));
9967                                                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9968                                                                                                                Process process = modelFactory.createProcess(stmt);
9969                                                                                                                relation.setProcess(process);
9970                                                                                                        }
9971                                                                                                        continue;
9972                                                                                                }
9973                                                                                                else {
9974                                                                                                        if (tableModel.getColumns().size() <= i) {
9975                                                                                                                continue;
9976                                                                                                        }
9977                                                                                                        tableColumn = tableModel.getColumns().get(i);
9978                                                                                                }
9979                                                                                        } else {
9980                                                                                                tableColumn = modelFactory.createInsertTableColumn(tableModel,
9981                                                                                                                fieldAttr);
9982                                                                                        }
9983                                                                                } else {
9984                                                                                        tableColumn = matchColumn(tableColumns, fieldAttr);
9985                                                                                        if (tableColumn == null) {
9986                                                                                                continue;
9987                                                                                        }
9988                                                                                }
9989                                                                                
9990                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9991                                                                                relation.setEffectType(effectType);
9992                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9993                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9994                                                                                Process process = modelFactory.createProcess(stmt);
9995                                                                                relation.setProcess(process);
9996                                                                        } else if (((TResultColumn) resultColumn.getColumnObject()).getExpr()
9997                                                                                        .getExpressionType() == EExpressionType.simple_constant_t) {
9998                                                                                if (!initColumn) {
9999                                                                                        TableColumn tableColumn;
10000                                                                                        if (tableModel.isCreateTable()
10001                                                                                                        && !containStarColumn(tableModel.getColumns())) {
10002                                                                                                if(resultColumn.getName().endsWith("*")) {
10003                                                                                                        for(TableColumn tableColumnItem: tableModel.getColumns()) {
10004                                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10005                                                                                                                relation.setEffectType(effectType);
10006                                                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumnItem));
10007                                                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
10008                                                                                                                Process process = modelFactory.createProcess(stmt);
10009                                                                                                                relation.setProcess(process);
10010                                                                                                        }
10011                                                                                                        continue;
10012                                                                                                }
10013                                                                                                else {
10014                                                                                                        if (tableModel.getColumns().size() <= i) {
10015                                                                                                                continue;
10016                                                                                                        }
10017                                                                                                        tableColumn = tableModel.getColumns().get(i);
10018                                                                                                }
10019                                                                                        } else {
10020                                                                                                tableColumn = modelFactory.createInsertTableColumn(tableModel,
10021                                                                                                                ((TResultColumn) resultColumn.getColumnObject()).getExpr()
10022                                                                                                                                .getConstantOperand(),
10023                                                                                                                i);
10024                                                                                        }
10025                                                                                        
10026                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10027                                                                                        relation.setEffectType(effectType);
10028                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
10029                                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
10030                                                                                        Process process = modelFactory.createProcess(stmt);
10031                                                                                        relation.setProcess(process);
10032                                                                                }
10033                                                                        } else {
10034                                                                                if (!initColumn) {
10035                                                                                        TableColumn tableColumn;
10036                                                                                        if (tableModel.isCreateTable()
10037                                                                                                        && !containStarColumn(tableModel.getColumns())) {
10038                                                                                                if(resultColumn.getName().endsWith("*")) {
10039                                                                                                        for(TableColumn tableColumnItem: tableModel.getColumns()) {
10040                                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10041                                                                                                                relation.setEffectType(effectType);
10042                                                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumnItem));
10043                                                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
10044                                                                                                                Process process = modelFactory.createProcess(stmt);
10045                                                                                                                relation.setProcess(process);
10046                                                                                                        }
10047                                                                                                        continue;
10048                                                                                                }
10049                                                                                                else {
10050                                                                                                        if (tableModel.getColumns().size() <= i) {
10051                                                                                                                continue;
10052                                                                                                        }
10053                                                                                                        tableColumn = tableModel.getColumns().get(i);
10054                                                                                                }
10055                                                                                        } else {
10056                                                                                                tableColumn = modelFactory.createInsertTableColumn(tableModel,
10057                                                                                                                ((TResultColumn) resultColumn.getColumnObject()).getExpr(),
10058                                                                                                                i);
10059                                                                                        }
10060                                                                                        
10061                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10062                                                                                        relation.setEffectType(effectType);
10063                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
10064                                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
10065                                                                                        Process process = modelFactory.createProcess(stmt);
10066                                                                                        relation.setProcess(process);
10067                                                                                }
10068                                                                        }
10069                                                                }
10070                                                        }
10071                                                }
10072                                        }
10073                                }
10074                                else if (procedure!=null && procedure.getProcedureObject() instanceof TObjectName) {
10075                                        TObjectName functionName = new TObjectName();
10076                                        functionName.setString(procedure.getName());
10077                                        Function function = (Function)createFunction(functionName);
10078                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10079                                        relation.setEffectType(effectType);
10080                                        TObjectName starColumn = new TObjectName();
10081                                        starColumn.setString("*");
10082                                        TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
10083                                                        starColumn);
10084                                        tableColumn.setExpandStar(false);
10085                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
10086                                        relation.addSource(new ResultColumnRelationshipElement(function.getColumns().get(0)));
10087                                        Process process = modelFactory.createProcess(stmt);
10088                                        relation.setProcess(process);
10089                                }
10090                        }
10091                }
10092                
10093                if(stmt.getOnDuplicateKeyUpdate()!=null) {
10094                        TTable table = stmt.getTargetTable();
10095                        Table tableModel = modelFactory.createTable(table);
10096                        for(TResultColumn column: stmt.getOnDuplicateKeyUpdate()) {
10097                                if(column.getExpr()==null || column.getExpr().getExpressionType() != EExpressionType.assignment_t) {
10098                                        continue;
10099                                }
10100                                TExpression left = column.getExpr().getLeftOperand();
10101                                TExpression right = column.getExpr().getRightOperand();
10102                                TObjectName columnObject = left.getObjectOperand();
10103                                if (columnObject != null) {
10104                                        TableColumn tableColumn = modelFactory.createTableColumn(tableModel, columnObject, false);
10105                                        if (tableColumn != null) {
10106                                                columnsInExpr visitor = new columnsInExpr();
10107                                                right.inOrderTraverse(visitor);
10108                                                List<TObjectName> objectNames = visitor.getObjectNames();
10109                                                List<TParseTreeNode> functions = visitor.getFunctions();
10110                                                List<TParseTreeNode> constants = visitor.getConstants();
10111                                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
10112
10113                                                if (functions != null && !functions.isEmpty()) {
10114                                                        analyzeFunctionDataFlowRelation(tableColumn, functions, EffectType.update);
10115                                                }
10116                                                if (subquerys != null && !subquerys.isEmpty()) {
10117                                                        analyzeSubqueryDataFlowRelation(tableColumn, subquerys, EffectType.update);
10118                                                }
10119                                                if (objectNames != null && !objectNames.isEmpty()) {
10120                                                        analyzeDataFlowRelation(tableColumn, objectNames, EffectType.update, functions);
10121                                                }
10122                                                if (constants != null && !constants.isEmpty()) {
10123                                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.update, functions);
10124                                                }
10125                                        }
10126                                }
10127                        }
10128                }
10129
10130                if (!expressions.isEmpty() && stmt.getSubQuery() != null) {
10131                        analyzeInsertImpactRelation(stmt.getSubQuery(), tableColumnMap, expressions, effectType);
10132                }
10133        }
10134
10135        private List<TSelectSqlStatement> getLastSelectStmt(TStoredProcedureSqlStatement procedureStmt) {
10136                List<TSelectSqlStatement> stmts = new ArrayList<TSelectSqlStatement>();
10137                if (procedureStmt.getBodyStatements().size() > 0) {
10138                        for (int j = procedureStmt.getBodyStatements().size() - 1; j >= 0; j--) {
10139                                TCustomSqlStatement stmtItem = procedureStmt.getBodyStatements().get(j);
10140                                if (stmtItem instanceof TReturnStmt || stmtItem instanceof TMssqlReturn) {
10141                                        if (stmtItem.getStatements() != null) {
10142                                                List<TSelectSqlStatement> item = getLastSelectStmt(stmtItem);
10143                                                if (item != null && !item.isEmpty()) {
10144                                                        stmts.addAll(item);
10145                                                        if(option.getVendor()!=EDbVendor.dbvmssql && option.getVendor()!=EDbVendor.dbvazuresql) {
10146                                                                break;
10147                                                        }
10148                                                }
10149                                        }
10150                                        break;
10151                                }
10152                                if (stmtItem instanceof TSelectSqlStatement) {
10153                                        stmts.add((TSelectSqlStatement) stmtItem);
10154                                        if(option.getVendor()!=EDbVendor.dbvmssql && option.getVendor()!=EDbVendor.dbvazuresql) {
10155                                                break;
10156                                        }
10157                                } else if (stmtItem.getStatements() != null) {
10158                                        List<TSelectSqlStatement> item = getLastSelectStmt(stmtItem);
10159                                        if (item != null && !item.isEmpty()) {
10160                                                stmts.addAll(item);
10161                                                if(option.getVendor()!=EDbVendor.dbvmssql && option.getVendor()!=EDbVendor.dbvazuresql) {
10162                                                        break;
10163                                                }
10164                                        }
10165                                }
10166                        }
10167                }
10168                return stmts;
10169        }
10170
10171        private List<TSelectSqlStatement> getLastSelectStmt(TCustomSqlStatement stmt) {
10172                List<TSelectSqlStatement> stmts = new ArrayList<TSelectSqlStatement>();
10173                for (int j = stmt.getStatements().size() - 1; j >= 0; j--) {
10174                        TCustomSqlStatement stmtItem = stmt.getStatements().get(j);
10175                        if (stmtItem instanceof TReturnStmt || stmtItem instanceof TMssqlReturn) {
10176                                if (stmtItem.getStatements() != null) {
10177                                        List<TSelectSqlStatement> item = getLastSelectStmt(stmtItem);
10178                                        if (item != null && !item.isEmpty()) {
10179                                                stmts.addAll(item);
10180                                                if(option.getVendor()!=EDbVendor.dbvmssql && option.getVendor()!=EDbVendor.dbvazuresql) {
10181                                                        break;
10182                                                }
10183                                        }
10184                                }
10185                                break;
10186                        }
10187                        if (stmtItem instanceof TSelectSqlStatement) {
10188                                stmts.add((TSelectSqlStatement) stmtItem);
10189                                if(option.getVendor()!=EDbVendor.dbvmssql && option.getVendor()!=EDbVendor.dbvazuresql) {
10190                                        break;
10191                                }
10192                        } else if (stmtItem.getStatements() != null) {
10193                                List<TSelectSqlStatement> item = getLastSelectStmt(stmtItem);
10194                                if (item != null && !item.isEmpty()) {
10195                                        stmts.addAll(item);
10196                                        if(option.getVendor()!=EDbVendor.dbvmssql && option.getVendor()!=EDbVendor.dbvazuresql) {
10197                                                break;
10198                                        }
10199                                }
10200                        }
10201                }
10202                return stmts;
10203        }
10204
10205        private TableColumn getStarColumn(List<TableColumn> columns) {
10206                for (TableColumn column : columns) {
10207                        if (column.getName().endsWith("*")) {
10208                                return column;
10209                        }
10210                }
10211                return null;
10212        }
10213
10214        private boolean containStarColumn(List<TableColumn> columns) {
10215                if (columns == null)
10216                        return false;
10217                for (TableColumn column : columns) {
10218                        if (column.getName().endsWith("*")) {
10219                                return true;
10220                        }
10221                }
10222                return false;
10223        }
10224
10225        private boolean containStarColumn(ResultSet resultSet) {
10226                if (resultSet == null || resultSet.getColumns() == null)
10227                        return false;
10228                for (ResultColumn column : resultSet.getColumns()) {
10229                        if (column.getName().endsWith("*")) {
10230                                return true;
10231                        }
10232                }
10233                return false;
10234        }
10235
10236        private int indexOfColumn(List<TResultColumn> columns, TObjectName objectName) {
10237                for (int i = 0; i < columns.size(); i++) {
10238                        if (columns.get(i).toString().trim().equalsIgnoreCase(objectName.toString().trim())) {
10239                                return i;
10240                        }
10241                }
10242                return -1;
10243        }
10244
10245        private boolean isEmptyCollection(Collection<?> keyMap) {
10246                return keyMap == null || keyMap.isEmpty();
10247        }
10248
10249        private void analyzeInsertImpactRelation(TSelectSqlStatement stmt, Map<String, List<TableColumn>> insertMap,
10250                        List<TExpression> expressions, EffectType effectType) {
10251                List<TObjectName> objectNames = new ArrayList<TObjectName>();
10252                for (int i = 0; i < expressions.size(); i++) {
10253                        TExpression condition = expressions.get(i);
10254                        columnsInExpr visitor = new columnsInExpr();
10255                        condition.inOrderTraverse(visitor);
10256                        objectNames.addAll(visitor.getObjectNames());
10257                }
10258
10259                Iterator<String> iter = insertMap.keySet().iterator();
10260                while (iter.hasNext()) {
10261                        String table = iter.next();
10262                        List<TableColumn> tableColumns = insertMap.get(table);
10263                        for (int i = 0; i < tableColumns.size(); i++) {
10264
10265                                TableColumn column = tableColumns.get(i);
10266                                ImpactRelationship relation = modelFactory.createImpactRelation();
10267                                relation.setEffectType(effectType);
10268                                relation.setTarget(new TableColumnRelationshipElement(column));
10269
10270                                for (int j = 0; j < objectNames.size(); j++) {
10271                                        TObjectName columnName = objectNames.get(j);
10272                                        Object model = modelManager.getModel(stmt);
10273                                        if (model instanceof SelectResultSet) {
10274                                                SelectResultSet queryTable = (SelectResultSet) model;
10275                                                List<ResultColumn> columns = queryTable.getColumns();
10276                                                for (int k = 0; k < columns.size(); k++) {
10277                                                        ResultColumn resultColumn = columns.get(k);
10278                                                        if (resultColumn.getAlias() != null
10279                                                                        && columnName.toString().equalsIgnoreCase(resultColumn.getAlias())) {
10280                                                                relation.addSource(
10281                                                                                new ResultColumnRelationshipElement(resultColumn, columnName.getLocation()));
10282                                                        } else if (resultColumn.getName() != null
10283                                                                        && columnName.toString().equalsIgnoreCase(resultColumn.getName())) {
10284                                                                relation.addSource(
10285                                                                                new ResultColumnRelationshipElement(resultColumn, columnName.getLocation()));
10286                                                        }
10287                                                }
10288                                        }
10289                                }
10290                        }
10291                }
10292        }
10293
10294        private void analyzeUpdateStmt(TUpdateSqlStatement stmt) {
10295                if (stmt.getResultColumnList() == null)
10296                        return;
10297
10298                TTable table = stmt.getTargetTable();
10299                while (table.getCTE() != null || table.getSubquery() != null || (table.getLinkTable() != null && table.getLinkTable().getSubquery() != null)) {
10300                        if (table.getCTE() != null) {
10301                                table = table.getCTE().getSubquery().getTables().getTable(0);
10302                        } else if (table.getLinkTable() != null && table.getLinkTable().getSubquery() != null) {
10303                                table = table.getLinkTable().getSubquery().getTables().getTable(0);
10304                        } else if (table.getSubquery() != null) {
10305                                table = table.getSubquery().getTables().getTable(0);
10306                        }
10307                }
10308                Table tableModel = modelFactory.createTable(table);
10309                Process process = modelFactory.createProcess(stmt);
10310                tableModel.addProcess(process);
10311
10312                for (int i = 0; i < stmt.tables.size(); i++) {
10313                        TTable tableElement = stmt.tables.getTable(i);
10314                        if (tableElement.getSubquery() != null) {
10315                                QueryTable queryTable = modelFactory.createQueryTable(tableElement);
10316                                TSelectSqlStatement subquery = tableElement.getSubquery();
10317                                analyzeSelectStmt(subquery);
10318
10319                                if (subquery.getSetOperatorType() != ESetOperatorType.none) {
10320                                        SelectSetResultSet selectSetResultSetModel = (SelectSetResultSet) modelManager.getModel(subquery);
10321                                        for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) {
10322                                                ResultColumn sourceColumn = selectSetResultSetModel.getColumns().get(j);
10323                                                ResultColumn targetColumn = modelFactory.createSelectSetResultColumn(queryTable, sourceColumn);
10324                                                DataFlowRelationship selectSetRalation = modelFactory.createDataFlowRelation();
10325                                                selectSetRalation.setEffectType(EffectType.select);
10326                                                selectSetRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
10327                                                selectSetRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
10328                                                selectSetRalation.setProcess(process);
10329                                        }
10330                                }
10331
10332                                ResultSet resultSetModel = (ResultSet) modelManager.getModel(tableElement.getSubquery());
10333                                if (resultSetModel != null && resultSetModel != queryTable
10334                                                && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
10335                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
10336                                        impactRelation.setEffectType(EffectType.update);
10337                                        impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
10338                                                        resultSetModel.getRelationRows()));
10339                                        impactRelation.setTarget(
10340                                                        new RelationRowsRelationshipElement<ResultSetRelationRows>(queryTable.getRelationRows()));
10341                                }
10342
10343                        } else if (tableElement.getCTE() != null) {
10344                                QueryTable queryTable = modelFactory.createQueryTable(tableElement);
10345
10346                                TObjectNameList cteColumns = tableElement.getCTE().getColumnList();
10347                                if (cteColumns != null) {
10348                                        for (int j = 0; j < cteColumns.size(); j++) {
10349                                                modelFactory.createResultColumn(queryTable, cteColumns.getObjectName(j));
10350                                        }
10351                                }
10352                                TSelectSqlStatement subquery = tableElement.getCTE().getSubquery();
10353                                if (subquery != null && !stmtStack.contains(subquery)) {
10354                                        analyzeSelectStmt(subquery);
10355
10356                                        ResultSet resultSetModel = (ResultSet) modelManager.getModel(subquery);
10357                                        if (resultSetModel != null && resultSetModel != queryTable
10358                                                        && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
10359                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
10360                                                impactRelation.setEffectType(EffectType.select);
10361                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
10362                                                                resultSetModel.getRelationRows()));
10363                                                impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
10364                                                                queryTable.getRelationRows()));
10365                                        }
10366
10367                                        if (subquery.getSetOperatorType() != ESetOperatorType.none) {
10368                                                SelectSetResultSet selectSetResultSetModel = (SelectSetResultSet) modelManager
10369                                                                .getModel(subquery);
10370                                                for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) {
10371                                                        ResultColumn sourceColumn = selectSetResultSetModel.getColumns().get(j);
10372                                                        ResultColumn targetColumn = null;
10373                                                        if (cteColumns != null) {
10374                                                                targetColumn = queryTable.getColumns().get(j);
10375                                                        } else {
10376                                                                targetColumn = modelFactory.createSelectSetResultColumn(queryTable, sourceColumn);
10377                                                        }
10378                                                        for (Set<TObjectName> starLinkColumns : sourceColumn.getStarLinkColumns().values()) {
10379                                                                for (TObjectName starLinkColumn : starLinkColumns) {
10380                                                                        targetColumn.bindStarLinkColumn(starLinkColumn);
10381                                                                }
10382                                                        }
10383                                                        DataFlowRelationship selectSetRalation = modelFactory.createDataFlowRelation();
10384                                                        selectSetRalation.setEffectType(EffectType.select);
10385                                                        selectSetRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
10386                                                        selectSetRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
10387                                                        selectSetRalation.setProcess(process);
10388                                                }
10389                                        } else {
10390                                                for (int j = 0; j < resultSetModel.getColumns().size(); j++) {
10391                                                        ResultColumn sourceColumn = resultSetModel.getColumns().get(j);
10392                                                        ResultColumn targetColumn = null;
10393                                                        if (cteColumns != null) {
10394                                                                targetColumn = queryTable.getColumns().get(j);
10395                                                        } else {
10396                                                                targetColumn = modelFactory.createSelectSetResultColumn(queryTable, sourceColumn);
10397                                                        }
10398                                                        for (TObjectName starLinkColumn : sourceColumn.getStarLinkColumnList()) {
10399                                                                targetColumn.bindStarLinkColumn(starLinkColumn);
10400                                                        }
10401                                                        DataFlowRelationship selectSetRalation = modelFactory.createDataFlowRelation();
10402                                                        selectSetRalation.setEffectType(EffectType.select);
10403                                                        selectSetRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
10404                                                        selectSetRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
10405                                                        selectSetRalation.setProcess(process);
10406                                                }
10407                                        }
10408                                } else if (tableElement.getCTE().getUpdateStmt() != null) {
10409                                        analyzeCustomSqlStmt(tableElement.getCTE().getUpdateStmt());
10410                                } else if (tableElement.getCTE().getInsertStmt() != null) {
10411                                        analyzeCustomSqlStmt(tableElement.getCTE().getInsertStmt());
10412                                } else if (tableElement.getCTE().getDeleteStmt() != null) {
10413                                        analyzeCustomSqlStmt(tableElement.getCTE().getDeleteStmt());
10414                                }
10415                        } else {
10416                                modelFactory.createTable(stmt.tables.getTable(i));
10417                        }
10418                }
10419
10420                for (int i = 0; i < stmt.getResultColumnList().size(); i++) {
10421                        TResultColumn field = stmt.getResultColumnList().getResultColumn(i);
10422
10423                        if (field.getExpr().getExpressionType() == EExpressionType.function_t) {
10424                                // Handle SQL Server XML modify() method for data lineage
10425                                TFunctionCall funcCall = field.getExpr().getFunctionCall();
10426                                if (funcCall != null && funcCall.getFunctionType() == EFunctionType.xmlmodify_t) {
10427                                        analyzeXmlModifyFunction(stmt, tableModel, process, funcCall);
10428                                }
10429                                continue;
10430                        }
10431
10432                        TExpression expression = field.getExpr().getLeftOperand();
10433                        if (expression == null) {
10434                                ErrorInfo errorInfo = new ErrorInfo();
10435                                errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
10436                                errorInfo.setErrorMessage(
10437                                                "Can't get result column expression. Expression is " + field.getExpr().toString());
10438                                errorInfo.setStartPosition(new Pair3<Long, Long, String>(field.getExpr().getStartToken().lineNo,
10439                                                field.getExpr().getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
10440                                errorInfo.setEndPosition(new Pair3<Long, Long, String>(field.getExpr().getEndToken().lineNo,
10441                                                field.getExpr().getEndToken().columnNo + field.getExpr().getEndToken().getAstext().length(),
10442                                                ModelBindingManager.getGlobalHash()));
10443                                errorInfo.fillInfo(this);
10444                                errorInfos.add(errorInfo);
10445                                continue;
10446                        }
10447                        if (expression.getExpressionType() == EExpressionType.list_t) {
10448                                TExpression setExpression = field.getExpr().getRightOperand();
10449                                if (setExpression != null && setExpression.getSubQuery() != null) {
10450                                        TSelectSqlStatement query = setExpression.getSubQuery();
10451                                        analyzeSelectStmt(query);
10452
10453                                        SelectResultSet resultSetModel = (SelectResultSet) modelManager
10454                                                        .getModel(query.getResultColumnList());
10455
10456                                        if (!resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
10457                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
10458                                                impactRelation.setEffectType(EffectType.update);
10459                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
10460                                                                resultSetModel.getRelationRows()));
10461                                                impactRelation.setTarget(
10462                                                                new RelationRowsRelationshipElement<TableRelationRows>(tableModel.getRelationRows()));
10463                                        }
10464
10465                                        TExpressionList columnList = expression.getExprList();
10466                                        for (int j = 0; j < columnList.size(); j++) {
10467                                                TObjectName column = columnList.getExpression(j).getObjectOperand();
10468
10469                                                if (column.getDbObjectType() == EDbObjectType.variable) {
10470                                                        continue;
10471                                                }
10472
10473                                                if (column.getColumnNameOnly().startsWith("@") && (option.getVendor() == EDbVendor.dbvmssql
10474                                                                || option.getVendor() == EDbVendor.dbvazuresql)) {
10475                                                        continue;
10476                                                }
10477
10478                                                if (column.getColumnNameOnly().startsWith(":") && (option.getVendor() == EDbVendor.dbvhana
10479                                                                || option.getVendor() == EDbVendor.dbvteradata)) {
10480                                                        continue;
10481                                                }
10482
10483                                                ResultColumn resultColumn = resultSetModel.getColumns().get(j);
10484                                                TableColumn tableColumn = modelFactory.createTableColumn(tableModel, column, false);
10485                                                if (tableColumn != null) {
10486                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10487                                                        relation.setEffectType(EffectType.update);
10488                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
10489                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
10490                                                        relation.setProcess(process);
10491                                                }
10492
10493                                        }
10494                                }
10495                        } else if (expression.getExpressionType() == EExpressionType.simple_object_name_t) {
10496                                TExpression setExpression = field.getExpr().getRightOperand();
10497                                if (setExpression != null && setExpression.getSubQuery() != null) {
10498                                        TSelectSqlStatement query = setExpression.getSubQuery();
10499                                        analyzeSelectStmt(query);
10500
10501                                        SelectResultSet resultSetModel = (SelectResultSet) modelManager
10502                                                        .getModel(query.getResultColumnList());
10503
10504                                        if (!resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
10505                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
10506                                                impactRelation.setEffectType(EffectType.update);
10507                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
10508                                                                resultSetModel.getRelationRows()));
10509                                                impactRelation.setTarget(
10510                                                                new RelationRowsRelationshipElement<TableRelationRows>(tableModel.getRelationRows()));
10511                                        }
10512
10513                                        TObjectName column = expression.getObjectOperand();
10514                                        ResultColumn resultColumn = resultSetModel.getColumns().get(0);
10515                                        TableColumn tableColumn = modelFactory.createTableColumn(tableModel, column, false);
10516                                        if (tableColumn != null) {
10517                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10518                                                relation.setEffectType(EffectType.update);
10519                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
10520                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
10521                                                relation.setProcess(process);
10522                                        }
10523                                } else if (setExpression != null) {
10524                                        // ResultSet resultSet = modelFactory.createResultSet(stmt,
10525                                        // true);
10526
10527                                        ResultSet resultSet = modelFactory.createResultSet(stmt, false);
10528
10529                                        createPseudoImpactRelation(stmt, resultSet, EffectType.update);
10530
10531                                        TObjectName columnObject = expression.getObjectOperand();
10532
10533                                        ResultColumn updateColumn = modelFactory.createUpdateResultColumn(resultSet, columnObject);
10534
10535                                        columnsInExpr visitor = new columnsInExpr();
10536                                        field.getExpr().getRightOperand().inOrderTraverse(visitor);
10537
10538                                        List<TObjectName> objectNames = visitor.getObjectNames();
10539                                        List<TParseTreeNode> functions = visitor.getFunctions();
10540                                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
10541
10542                                        if (functions != null && !functions.isEmpty()) {
10543                                                analyzeFunctionDataFlowRelation(updateColumn, functions, EffectType.update);
10544                                        }
10545
10546                                        if (subquerys != null && !subquerys.isEmpty()) {
10547                                                analyzeSubqueryDataFlowRelation(updateColumn, subquerys, EffectType.update);
10548                                        }
10549
10550                                        Transform transform = new Transform();
10551                                        transform.setType(Transform.EXPRESSION);
10552                                        transform.setCode(setExpression);
10553                                        updateColumn.setTransform(transform);
10554                                        analyzeDataFlowRelation(updateColumn, objectNames, EffectType.update, functions);
10555
10556                                        List<TParseTreeNode> constants = visitor.getConstants();
10557                                        analyzeConstantDataFlowRelation(updateColumn, constants, EffectType.update, functions);
10558
10559                                        TableColumn tableColumn = modelFactory.createTableColumn(tableModel, columnObject, false);
10560                                        if(tableColumn!=null) {
10561                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10562                                                relation.setEffectType(EffectType.update);
10563                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
10564                                                relation.addSource(new ResultColumnRelationshipElement(updateColumn));
10565                                                relation.setProcess(process);
10566                                        }
10567                                        
10568                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
10569                                        impactRelation.setEffectType(EffectType.update);
10570                                        impactRelation.addSource(
10571                                                        new RelationRowsRelationshipElement<ResultSetRelationRows>(resultSet.getRelationRows()));
10572                                        impactRelation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(
10573                                                        tableModel.getRelationRows()));
10574                                }
10575                        }
10576                }
10577
10578                if (stmt.getJoins() != null && stmt.getJoins().size() > 0) {
10579                        for (int i = 0; i < stmt.getJoins().size(); i++) {
10580                                TJoin join = stmt.getJoins().getJoin(i);
10581                                if (join.getJoinItems() != null) {
10582                                        for (int j = 0; j < join.getJoinItems().size(); j++) {
10583                                                TJoinItem joinItem = join.getJoinItems().getJoinItem(j);
10584                                                TExpression expr = joinItem.getOnCondition();
10585                                                analyzeFilterCondition(null, expr, joinItem.getJoinType(), JoinClauseType.on,
10586                                                                EffectType.update);
10587                                        }
10588                                }
10589                        }
10590                }
10591
10592                if (stmt.getWhereClause() != null && stmt.getWhereClause().getCondition() != null) {
10593                        analyzeFilterCondition(null, stmt.getWhereClause().getCondition(), null, JoinClauseType.where,
10594                                        EffectType.update);
10595                }
10596
10597                if (stmt.getOutputClause() != null) {
10598                        TOutputClause outputClause = stmt.getOutputClause();
10599                        if (outputClause.getSelectItemList() != null) {
10600                                ResultSet resultSet = modelFactory.createResultSet(outputClause, false);
10601                                for (int j = 0; j < outputClause.getSelectItemList().size(); j++) {
10602                                        TResultColumn sourceColumn = outputClause.getSelectItemList().getResultColumn(j);
10603                                        ResultColumn sourceColumnModel = modelFactory.createResultColumn(resultSet, sourceColumn);
10604                                        analyzeResultColumn(sourceColumn, EffectType.select);
10605
10606                                        if (outputClause.getIntoTable() != null) {
10607                                                Table intoTableModel = modelFactory.createTableByName(outputClause.getIntoTable());
10608                                                intoTableModel.addProcess(process);
10609                                                if (outputClause.getIntoColumnList() != null) {
10610                                                        TableColumn intoTableColumn = modelFactory.createInsertTableColumn(intoTableModel,
10611                                                                        outputClause.getIntoColumnList().getObjectName(j));
10612
10613                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10614                                                        relation.setEffectType(EffectType.insert);
10615                                                        relation.setTarget(new TableColumnRelationshipElement(intoTableColumn));
10616                                                        relation.addSource(new ResultColumnRelationshipElement(sourceColumnModel));
10617                                                } else if (sourceColumn.getAliasClause() != null
10618                                                                || sourceColumn.getExpr().getObjectOperand() != null) {
10619                                                        TObjectName tableColumnObject = null;
10620                                                        if (sourceColumn.getAliasClause() != null) {
10621                                                                tableColumnObject = sourceColumn.getAliasClause().getAliasName();
10622                                                        } else {
10623                                                                tableColumnObject = sourceColumn.getExpr().getObjectOperand();
10624                                                        }
10625
10626                                                        TableColumn intoTableColumn = modelFactory.createInsertTableColumn(intoTableModel,
10627                                                                        tableColumnObject);
10628                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10629                                                        relation.setEffectType(EffectType.insert);
10630                                                        relation.setTarget(new TableColumnRelationshipElement(intoTableColumn));
10631                                                        relation.addSource(new ResultColumnRelationshipElement(sourceColumnModel));
10632                                                }
10633                                        }
10634                                }
10635                        }
10636                }
10637        }
10638
10639        /**
10640         * Analyzes SQL Server XML modify() function to extract data lineage from sql:column() references
10641         * in XQuery expressions.
10642         *
10643         * Example SQL:
10644         * SET [Demographics].modify('... sql:column("deleted.LineTotal") ...')
10645         *
10646         * This extracts:
10647         * - Target column: Demographics (from the XML column being modified)
10648         * - Source column: deleted.LineTotal (from sql:column() reference in XQuery)
10649         */
10650        private void analyzeXmlModifyFunction(TUpdateSqlStatement stmt, Table tableModel, Process process, TFunctionCall funcCall) {
10651                TObjectName funcName = funcCall.getFunctionName();
10652                if (funcName == null || funcName.getPartToken() == null) {
10653                        return;
10654                }
10655
10656                // Get the XML column being modified (e.g., [Demographics] from "[Demographics].modify")
10657                String xmlColumnName = funcName.getPartToken().astext;
10658
10659                // Get the XQuery argument
10660                if (funcCall.getArgs() == null || funcCall.getArgs().size() == 0) {
10661                        return;
10662                }
10663
10664                String xqueryString = funcCall.getArgs().getExpression(0).toString();
10665
10666                // Extract sql:column() references from the XQuery string
10667                List<String> sqlColumnRefs = extractSqlColumnReferences(xqueryString);
10668                if (sqlColumnRefs.isEmpty()) {
10669                        return;
10670                }
10671
10672                // Extract XPath target from XQuery (e.g., /IndividualSurvey/TotalPurchaseYTD)
10673                String xpathTarget = extractXPathTarget(xqueryString);
10674
10675                // Build full target column name including XML path
10676                String fullTargetColumnName = xmlColumnName;
10677                if (xpathTarget != null && !xpathTarget.isEmpty()) {
10678                        fullTargetColumnName = xmlColumnName + "." + xpathTarget;
10679                }
10680
10681                // Create the target table column for the XML column being modified
10682                TableColumn targetTableColumn = modelFactory.createInsertTableColumn(tableModel, fullTargetColumnName);
10683
10684                // Create source-to-target relationships for each sql:column reference
10685                for (String sqlColRef : sqlColumnRefs) {
10686                        // Parse the column reference (e.g., "deleted.LineTotal" -> table="deleted", column="LineTotal")
10687                        String[] parts = sqlColRef.split("\\.", 2);
10688                        String sourceTableName = parts.length > 1 ? parts[0] : null;
10689                        String sourceColumnName = parts.length > 1 ? parts[1] : parts[0];
10690
10691                        // Find the source table in the statement's tables
10692                        TTable sourceTable = null;
10693                        if (sourceTableName != null && stmt.tables != null) {
10694                                for (int j = 0; j < stmt.tables.size(); j++) {
10695                                        TTable t = stmt.tables.getTable(j);
10696                                        String tableName = t.getTableName().toString();
10697                                        String alias = t.getAliasName();
10698                                        if (tableName.equalsIgnoreCase(sourceTableName) ||
10699                                                (alias != null && alias.equalsIgnoreCase(sourceTableName))) {
10700                                                sourceTable = t;
10701                                                break;
10702                                        }
10703                                }
10704                        }
10705
10706                        if (sourceTable != null && targetTableColumn != null) {
10707                                // Create a table model for the source if needed
10708                                Table sourceTableModel = modelFactory.createTable(sourceTable);
10709                                TableColumn sourceColumn = modelFactory.createInsertTableColumn(sourceTableModel, sourceColumnName);
10710
10711                                if (sourceColumn != null) {
10712                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10713                                        relation.setEffectType(EffectType.update);
10714                                        relation.setTarget(new TableColumnRelationshipElement(targetTableColumn));
10715                                        relation.addSource(new TableColumnRelationshipElement(sourceColumn));
10716                                        relation.setProcess(process);
10717                                        relation.setFunction("modify");
10718                                }
10719                        }
10720                }
10721        }
10722
10723        /**
10724         * Extracts sql:column() references from an XQuery string.
10725         * Example: 'sql:column("deleted.LineTotal")' -> ["deleted.LineTotal"]
10726         */
10727        private List<String> extractSqlColumnReferences(String xquery) {
10728                List<String> refs = new ArrayList<>();
10729                if (xquery == null) {
10730                        return refs;
10731                }
10732
10733                int startIdx = 0;
10734                while ((startIdx = xquery.indexOf("sql:column(", startIdx)) >= 0) {
10735                        int parenStart = startIdx + "sql:column(".length();
10736                        int parenEnd = xquery.indexOf(")", parenStart);
10737                        if (parenEnd < 0) {
10738                                break;
10739                        }
10740
10741                        String arg = xquery.substring(parenStart, parenEnd).trim();
10742                        // Remove quotes (single or double)
10743                        if ((arg.startsWith("\"") && arg.endsWith("\"")) ||
10744                                (arg.startsWith("'") && arg.endsWith("'"))) {
10745                                arg = arg.substring(1, arg.length() - 1);
10746                        }
10747
10748                        if (!arg.isEmpty()) {
10749                                refs.add(arg);
10750                        }
10751
10752                        startIdx = parenEnd + 1;
10753                }
10754
10755                return refs;
10756        }
10757
10758        /**
10759         * Extracts the XPath target from an XQuery modify expression.
10760         * Example: 'replace value of (/IndividualSurvey/TotalPurchaseYTD)[1]' -> "IndividualSurvey.TotalPurchaseYTD"
10761         */
10762        private String extractXPathTarget(String xquery) {
10763                if (xquery == null) {
10764                        return null;
10765                }
10766
10767                // Look for patterns like "(/path/to/element)" or "(/path/to/element)[1]"
10768                int replaceIdx = xquery.indexOf("replace value of");
10769                if (replaceIdx < 0) {
10770                        return null;
10771                }
10772
10773                int parenStart = xquery.indexOf("(/", replaceIdx);
10774                if (parenStart < 0) {
10775                        return null;
10776                }
10777
10778                int parenEnd = xquery.indexOf(")", parenStart);
10779                if (parenEnd < 0) {
10780                        return null;
10781                }
10782
10783                String xpath = xquery.substring(parenStart + 1, parenEnd);
10784                // Remove any predicates like [1]
10785                int bracketIdx = xpath.indexOf("[");
10786                if (bracketIdx > 0) {
10787                        xpath = xpath.substring(0, bracketIdx);
10788                }
10789
10790                // Convert XPath to dot notation (e.g., /IndividualSurvey/TotalPurchaseYTD -> IndividualSurvey.TotalPurchaseYTD)
10791                if (xpath.startsWith("/")) {
10792                        xpath = xpath.substring(1);
10793                }
10794                xpath = xpath.replace("/", ".");
10795
10796                return xpath;
10797        }
10798
10799        private void analyzeConstantDataFlowRelation(Object modelObject, List<TParseTreeNode> constants,
10800                        EffectType effectType, List<TParseTreeNode> functions) {
10801                analyzeConstantDataFlowRelation(modelObject, constants, effectType, functions, null);
10802        }
10803
10804        private void analyzeConstantDataFlowRelation(Object modelObject, List<TParseTreeNode> constants,
10805                        EffectType effectType, List<TParseTreeNode> functions, Process process) {
10806                if (constants == null || constants.size() == 0)
10807                        return;
10808
10809                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10810                relation.setEffectType(effectType);
10811                relation.setProcess(process);
10812
10813                if (functions != null && !functions.isEmpty()) {
10814                        relation.setFunction(getFunctionName(functions.get(0)));
10815                }
10816
10817                if (modelObject instanceof ResultColumn) {
10818                        relation.setTarget(new ResultColumnRelationshipElement((ResultColumn) modelObject));
10819
10820                } else if (modelObject instanceof TableColumn) {
10821                        relation.setTarget(new TableColumnRelationshipElement((TableColumn) modelObject));
10822
10823                } else {
10824                        throw new UnsupportedOperationException();
10825                }
10826
10827                if (option.isShowConstantTable()) {
10828                        Table constantTable = null;
10829                        if(modelObject instanceof FunctionResultColumn && ((FunctionResultColumn)modelObject).getFunction() instanceof TFunctionCall) {
10830                                TFunctionCall function = (TFunctionCall)((FunctionResultColumn)modelObject).getFunction();
10831                                if(function.getFunctionType() == EFunctionType.struct_t) {
10832                                        constantTable = modelFactory.createConstantsTable(String.valueOf(function.toString().hashCode()));
10833                                }
10834                        }
10835                        if(constantTable == null) {
10836                                constantTable = modelFactory.createConstantsTable(stmtStack.peek());
10837                        }
10838                        for (int i = 0; i < constants.size(); i++) {
10839                                TParseTreeNode constant = constants.get(i);
10840                                if (constant instanceof TConstant) {
10841                                        TableColumn constantColumn = modelFactory.createTableColumn(constantTable, (TConstant) constant);
10842                                        relation.addSource(new ConstantRelationshipElement(constantColumn));
10843                                } else if (constant instanceof TObjectName) {
10844                                        TableColumn constantColumn = modelFactory.createTableColumn(constantTable, (TObjectName) constant,
10845                                                        false);
10846                                        if(constantColumn == null) {
10847                                                continue;
10848                                        }
10849                                        relation.addSource(new ConstantRelationshipElement(constantColumn));
10850                                }
10851                        }
10852                }
10853
10854        }
10855
10856        private String getFunctionName(TParseTreeNode parseTreeNode) {
10857                if (parseTreeNode instanceof TFunctionCall) {
10858                        return ((TFunctionCall) parseTreeNode).getFunctionName().toString();
10859                }
10860                if (parseTreeNode instanceof TCaseExpression) {
10861                        return "case-when";
10862                }
10863                return null;
10864        }
10865
10866        private void analyzeCreateViewStmt(TCustomSqlStatement stmt, TSelectSqlStatement subquery,
10867                        TViewAliasClause viewAlias, TObjectName viewName) {
10868
10869                if (subquery != null) {
10870                        TTableList tables = subquery.getTables();
10871                        if (tables != null) {
10872                                for (int i = 0; i < tables.size(); i++) {
10873                                        TTable table = tables.getTable(i);
10874                                        TCustomSqlStatement createView = viewDDLMap
10875                                                        .get(DlineageUtil.getTableFullName(table.getTableName().toString()));
10876                                        if (createView != null) {
10877                                                analyzeCustomSqlStmt(createView);
10878                                        }
10879                                }
10880                        }
10881                        analyzeSelectStmt(subquery);
10882                }
10883
10884        
10885                if (viewAlias != null && viewAlias.getViewAliasItemList() != null) {
10886                        TViewAliasItemList viewItems = viewAlias.getViewAliasItemList();
10887                        Table viewModel = modelFactory.createView(stmt, viewName, true);
10888                        viewModel.setFromDDL(true);
10889                        viewModel.setDetermined(true);
10890                        Process process = modelFactory.createProcess(stmt);
10891                        viewModel.addProcess(process);
10892                        ResultSet resultSetModel = (ResultSet) modelManager.getModel(subquery);
10893                        if (resultSetModel != null) {
10894                                int resultSetSize = resultSetModel.getColumns().size();
10895                                int viewItemSize = viewItems.size();
10896                                int j = 0;
10897                                int viewColumnSize = viewItemSize;
10898                                if (resultSetModel.isDetermined() && resultSetSize > viewColumnSize) {
10899                                        viewColumnSize = resultSetSize;
10900                                }
10901                                for (int i = 0; i < viewColumnSize && j < resultSetSize; i++) {
10902                                        ResultColumn resultColumn = resultSetModel.getColumns().get(j);                                 
10903                                        if (i < viewItemSize) {
10904                                                TObjectName alias = viewItems.getViewAliasItem(i).getAlias();
10905
10906                                                if (!resultSetModel.getColumns().get(j).getName().contains("*")) {
10907                                                        j++;
10908                                                } else {
10909                                                        if (resultSetSize - j == viewItems.size() - i) {
10910                                                                j++;
10911                                                        }
10912                                                }
10913
10914                                                if (alias != null) {
10915                                                        TableColumn viewColumn = modelFactory.createViewColumn(viewModel, alias, i, true);
10916                                                        appendTableColumnToSQLEnv(viewModel, viewColumn);
10917                                                        if (resultColumn != null) {
10918                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10919                                                                relation.setEffectType(EffectType.create_view);
10920                                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
10921                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
10922                                                                relation.setProcess(process);
10923                                                        }
10924                                                } else if (resultColumn.getColumnObject() instanceof TObjectName) {
10925                                                        TableColumn viewColumn = modelFactory.createViewColumn(viewModel,
10926                                                                        (TObjectName) resultColumn.getColumnObject(), i, true);
10927                                                        appendTableColumnToSQLEnv(viewModel, viewColumn);
10928                                                        if (resultColumn != null) {
10929                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10930                                                                relation.setEffectType(EffectType.create_view);
10931                                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
10932                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
10933                                                                relation.setProcess(process);
10934                                                        }
10935                                                } else if (resultColumn.getColumnObject() instanceof TResultColumn) {
10936                                                        TableColumn viewColumn = modelFactory.createViewColumn(viewModel,
10937                                                                        ((TResultColumn) resultColumn.getColumnObject()).getFieldAttr(), i, true);
10938                                                        appendTableColumnToSQLEnv(viewModel, viewColumn);
10939                                                        ResultColumn column = (ResultColumn) modelManager.getModel(resultColumn.getColumnObject());
10940                                                        if (column != null && !column.getStarLinkColumns().isEmpty()) {
10941                                                                viewColumn.bindStarLinkColumns(column.getStarLinkColumns());
10942                                                        }
10943                                                        if (resultColumn != null) {
10944                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10945                                                                relation.setEffectType(EffectType.create_view);
10946                                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
10947                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
10948                                                                relation.setProcess(process);
10949                                                        }
10950                                                }
10951                                        }
10952                                        else if(resultSetModel.isDetermined()){
10953                                                TObjectName viewColumnName = new TObjectName();
10954                                                viewColumnName.setString(resultColumn.getName());
10955                                                TableColumn viewColumn = modelFactory.createViewColumn(viewModel, viewColumnName, viewModel.getColumns().size(), true);
10956                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10957                                                relation.setEffectType(EffectType.create_view);
10958                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
10959                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
10960                                                relation.setProcess(process);
10961                                                j++;
10962                                        }
10963                                }
10964                                if (resultSetModel != null && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
10965                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
10966                                        impactRelation.setEffectType(EffectType.create_view);
10967                                        impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
10968                                                        resultSetModel.getRelationRows()));
10969                                        impactRelation.setTarget(
10970                                                        new RelationRowsRelationshipElement<TableRelationRows>(viewModel.getRelationRows()));
10971                                }
10972                        }
10973
10974                        if (subquery.getResultColumnList() == null && subquery.getValueClause() != null
10975                                        && subquery.getValueClause().getValueRows().size() == viewItems.size()) {
10976                                for (int i = 0; i < viewItems.size(); i++) {
10977                                        TObjectName alias = viewItems.getViewAliasItem(i).getAlias();
10978
10979                                        if (alias != null) {
10980                                                TableColumn viewColumn = modelFactory.createViewColumn(viewModel, alias, i, true);
10981                                                appendTableColumnToSQLEnv(viewModel, viewColumn);
10982                                                TExpression expression = subquery.getValueClause().getValueRows().getValueRowItem(i).getExpr();
10983
10984                                                columnsInExpr visitor = new columnsInExpr();
10985                                                expression.inOrderTraverse(visitor);
10986                                                List<TObjectName> objectNames = visitor.getObjectNames();
10987                                                List<TParseTreeNode> functions = visitor.getFunctions();
10988
10989                                                if (functions != null && !functions.isEmpty()) {
10990                                                        analyzeFunctionDataFlowRelation(viewColumn, functions, EffectType.select);
10991
10992                                                }
10993
10994                                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
10995                                                if (subquerys != null && !subquerys.isEmpty()) {
10996                                                        analyzeSubqueryDataFlowRelation(viewColumn, subquerys, EffectType.select);
10997                                                }
10998
10999                                                analyzeDataFlowRelation(viewColumn, objectNames, EffectType.select, functions);
11000                                                List<TParseTreeNode> constants = visitor.getConstants();
11001                                                analyzeConstantDataFlowRelation(viewColumn, constants, EffectType.select, functions);
11002                                        }
11003                                }
11004                        }
11005
11006                } else {
11007                        if (viewName == null) {
11008                                ErrorInfo errorInfo = new ErrorInfo();
11009                                errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
11010                                errorInfo.setErrorMessage("Can't get view name. CreateView is " + stmt.toString());
11011                                errorInfo.setStartPosition(new Pair3<Long, Long, String>(stmt.getStartToken().lineNo,
11012                                                stmt.getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
11013                                errorInfo.setEndPosition(new Pair3<Long, Long, String>(stmt.getEndToken().lineNo,
11014                                                stmt.getEndToken().columnNo + stmt.getEndToken().getAstext().length(),
11015                                                ModelBindingManager.getGlobalHash()));
11016                                errorInfo.fillInfo(this);
11017                                errorInfos.add(errorInfo);
11018                                return;
11019                        }
11020                        Table viewModel = modelFactory.createView(stmt, viewName);
11021                        Process process = modelFactory.createProcess(stmt);
11022                        viewModel.addProcess(process);
11023                        if (subquery != null && !subquery.isCombinedQuery()) {
11024                                SelectResultSet resultSetModel = (SelectResultSet) modelManager
11025                                                .getModel(subquery.getResultColumnList());
11026
11027                                boolean determined = false;
11028                                if (!containStarColumn(resultSetModel)) {
11029                                        viewModel.setCreateTable(true);
11030                                        determined = true;
11031                                        viewModel.setFromDDL(true);
11032                                }
11033                                for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
11034                                        ResultColumn resultColumn = resultSetModel.getColumns().get(i);
11035                                        if (resultColumn.getColumnObject() instanceof TResultColumn) {
11036                                                TResultColumn columnObject = ((TResultColumn) resultColumn.getColumnObject());
11037
11038                                                TAliasClause alias = ((TResultColumn) resultColumn.getColumnObject()).getAliasClause();
11039                                                if (alias != null && alias.getAliasName() != null) {
11040                                                        TableColumn viewColumn = modelFactory.createViewColumn(viewModel, alias.getAliasName(), i,
11041                                                                        determined);
11042                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11043                                                        relation.setEffectType(EffectType.create_view);
11044                                                        relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
11045                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11046                                                        relation.setProcess(process);
11047                                                        if (determined) {
11048                                                                appendTableColumnToSQLEnv(viewModel, viewColumn);
11049                                                        }
11050                                                } else if (columnObject.getFieldAttr() != null) {
11051                                                        TObjectName viewColumnObject = columnObject.getFieldAttr();
11052                                                        TableColumn viewColumn;
11053                                                        Object model = modelManager.getModel(resultColumn.getColumnObject());
11054                                                        if ("*".equals(viewColumnObject.getColumnNameOnly())) {
11055                                                                if (model instanceof LinkedHashMap) {
11056                                                                        String columnName = getColumnNameOnly(resultColumn.getName());
11057                                                                        LinkedHashMap<String, ResultColumn> resultColumns = (LinkedHashMap<String, ResultColumn>) model;
11058                                                                        if ("*".equals(columnName)) {
11059                                                                                for (String key : resultColumns.keySet()) {
11060                                                                                        ResultColumn column = resultColumns.get(key);
11061                                                                                        viewColumn = modelFactory.createInsertTableColumn(viewModel, column.getName());
11062                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11063                                                                                        relation.setEffectType(EffectType.create_view);
11064                                                                                        relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
11065                                                                                        relation.addSource(new ResultColumnRelationshipElement(column));
11066                                                                                        relation.setProcess(process);
11067                                                                                }
11068                                                                                continue;
11069                                                                        } else if (resultColumns.containsKey(columnName)) {
11070                                                                                ResultColumn column = resultColumns.get(columnName);
11071                                                                                viewColumn = modelFactory.createInsertTableColumn(viewModel, column.getName());
11072                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11073                                                                                relation.setEffectType(EffectType.create_view);
11074                                                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
11075                                                                                relation.addSource(new ResultColumnRelationshipElement(column));
11076                                                                                relation.setProcess(process);
11077                                                                                continue;
11078                                                                        }
11079                                                                }
11080                                                        }
11081                                                        
11082                                                        if (!SQLUtil.isEmpty(viewColumnObject.getColumnNameOnly())
11083                                                                        && viewColumnObject.getPropertyToken() != null) {
11084                                                                TObjectName object = new TObjectName();
11085                                                                // object.setString(viewColumnObject.getPropertyToken().astext);
11086                                                                object.setPartToken(viewColumnObject.getPropertyToken());
11087                                                                object.setStartToken(viewColumnObject.getPropertyToken());
11088                                                                object.setEndToken(viewColumnObject.getPropertyToken());
11089                                                                viewColumn = modelFactory.createViewColumn(viewModel, object, i, determined);
11090                                                        } else {
11091                                                                viewColumn = modelFactory.createViewColumn(viewModel, columnObject.getFieldAttr(),
11092                                                                                i, determined);
11093                                                        }
11094                                                        
11095                                                        if(determined) {
11096                                                                appendTableColumnToSQLEnv(viewModel, viewColumn);
11097                                                        }
11098                                                        
11099                                                        if (model instanceof ResultColumn) {
11100                                                                ResultColumn column = (ResultColumn) modelManager
11101                                                                                .getModel(resultColumn.getColumnObject());
11102                                                                if (column != null && !column.getStarLinkColumns().isEmpty()) {
11103                                                                        viewColumn.bindStarLinkColumns(column.getStarLinkColumns());
11104                                                                }
11105                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11106                                                                relation.setEffectType(EffectType.create_view);
11107                                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
11108                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11109                                                                if (sqlenv == null) {
11110                                                                        if (resultColumn.isShowStar()) {
11111                                                                                relation.setShowStarRelation(true);
11112                                                                                viewColumn.setShowStar(true);
11113                                                                                resultColumn.setShowStar(true);
11114                                                                                setSourceShowStar(resultColumn);
11115                                                                        }
11116                                                                }
11117                                                                if (viewColumn.getName().endsWith("*") && resultColumn.getName().endsWith("*")) {
11118                                                                        viewModel.setStarStmt("create_view");
11119                                                                }
11120                                                                relation.setProcess(process);
11121                                                        }
11122                                                } else if (resultColumn.getAlias() != null && columnObject.getExpr()
11123                                                                .getExpressionType() == EExpressionType.sqlserver_proprietary_column_alias_t) {
11124                                                        TableColumn viewColumn = modelFactory.createViewColumn(viewModel,
11125                                                                        columnObject.getExpr().getLeftOperand().getObjectOperand(), i, determined);
11126                                                        if(determined) {
11127                                                                appendTableColumnToSQLEnv(viewModel, viewColumn);
11128                                                        }
11129                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11130                                                        relation.setEffectType(EffectType.create_view);
11131                                                        relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
11132                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11133                                                        relation.setProcess(process);
11134                                                } else {
11135                                                        TGSqlParser parser = columnObject.getGsqlparser();
11136                                                        TObjectName viewColumnName = parser
11137                                                                        .parseObjectName(generateQuotedName(parser, columnObject.toString()));
11138                                                        if (viewColumnName == null) {
11139                                                                ErrorInfo errorInfo = new ErrorInfo();
11140                                                                errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
11141                                                                errorInfo.setErrorMessage(
11142                                                                                "Can't parse view column. Column is " + columnObject.toString());
11143                                                                errorInfo.setStartPosition(new Pair3<Long, Long, String>(
11144                                                                                columnObject.getStartToken().lineNo, columnObject.getStartToken().columnNo,
11145                                                                                ModelBindingManager.getGlobalHash()));
11146                                                                errorInfo
11147                                                                                .setEndPosition(new Pair3<Long, Long, String>(columnObject.getEndToken().lineNo,
11148                                                                                                columnObject.getEndToken().columnNo
11149                                                                                                                + columnObject.getEndToken().getAstext().length(),
11150                                                                                                ModelBindingManager.getGlobalHash()));
11151                                                                errorInfo.fillInfo(this);
11152                                                                errorInfos.add(errorInfo);
11153                                                        } else {
11154                                                                TableColumn viewColumn = modelFactory.createViewColumn(viewModel, viewColumnName, i,
11155                                                                                determined);
11156                                                                if(determined) {
11157                                                                        appendTableColumnToSQLEnv(viewModel, viewColumn);
11158                                                                }
11159                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11160                                                                relation.setEffectType(EffectType.create_view);
11161                                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
11162                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11163                                                                relation.setProcess(process);
11164                                                        }
11165                                                }
11166                                        } else if (resultColumn.getColumnObject() instanceof TObjectName) {
11167                                                TableColumn viewColumn = modelFactory.createViewColumn(viewModel,
11168                                                                (TObjectName) resultColumn.getColumnObject(), i, determined);
11169                                                if(determined) {
11170                                                        appendTableColumnToSQLEnv(viewModel, viewColumn);
11171                                                }
11172                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11173                                                relation.setEffectType(EffectType.create_view);
11174                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
11175                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11176                                                relation.setProcess(process);
11177                                        }
11178                                }
11179                                if (!resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
11180                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
11181                                        impactRelation.setEffectType(EffectType.create_view);
11182                                        impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
11183                                                        resultSetModel.getRelationRows()));
11184                                        impactRelation.setTarget(
11185                                                        new RelationRowsRelationshipElement<TableRelationRows>(viewModel.getRelationRows()));
11186                                }
11187                        } else if (subquery != null && subquery.isCombinedQuery()) {
11188                                SelectSetResultSet resultSetModel = (SelectSetResultSet) modelManager.getModel(subquery);
11189
11190                                boolean determined = false;
11191                                if (!containStarColumn(resultSetModel)) {
11192                                        viewModel.setCreateTable(true);
11193                                        viewModel.setFromDDL(true);
11194                                        determined = true;
11195                                }
11196
11197                                for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
11198                                        ResultColumn resultColumn = resultSetModel.getColumns().get(i);
11199
11200                                        if (resultColumn.getColumnObject() instanceof TResultColumn) {
11201                                                TResultColumn columnObject = ((TResultColumn) resultColumn.getColumnObject());
11202
11203                                                TAliasClause alias = columnObject.getAliasClause();
11204                                                if (alias != null && alias.getAliasName() != null) {
11205                                                        TableColumn viewColumn = modelFactory.createViewColumn(viewModel, alias.getAliasName(), i,
11206                                                                        determined);
11207                                                        if(determined) {
11208                                                                appendTableColumnToSQLEnv(viewModel, viewColumn);
11209                                                        }
11210                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11211                                                        relation.setEffectType(EffectType.create_view);
11212                                                        relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
11213                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11214                                                        relation.setProcess(process);
11215                                                } else if (columnObject.getFieldAttr() != null) {
11216                                                        TableColumn viewColumn = modelFactory.createViewColumn(viewModel,
11217                                                                        columnObject.getFieldAttr(), i, determined);
11218                                                        if(determined) {
11219                                                                appendTableColumnToSQLEnv(viewModel, viewColumn);
11220                                                        }
11221                                                        ResultColumn column = (ResultColumn) modelManager.getModel(resultColumn.getColumnObject());
11222                                                        if (column != null && !column.getStarLinkColumns().isEmpty()) {
11223                                                                viewColumn.bindStarLinkColumns(column.getStarLinkColumns());
11224                                                        }
11225                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11226                                                        relation.setEffectType(EffectType.create_view);
11227                                                        relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
11228                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11229                                                        relation.setProcess(process);
11230                                                } else if (resultColumn.getAlias() != null && columnObject.getExpr()
11231                                                                .getExpressionType() == EExpressionType.sqlserver_proprietary_column_alias_t) {
11232                                                        TableColumn viewColumn = modelFactory.createViewColumn(viewModel,
11233                                                                        columnObject.getExpr().getLeftOperand().getObjectOperand(), i, determined);
11234                                                        if(determined) {
11235                                                                appendTableColumnToSQLEnv(viewModel, viewColumn);
11236                                                        }
11237                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11238                                                        relation.setEffectType(EffectType.create_view);
11239                                                        relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
11240                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11241                                                        relation.setProcess(process);
11242                                                } else {
11243                                                        TGSqlParser parser = columnObject.getGsqlparser();
11244                                                        TObjectName viewColumnName = parser
11245                                                                        .parseObjectName(generateQuotedName(parser, columnObject.toString()));
11246                                                        TableColumn viewColumn = modelFactory.createViewColumn(viewModel, viewColumnName, i,
11247                                                                        determined);
11248                                                        if(determined) {
11249                                                                appendTableColumnToSQLEnv(viewModel, viewColumn);
11250                                                        }
11251                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11252                                                        relation.setEffectType(EffectType.create_view);
11253                                                        relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
11254                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11255                                                        relation.setProcess(process);
11256                                                }
11257                                        } else if (resultColumn.getColumnObject() instanceof TObjectName) {
11258                                                TableColumn viewColumn = modelFactory.createViewColumn(viewModel,
11259                                                                (TObjectName) resultColumn.getColumnObject(), i, determined);
11260                                                if(viewColumn!=null) {
11261                                                        if(determined) {
11262                                                                appendTableColumnToSQLEnv(viewModel, viewColumn);
11263                                                        }
11264                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11265                                                        relation.setEffectType(EffectType.create_view);
11266                                                        relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
11267                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11268                                                        relation.setProcess(process);
11269                                                }
11270                                        }
11271                                }
11272                                if (!resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
11273                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
11274                                        impactRelation.setEffectType(EffectType.create_view);
11275                                        impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
11276                                                        resultSetModel.getRelationRows()));
11277                                        impactRelation.setTarget(
11278                                                        new RelationRowsRelationshipElement<TableRelationRows>(viewModel.getRelationRows()));
11279                                }
11280                        }
11281                }
11282        }
11283
11284        private void setSourceShowStar(Object resultColumn) {
11285                for (Relationship relation : modelManager.getRelations()) {
11286                        Collection<RelationshipElement<?>> sources = (Collection<RelationshipElement<?>>)relation.getSources();
11287                        if (relation.getTarget().getElement() == resultColumn && sources != null) {
11288
11289                                ((AbstractRelationship) relation).setShowStarRelation(true);
11290                                for (RelationshipElement<?> source : sources) {
11291                                        Object column = source.getElement();
11292                                        if (column instanceof TableColumn) {
11293                                                if (((TableColumn) column).isShowStar())
11294                                                        continue;
11295                                                ((TableColumn) column).setShowStar(true);
11296                                        }
11297                                        if (column instanceof ResultColumn) {
11298                                                if (((ResultColumn) column).isShowStar())
11299                                                        continue;
11300                                                ((ResultColumn) column).setShowStar(true);
11301                                        }
11302                                        setSourceShowStar(column);
11303                                }
11304                        }
11305                }
11306        }
11307
11308        private String generateQuotedName(TGSqlParser parser, String name) {
11309                return "\"" + name + "\"";
11310        }
11311
11312        private void appendRelations(dataflow dataflow) {
11313                Relationship[] relations = modelManager.getRelations();
11314
11315                appendRelation(dataflow, relations, DataFlowRelationship.class);
11316                appendRelation(dataflow, relations, IndirectImpactRelationship.class);
11317                appendRecordSetRelation(dataflow, relations);
11318                appendCallRelation(dataflow, relations);
11319                appendRelation(dataflow, relations, ImpactRelationship.class);
11320                appendRelation(dataflow, relations, JoinRelationship.class);
11321                appendRelation(dataflow, relations, ERRelationship.class);
11322                appendRelation(dataflow, relations, CrudRelationship.class);
11323        }
11324        
11325        
11326        private relationship cloneRelationshipWithSingleSource(relationship orig, sourceColumn src) {
11327                relationship rel = new relationship();
11328                rel.setType(orig.getType());
11329                rel.setEffectType(orig.getEffectType());
11330                rel.setId(orig.getId() + "_" + src.getColumn());
11331                rel.setSqlHash(orig.getSqlHash());
11332                rel.setSqlComment(orig.getSqlComment());
11333                rel.setProcessId(orig.getProcessId());
11334                rel.setProcessType(orig.getProcessType());
11335                rel.setFunction(orig.getFunction());
11336                rel.setProcedureId(orig.getProcedureId());
11337
11338                targetColumn t = new targetColumn();
11339                t.setId(orig.getTarget().getId());
11340                t.setColumn(orig.getTarget().getColumn());
11341                t.setParent_id(orig.getTarget().getParent_id());
11342                t.setParent_name(orig.getTarget().getParent_name());
11343                t.setParent_alias(orig.getTarget().getParent_alias());
11344                t.setCoordinate(orig.getTarget().getCoordinate());
11345                rel.setTarget(t);
11346
11347                sourceColumn s = new sourceColumn();
11348                s.setId(src.getId());
11349                s.setColumn(src.getColumn());
11350                s.setParent_id(src.getParent_id());
11351                s.setParent_name(src.getParent_name());
11352                s.setCoordinate(src.getCoordinate());
11353                rel.addSource(s);
11354                return rel;
11355        }
11356
11357        private Set<String> appendStarColumns = new HashSet<String>();
11358        private void appendRelation(dataflow dataflow, Relationship[] relations, Class<? extends Relationship> clazz) {
11359                for (int i = 0; i < relations.length; i++) {
11360                        AbstractRelationship relation = (AbstractRelationship) relations[i];
11361                        if (relation.getClass() == clazz) {
11362                                if (relation.getSources() == null || relation.getTarget() == null) {
11363                                        continue;
11364                                }
11365                                Object targetElement = relation.getTarget().getElement();
11366                                TObjectName targetColumnName = null;
11367                                if(relation.getTarget() instanceof ResultColumnRelationshipElement) {
11368                                        targetColumnName = ((ResultColumnRelationshipElement) relation.getTarget()).getColumnName();
11369                                }
11370                                if (targetElement instanceof ResultColumn) {
11371                                        ResultColumn targetColumn = (ResultColumn) targetElement;
11372                                        if (!targetColumn.isPseduo()) 
11373                                        {
11374                                                if (targetColumnName == null) 
11375                                                {
11376                                                        if ("*".equals(targetColumn.getName())) {
11377                                                                updateResultColumnStarLinks(dataflow, relation, -1);
11378                                                        }
11379
11380                                                        if (targetColumn.hasStarLinkColumn()) {
11381                                                                for (int j = 0; j < targetColumn.getStarLinkColumnNames().size(); j++) {
11382                                                                        appendStarRelation(dataflow, relation, j);
11383                                                                }
11384
11385                                                                if (!containsStar(relation.getSources())) {
11386                                                                        continue;
11387                                                                }
11388                                                        }
11389                                                }
11390                                                else {
11391                                                        String columnName = DlineageUtil.getColumnName(targetColumnName);
11392                                                        int index = targetColumn.getStarLinkColumnNames().indexOf(columnName);
11393                                                        if (index != -1) {
11394                                                                int size = targetColumn.getStarLinkColumnNames().size();
11395                                                                if ("*".equals(targetColumn.getName())) {
11396                                                                        if (appendStarColumns.contains(columnName)) {
11397                                                                                updateResultColumnStarLinks(dataflow, relation, index);
11398                                                                        }
11399                                                                        else {
11400                                                                                updateResultColumnStarLinks(dataflow, relation, -1);
11401                                                                                appendStarColumns.add(columnName);
11402                                                                        }
11403                                                                }
11404                                                                appendStarRelation(dataflow, relation, index);
11405                                                                for(int j= size; j < targetColumn.getStarLinkColumnNames().size(); j++) {
11406                                                                        appendStarRelation(dataflow, relation, j);
11407                                                                }
11408                                                                if (!containsStar(relation.getSources())) {
11409                                                                        continue;
11410                                                                }
11411                                                        }
11412                                                }
11413                                        }
11414                                } else if (targetElement instanceof TableColumn) {
11415                                        TableColumn targetColumn = (TableColumn) targetElement;
11416                                        if (!targetColumn.isPseduo()) {
11417                                                if ("*".equals(targetColumn.getName())) {
11418                                                        updateTableColumnStarLinks(dataflow, relation);
11419                                                }
11420
11421                                                if (targetColumn.hasStarLinkColumn() && !targetColumn.isVariant()
11422                                                                && targetColumn.isExpandStar()) {
11423                                                        for (int j = 0; j < targetColumn.getStarLinkColumnNames().size(); j++) {
11424                                                                appendStarRelation(dataflow, relation, j);
11425                                                        }
11426                                                        if (!containsStar(relation.getSources())) {
11427                                                                continue;
11428                                                        }
11429                                                }
11430                                        }
11431                                }
11432
11433                                relationship relationElement = new relationship();
11434                                relationElement.setType(relation.getRelationshipType().name());
11435                                if (relation.getEffectType() != null) {
11436                                        relationElement.setEffectType(relation.getEffectType().name());
11437                                }
11438                                if (relation.getFunction() != null) {
11439                                        relationElement.setFunction(relation.getFunction());
11440                                }
11441                                relationElement.setSqlHash(relation.getSqlHash());
11442                                relationElement.setSqlComment(relation.getSqlComment());
11443
11444                                if (relation.getProcedureId() != null) {
11445                                        relationElement.setProcedureId(String.valueOf(relation.getProcedureId()));
11446                                }
11447                                relationElement.setId(String.valueOf(relation.getId()));
11448                                if (relation.getProcess() != null) {
11449                                        relationElement.setProcessId(String.valueOf(relation.getProcess().getId()));
11450                                        if (relation.getProcess().getGspObject() != null) {
11451                                                relationElement.setProcessType(relation.getProcess().getGspObject().sqlstatementtype.name());
11452                                        }
11453                                }
11454
11455                                if (relation.getPartition() != null) {
11456                                        relationElement.setPartition(relation.getPartition());
11457                                }
11458                                relationElement.setSqlHash(relation.getSqlHash());
11459                                relationElement.setSqlComment(relation.getSqlComment());
11460                                
11461                                if (relation.getProcedureId() != null) {
11462                                        relationElement.setProcedureId(String.valueOf(relation.getProcedureId()));
11463                                }
11464                                
11465                                if (relation instanceof JoinRelationship) {
11466                                        relationElement.setCondition(((JoinRelationship) relation).getJoinCondition());
11467                                        relationElement.setJoinType(((JoinRelationship) relation).getJoinType().name());
11468                                        relationElement.setClause(((JoinRelationship) relation).getJoinClauseType().name());
11469                                }
11470
11471                                if(relation instanceof ImpactRelationship){
11472                                        ImpactRelationship impactRelationship = (ImpactRelationship)relation;
11473                                        if(impactRelationship.getJoinClauseType()!=null){
11474                                                relationElement.setClause(impactRelationship.getJoinClauseType().name());
11475                                        }
11476                                }
11477
11478                                String targetName = null;
11479                                Object columnObject = null;
11480                                List<TObjectName> targetObjectNames = null;
11481
11482                                if (targetElement instanceof ResultSetRelationRows) {
11483                                        ResultSetRelationRows targetColumn = (ResultSetRelationRows) targetElement;
11484                                        targetColumn target = new targetColumn();
11485                                        target.setId(String.valueOf(targetColumn.getId()));
11486                                        target.setColumn(targetColumn.getName());
11487                                        target.setParent_id(String.valueOf(targetColumn.getHolder().getId()));
11488                                        target.setParent_name(getResultSetName(targetColumn.getHolder()));
11489                                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
11490                                                target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
11491                                                                + convertCoordinate(targetColumn.getEndPosition()));
11492                                        }
11493                                        if (relation instanceof RecordSetRelationship) {
11494                                                target.setFunction(((RecordSetRelationship) relation).getAggregateFunction());
11495                                        }
11496                                        target.setSource("system");
11497                                        targetName = targetColumn.getName();
11498                                        relationElement.setTarget(target);
11499                                } else if (targetElement instanceof TableRelationRows) {
11500                                        TableRelationRows targetColumn = (TableRelationRows) targetElement;
11501                                        targetColumn target = new targetColumn();
11502                                        target.setId(String.valueOf(targetColumn.getId()));
11503                                        target.setColumn(targetColumn.getName());
11504                                        target.setParent_id(String.valueOf(targetColumn.getHolder().getId()));
11505                                        target.setParent_name(getTableName(targetColumn.getHolder()));
11506                                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
11507                                                target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
11508                                                                + convertCoordinate(targetColumn.getEndPosition()));
11509                                        }
11510                                        if (relation instanceof RecordSetRelationship) {
11511                                                target.setFunction(((RecordSetRelationship) relation).getAggregateFunction());
11512                                        }
11513                                        target.setSource("system");
11514                                        targetName = targetColumn.getName();
11515                                        relationElement.setTarget(target);
11516                                } else if (targetElement instanceof ResultColumn) {
11517                                        ResultColumn targetColumn = (ResultColumn) targetElement;
11518                                        targetColumn target = new targetColumn();
11519                                        target.setId(String.valueOf(targetColumn.getId()));
11520                                        target.setColumn(targetColumn.getName());
11521                                        target.setStruct(targetColumn.isStruct());
11522                                        target.setParent_id(String.valueOf(targetColumn.getResultSet().getId()));
11523                                        target.setParent_name(getResultSetName(targetColumn.getResultSet()));
11524                                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
11525                                                target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
11526                                                                + convertCoordinate(targetColumn.getEndPosition()));
11527                                        }
11528                                        if (relation instanceof RecordSetRelationship) {
11529                                                target.setFunction(((RecordSetRelationship) relation).getAggregateFunction());
11530                                        }
11531                                        targetName = targetColumn.getName();
11532                                        if (((ResultColumn) targetColumn).getColumnObject() instanceof TResultColumn) {
11533                                                columnsInExpr visitor = new columnsInExpr();
11534                                                ((TResultColumn) ((ResultColumn) targetColumn).getColumnObject()).getExpr()
11535                                                                .inOrderTraverse(visitor);
11536                                                targetObjectNames = visitor.getObjectNames();
11537                                        }
11538
11539                                        if (targetElement instanceof FunctionResultColumn) {
11540                                                columnObject = ((FunctionResultColumn) targetElement).getColumnObject();
11541                                        }
11542                                        if(targetColumn.isPseduo()) {
11543                                                target.setSource("system");
11544                                        }
11545                                        relationElement.setTarget(target);
11546                                } else if (targetElement instanceof TableColumn) {
11547                                        TableColumn targetColumn = (TableColumn) targetElement;
11548                                        targetColumn target = new targetColumn();
11549                                        target.setId(String.valueOf(targetColumn.getId()));
11550                                        target.setColumn(targetColumn.getName());
11551                                        target.setStruct(targetColumn.isStruct());
11552                                        target.setParent_id(String.valueOf(targetColumn.getTable().getId()));
11553                                        target.setParent_name(getTableName(targetColumn.getTable()));
11554                                        if (relation.getTarget() instanceof TableColumnRelationshipElement) {
11555                                                target.setParent_alias(((TableColumnRelationshipElement) relation.getTarget()).getTableAlias());
11556                                        }
11557                                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
11558                                                target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
11559                                                                + convertCoordinate(targetColumn.getEndPosition()));
11560                                        }
11561                                        if (relation instanceof RecordSetRelationship) {
11562                                                target.setFunction(((RecordSetRelationship) relation).getAggregateFunction());
11563                                        }
11564                                        if(targetColumn.isPseduo()) {
11565                                                target.setSource("system");
11566                                        }
11567                                        targetName = targetColumn.getName();
11568                                        relationElement.setTarget(target);
11569                                } else if (targetElement instanceof Argument) {
11570                                        Argument targetColumn = (Argument) targetElement;
11571                                        targetColumn target = new targetColumn();
11572                                        target.setId(String.valueOf(targetColumn.getId()));
11573                                        target.setColumn(targetColumn.getName());
11574                                        target.setParent_id(String.valueOf(targetColumn.getProcedure().getId()));
11575                                        target.setParent_name(targetColumn.getProcedure().getName());
11576                                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
11577                                                target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
11578                                                                + convertCoordinate(targetColumn.getEndPosition()));
11579                                        }
11580                                        if (relation instanceof RecordSetRelationship) {
11581                                                target.setFunction(((RecordSetRelationship) relation).getAggregateFunction());
11582                                        }
11583                                        targetName = targetColumn.getName();
11584                                        relationElement.setTarget(target);
11585                                } else if (targetElement instanceof Table) {
11586                                        Table table = (Table) targetElement;
11587                                        targetColumn target = new targetColumn();
11588                                        target.setTarget_id(String.valueOf(table.getId()));
11589                                        target.setTarget_name(getTableName(table));
11590                                        if (table.getStartPosition() != null && table.getEndPosition() != null) {
11591                                                target.setCoordinate(convertCoordinate(table.getStartPosition()) + ","
11592                                                                + convertCoordinate(table.getEndPosition()));
11593                                        }
11594                                        relationElement.setTarget(target);
11595                                } else {
11596                                        continue;
11597                                }
11598
11599                                Collection<RelationshipElement<?>> sourceElements = relation.getSources();
11600                                if (sourceElements.size() == 0) {
11601                                        if(clazz == CrudRelationship.class && option.getAnalyzeMode() == AnalyzeMode.crud) {
11602                                                dataflow.getRelationships().add(relationElement);
11603                                        }
11604                                        continue;
11605                                }
11606
11607                                boolean append = false;
11608                                for (RelationshipElement<?> sourceItem: relation.getSources()) {
11609                                        Object sourceElement = sourceItem.getElement();
11610                                        TObjectName sourceColumnName = null;
11611                                        if (sourceItem instanceof ResultColumnRelationshipElement) {
11612                                                sourceColumnName = ((ResultColumnRelationshipElement) sourceItem).getColumnName();
11613                                        }
11614//                                      if (sourceItem instanceof TableColumnRelationElement
11615//                                                      && (((TableColumnRelationElement) sourceItem).getColumnIndex() != null)) {
11616//                                              TableColumnRelationElement tableColumnRelationElement = (TableColumnRelationElement) sourceItem;
11617//                                              sourceElement = tableColumnRelationElement.getElement().getTable().getColumns()
11618//                                                              .get(tableColumnRelationElement.getColumnIndex() + 1);
11619//                                      }
11620                                        if (sourceElement instanceof ResultColumn) {
11621                                                ResultColumn sourceColumn = (ResultColumn) sourceElement;
11622                                                if (sourceColumn.hasStarLinkColumn() && !relation.isShowStarRelation() && !sourceColumn.isPseduo()) {
11623                                                        List<String> sourceStarColumnNames = sourceColumn.getStarLinkColumnNames();
11624                                                        sourceColumn source = new sourceColumn();
11625                                                        if (targetObjectNames != null && !targetObjectNames.isEmpty()) {
11626
11627                                                                for (int k = 0; k < targetObjectNames.size(); k++) {
11628                                                                        String targetObjectName = getColumnName(targetObjectNames.get(k));
11629                                                                        if(sourceColumn.getResultSet()!=null && sourceColumn.getResultSet().getColumns()!=null) {
11630                                                                                boolean find = false;
11631                                                                                for(ResultColumn column: sourceColumn.getResultSet().getColumns()) {
11632                                                                                        if(column.getName().equalsIgnoreCase(targetObjectName)) {
11633                                                                                                source = new sourceColumn();
11634                                                                                                source.setId(String.valueOf(column.getId()));
11635                                                                                                source.setColumn(targetObjectName);
11636                                                                                                source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
11637                                                                                                source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
11638                                                                                                if (sourceColumn.getStartPosition() != null
11639                                                                                                                && sourceColumn.getEndPosition() != null) {
11640                                                                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition())
11641                                                                                                                        + "," + convertCoordinate(sourceColumn.getEndPosition()));
11642                                                                                                }
11643                                                                                                append = true;
11644                                                                                                relationElement.addSource(source);
11645                                                                                                find = true;
11646                                                                                                break;
11647                                                                                        }
11648                                                                                }
11649                                                                                if(find) {
11650                                                                                        continue;
11651                                                                                }
11652                                                                        }
11653                                                                        if (sourceColumn.getStarLinkColumns().containsKey(targetObjectName)) {
11654                                                                                source = new sourceColumn();
11655                                                                                source.setId(String.valueOf(sourceColumn.getId()) + "_"
11656                                                                                                + sourceStarColumnNames.indexOf(targetObjectName));
11657                                                                                source.setColumn(targetObjectName);
11658                                                                                source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
11659                                                                                source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
11660                                                                                if (sourceColumn.getStartPosition() != null
11661                                                                                                && sourceColumn.getEndPosition() != null) {
11662                                                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition())
11663                                                                                                        + "," + convertCoordinate(sourceColumn.getEndPosition()));
11664                                                                                }
11665                                                                                append = true;
11666                                                                                relationElement.addSource(source);
11667                                                                        } else {
11668                                                                                source = new sourceColumn();
11669                                                                                source.setId(String.valueOf(sourceColumn.getId()));
11670                                                                                source.setColumn(relationElement.getTarget().getColumn());
11671                                                                                source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
11672                                                                                source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
11673                                                                                if (sourceColumn.getStartPosition() != null
11674                                                                                                && sourceColumn.getEndPosition() != null) {
11675                                                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition())
11676                                                                                                        + "," + convertCoordinate(sourceColumn.getEndPosition()));
11677                                                                                }
11678                                                                                append = true;
11679                                                                                relationElement.addSource(source);
11680                                                                        }
11681                                                                }
11682                                                        } else {
11683                                                                if (columnObject instanceof TWhenClauseItemList) {
11684                                                                        TCaseExpression expr = (TCaseExpression)((FunctionResultColumn) targetElement).getResultSet().getGspObject();
11685                                                                        List<TExpression> directExpressions = new ArrayList<TExpression>();
11686                                                                        
11687//                                                                      TExpression inputExpr = expr.getInput_expr();
11688//                                                                      if (inputExpr != null) {
11689//                                                                              directExpressions.add(inputExpr);
11690//                                                                      }
11691                                                                        TExpression defaultExpr = expr.getElse_expr();
11692                                                                        if (defaultExpr != null) {
11693                                                                                directExpressions.add(defaultExpr);
11694                                                                        }
11695                                                                        TWhenClauseItemList list = expr.getWhenClauseItemList();
11696                                                                        for (int k = 0; k < list.size(); k++) {
11697                                                                                TWhenClauseItem element = list.getWhenClauseItem(k);
11698                                                                                directExpressions.add(element.getReturn_expr());
11699                                                                        }
11700                                                                        
11701                                                                        for (int k = 0; k < directExpressions.size(); k++) {
11702                                                                                columnsInExpr visitor = new columnsInExpr();
11703                                                                                directExpressions.get(k).inOrderTraverse(visitor);
11704                                                                                List<TObjectName> objectNames = visitor.getObjectNames();
11705                                                                                if (objectNames == null) {
11706                                                                                        continue;
11707                                                                                }
11708                                                                                for (int x = 0; x < objectNames.size(); x++) {
11709                                                                                        String objectName = getColumnName(objectNames.get(x));
11710                                                                                        if (sourceColumn.getStarLinkColumns().containsKey(objectName)) {
11711
11712                                                                                                if(sourceColumn.getResultSet()!=null && sourceColumn.getResultSet().getColumns()!=null) {
11713                                                                                                        boolean find = false;
11714                                                                                                        for(ResultColumn column: sourceColumn.getResultSet().getColumns()) {
11715                                                                                                                if(column.getName().equalsIgnoreCase(objectName)) {
11716                                                                                                                        source = new sourceColumn();
11717                                                                                                                        source.setId(String.valueOf(column.getId()));
11718                                                                                                                        source.setColumn(objectName);
11719                                                                                                                        source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
11720                                                                                                                        source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
11721                                                                                                                        if (sourceColumn.getStartPosition() != null
11722                                                                                                                                        && sourceColumn.getEndPosition() != null) {
11723                                                                                                                                source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition())
11724                                                                                                                                                + "," + convertCoordinate(sourceColumn.getEndPosition()));
11725                                                                                                                        }
11726                                                                                                                        append = true;
11727                                                                                                                        relationElement.addSource(source);
11728                                                                                                                        find = true;
11729                                                                                                                        break;
11730                                                                                                                }
11731                                                                                                        }
11732                                                                                                        if(find) {
11733                                                                                                                continue;
11734                                                                                                        }
11735                                                                                                }
11736                                                                                                
11737                                                                                                source.setId(String.valueOf(sourceColumn.getId()) + "_"
11738                                                                                                                + sourceStarColumnNames.indexOf(objectName));
11739                                                                                                source.setColumn(objectName);
11740                                                                                        } else {
11741                                                                                                source.setId(String.valueOf(sourceColumn.getId()));
11742                                                                                                source.setColumn(sourceColumn.getName());
11743                                                                                        }
11744                                                                                        source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
11745                                                                                        source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
11746                                                                                        if (sourceColumn.getStartPosition() != null
11747                                                                                                        && sourceColumn.getEndPosition() != null) {
11748                                                                                                source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition())
11749                                                                                                                + "," + convertCoordinate(sourceColumn.getEndPosition()));
11750                                                                                        }
11751                                                                                        append = true;
11752                                                                                        relationElement.addSource(source);
11753                                                                                }
11754                                                                        }
11755                                                                } else {
11756                                                                        String objectName = getColumnName(targetName);
11757                                                                        boolean find = false;
11758                                                                        
11759                                                                        if (!find && sourceColumn.getStarLinkColumns().containsKey(objectName)) {
11760                                                                                source.setId(String.valueOf(sourceColumn.getId()) + "_"
11761                                                                                                + sourceStarColumnNames.indexOf(objectName));
11762                                                                                source.setColumn(objectName);
11763                                                                                find = true;
11764                                                                        }
11765
11766                                                                        if (!find && sourceColumnName != null) {
11767                                                                                objectName = getColumnName(sourceColumnName);
11768                                                                                
11769                                                                                if(sourceColumn.getResultSet()!=null && sourceColumn.getResultSet().getColumns()!=null) {
11770                                                                                        for(ResultColumn column: sourceColumn.getResultSet().getColumns()) {
11771                                                                                                if(column.getName().equalsIgnoreCase(objectName)) {
11772                                                                                                        source = new sourceColumn();
11773                                                                                                        source.setId(String.valueOf(column.getId()));
11774                                                                                                        source.setColumn(objectName);
11775                                                                                                        source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
11776                                                                                                        source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
11777                                                                                                        if (sourceColumn.getStartPosition() != null
11778                                                                                                                        && sourceColumn.getEndPosition() != null) {
11779                                                                                                                source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition())
11780                                                                                                                                + "," + convertCoordinate(sourceColumn.getEndPosition()));
11781                                                                                                        }
11782                                                                                                        append = true;
11783                                                                                                        relationElement.addSource(source);
11784                                                                                                        find = true;
11785                                                                                                        break;
11786                                                                                                }
11787                                                                                        }
11788                                                                                        if(find) {
11789                                                                                                continue;
11790                                                                                        }
11791                                                                                }
11792                                                                                
11793                                                                                if (sourceColumn.getStarLinkColumns().containsKey(objectName)) {
11794                                                                                        source.setId(String.valueOf(sourceColumn.getId()) + "_"
11795                                                                                                        + sourceStarColumnNames.indexOf(objectName));
11796                                                                                        source.setColumn(objectName);
11797                                                                                        find = true;
11798                                                                                }
11799                                                                        }
11800                                                                        
11801                                                                        if (!find && sourceItem instanceof ResultColumnRelationshipElement) {
11802                                                                                int starIndex = ((ResultColumnRelationshipElement) sourceItem)
11803                                                                                                .getStarIndex();
11804                                                                                if (starIndex > -1 && sourceColumn.getStarLinkColumnList().size() > starIndex) {
11805                                                                                        objectName = getColumnName(sourceColumn.getStarLinkColumnList().get(starIndex));
11806                                                                                        source.setId(String.valueOf(sourceColumn.getId()) + "_"
11807                                                                                                        + starIndex);
11808                                                                                        source.setColumn(objectName);
11809                                                                                        find = true;
11810                                                                                }
11811                                                                        }
11812
11813                                                                        if (!find) {
11814                                                                                source.setId(String.valueOf(sourceColumn.getId()));
11815                                                                                source.setColumn(sourceColumn.getName());
11816                                                                        }
11817
11818                                                                        source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
11819                                                                        source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
11820                                                                        if (sourceColumn.getStartPosition() != null
11821                                                                                        && sourceColumn.getEndPosition() != null) {
11822                                                                                source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
11823                                                                                                + convertCoordinate(sourceColumn.getEndPosition()));
11824                                                                        }
11825                                                                        append = true;
11826                                                                        relationElement.addSource(source);
11827                                                                }
11828                                                        }
11829
11830                                                } else {
11831                                                        sourceColumn source = new sourceColumn();
11832                                                        source.setId(String.valueOf(sourceColumn.getId()));
11833                                                        source.setColumn(sourceColumn.getName());
11834                                                        source.setStruct(sourceColumn.isStruct());
11835                                                        source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
11836                                                        source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
11837                                                        if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
11838                                                                source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
11839                                                                                + convertCoordinate(sourceColumn.getEndPosition()));
11840                                                        }
11841                                                        append = true;
11842                                                        if (sourceItem.getTransforms() != null) {
11843                                                                for (Transform transform : sourceItem.getTransforms()) {
11844                                                                        source.addTransform(transform);
11845                                                                }
11846                                                        }
11847                                                        if(sourceColumn.isPseduo()) {
11848                                                                source.setSource("system");
11849                                                        }
11850                                                        relationElement.addSource(source);
11851                                                }
11852                                        } else if (sourceElement instanceof TableColumn) {
11853                                                TableColumn sourceColumn = (TableColumn) sourceElement;
11854                                                if (!sourceColumn.isPseduo() && sourceColumn.hasStarLinkColumn()
11855                                                                && sourceItem instanceof TableColumnRelationshipElement) {
11856                                                        sourceColumn source = new sourceColumn();
11857                                                        boolean find = false;
11858                                                        if (((TableColumnRelationshipElement) sourceItem).getColumnIndex() != null) {
11859                                                                int columnIndex = ((TableColumnRelationshipElement) sourceItem).getColumnIndex();
11860                                                                if (sourceColumn.getStarLinkColumns().size() > columnIndex) {
11861                                                                        source = new sourceColumn();
11862                                                                        source.setId(String.valueOf(sourceColumn.getId()) + "_" + columnIndex);
11863                                                                        String targetObjectName = getColumnName(
11864                                                                                        sourceColumn.getStarLinkColumnList().get(columnIndex));
11865                                                                        source.setColumn(targetObjectName);
11866                                                                        source.setParent_id(String.valueOf(sourceColumn.getTable().getId()));
11867                                                                        source.setParent_name(sourceColumn.getTable().getName());
11868                                                                        if (sourceItem instanceof TableColumnRelationshipElement) {
11869                                                                                source.setParent_alias(
11870                                                                                                ((TableColumnRelationshipElement) sourceItem).getTableAlias());
11871                                                                        }
11872                                                                        if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
11873                                                                                source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
11874                                                                                                + convertCoordinate(sourceColumn.getEndPosition()));
11875                                                                        }
11876                                                                        append = true;
11877                                                                        relationElement.addSource(source);
11878                                                                        find = true;
11879                                                                }
11880                                                        }
11881
11882                                                        if (!find) {
11883                                                                String objectName = getColumnName(targetName);
11884
11885                                                                table tableElement = null;
11886                                                                if (dataflow.getTables() != null) {
11887                                                                        for (table t : dataflow.getTables()) {
11888                                                                                if (t.getId().equals(String.valueOf(sourceColumn.getTable().getId()))) {
11889                                                                                        tableElement = t;
11890                                                                                        break;
11891                                                                                }
11892                                                                        }
11893                                                                }
11894                                                                if (tableElement == null && dataflow.getViews() != null) {
11895                                                                        for (table t : dataflow.getViews()) {
11896                                                                                if (t.getId().equals(String.valueOf(sourceColumn.getTable().getId()))) {
11897                                                                                        tableElement = t;
11898                                                                                        break;
11899                                                                                }
11900                                                                        }
11901                                                                }
11902                                                                if (tableElement == null && dataflow.getVariables() != null) {
11903                                                                        for (table t : dataflow.getVariables()) {
11904                                                                                if (t.getId().equals(String.valueOf(sourceColumn.getTable().getId()))) {
11905                                                                                        tableElement = t;
11906                                                                                        break;
11907                                                                                }
11908                                                                        }
11909                                                                }
11910
11911                                                                if(tableElement!=null) {
11912                                                                        for (column column : tableElement.getColumns()) {
11913                                                                                if (column.getName() != null && column.getName().equalsIgnoreCase(objectName)) {
11914                                                                                        source = new sourceColumn();
11915                                                                                        source.setId(String.valueOf(column.getId()));
11916                                                                                        source.setColumn(objectName);
11917                                                                                        source.setParent_id(String.valueOf(sourceColumn.getTable().getId()));
11918                                                                                        source.setParent_name(sourceColumn.getTable().getName());
11919                                                                                        if (sourceItem instanceof TableColumnRelationshipElement) {
11920                                                                                                source.setParent_alias(
11921                                                                                                                ((TableColumnRelationshipElement) sourceItem).getTableAlias());
11922                                                                                        }
11923                                                                                        if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
11924                                                                                                source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
11925                                                                                                                + convertCoordinate(sourceColumn.getEndPosition()));
11926                                                                                        }
11927                                                                                        if (sourceItem.getTransforms() != null) {
11928                                                                                                for (Transform transform : sourceItem.getTransforms()) {
11929                                                                                                        source.addTransform(transform);
11930                                                                                                }
11931                                                                                        }
11932                                                                                        if (sourceColumn.getCandidateParents() != null) {
11933                                                                                                for(Object item: sourceColumn.getCandidateParents()) {
11934                                                                                                        candidateTable candidateParent = new candidateTable();
11935                                                                                                        if(item instanceof Table) {
11936                                                                                                                candidateParent.setId(String.valueOf(((Table)item).getId()));
11937                                                                                                                candidateParent.setName(getTableName((Table)item));
11938                                                                                                                source.addCandidateParent(candidateParent);
11939                                                                                                        }
11940                                                                                                        else if(item instanceof ResultSet) {
11941                                                                                                                candidateParent.setId(String.valueOf(((ResultSet)item).getId()));
11942                                                                                                                candidateParent.setName(getResultSetName((ResultSet)item));
11943                                                                                                                source.addCandidateParent(candidateParent);
11944                                                                                                        }
11945                                                                                                }
11946                                                                                        }
11947                                                                                        append = true;
11948                                                                                        relationElement.addSource(source);
11949                                                                                        find = true;
11950                                                                                        break;
11951                                                                                }
11952                                                                        }
11953                                                                }
11954                                                        }
11955
11956                                                        if(!find) {
11957                                                                source = new sourceColumn();
11958                                                                source.setId(String.valueOf(sourceColumn.getId()));
11959                                                                source.setColumn(sourceColumn.getName());
11960                                                                source.setParent_id(String.valueOf(sourceColumn.getTable().getId()));
11961                                                                source.setParent_name(sourceColumn.getTable().getName());
11962                                                                if (sourceItem instanceof TableColumnRelationshipElement) {
11963                                                                        source.setParent_alias(
11964                                                                                        ((TableColumnRelationshipElement) sourceItem).getTableAlias());
11965                                                                }
11966                                                                if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
11967                                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
11968                                                                                        + convertCoordinate(sourceColumn.getEndPosition()));
11969                                                                }
11970                                                                if (sourceItem.getTransforms() != null) {
11971                                                                        for (Transform transform : sourceItem.getTransforms()) {
11972                                                                                source.addTransform(transform);
11973                                                                        }
11974                                                                }
11975                                                                if (sourceColumn.getCandidateParents() != null) {
11976                                                                        for(Object item: sourceColumn.getCandidateParents()) {
11977                                                                                candidateTable candidateParent = new candidateTable();
11978                                                                                if(item instanceof Table) {
11979                                                                                        candidateParent.setId(String.valueOf(((Table)item).getId()));
11980                                                                                        candidateParent.setName(getTableName((Table)item));
11981                                                                                        source.addCandidateParent(candidateParent);
11982                                                                                }
11983                                                                                else if(item instanceof ResultSet) {
11984                                                                                        candidateParent.setId(String.valueOf(((ResultSet)item).getId()));
11985                                                                                        candidateParent.setName(getResultSetName((ResultSet)item));
11986                                                                                        source.addCandidateParent(candidateParent);
11987                                                                                }
11988                                                                        }
11989                                                                }
11990                                                                append = true;
11991                                                                relationElement.addSource(source);
11992                                                        }
11993
11994                                                } else {
11995                                                        sourceColumn source = new sourceColumn();
11996                                                        source.setId(String.valueOf(sourceColumn.getId()));
11997                                                        source.setColumn(sourceColumn.getName());
11998                                                        source.setStruct(sourceColumn.isStruct());
11999                                                        source.setParent_id(String.valueOf(sourceColumn.getTable().getId()));
12000                                                        source.setParent_name(getTableName(sourceColumn.getTable()));
12001                                                        if (sourceItem instanceof TableColumnRelationshipElement) {
12002                                                                source.setParent_alias(
12003                                                                                ((TableColumnRelationshipElement) sourceItem).getTableAlias());
12004                                                        }
12005                                                        if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
12006                                                                source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
12007                                                                                + convertCoordinate(sourceColumn.getEndPosition()));
12008                                                        }
12009                                                        if (sourceItem.getTransforms() != null) {
12010                                                                for (Transform transform : sourceItem.getTransforms()) {
12011                                                                        source.addTransform(transform);
12012                                                                }
12013                                                        }
12014                                                        if(sourceColumn.isPseduo()) {
12015                                                                source.setSource("system");
12016                                                        }
12017                                                        if (sourceColumn.getCandidateParents() != null) {
12018                                                                for(Object item: sourceColumn.getCandidateParents()) {
12019                                                                        candidateTable candidateParent = new candidateTable();
12020                                                                        if(item instanceof Table) {
12021                                                                                candidateParent.setId(String.valueOf(((Table)item).getId()));
12022                                                                                candidateParent.setName(getTableName((Table)item));
12023                                                                                source.addCandidateParent(candidateParent);
12024                                                                        }
12025                                                                        else if(item instanceof ResultSet) {
12026                                                                                candidateParent.setId(String.valueOf(((ResultSet)item).getId()));
12027                                                                                candidateParent.setName(getResultSetName((ResultSet)item));
12028                                                                                source.addCandidateParent(candidateParent);
12029                                                                        }
12030                                                                }
12031                                                        }
12032                                                        append = true;
12033                                                        relationElement.addSource(source);
12034                                                }
12035                                        } else if (sourceElement instanceof Argument) {
12036                                                Argument sourceColumn = (Argument) sourceElement;
12037                                                sourceColumn source = new sourceColumn();
12038                                                source.setId(String.valueOf(sourceColumn.getId()));
12039                                                source.setColumn(sourceColumn.getName());
12040                                                source.setParent_id(String.valueOf(sourceColumn.getProcedure().getId()));
12041                                                source.setParent_name(sourceColumn.getProcedure().getName());
12042                                                if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
12043                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
12044                                                                        + convertCoordinate(sourceColumn.getEndPosition()));
12045                                                }
12046                                                append = true;
12047                                                relationElement.addSource(source);
12048                                        } else if (sourceElement instanceof TableRelationRows) {
12049                                                TableRelationRows sourceColumn = (TableRelationRows) sourceElement;
12050                                                sourceColumn source = new sourceColumn();
12051                                                source.setId(String.valueOf(sourceColumn.getId()));
12052                                                source.setColumn(sourceColumn.getName());
12053                                                source.setParent_id(String.valueOf(sourceColumn.getHolder().getId()));
12054                                                source.setParent_name(getTableName(sourceColumn.getHolder()));
12055                                                if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
12056                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
12057                                                                        + convertCoordinate(sourceColumn.getEndPosition()));
12058                                                }
12059                                                source.setSource("system");
12060                                                append = true;
12061                                                relationElement.addSource(source);
12062                                        } else if (sourceElement instanceof ResultSetRelationRows) {
12063                                                ResultSetRelationRows sourceColumn = (ResultSetRelationRows) sourceElement;
12064                                                sourceColumn source = new sourceColumn();
12065                                                source.setId(String.valueOf(sourceColumn.getId()));
12066                                                source.setColumn(sourceColumn.getName());
12067                                                source.setParent_id(String.valueOf(sourceColumn.getHolder().getId()));
12068                                                source.setParent_name(getResultSetName(sourceColumn.getHolder()));
12069                                                if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
12070                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
12071                                                                        + convertCoordinate(sourceColumn.getEndPosition()));
12072                                                }
12073                                                source.setSource("system");
12074                                                append = true;
12075                                                relationElement.addSource(source);
12076                                        } else if (sourceElement instanceof Constant) {
12077                                                Constant sourceColumn = (Constant) sourceElement;
12078                                                sourceColumn source = new sourceColumn();
12079                                                source.setId(String.valueOf(sourceColumn.getId()));
12080                                                source.setColumn(sourceColumn.getName());
12081                                                source.setColumn_type("constant");
12082                                                if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
12083                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
12084                                                                        + convertCoordinate(sourceColumn.getEndPosition()));
12085                                                }
12086                                                append = true;
12087                                                relationElement.addSource(source);
12088                                        } else if (sourceElement instanceof Table) {
12089                                                Table table = (Table) sourceElement;
12090                                                sourceColumn source = new sourceColumn();
12091                                                source.setSource_id(String.valueOf(table.getId()));
12092                                                source.setSource_name(getTableName(table));
12093                                                if (table.getStartPosition() != null && table.getEndPosition() != null) {
12094                                                        source.setCoordinate(convertCoordinate(table.getStartPosition()) + ","
12095                                                                        + convertCoordinate(table.getEndPosition()));
12096                                                }
12097                                                append = true;
12098                                                relationElement.addSource(source);
12099                                        }
12100
12101                                        if (relation instanceof ImpactRelationship) {
12102                                                ESqlClause clause = getSqlClause(sourceItem);
12103                                                if (clause != null
12104                                                                && (relationElement.getSources() != null && !relationElement.getSources().isEmpty())) {
12105                                                        relationElement.getSources().get(relationElement.getSources().size() - 1)
12106                                                                        .setClauseType(clause.name());
12107                                                }
12108                                        }
12109                                }
12110                                if (append)
12111                                        dataflow.getRelationships().add(relationElement);
12112                        }
12113                }
12114        }
12115
12116        private boolean containsStar(Collection<RelationshipElement<?>> elements) {
12117                if (elements == null || elements.size() == 0)
12118                        return false;
12119
12120                for (RelationshipElement<?> element : elements) {
12121                        if (element.getElement() instanceof ResultColumn) {
12122                                ResultColumn object = (ResultColumn) element.getElement();
12123                                if (object.getName().endsWith("*") && object.isShowStar())
12124                                        return true;
12125                        } else if (element.getElement() instanceof TableColumn) {
12126                                TableColumn object = (TableColumn) element.getElement();
12127                                if (object.getName().endsWith("*") && object.isShowStar())
12128                                        return true;
12129                        }
12130                }
12131                return false;
12132        }
12133
12134        private Map<String, Set<String>> appendTableStarColumns = new HashMap<String, Set<String>>();
12135
12136        // Tracks column names that have explicit (non-star) sources per target star column.
12137        // Used across relationships to prevent phantom star expansions.
12138        // Key: target star column ID, Value: set of column names with explicit sources.
12139        private Map<Long, Set<String>> explicitStarTargetColumns = new HashMap<Long, Set<String>>();
12140        private void updateResultColumnStarLinks(dataflow dataflow, AbstractRelationship relation, int index) {
12141        try {
12142            if (option.getAnalyzeMode() == AnalyzeMode.crud) {
12143                return;
12144            }
12145
12146            ResultColumn targetColumn = (ResultColumn) relation.getTarget().getElement();
12147
12148            Collection<RelationshipElement<?>> sourceElements = (Collection<RelationshipElement<?>>) relation.getSources();
12149            if (sourceElements == null || sourceElements.size() == 0)
12150                return;
12151
12152            for (RelationshipElement<?> sourceItem : sourceElements) {
12153                Object sourceElement = sourceItem.getElement();
12154                if (sourceElement instanceof ResultColumn) {
12155                    ResultColumn source = (ResultColumn) sourceElement;
12156                    if (source.hasStarLinkColumn()) {
12157                        for (Map.Entry<String, Set<TObjectName>> item : source.getStarLinkColumns().entrySet()) {
12158                            if (!targetColumn.getStarLinkColumns().containsKey(item.getKey())) {
12159                                targetColumn.getStarLinkColumns().put(item.getKey(), new LinkedHashSet<TObjectName>());
12160                            }
12161                            targetColumn.getStarLinkColumns().get(item.getKey()).addAll(item.getValue());
12162                        }
12163                        if (!source.isShowStar()) {
12164                            targetColumn.setShowStar(false);
12165                            relation.setShowStarRelation(false);
12166                        }
12167                    } else if (!"*".equals(source.getName())) {
12168                        if (source.getColumnObject() instanceof TObjectName) {
12169                            if (source instanceof FunctionResultColumn) {
12170
12171                            } else {
12172                                targetColumn.bindStarLinkColumn((TObjectName) source.getColumnObject());
12173                            }
12174                        } else if (source.getColumnObject() instanceof TResultColumn) {
12175                            TResultColumn sourceColumn = (TResultColumn) source.getColumnObject();
12176                            if (sourceColumn.getAliasClause() != null) {
12177                                targetColumn.bindStarLinkColumn(sourceColumn.getAliasClause().getAliasName());
12178                            } else if (sourceColumn.getFieldAttr() != null) {
12179                                targetColumn.bindStarLinkColumn(sourceColumn.getFieldAttr());
12180                            } else {
12181                                TObjectName column = new TObjectName();
12182                                if (sourceColumn.getExpr().getExpressionType() == EExpressionType.typecast_t) {
12183                                    column.setString(sourceColumn.getExpr().getLeftOperand().toString());
12184                                } else {
12185                                    column.setString(sourceColumn.toString());
12186                                }
12187                                targetColumn.bindStarLinkColumn(column);
12188                            }
12189                        }
12190                    }
12191                } else if (sourceElement instanceof TableColumn && !targetColumn.isStruct()) {
12192                    TableColumn source = (TableColumn) sourceElement;
12193                    if (!source.isPseduo() && source.hasStarLinkColumn()) {
12194                                                for (Map.Entry<String, Set<TObjectName>> item : source.getStarLinkColumns().entrySet()) {
12195                                                        if (!targetColumn.getStarLinkColumns().containsKey(item.getKey())) {
12196                                                                targetColumn.getStarLinkColumns().put(item.getKey(),
12197                                                                                new LinkedHashSet<TObjectName>());
12198                                                        }
12199                                                        targetColumn.getStarLinkColumns().get(item.getKey()).addAll(item.getValue());
12200                                                }
12201                                                if ((source.getTable().isCreateTable() || source.getTable().hasSQLEnv())
12202                                                                && !source.isShowStar()) {
12203                                                        targetColumn.setShowStar(false);
12204                                                        relation.setShowStarRelation(false);
12205                                                }
12206                    } else if (!"*".equals(source.getName())) {
12207                        targetColumn.bindStarLinkColumn(source.getColumnObject());
12208                    }
12209                }
12210            }
12211
12212            if (targetColumn.hasStarLinkColumn()) {
12213                table resultSetElement = null;
12214                for (table t : dataflow.getResultsets()) {
12215                    if (t.getId().equals(String.valueOf(targetColumn.getResultSet().getId()))) {
12216                        resultSetElement = t;
12217                        break;
12218                    }
12219                }
12220
12221                int starColumnCount = 0;
12222                for (column item : resultSetElement.getColumns()) {
12223                    if (item.getName() != null && item.getName().endsWith("*")) {
12224                        starColumnCount += 1;
12225                    }
12226                }
12227
12228                if (resultSetElement != null && starColumnCount <= 1) {
12229                    List<String> columns = targetColumn.getStarLinkColumnNames();
12230                    if (index == -1) {
12231                        for (int k = 0; k < columns.size(); k++) {
12232                            String columnName = columns.get(k);
12233                            String id = String.valueOf(targetColumn.getId()) + "_" + k;
12234                            if (appendTableStarColumns.containsKey(resultSetElement.getId()) && appendTableStarColumns.get(resultSetElement.getId()).contains(id)) {
12235                                continue;
12236                            }
12237                            column columnElement = new column();
12238                            columnElement.setId(id);
12239                            columnElement.setName(columnName);
12240                            if (targetColumn.isFunction()) {
12241                                columnElement.setIsFunction(String.valueOf(targetColumn.isFunction()));
12242                            }
12243                            if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
12244                                columnElement.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
12245                                        + convertCoordinate(targetColumn.getEndPosition()));
12246                            }
12247
12248                            if (targetColumn.getResultSet() != null && targetColumn.getResultSet().getColumns() != null) {
12249                                boolean find = false;
12250                                for (int i = 0; i < targetColumn.getResultSet().getColumns().size(); i++) {
12251                                    ResultColumn columnModel = targetColumn.getResultSet().getColumns().get(i);
12252                                    if (DlineageUtil.getIdentifierNormalColumnName(columnModel.getName())
12253                                            .equals(columnName)) {
12254                                        find = true;
12255                                        break;
12256                                    }
12257                                }
12258                                if (find) {
12259                                    continue;
12260                                }
12261                            }
12262
12263                            if (!resultSetElement.getColumns().contains(columnElement)) {
12264                                resultSetElement.getColumns().add(columnElement);
12265                                appendTableStarColumns.putIfAbsent(resultSetElement.getId(), new HashSet<String>());
12266                                appendTableStarColumns.get(resultSetElement.getId()).add(id);
12267                            }
12268                        }
12269                    } else {
12270                        int k = index;
12271                        String columnName = columns.get(k);
12272                        column columnElement = new column();
12273                        columnElement.setId(String.valueOf(targetColumn.getId()) + "_" + k);
12274                        columnElement.setName(columnName);
12275                        if (targetColumn.isFunction()) {
12276                            columnElement.setIsFunction(String.valueOf(targetColumn.isFunction()));
12277                        }
12278                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
12279                            columnElement.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
12280                                    + convertCoordinate(targetColumn.getEndPosition()));
12281                        }
12282
12283                        if (targetColumn.getResultSet() != null && targetColumn.getResultSet().getColumns() != null) {
12284                            boolean find = false;
12285                            for (int i = 0; i < targetColumn.getResultSet().getColumns().size(); i++) {
12286                                ResultColumn columnModel = targetColumn.getResultSet().getColumns().get(i);
12287                                if (DlineageUtil.getIdentifierNormalColumnName(columnModel.getName()).equals(columnName)) {
12288                                    find = true;
12289                                    break;
12290                                }
12291                            }
12292                            if (find) {
12293                                return;
12294                            }
12295                        }
12296
12297                        if (!resultSetElement.getColumns().contains(columnElement)) {
12298                            resultSetElement.getColumns().add(columnElement);
12299                        }
12300                    }
12301                }
12302            }
12303        } catch (Exception e) {
12304            logger.error("updateResultColumnStarLinks occurs unknown exceptions.", e);
12305        }
12306        }
12307
12308        private void updateTableColumnStarLinks(dataflow dataflow, AbstractRelationship relation) {
12309                TableColumn targetColumn = (TableColumn) relation.getTarget().getElement();             
12310                Collection<RelationshipElement<?>> sourceElements = (Collection<RelationshipElement<?>>) relation.getSources();
12311                if (sourceElements == null || sourceElements.size() == 0)
12312                        return;
12313
12314                TableColumn sourceStarColumn = null;
12315
12316                for (RelationshipElement<?> sourceItem: sourceElements) {
12317                        Object sourceElement = sourceItem.getElement();
12318                        if (sourceElement instanceof ResultColumn) {
12319                                ResultColumn source = (ResultColumn) sourceElement;
12320                                if(source.getResultSet() instanceof Function) {
12321                                        continue;
12322                                }
12323                                if (source.hasStarLinkColumn()) {
12324                                        for (Map.Entry<String, Set<TObjectName>> item : source.getStarLinkColumns().entrySet()) {
12325                                                if (!targetColumn.getStarLinkColumns().containsKey(item.getKey())) {
12326                                                        targetColumn.getStarLinkColumns().put(item.getKey(), new LinkedHashSet<TObjectName>());
12327                                                }
12328                                                targetColumn.getStarLinkColumns().get(item.getKey()).addAll(item.getValue());
12329                                        }
12330                                        if (!source.isShowStar()) {
12331                                                targetColumn.setShowStar(false);
12332                                                relation.setShowStarRelation(false);
12333                                        }
12334                                } else if (!"*".equals(source.getName())) {
12335                                        if (source.getColumnObject() instanceof TObjectName) {
12336                                                targetColumn.bindStarLinkColumn((TObjectName) source.getColumnObject());
12337                                        } else if (source.getColumnObject() instanceof TResultColumn) {
12338                                                if (((TResultColumn) source.getColumnObject()).getAliasClause() != null) {
12339                                                        TObjectName field = ((TResultColumn) source.getColumnObject()).getAliasClause()
12340                                                                        .getAliasName();
12341                                                        if (field != null) {
12342                                                                targetColumn.bindStarLinkColumn(field);
12343                                                        }
12344                                                } else {
12345                                                        TObjectName field = ((TResultColumn) source.getColumnObject()).getFieldAttr();
12346                                                        if (field != null) {
12347                                                                targetColumn.bindStarLinkColumn(field);
12348                                                        } else {
12349                                                                TObjectName column = new TObjectName();
12350                                                                if (((TResultColumn) source.getColumnObject()).getExpr()
12351                                                                                .getExpressionType() == EExpressionType.typecast_t) {
12352                                                                        column.setString(((TResultColumn) source.getColumnObject()).getExpr()
12353                                                                                        .getLeftOperand().toString());
12354                                                                } else {
12355                                                                        column.setString(((TResultColumn) source.getColumnObject()).toString());
12356                                                                }
12357                                                                targetColumn.bindStarLinkColumn(column);
12358                                                        }
12359                                                }
12360                                        }
12361                                }
12362                        } else if (sourceElement instanceof TableColumn) {
12363                                TableColumn source = (TableColumn) sourceElement;
12364                                if (source.hasStarLinkColumn()) {
12365                                        for (Map.Entry<String, Set<TObjectName>> item : source.getStarLinkColumns().entrySet()) {
12366                                                if (!targetColumn.getStarLinkColumns().containsKey(item.getKey())) {
12367                                                        targetColumn.getStarLinkColumns().put(item.getKey(), new LinkedHashSet<TObjectName>());
12368                                                }
12369                                                targetColumn.getStarLinkColumns().get(item.getKey()).addAll(item.getValue());
12370                                        }
12371                                        for (Map.Entry<String, Set<TObjectName>> item : targetColumn.getStarLinkColumns().entrySet()) {
12372                                                if (!source.getStarLinkColumns().containsKey(item.getKey())) {
12373                                                        source.getStarLinkColumns().put(item.getKey(), new LinkedHashSet<TObjectName>());
12374                                                }
12375                                                source.getStarLinkColumns().get(item.getKey()).addAll(item.getValue());
12376                                        }
12377
12378                                        sourceStarColumn = source;
12379
12380                                        if (source.getTable().isCreateTable() && !source.isShowStar()) {
12381                                                targetColumn.setShowStar(false);
12382                                                relation.setShowStarRelation(false);
12383                                        }
12384                                } else if (!"*".equals(source.getName())) {
12385                                        if (source.isStruct()) {
12386                                                targetColumn.bindStarLinkColumn(source.getColumnObject());
12387                                        }
12388                                        else {
12389                                                TObjectName objectName = new TObjectName();
12390                                                objectName.setString(DlineageUtil.getColumnNameOnly(source.getName()));
12391                                                targetColumn.bindStarLinkColumn(objectName);
12392                                        }
12393                                }
12394                        }
12395                }
12396
12397                if (targetColumn.hasStarLinkColumn()) {
12398                        table tableElement = null;
12399                        if (dataflow.getTables() != null) {
12400                                for (table t : dataflow.getTables()) {
12401                                        if (t.getId().equals(String.valueOf(targetColumn.getTable().getId()))) {
12402                                                tableElement = t;
12403                                                break;
12404                                        }
12405                                }
12406                        }
12407                        if (tableElement == null && dataflow.getViews() != null) {
12408                                for (table t : dataflow.getViews()) {
12409                                        if (t.getId().equals(String.valueOf(targetColumn.getTable().getId()))) {
12410                                                tableElement = t;
12411                                                break;
12412                                        }
12413                                }
12414                        }
12415                        if (tableElement == null && dataflow.getVariables() != null) {
12416                                for (table t : dataflow.getVariables()) {
12417                                        if (t.getId().equals(String.valueOf(targetColumn.getTable().getId()))) {
12418                                                tableElement = t;
12419                                                break;
12420                                        }
12421                                }
12422                        }
12423
12424                        if (tableElement != null) {
12425                                List<String> columns = new ArrayList<String>(targetColumn.getStarLinkColumns().keySet());
12426                                for (int k = 0; k < columns.size(); k++) {
12427                                        String columnName = columns.get(k);
12428                                        if (containStarColumn(targetColumn.getTable().getColumns(), columnName)) {
12429                                                continue;
12430                                        }
12431                                        column columnElement = new column();
12432                                        columnElement.setId(String.valueOf(targetColumn.getId()) + "_" + k);
12433                                        columnElement.setName(columnName);
12434                                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
12435                                                columnElement.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
12436                                                                + convertCoordinate(targetColumn.getEndPosition()));
12437                                        }
12438                                        if (!tableElement.getColumns().contains(columnElement)) {
12439                                                tableElement.getColumns().add(columnElement);
12440                                        }
12441                                }
12442                        }
12443                }
12444
12445                if (sourceStarColumn != null) {
12446                        table tableElement = null;
12447                        if (dataflow.getTables() != null) {
12448                                for (table t : dataflow.getTables()) {
12449                                        if (t.getId().equals(String.valueOf(sourceStarColumn.getTable().getId()))) {
12450                                                tableElement = t;
12451                                                break;
12452                                        }
12453                                }
12454                        }
12455                        if (tableElement == null && dataflow.getViews() != null) {
12456                                for (table t : dataflow.getViews()) {
12457                                        if (t.getId().equals(String.valueOf(sourceStarColumn.getTable().getId()))) {
12458                                                tableElement = t;
12459                                                break;
12460                                        }
12461                                }
12462                        }
12463                        if (tableElement == null && dataflow.getVariables() != null) {
12464                                for (table t : dataflow.getVariables()) {
12465                                        if (t.getId().equals(String.valueOf(sourceStarColumn.getTable().getId()))) {
12466                                                tableElement = t;
12467                                                break;
12468                                        }
12469                                }
12470                        }
12471
12472                        if (tableElement != null) {
12473                                List<String> columns = new ArrayList<String>(sourceStarColumn.getStarLinkColumns().keySet());
12474                                for (int k = 0; k < columns.size(); k++) {
12475                                        String columnName = columns.get(k);
12476                                        if (containStarColumn(sourceStarColumn.getTable().getColumns(), columnName)) {
12477                                                continue;
12478                                        }
12479                                        column columnElement = new column();
12480                                        columnElement.setId(sourceStarColumn.getId() + "_" + k);
12481                                        columnElement.setName(columnName);
12482                                        if (sourceStarColumn.getStartPosition() != null && sourceStarColumn.getEndPosition() != null) {
12483                                                columnElement.setCoordinate(convertCoordinate(sourceStarColumn.getStartPosition()) + ","
12484                                                                + convertCoordinate(sourceStarColumn.getEndPosition()));
12485                                        }
12486                                        if (!tableElement.getColumns().contains(columnElement)) {
12487                                                tableElement.getColumns().add(columnElement);
12488                                        }
12489                                }
12490                        }
12491                }
12492        }
12493
12494        private ESqlClause getSqlClause(RelationshipElement<?> relationshipElement) {
12495                if (relationshipElement instanceof TableColumnRelationshipElement) {
12496                        return ((TableColumnRelationshipElement) relationshipElement).getRelationLocation();
12497                } else if (relationshipElement instanceof ResultColumnRelationshipElement) {
12498                        return ((ResultColumnRelationshipElement) relationshipElement).getRelationLocation();
12499                }
12500                return null;
12501        }
12502
12503        private void appendStarRelation(dataflow dataflow, AbstractRelationship relation, int index) {
12504                if(option.getAnalyzeMode() == AnalyzeMode.crud) {
12505                        return;
12506                }
12507                
12508                Object targetElement = relation.getTarget().getElement();
12509
12510                relationship relationElement = new relationship();
12511                relationElement.setType(relation.getRelationshipType().name());
12512                if (relation.getEffectType() != null) {
12513                        relationElement.setEffectType(relation.getEffectType().name());
12514                }
12515                relationElement.setSqlHash(relation.getSqlHash());
12516                relationElement.setSqlComment(relation.getSqlComment());
12517
12518                if (relation.getProcedureId() != null) {
12519                        relationElement.setProcedureId(String.valueOf(relation.getProcedureId()));
12520                }
12521                relationElement.setId(String.valueOf(relation.getId()) + "_" + index);
12522                if (relation.getProcess() != null) {
12523                        relationElement.setProcessId(String.valueOf(relation.getProcess().getId()));
12524                        if (relation.getProcess().getGspObject() != null) {
12525                                relationElement.setProcessType(relation.getProcess().getGspObject().sqlstatementtype.name());
12526                        }
12527                }
12528                if (relation instanceof DataFlowRelationship) {
12529                        relationElement.setSqlHash(((DataFlowRelationship) relation).getSqlHash());
12530                        relationElement.setSqlComment(((DataFlowRelationship) relation).getSqlComment());
12531                
12532                        if (relation.getProcedureId() != null) {
12533                                relationElement.setProcedureId(String.valueOf(relation.getProcedureId()));
12534                        }
12535                }
12536                String targetName = "";
12537
12538                if (targetElement instanceof ResultColumn) {
12539                        ResultColumn targetColumn = (ResultColumn) targetElement;
12540
12541                        targetName = targetColumn.getStarLinkColumnNames().get(index);
12542
12543                        
12544                        targetColumn target = new targetColumn();
12545                        target.setId(String.valueOf(targetColumn.getId()) + "_" + index);
12546                        if(targetColumn.getResultSet()!=null && targetColumn.getResultSet().getColumns()!=null) {
12547                                for (int i = 0; i < targetColumn.getResultSet().getColumns().size(); i++) {
12548                                        ResultColumn columnModel = targetColumn.getResultSet().getColumns().get(i);
12549                                        if (DlineageUtil.getIdentifierNormalColumnName(columnModel.getName()).equals(targetName)) {
12550                                                target.setId(String.valueOf(columnModel.getId()));
12551                                                break;
12552                                        }
12553                                }
12554                        }
12555                        target.setColumn(targetName);
12556                        target.setStruct(targetColumn.isStruct());
12557                        target.setParent_id(String.valueOf(targetColumn.getResultSet().getId()));
12558                        target.setParent_name(getResultSetName(targetColumn.getResultSet()));
12559                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
12560                                target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
12561                                                + convertCoordinate(targetColumn.getEndPosition()));
12562                        }
12563                        relationElement.setTarget(target);
12564                } else if (targetElement instanceof TableColumn) {
12565                        TableColumn targetColumn = (TableColumn) targetElement;
12566
12567                        targetName = targetColumn.getStarLinkColumnNames().get(index);
12568
12569                        TableColumn tableColumn = searchTableColumn(targetColumn.getTable().getColumns(), targetName);
12570
12571                        targetColumn target = new targetColumn();
12572                        if (tableColumn == null) {
12573                                target.setId(targetColumn.getId() + "_" + index);
12574                        } else {
12575                                target.setId(String.valueOf(tableColumn.getId()));
12576                        }
12577                        target.setStruct(targetColumn.isStruct());
12578                        target.setColumn(targetName);
12579                        target.setParent_id(String.valueOf(targetColumn.getTable().getId()));
12580                        target.setParent_name(targetColumn.getTable().getName());
12581                        if (relation.getTarget() instanceof TableColumnRelationshipElement) {
12582                                target.setParent_alias(((TableColumnRelationshipElement) relation.getTarget()).getTableAlias());
12583                        }
12584                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
12585                                target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
12586                                                + convertCoordinate(targetColumn.getEndPosition()));
12587                        }
12588                        relationElement.setTarget(target);
12589                } else {
12590                        return;
12591                }
12592
12593                Collection<RelationshipElement<?>> sourceElements = (Collection<RelationshipElement<?>>) relation.getSources();
12594                if (sourceElements.size() == 0) {
12595                        return;
12596                }
12597
12598                String targetIdentifierName = DlineageUtil.getIdentifierNormalColumnName(targetName);
12599                if (targetIdentifierName == null) {
12600                        return;
12601                }
12602
12603                // Track explicit (non-star) source columns per target star column to prevent
12604                // phantom star expansions. Skip for UNION targets where branches contribute independently.
12605                long targetColumnId = -1;
12606                boolean isUnionTarget = false;
12607                if (targetElement instanceof ResultColumn) {
12608                        ResultColumn tc = (ResultColumn) targetElement;
12609                        targetColumnId = tc.getId();
12610                        isUnionTarget = (tc.getResultSet() instanceof SelectSetResultSet);
12611                } else if (targetElement instanceof TableColumn) {
12612                        targetColumnId = ((TableColumn) targetElement).getId();
12613                }
12614
12615                if (!isUnionTarget && targetColumnId != -1) {
12616                        // Check if any source in this relationship is a non-star source matching the target
12617                        // column name, AND whose parent does NOT also contribute a star source.
12618                        // This distinguishes:
12619                        //   - "b.col_a" (definitive: b only provides explicit cols, not *) => suppress star expansion
12620                        //   - "aTab.id" (not definitive: aTab also provides *) => don't suppress
12621                        boolean hasDefinitiveExplicitSource = hasDefinitiveNonStarSource(sourceElements, targetIdentifierName);
12622                        if (hasDefinitiveExplicitSource) {
12623                                if (!explicitStarTargetColumns.containsKey(targetColumnId)) {
12624                                        explicitStarTargetColumns.put(targetColumnId, new HashSet<String>());
12625                                }
12626                                explicitStarTargetColumns.get(targetColumnId).add(targetIdentifierName);
12627                        }
12628                }
12629
12630                // Column names that have explicit (non-star) sources for this target, across all relationships
12631                Set<String> targetExplicitNames = isUnionTarget ? null : explicitStarTargetColumns.get(targetColumnId);
12632
12633                for (RelationshipElement<?> sourceItem: sourceElements) {
12634                        Object sourceElement = sourceItem.getElement();
12635                        if (sourceElement instanceof ResultColumn) {
12636                                ResultColumn sourceColumn = (ResultColumn) sourceElement;
12637                                if (sourceColumn.hasStarLinkColumn()) {
12638                                        List<String> linkColumnNames = sourceColumn.getStarLinkColumnNames();
12639                                        int linkColumnNameSize = linkColumnNames.size();
12640                                        for (int k = 0; k < linkColumnNameSize; k++) {
12641                                                String sourceName = linkColumnNames.get(k);
12642                                                if (relation.getRelationshipType() == RelationshipType.fdd) {
12643                                                        if (!targetIdentifierName.equalsIgnoreCase(sourceName) && !"*".equals(sourceName))
12644                                                                continue;
12645                                                }
12646                                                // Skip star-expanded source when an explicit (non-star) source provides
12647                                                // the same column, either in this relationship or a prior one.
12648                                                if (targetExplicitNames != null && targetExplicitNames.contains(sourceName)) {
12649                                                        continue;
12650                                                }
12651                                                sourceColumn source = new sourceColumn();                                               
12652                                                
12653                                                boolean find = false;
12654                                                if(sourceColumn.getResultSet()!=null && sourceColumn.getResultSet().getColumns()!=null) {
12655                                                        for (int i = 0; i < sourceColumn.getResultSet().getColumns().size(); i++) {
12656                                                                ResultColumn columnModel = sourceColumn.getResultSet().getColumns().get(i);
12657                                                                if (DlineageUtil.getIdentifierNormalColumnName(columnModel.getName()).equals(sourceName)) {
12658                                                                        source.setId(String.valueOf(columnModel.getId()));
12659                                                                        find = true;
12660                                                                        break;
12661                                                                }
12662                                                        }
12663                                                }
12664                                                if(!find) {
12665                                                        source.setId(String.valueOf(sourceColumn.getId()) + "_" + k);
12666                                                }
12667                                                source.setColumn(sourceName);
12668                                                source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
12669                                                source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
12670                                                if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
12671                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
12672                                                                        + convertCoordinate(sourceColumn.getEndPosition()));
12673                                                }
12674                                                if (sourceItem.getTransforms() != null) {
12675                                                        for (Transform transform : sourceItem.getTransforms()) {
12676                                                                source.addTransform(transform);
12677                                                        }
12678                                                }
12679                                                relationElement.addSource(source);
12680                                        }
12681                                        if(relationElement.getSources().isEmpty()
12682                                                && !(targetExplicitNames != null && targetExplicitNames.contains(targetIdentifierName))
12683                                                && sourceColumn.getResultSet()!=null && sourceColumn.getResultSet().getColumns()!=null) {
12684                                                for (int i = 0; i < sourceColumn.getResultSet().getColumns().size(); i++) {
12685                                                        ResultColumn columnModel = sourceColumn.getResultSet().getColumns().get(i);
12686                                                        if (DlineageUtil.getIdentifierNormalColumnName(columnModel.getName()).equalsIgnoreCase(targetIdentifierName)) {
12687                                                                sourceColumn source = new sourceColumn();
12688                                                                source.setId(String.valueOf(columnModel.getId()));
12689                                                                source.setColumn(columnModel.getName());
12690                                                                source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
12691                                                                source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
12692                                                                if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
12693                                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
12694                                                                                        + convertCoordinate(sourceColumn.getEndPosition()));
12695                                                                }
12696                                                                if (sourceItem.getTransforms() != null) {
12697                                                                        for (Transform transform : sourceItem.getTransforms()) {
12698                                                                                source.addTransform(transform);
12699                                                                        }
12700                                                                }
12701                                                                relationElement.addSource(source);
12702                                                                break;
12703                                                        }
12704                                                }
12705                                        }
12706                                        if (relationElement.getSources().isEmpty()
12707                                                && !(targetExplicitNames != null && targetExplicitNames.contains(targetIdentifierName))) {
12708                                                TObjectName sourceStarLinkColumn = new TObjectName();
12709                                                sourceStarLinkColumn.setString(targetName);
12710                                                boolean newBinding = sourceColumn.bindStarLinkColumn(sourceStarLinkColumn);
12711                                                sourceColumn source = new sourceColumn();
12712                                                String sourceName = DlineageUtil.getColumnName(sourceStarLinkColumn);
12713                                                if (!newBinding) {
12714                                                        source.setId(String.valueOf(sourceColumn.getId()) + "_"
12715                                                                        + sourceColumn.indexOfStarLinkColumn(sourceStarLinkColumn));
12716                                                } else {
12717                                                        source.setId(String.valueOf(sourceColumn.getId()) + "_" + linkColumnNameSize);
12718                                                }
12719                                                source.setColumn(sourceName);
12720                                                source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
12721                                                source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
12722                                                if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
12723                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
12724                                                                        + convertCoordinate(sourceColumn.getEndPosition()));
12725                                                }
12726                                                if (sourceItem.getTransforms() != null) {
12727                                                        for (Transform transform : sourceItem.getTransforms()) {
12728                                                                source.addTransform(transform);
12729                                                        }
12730                                                }
12731                                                relationElement.addSource(source);
12732
12733                                                if (newBinding) {
12734                                                        table resultSetElement = null;
12735                                                        for (table t : dataflow.getResultsets()) {
12736                                                                if (t.getId().equals(String.valueOf(sourceColumn.getResultSet().getId()))) {
12737                                                                        resultSetElement = t;
12738                                                                        break;
12739                                                                }
12740                                                        }
12741                                                        if (resultSetElement != null) {
12742                                                                column columnElement = new column();
12743                                                                columnElement.setId(String.valueOf(sourceColumn.getId()) + "_" + linkColumnNameSize);
12744                                                                columnElement.setName(sourceName);
12745                                                                if (sourceColumn.isFunction()) {
12746                                                                        columnElement.setIsFunction(String.valueOf(sourceColumn.isFunction()));
12747                                                                }
12748                                                                if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
12749                                                                        columnElement.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
12750                                                                                        + convertCoordinate(sourceColumn.getEndPosition()));
12751                                                                }
12752                                                                resultSetElement.getColumns().add(columnElement);
12753                                                        }
12754                                                }
12755                                        }
12756                                } else {
12757                                        sourceColumn source = new sourceColumn();
12758                                        source.setId(String.valueOf(sourceColumn.getId()));
12759                                        source.setColumn(sourceColumn.getName());
12760                                        source.setStruct(sourceColumn.isStruct());
12761                                        source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
12762                                        source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
12763                                        if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
12764                                                source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
12765                                                                + convertCoordinate(sourceColumn.getEndPosition()));
12766                                        }
12767                                        if (relation.getRelationshipType() == RelationshipType.fdd) {
12768                                                if (!targetIdentifierName
12769                                                                .equalsIgnoreCase(DlineageUtil.getIdentifierNormalColumnName(sourceColumn.getName()))) {
12770                                                        if (!"*".equals(sourceColumn.getName())) {
12771                                                                continue;
12772                                                        }
12773                                                        else {
12774                                                                boolean flag = false;
12775                                                                for(ResultColumn column: sourceColumn.getResultSet().getColumns()) {
12776                                                                        if(targetIdentifierName
12777                                                                                        .equalsIgnoreCase(DlineageUtil.getIdentifierNormalColumnName(column.getName()))) {
12778                                                                                flag = true;
12779                                                                                break;
12780                                                                        }
12781                                                                }
12782                                                                if(flag) {
12783                                                                        continue;
12784                                                                }
12785                                                        }
12786                                                }
12787                                        }
12788                                        if (sourceItem.getTransforms() != null) {
12789                                                for (Transform transform : sourceItem.getTransforms()) {
12790                                                        source.addTransform(transform);
12791                                                }
12792                                        }
12793                                        relationElement.addSource(source);
12794                                }
12795                        } else if (sourceElement instanceof TableColumn) {
12796                                TableColumn sourceColumn = (TableColumn) sourceElement;
12797                                if (!sourceColumn.isPseduo() && sourceColumn.hasStarLinkColumn()) {
12798                                        List<String> linkColumnNames = sourceColumn.getStarLinkColumnNames();
12799                                        int linkColumnNameSize = linkColumnNames.size();
12800                                        for (int k = 0; k < linkColumnNameSize; k++) {
12801                                                String sourceName = linkColumnNames.get(k);
12802                                                if (relation.getRelationshipType() == RelationshipType.fdd) {
12803                                                        if (!targetIdentifierName.equalsIgnoreCase(sourceName) && !"*".equals(sourceName))
12804                                                                continue;
12805                                                }
12806
12807                                                TableColumn tableColumn = searchTableColumn(sourceColumn.getTable().getColumns(), sourceName);
12808
12809                                                sourceColumn source = new sourceColumn();
12810                                                if (tableColumn == null) {
12811                                                        source.setId(sourceColumn.getId() + "_" + k);
12812                                                } else {
12813                                                        source.setId(String.valueOf(tableColumn.getId()));
12814                                                }
12815                                                source.setColumn(sourceName);
12816                                                source.setStruct(sourceColumn.isStruct());
12817                                                if (containStarColumn(sourceElements, sourceName)) {
12818                                                        continue;
12819                                                }
12820                                                // Cross-relationship check: skip if explicit source was found in prior relationship
12821                                                if (targetExplicitNames != null && targetExplicitNames.contains(sourceName)) {
12822                                                        continue;
12823                                                }
12824                                                if (sourceColumn.getTable().getColumns().size() > 1) {
12825                                                        for (int y = 0; y < sourceColumn.getTable().getColumns().size(); y++) {
12826                                                                if (sourceColumn.getTable().getColumns().get(y).getName()
12827                                                                                .equalsIgnoreCase(sourceName)) {
12828                                                                        source.setId(String.valueOf(sourceColumn.getTable().getColumns().get(y).getId()));
12829                                                                        break;
12830                                                                }
12831                                                        }
12832                                                }
12833                                                source.setParent_id(String.valueOf(sourceColumn.getTable().getId()));
12834                                                source.setParent_name(getTableName(sourceColumn.getTable()));
12835                                                if (sourceItem instanceof TableColumnRelationshipElement) {
12836                                                        source.setParent_alias(
12837                                                                        ((TableColumnRelationshipElement) sourceItem).getTableAlias());
12838                                                }
12839                                                if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
12840                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
12841                                                                        + convertCoordinate(sourceColumn.getEndPosition()));
12842                                                }
12843                                                if (sourceItem.getTransforms() != null) {
12844                                                        for (Transform transform : sourceItem.getTransforms()) {
12845                                                                source.addTransform(transform);
12846                                                        }
12847                                                }
12848                                                relationElement.addSource(source);
12849                                        }
12850                                } else {
12851                                        sourceColumn source = new sourceColumn();
12852                                        source.setId(String.valueOf(sourceColumn.getId()));
12853                                        source.setColumn(sourceColumn.getName());
12854                                        source.setStruct(sourceColumn.isStruct());
12855                                        source.setParent_id(String.valueOf(sourceColumn.getTable().getId()));
12856                                        source.setParent_name(getTableName(sourceColumn.getTable()));
12857                                        if (sourceItem instanceof TableColumnRelationshipElement) {
12858                                                source.setParent_alias(((TableColumnRelationshipElement) sourceItem).getTableAlias());
12859                                        }
12860                                        if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
12861                                                source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
12862                                                                + convertCoordinate(sourceColumn.getEndPosition()));
12863                                        }
12864                                        if (relation.getRelationshipType() == RelationshipType.fdd) {
12865                                                if (!targetIdentifierName
12866                                                                .equalsIgnoreCase(DlineageUtil.getIdentifierNormalColumnName(sourceColumn.getName()))
12867                                                                && !"*".equals(sourceColumn.getName()))
12868                                                        continue;
12869                                        }
12870                                        if (sourceItem.getTransforms() != null) {
12871                                                for (Transform transform : sourceItem.getTransforms()) {
12872                                                        source.addTransform(transform);
12873                                                }
12874                                        }
12875                                        relationElement.addSource(source);
12876                                }
12877                        }
12878                }
12879
12880                if (relationElement.getTarget() != null && relationElement.getSources() != null
12881                                && !relationElement.getSources().isEmpty()) {
12882                        dataflow.getRelationships().add(relationElement);
12883                }
12884        }
12885
12886        private String getColumnName(TObjectName column) {
12887                if (column == null) {
12888                        return null;
12889                }
12890                String name = column.getColumnNameOnly();
12891                if (name == null || "".equals(name.trim())) {
12892                        return DlineageUtil.getIdentifierNormalColumnName(column.toString().trim());
12893                } else
12894                        return DlineageUtil.getIdentifierNormalColumnName(name.trim());
12895        }
12896
12897        /**
12898         * For BigQuery/Redshift struct field access, returns the full struct path
12899         * (e.g., "customer.name" from ColumnSource with exposedName="customer", fieldPath=["name"]).
12900         * Checks StructFieldHint first (for 3+ part no-alias), then ColumnSource (for 2-part/alias).
12901         * Returns null if this is not a struct field access.
12902         */
12903        private String getStructFieldFullName(TObjectName column) {
12904                if (column == null) return null;
12905                if (getOption().getVendor() != EDbVendor.dbvbigquery
12906                        && getOption().getVendor() != EDbVendor.dbvredshift) return null;
12907                // Priority 1: StructFieldHint (side-channel, for 3+ part no-alias deep struct access)
12908                gudusoft.gsqlparser.resolver2.model.StructFieldHint hint = column.getStructFieldHint();
12909                if (hint != null && hint.getFieldPath() != null && !hint.getFieldPath().isEmpty()) {
12910                        return hint.toFullReference();
12911                }
12912                // Priority 2: ColumnSource (main resolution, for 2-part and alias cases)
12913                gudusoft.gsqlparser.resolver2.model.ColumnSource source = column.getColumnSource();
12914                if (source != null && source.isStructFieldAccess() && source.hasFieldPath()) {
12915                        return source.getFieldPath().toFullReference(source.getExposedName());
12916                }
12917                return null;
12918        }
12919
12920        /**
12921         * Get the base column name for a struct field access column.
12922         * Checks StructFieldHint first (3+ part no-alias), then ColumnSource (2-part/alias).
12923         * Returns null if not a struct field access.
12924         */
12925        private String getStructFieldBaseName(TObjectName column) {
12926                if (column == null) return null;
12927                gudusoft.gsqlparser.resolver2.model.StructFieldHint hint = column.getStructFieldHint();
12928                if (hint != null && hint.getBaseColumn() != null) {
12929                        return hint.getBaseColumn();
12930                }
12931                gudusoft.gsqlparser.resolver2.model.ColumnSource source = column.getColumnSource();
12932                if (source != null && source.isStructFieldAccess()) {
12933                        return source.getExposedName();
12934                }
12935                return null;
12936        }
12937
12938        private String getColumnName(String column) {
12939                if (column == null) {
12940                        return null;
12941                }
12942                String name = column.substring(column.lastIndexOf(".") + 1);
12943                if (name == null || "".equals(name.trim())) {
12944                        return DlineageUtil.getIdentifierNormalColumnName(column.toString().trim());
12945                } else
12946                        return DlineageUtil.getIdentifierNormalColumnName(name.trim());
12947        }
12948        
12949        private String getColumnNameOnly(String column) {
12950                if (column == null) {
12951                        return null;
12952                }
12953                return DlineageUtil.getColumnNameOnly(column);
12954        }
12955
12956        private void appendRecordSetRelation(dataflow dataflow, Relationship[] relations) {
12957                for (int i = 0; i < relations.length; i++) {
12958                        AbstractRelationship relation = (AbstractRelationship) relations[i];
12959                        relationship relationElement = new relationship();
12960                        relationElement.setType(relation.getRelationshipType().name());
12961                        if (relation.getFunction() != null) {
12962                                relationElement.setFunction(relation.getFunction());
12963                        }
12964                        if (relation.getEffectType() != null) {
12965                                relationElement.setEffectType(relation.getEffectType().name());
12966                        }
12967                        relationElement.setSqlHash(relation.getSqlHash());
12968                        relationElement.setSqlComment(relation.getSqlComment());
12969
12970                        if (relation.getProcedureId() != null) {
12971                                relationElement.setProcedureId(String.valueOf(relation.getProcedureId()));
12972                        }
12973                        relationElement.setId(String.valueOf(relation.getId()));
12974
12975                        if (relation instanceof RecordSetRelationship) {
12976                                RecordSetRelationship recordCountRelation = (RecordSetRelationship) relation;
12977
12978                                Object targetElement = recordCountRelation.getTarget().getElement();
12979                                if (targetElement instanceof ResultColumn) {
12980                                        ResultColumn targetColumn = (ResultColumn) targetElement;
12981                                        targetColumn target = new targetColumn();
12982                                        target.setId(String.valueOf(targetColumn.getId()));
12983                                        target.setColumn(targetColumn.getName());
12984                                        target.setStruct(targetColumn.isStruct());
12985                                        target.setFunction(recordCountRelation.getAggregateFunction());
12986                                        target.setParent_id(String.valueOf(targetColumn.getResultSet().getId()));
12987                                        target.setParent_name(getResultSetName(targetColumn.getResultSet()));
12988                                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
12989                                                target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
12990                                                                + convertCoordinate(targetColumn.getEndPosition()));
12991                                        }
12992                                        relationElement.setTarget(target);
12993                                } else if (targetElement instanceof TableColumn) {
12994                                        TableColumn targetColumn = (TableColumn) targetElement;
12995                                        targetColumn target = new targetColumn();
12996                                        target.setId(String.valueOf(targetColumn.getId()));
12997                                        target.setColumn(targetColumn.getName());
12998                                        target.setStruct(targetColumn.isStruct());
12999                                        target.setFunction(recordCountRelation.getAggregateFunction());
13000                                        target.setParent_id(String.valueOf(targetColumn.getTable().getId()));
13001                                        target.setParent_name(getTableName(targetColumn.getTable()));
13002                                        if (recordCountRelation.getTarget() instanceof TableColumnRelationshipElement) {
13003                                                target.setParent_alias(
13004                                                                ((TableColumnRelationshipElement) recordCountRelation.getTarget()).getTableAlias());
13005                                        }
13006                                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
13007                                                target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
13008                                                                + convertCoordinate(targetColumn.getEndPosition()));
13009                                        }
13010                                        relationElement.setTarget(target);
13011                                } else if (targetElement instanceof ResultSetRelationRows) {
13012                                        ResultSetRelationRows targetColumn = (ResultSetRelationRows) targetElement;
13013                                        targetColumn target = new targetColumn();
13014                                        target.setId(String.valueOf(targetColumn.getId()));
13015                                        target.setColumn(targetColumn.getName());
13016                                        target.setParent_id(String.valueOf(targetColumn.getHolder().getId()));
13017                                        target.setParent_name(getResultSetName(targetColumn.getHolder()));
13018                                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
13019                                                target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
13020                                                                + convertCoordinate(targetColumn.getEndPosition()));
13021                                        }
13022                                        target.setSource("system");
13023                                        relationElement.setTarget(target);
13024                                } else {
13025                                        continue;
13026                                }
13027
13028                                Collection<RelationshipElement<?>> sourceElements = (Collection<RelationshipElement<?>>)recordCountRelation.getSources();
13029                                if (sourceElements.size() == 0) {
13030                                        continue;
13031                                }
13032
13033                                boolean append = false;
13034                                for (RelationshipElement<?> sourceItem: sourceElements) {
13035                                        Object sourceElement = sourceItem.getElement();
13036                                        if (sourceElement instanceof Table) {
13037                                                Table table = (Table) sourceElement;
13038                                                sourceColumn source = new sourceColumn();
13039                                                source.setSource_id(String.valueOf(table.getId()));
13040                                                source.setSource_name(getTableName(table));
13041                                                if (table.getStartPosition() != null && table.getEndPosition() != null) {
13042                                                        source.setCoordinate(convertCoordinate(table.getStartPosition()) + ","
13043                                                                        + convertCoordinate(table.getEndPosition()));
13044                                                }
13045                                                append = true;
13046                                                relationElement.addSource(source);
13047                                        } else if (sourceElement instanceof QueryTable) {
13048                                                QueryTable table = (QueryTable) sourceElement;
13049                                                sourceColumn source = new sourceColumn();
13050                                                source.setSource_id(String.valueOf(table.getId()));
13051                                                source.setSource_name(getResultSetName(table));
13052                                                if (table.getStartPosition() != null && table.getEndPosition() != null) {
13053                                                        source.setCoordinate(convertCoordinate(table.getStartPosition()) + ","
13054                                                                        + convertCoordinate(table.getEndPosition()));
13055                                                }
13056                                                append = true;
13057                                                relationElement.addSource(source);
13058                                        } else if (sourceElement instanceof TableRelationRows) {
13059                                                TableRelationRows relationRows = (TableRelationRows) sourceElement;
13060                                                sourceColumn source = new sourceColumn();
13061                                                source.setId(String.valueOf(relationRows.getId()));
13062                                                source.setColumn(relationRows.getName());
13063                                                source.setParent_id(String.valueOf(relationRows.getHolder().getId()));
13064                                                source.setParent_name(getTableName(relationRows.getHolder()));
13065                                                if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
13066                                                        source.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
13067                                                                        + convertCoordinate(relationRows.getEndPosition()));
13068                                                }
13069                                                source.setSource("system");
13070                                                append = true;
13071                                                relationElement.addSource(source);
13072                                        } else if (sourceElement instanceof ResultSetRelationRows) {
13073                                                ResultSetRelationRows relationRows = (ResultSetRelationRows) sourceElement;
13074                                                sourceColumn source = new sourceColumn();
13075                                                source.setId(String.valueOf(relationRows.getId()));
13076                                                source.setColumn(relationRows.getName());
13077                                                source.setParent_id(String.valueOf(relationRows.getHolder().getId()));
13078                                                source.setParent_name(getResultSetName(relationRows.getHolder()));
13079                                                if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
13080                                                        source.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
13081                                                                        + convertCoordinate(relationRows.getEndPosition()));
13082                                                }
13083                                                source.setSource("system");
13084                                                append = true;
13085                                                relationElement.addSource(source);
13086                                        } else if (sourceElement instanceof TableColumn) {
13087                                                TableColumn sourceColumn = (TableColumn) sourceElement;
13088                                                sourceColumn source = new sourceColumn();
13089                                                source.setId(String.valueOf(sourceColumn.getId()));
13090                                                source.setColumn(sourceColumn.getName());
13091                                                source.setStruct(sourceColumn.isStruct());
13092                                                source.setParent_id(String.valueOf(sourceColumn.getTable().getId()));
13093                                                source.setParent_name(getTableName(sourceColumn.getTable()));
13094                                                if (sourceItem instanceof TableColumnRelationshipElement) {
13095                                                        source.setParent_alias(
13096                                                                        ((TableColumnRelationshipElement) sourceItem).getTableAlias());
13097                                                }
13098                                                if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
13099                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
13100                                                                        + convertCoordinate(sourceColumn.getEndPosition()));
13101                                                }
13102                                                append = true;
13103                                                relationElement.addSource(source);
13104                                        }
13105                                        if (sourceElement instanceof ResultColumn) {
13106                                                ResultColumn sourceColumn = (ResultColumn) sourceElement;
13107                                                sourceColumn source = new sourceColumn();
13108                                                source.setId(String.valueOf(sourceColumn.getId()));
13109                                                source.setColumn(sourceColumn.getName());
13110                                                source.setStruct(sourceColumn.isStruct());
13111                                                source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
13112                                                source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
13113                                                if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
13114                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
13115                                                                        + convertCoordinate(sourceColumn.getEndPosition()));
13116                                                }
13117                                                append = true;
13118                                                relationElement.addSource(source);
13119                                        }
13120                                }
13121
13122                                if (append)
13123                                        dataflow.getRelationships().add(relationElement);
13124                        }
13125                }
13126        }
13127
13128        private void appendCallRelation(dataflow dataflow, Relationship[] relations) {
13129                for (int i = 0; i < relations.length; i++) {
13130                        AbstractRelationship relation = (AbstractRelationship) relations[i];
13131                        relationship relationElement = new relationship();
13132                        relationElement.setType(relation.getRelationshipType().name());
13133                        if (relation.getFunction() != null) {
13134                                relationElement.setFunction(relation.getFunction());
13135                        }
13136                        if (relation.getEffectType() != null) {
13137                                relationElement.setEffectType(relation.getEffectType().name());
13138                        }
13139                        relationElement.setSqlHash(relation.getSqlHash());
13140                        relationElement.setSqlComment(relation.getSqlComment());
13141                        
13142                        if (relation.getProcedureId() != null) {
13143                                relationElement.setProcedureId(String.valueOf(relation.getProcedureId()));
13144                        }
13145                        relationElement.setId(String.valueOf(relation.getId()));
13146
13147                        if (relation instanceof CallRelationship) {
13148                                CallRelationship callRelation = (CallRelationship) relation;
13149                                
13150                                if (callRelation.getCallObject() != null) {
13151                                        relationElement.setCallStmt(callRelation.getCallObject().toString());
13152                                        if (callRelation.getStartPosition() != null && callRelation.getEndPosition() != null) {
13153                                                relationElement.setCallCoordinate(convertCoordinate(callRelation.getStartPosition()) + ","
13154                                                                + convertCoordinate(callRelation.getEndPosition()));
13155                                        }
13156                                }
13157                                
13158                                if (Boolean.TRUE.equals(callRelation.getBuiltIn())) {
13159                                        relationElement.setBuiltIn(true);
13160                                }
13161                                Object targetElement = callRelation.getTarget().getElement();
13162                                if (targetElement instanceof Procedure) {
13163                                        Procedure procedure = (Procedure) targetElement;
13164                                        targetColumn target = new targetColumn();
13165                                        target.setId(String.valueOf(procedure.getId()));
13166                                        target.setName(getProcedureName(procedure));
13167                                        if (procedure.getStartPosition() != null && procedure.getEndPosition() != null) {
13168                                                target.setCoordinate(convertCoordinate(procedure.getStartPosition()) + ","
13169                                                                + convertCoordinate(procedure.getEndPosition()));
13170                                        }
13171                                        String clazz = procedure.getProcedureObject().getClass().getSimpleName().toLowerCase();
13172                                        if (clazz.indexOf("function") != -1) {
13173                                                target.setType("function");
13174                                        } else if (clazz.indexOf("trigger") != -1) {
13175                                                target.setType("trigger");
13176                                        } else if (clazz.indexOf("macro") != -1) {
13177                                                target.setType("macro");
13178                                        } else {
13179                                                target.setType("procedure");
13180                                        }
13181                                        relationElement.setCaller(target);
13182                                } else {
13183                                        continue;
13184                                }
13185
13186                                Collection<RelationshipElement<?>> sourceElements = (Collection<RelationshipElement<?>>)callRelation.getSources();
13187                                if (sourceElements.size() == 0) {
13188                                        continue;
13189                                }
13190
13191                                boolean append = false;
13192                                for (RelationshipElement<?> sourceItem: sourceElements) {
13193                                        Object sourceElement = sourceItem.getElement();
13194                                        if (sourceElement instanceof Procedure) {
13195                                                Procedure procedure = (Procedure) sourceElement;
13196                                                sourceColumn source = new sourceColumn();
13197                                                source.setId(String.valueOf(procedure.getId()));
13198                                                source.setName(getProcedureName(procedure));
13199                                                if (procedure.getStartPosition() != null && procedure.getEndPosition() != null) {
13200                                                        source.setCoordinate(convertCoordinate(procedure.getStartPosition()) + ","
13201                                                                        + convertCoordinate(procedure.getEndPosition()));
13202                                                }
13203                                                String clazz = procedure.getProcedureObject().getClass().getSimpleName().toLowerCase();
13204                                                if (clazz.indexOf("function") != -1) {
13205                                                        source.setType("function");
13206                                                } else if (clazz.indexOf("trigger") != -1) {
13207                                                        source.setType("trigger");
13208                                                } else if (clazz.indexOf("macro") != -1) {
13209                                                        source.setType("macro");
13210                                                } else {
13211                                                        source.setType("procedure");
13212                                                }
13213                                                append = true;
13214                                                relationElement.getCallees().add(source);
13215                                        } else if (sourceElement instanceof Function) {
13216                                                Function function = (Function) sourceElement;
13217                                                sourceColumn source = new sourceColumn();
13218                                                source.setId(String.valueOf(function.getId()));
13219                                                source.setName(getFunctionName(function.getFunctionObject()));
13220                                                if (function.getStartPosition() != null && function.getEndPosition() != null) {
13221                                                        source.setCoordinate(convertCoordinate(function.getStartPosition()) + ","
13222                                                                        + convertCoordinate(function.getEndPosition()));
13223                                                }
13224                                                source.setType("function");
13225                                                append = true;
13226                                                relationElement.getCallees().add(source);
13227                                        }
13228                                }
13229
13230                                if (append)
13231                                        dataflow.getRelationships().add(relationElement);
13232                        }
13233                }
13234        }
13235
13236        private void appendResultSets(dataflow dataflow) {
13237                Set<ResultSet> resultSets = modelManager.getResultSets(); 
13238                for (ResultSet resultSet: resultSets) {
13239                        appendResultSet(dataflow, resultSet);
13240                }
13241        }
13242
13243        private void appendResultSet(dataflow dataflow, ResultSet resultSetModel) {
13244                if (!appendResultSets.contains(resultSetModel)) {
13245                        appendResultSets.add(resultSetModel);
13246                } else {
13247                        return;
13248                }
13249
13250                table resultSetElement = new table();
13251                resultSetElement.setId(String.valueOf(resultSetModel.getId()));
13252                resultSetElement.setServer(resultSetModel.getServer());
13253                if (!SQLUtil.isEmpty(resultSetModel.getDatabase())) {
13254                        resultSetElement.setDatabase(resultSetModel.getDatabase());
13255                }
13256                if (!SQLUtil.isEmpty(resultSetModel.getSchema())) {
13257                        resultSetElement.setSchema(resultSetModel.getSchema());
13258                }
13259                resultSetElement.setName(getResultSetName(resultSetModel));
13260                resultSetElement.setType(getResultSetType(resultSetModel));
13261                // if ((ignoreRecordSet || simpleOutput) && resultSetModel.isTarget()) {
13262                resultSetElement.setIsTarget(String.valueOf(resultSetModel.isTarget()));
13263                // }
13264                resultSetElement.setIsDetermined(String.valueOf(resultSetModel.isDetermined()));
13265                if (resultSetModel.getStartPosition() != null && resultSetModel.getEndPosition() != null) {
13266                        resultSetElement.setCoordinate(convertCoordinate(resultSetModel.getStartPosition()) + ","
13267                                        + convertCoordinate(resultSetModel.getEndPosition()));
13268                }
13269                dataflow.getResultsets().add(resultSetElement);
13270
13271                List<ResultColumn> columns = resultSetModel.getColumns();
13272
13273                Map<String, Integer> columnCounts = new HashMap<String, Integer>();
13274                for (ResultColumn column : columns) {
13275                        String columnName = DlineageUtil.getIdentifierNormalColumnName(column.getName());
13276                        if (!columnCounts.containsKey(columnName)) {
13277                                columnCounts.put(columnName, 0);
13278                        }
13279                        columnCounts.put(columnName, columnCounts.get(columnName) + 1);
13280                        // if (column.hasStarLinkColumn()) {
13281                        // List<String> starLinkColumns = column.getStarLinkColumnNames();
13282                        // for (int k = 0; k < starLinkColumns.size(); k++) {
13283                        // columnName = starLinkColumns.get(k);
13284                        // if (!columnCounts.containsKey(columnName)) {
13285                        // columnCounts.put(columnName, 0);
13286                        // }
13287                        // columnCounts.put(columnName, columnCounts.get(columnName) + 1);
13288                        // }
13289                        // }
13290                }
13291
13292                for (int j = 0; j < columns.size(); j++) {
13293                        ResultColumn columnModel = columns.get(j);
13294                        if (columnModel.hasStarLinkColumn()) {
13295                                // List<String> starLinkColumns =
13296                                // columnModel.getStarLinkColumnNames();
13297                                // for (int k = 0; k < starLinkColumns.size(); k++) {
13298                                // column columnElement = new column();
13299                                // columnElement.setId( String.valueOf(columnModel.getId()) +
13300                                // "_" + k);
13301                                // String columnName = starLinkColumns.get(k);
13302                                // columnElement.setName(columnName);
13303                                // if(columnModel.isFunction()){
13304                                // columnElement.setIsFunction(String.valueOf(columnModel.isFunction()));
13305                                // }
13306                                // if (columnModel.getStartPosition() != null &&
13307                                // columnModel.getEndPosition() != null) {
13308                                // columnElement.setCoordinate(
13309                                // columnModel.getStartPosition() + "," +
13310                                // columnModel.getEndPosition());
13311                                // }
13312                                // String identifier = columnName;
13313                                // if(columnCounts.containsKey(identifier) &&
13314                                // columnCounts.get(identifier)>1){
13315                                // TObjectName column =
13316                                // columnModel.getStarLinkColumns().get(columnName).iterator().next();
13317                                // if(!SQLUtil.isEmpty(getQualifiedTable(column))){
13318                                // columnElement.setQualifiedTable(getQualifiedTable(column));
13319                                // }
13320                                // }
13321                                // resultSetElement.getColumns().add(columnElement);
13322                                // }
13323                                if (columnModel.isShowStar()) {
13324                                        column columnElement = new column();
13325                                        columnElement.setId(String.valueOf(columnModel.getId()));
13326                                        columnElement.setName(columnModel.getName());
13327                                        if (columnModel.isFunction()) {
13328                                                columnElement.setIsFunction(String.valueOf(columnModel.isFunction()));
13329                                        }
13330                                        if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
13331                                                columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
13332                                                                + convertCoordinate(columnModel.getEndPosition()));
13333                                        }
13334
13335                                        String identifier = DlineageUtil.getIdentifierNormalColumnName(columnModel.getName());
13336                                        if (columnCounts.containsKey(identifier) && columnCounts.get(identifier) > 1) {
13337                                                String qualifiedTable = getQualifiedTable(columnModel);
13338                                                if (!SQLUtil.isEmpty(qualifiedTable)) {
13339                                                        columnElement.setQualifiedTable(qualifiedTable);
13340                                                }
13341                                        }
13342                                        resultSetElement.getColumns().add(columnElement);
13343                                }
13344                        } else {
13345                                column columnElement = new column();
13346                                columnElement.setId(String.valueOf(columnModel.getId()));
13347                                columnElement.setName(columnModel.getName());
13348                                if (columnModel.isFunction()) {
13349                                        columnElement.setIsFunction(String.valueOf(columnModel.isFunction()));
13350                                }
13351                                if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
13352                                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
13353                                                        + convertCoordinate(columnModel.getEndPosition()));
13354                                }
13355
13356                                String identifier = DlineageUtil.getIdentifierNormalColumnName(columnModel.getName());
13357                                if (columnCounts.containsKey(identifier) && columnCounts.get(identifier) > 1) {
13358                                        String qualifiedTable = getQualifiedTable(columnModel);
13359                                        if (!SQLUtil.isEmpty(qualifiedTable)) {
13360                                                columnElement.setQualifiedTable(qualifiedTable);
13361                                        }
13362                                }
13363                                resultSetElement.getColumns().add(columnElement);
13364                        }
13365                }
13366
13367                ResultSetRelationRows relationRows = resultSetModel.getRelationRows();
13368                if (relationRows.hasRelation()) {
13369                        column relationRowsElement = new column();
13370                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
13371                        relationRowsElement.setName(relationRows.getName());
13372                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
13373                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
13374                                                + convertCoordinate(relationRows.getEndPosition()));
13375                        }
13376                        relationRowsElement.setSource("system");
13377                        resultSetElement.getColumns().add(relationRowsElement);
13378                }
13379        }
13380
13381        private String getQualifiedTable(ResultColumn columnModel) {
13382                if (columnModel.getColumnObject() instanceof TObjectName) {
13383                        return getQualifiedTable((TObjectName) columnModel.getColumnObject());
13384                }
13385                if (columnModel.getColumnObject() instanceof TResultColumn) {
13386                        TObjectName field = ((TResultColumn) columnModel.getColumnObject()).getFieldAttr();
13387                        if (field != null) {
13388                                return getQualifiedTable(field);
13389                        }
13390                }
13391                return null;
13392        }
13393
13394        private String getQualifiedTable(TObjectName column) {
13395                if (column == null)
13396                        return null;
13397                String[] splits = column.toString().split("\\.");
13398                if (splits.length > 1) {
13399                        return splits[splits.length - 2];
13400                }
13401                return null;
13402        }
13403
13404        /**
13405         * Get the qualified prefix (schema.table) from a column name for 3-part names.
13406         * For example, for "sch.pk_constv2.c_cdsl", returns "sch.pk_constv2".
13407         * Returns null if the column doesn't have both schema and table parts.
13408         */
13409        private String getQualifiedPrefixFromColumn(TObjectName column) {
13410                if (column == null) return null;
13411
13412                // Check if both schema and table tokens are present (3-part name)
13413                String schemaStr = column.getSchemaString();
13414                String tableStr = column.getTableString();
13415
13416                if (schemaStr != null && !schemaStr.isEmpty() &&
13417                        tableStr != null && !tableStr.isEmpty()) {
13418                        return schemaStr + "." + tableStr;
13419                }
13420
13421                // Fallback: parse from toString() for complex cases
13422                String[] splits = column.toString().split("\\.");
13423                if (splits.length >= 3) {
13424                        // Return all parts except the last one (column name)
13425                        StringBuilder prefix = new StringBuilder();
13426                        for (int i = 0; i < splits.length - 1; i++) {
13427                                if (i > 0) prefix.append(".");
13428                                prefix.append(splits[i]);
13429                        }
13430                        return prefix.toString();
13431                }
13432
13433                return null;
13434        }
13435
13436        private String getResultSetType(ResultSet resultSetModel) {
13437                if (resultSetModel instanceof QueryTable) {
13438                        QueryTable table = (QueryTable) resultSetModel;
13439                        if (table.getTableObject().getCTE() != null) {
13440                                return "with_cte";
13441                        }
13442                }
13443
13444                if (resultSetModel instanceof SelectSetResultSet) {
13445                        ESetOperatorType type = ((SelectSetResultSet) resultSetModel).getSetOperatorType();
13446                        return "select_" + type.name();
13447                }
13448
13449                if (resultSetModel instanceof SelectResultSet) {
13450                        if (((SelectResultSet) resultSetModel).getSelectStmt().getParentStmt() instanceof TInsertSqlStatement) {
13451                                return "insert-select";
13452                        }
13453                        if (((SelectResultSet) resultSetModel).getSelectStmt().getParentStmt() instanceof TUpdateSqlStatement) {
13454                                return "update-select";
13455                        }
13456                }
13457
13458                if (resultSetModel.getGspObject() instanceof TMergeUpdateClause) {
13459                        return "merge-update";
13460                }
13461
13462                if (resultSetModel.getGspObject() instanceof TOutputClause) {
13463                        return ResultSetType.output.name();
13464                }
13465
13466                if (resultSetModel.getGspObject() instanceof TMergeInsertClause) {
13467                        return "merge-insert";
13468                }
13469
13470                if (resultSetModel.getGspObject() instanceof TUpdateSqlStatement) {
13471                        return "update-set";
13472                }
13473
13474                if (resultSetModel.getGspObject() instanceof TFunctionCall && ((TFunctionCall)resultSetModel.getGspObject()).getFunctionType() == EFunctionType.array_t) {
13475                        return ResultSetType.array.name();
13476                }
13477
13478                if (resultSetModel.getGspObject() instanceof TFunctionCall && ((TFunctionCall)resultSetModel.getGspObject()).getFunctionType() == EFunctionType.struct_t) {
13479                        return ResultSetType.struct.name();
13480                }
13481
13482                if (resultSetModel.getGspObject() instanceof TFunctionCall || resultSetModel instanceof Function) {
13483                        return ResultSetType.function.name();
13484                }
13485
13486                if (resultSetModel.getGspObject() instanceof TAliasClause) {
13487                        return ResultSetType.alias.name();
13488                }
13489
13490                if (resultSetModel.getGspObject() instanceof TCursorDeclStmt) {
13491                        return ResultSetType.cursor.name();
13492                }
13493
13494                if (resultSetModel instanceof PivotedTable) {
13495                        if (((PivotedTable) resultSetModel).isUnpivoted()) {
13496                                return ResultSetType.unpivot_table.name();
13497                        }
13498                        return ResultSetType.pivot_table.name();
13499                }
13500
13501                return "select_list";
13502        }
13503
13504        private String getTableName(Table tableModel) {
13505                if (modelManager.DISPLAY_NAME.containsKey(tableModel.getId())) {
13506                        return modelManager.DISPLAY_NAME.get(tableModel.getId());
13507                }
13508
13509                String tableName;
13510                if (tableModel.getFullName() != null && tableModel.getFullName().trim().length() > 0) {
13511                        return tableModel.getFullName();
13512                }
13513                if (tableModel.getAlias() != null && tableModel.getAlias().trim().length() > 0) {
13514                        tableName = getResultSetWithId("RESULT_OF_" + tableModel.getAlias());
13515
13516                } else {
13517                        tableName = getResultSetDisplayId("RS");
13518                }
13519                modelManager.DISPLAY_NAME.put(tableModel.getId(), tableName);
13520                return tableName;
13521        }
13522
13523        private String getProcedureName(Procedure procedureModel) {
13524                if (modelManager.DISPLAY_NAME.containsKey(procedureModel.getId())) {
13525                        return modelManager.DISPLAY_NAME.get(procedureModel.getId());
13526                }
13527
13528                String procedureName = procedureModel.getFullName();
13529
13530                modelManager.DISPLAY_NAME.put(procedureModel.getId(), procedureName);
13531                return procedureName;
13532        }
13533
13534        private String getProcessName(Process processModel) {
13535                if (modelManager.DISPLAY_NAME.containsKey(processModel.getId())) {
13536                        return modelManager.DISPLAY_NAME.get(processModel.getId());
13537                } else {
13538                        if (processModel.getCustomType() != null) {
13539                                String name = processModel.getCustomType();
13540                                modelManager.DISPLAY_NAME.put(processModel.getId(), name);
13541                                return name;
13542                        }
13543                        String name = processModel.getType();
13544                        String procedureName = getProcedureParentName(processModel.getGspObject());
13545                        if (procedureName != null) {
13546                                name = getResultSetDisplayId(procedureName + " " + name);
13547                        } else {
13548                                name = getResultSetDisplayId("Query " + name);
13549                        }
13550                        modelManager.DISPLAY_NAME.put(processModel.getId(), name);
13551                        return name;
13552                }
13553        }
13554
13555        private String getDisplayIdByType(String type) {
13556                if (!modelManager.DISPLAY_ID.containsKey(type)) {
13557                        modelManager.DISPLAY_ID.put(type, option.getStartId() + 1);
13558                        return type + "-" + (option.getStartId() + 1);
13559                } else {
13560                        long id = modelManager.DISPLAY_ID.get(type);
13561                        modelManager.DISPLAY_ID.put(type, id + 1);
13562                        return type + "-" + (id + 1);
13563                }
13564        }
13565        
13566        private String getDisplayIdByTypeFromZero(String type) {
13567                if (!modelManager.DISPLAY_ID.containsKey(type)) {
13568                        modelManager.DISPLAY_ID.put(type, option.getStartId());
13569                        if(option.getStartId() == 0) {
13570                                return type;
13571                        }
13572                        return type + "-" + (option.getStartId() + 1);
13573                } else {
13574                        long id = modelManager.DISPLAY_ID.get(type);
13575                        modelManager.DISPLAY_ID.put(type, id + 1);
13576                        return type + "-" + (id + 1);
13577                }
13578        }
13579
13580        private String getConstantName(Table tableModel) {
13581                if (modelManager.DISPLAY_NAME.containsKey(tableModel.getId())) {
13582                        return modelManager.DISPLAY_NAME.get(tableModel.getId());
13583                } else {
13584                        String name = getDisplayIdByType("SQL_CONSTANTS");
13585                        modelManager.DISPLAY_NAME.put(tableModel.getId(), name);
13586                        return name;
13587                }
13588        }
13589        
13590        private String getTempTableName(TTable table) {
13591                if (modelManager.DISPLAY_NAME.containsKey((long)System.identityHashCode(table))) {
13592                        return modelManager.DISPLAY_NAME.get((long)System.identityHashCode(table));
13593                } else {
13594                        String name = getDisplayIdByTypeFromZero(table.getName());
13595                        modelManager.DISPLAY_NAME.put((long)System.identityHashCode(table), name);
13596                        return name;
13597                }
13598        }
13599
13600        private String getResultSetName(ResultSet resultSetModel) {
13601
13602                if (modelManager.DISPLAY_NAME.containsKey(resultSetModel.getId())) {
13603                        return modelManager.DISPLAY_NAME.get(resultSetModel.getId());
13604                }
13605                
13606                if (resultSetModel.getGspObject() instanceof TFunctionCall && ((TFunctionCall)resultSetModel.getGspObject()).getFunctionType() == EFunctionType.array_t) {
13607                        String name = getResultSetDisplayId("ARRAY");
13608                        modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
13609                        if(option.containsResultSetType(ResultSetType.array)) {
13610                                resultSetModel.setTarget(true);
13611                        }
13612                        return name;
13613                }
13614                
13615                if (resultSetModel.getGspObject() instanceof TFunctionCall && ((TFunctionCall)resultSetModel.getGspObject()).getFunctionType() == EFunctionType.struct_t) {
13616                        String name = getResultSetDisplayId("STRUCT");
13617                        modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
13618                        if(option.containsResultSetType(ResultSetType.struct)) {
13619                                resultSetModel.setTarget(true);
13620                        }
13621                        return name;
13622                }
13623
13624                if (resultSetModel instanceof QueryTable) {
13625                        QueryTable table = (QueryTable) resultSetModel;
13626                        if (table.getAlias() != null && table.getAlias().trim().length() > 0) {
13627                                String name = getResultSetWithId("RESULT_OF_" + table.getAlias().trim());
13628                                if (table.getTableObject().getCTE() != null) {
13629                                        name = getResultSetWithId("RESULT_OF_" + table.getTableObject().getCTE().getTableName().toString()
13630                                                        + "_" + table.getAlias().trim());
13631                                }
13632                                modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
13633                                if(option.containsResultSetType(ResultSetType.result_of)) {
13634                                        resultSetModel.setTarget(true);
13635                                }
13636                                return name;
13637                        } else if (table.getTableObject().getCTE() != null) {
13638                                String name = getResultSetWithId("CTE-" + table.getTableObject().getCTE().getTableName().toString());
13639                                modelManager.DISPLAY_NAME.put(table.getId(), name);
13640                                if(option.containsResultSetType(ResultSetType.cte)) {
13641                                        resultSetModel.setTarget(true);
13642                                }
13643                                return name;
13644                        }
13645                }
13646
13647                if (resultSetModel instanceof SelectResultSet) {
13648                        if (((SelectResultSet) resultSetModel).getSelectStmt().getParentStmt() instanceof TInsertSqlStatement) {
13649                                String name = getResultSetDisplayId("INSERT-SELECT");
13650                                modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
13651                                if(option.containsResultSetType(ResultSetType.insert_select)) {
13652                                        resultSetModel.setTarget(true);
13653                                }
13654                                return name;
13655                        }
13656
13657                        if (((SelectResultSet) resultSetModel).getSelectStmt().getParentStmt() instanceof TUpdateSqlStatement) {
13658                                String name = getResultSetDisplayId("UPDATE-SELECT");
13659                                modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
13660                                if(option.containsResultSetType(ResultSetType.update_select)) {
13661                                        resultSetModel.setTarget(true);
13662                                }
13663                                return name;
13664                        }
13665                }
13666
13667                if (resultSetModel instanceof SelectSetResultSet) {
13668                        ESetOperatorType type = ((SelectSetResultSet) resultSetModel).getSetOperatorType();
13669                        String name = getResultSetDisplayId("RESULT_OF_" + type.name().toUpperCase());
13670                        modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
13671                        if(option.containsResultSetType(ResultSetType.result_of)) {
13672                                resultSetModel.setTarget(true);
13673                        }
13674                        return name;
13675                }
13676
13677                if (resultSetModel.getGspObject() instanceof TMergeUpdateClause) {
13678                        String name = getResultSetDisplayId("MERGE-UPDATE");
13679                        modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
13680                        if(option.containsResultSetType(ResultSetType.merge_update)) {
13681                                resultSetModel.setTarget(true);
13682                        }
13683                        return name;
13684                }
13685
13686                if (resultSetModel.getGspObject() instanceof TOutputClause) {
13687                        String name = getResultSetDisplayId("OUTPUT");
13688                        modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
13689                        if(option.containsResultSetType(ResultSetType.output)) {
13690                                resultSetModel.setTarget(true);
13691                        }
13692                        return name;
13693                }
13694
13695                if (resultSetModel.getGspObject() instanceof TMergeInsertClause) {
13696                        String name = getResultSetDisplayId("MERGE-INSERT");
13697                        modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
13698                        if(option.containsResultSetType(ResultSetType.merge_insert)) {
13699                                resultSetModel.setTarget(true);
13700                        }
13701                        return name;
13702                }
13703
13704                if (resultSetModel.getGspObject() instanceof TUpdateSqlStatement) {
13705                        String name = getResultSetDisplayId("UPDATE-SET");
13706                        modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
13707                        if(option.containsResultSetType(ResultSetType.update_set)) {
13708                                resultSetModel.setTarget(true);
13709                        }
13710                        return name;
13711                }
13712
13713                if (resultSetModel.getGspObject() instanceof TCaseExpression) {
13714                        String name = ((Function) resultSetModel).getFunctionName();
13715                        modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
13716                        if (option.containsResultSetType(ResultSetType.case_when) || option.containsResultSetType(ResultSetType.function)) {
13717                                resultSetModel.setTarget(true);
13718                        }
13719                        return name;
13720                }
13721                
13722                if (resultSetModel.getGspObject() instanceof TFunctionCall || resultSetModel instanceof Function) {
13723                        String name = ((Function) resultSetModel).getFunctionName();
13724                        modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
13725                        if(option.containsResultSetType(ResultSetType.function)) {
13726                                resultSetModel.setTarget(true);
13727                        }
13728                        return name;
13729                }
13730
13731                if (resultSetModel instanceof PivotedTable) {
13732                        String name = getResultSetDisplayId("PIVOT-TABLE");
13733                        if (((PivotedTable) resultSetModel).isUnpivoted()) {
13734                                name = getResultSetDisplayId("UNPIVOT-TABLE");
13735                                if(option.containsResultSetType(ResultSetType.unpivot_table)) {
13736                                        resultSetModel.setTarget(true);
13737                                }
13738                        }
13739                        else {
13740                                if(option.containsResultSetType(ResultSetType.pivot_table)) {
13741                                        resultSetModel.setTarget(true);
13742                                }
13743                        }
13744                        modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
13745                        return name;
13746                }
13747
13748                if (resultSetModel instanceof Alias) {
13749                        String name = getResultSetDisplayId("ALIAS");
13750                        modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
13751                        if(option.containsResultSetType(ResultSetType.alias)) {
13752                                resultSetModel.setTarget(true);
13753                        }
13754                        return name;
13755                }
13756
13757                String name = getResultSetDisplayId("RS");
13758                modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
13759                if(option.containsResultSetType(ResultSetType.select_list)) {
13760                        resultSetModel.setTarget(true);
13761                }
13762                return name;
13763        }
13764
13765        private String getResultSetWithId(String type) {
13766                type = DlineageUtil.getIdentifierNormalTableName(type);
13767                if (!modelManager.DISPLAY_ID.containsKey(type)) {
13768                        modelManager.DISPLAY_ID.put(type, option.getStartId() + 1);
13769                        return type + "-" + (option.getStartId() + 1);
13770                } else {
13771                        long id = modelManager.DISPLAY_ID.get(type);
13772                        modelManager.DISPLAY_ID.put(type, id + 1);
13773                        return type + "-" + (id + 1);
13774                }
13775        }
13776
13777        private String getResultSetDisplayId(String type) {
13778                if (!modelManager.DISPLAY_ID.containsKey(type)) {
13779                        modelManager.DISPLAY_ID.put(type, option.getStartId() + 1);
13780                        return type + "-" + (option.getStartId() + 1);
13781                } else {
13782                        long id = modelManager.DISPLAY_ID.get(type);
13783                        modelManager.DISPLAY_ID.put(type, id + 1);
13784                        return type + "-" + (id + 1);
13785                }
13786        }
13787
13788        private void appendViews(dataflow dataflow) {
13789                List<TCustomSqlStatement> views = modelManager.getViews();
13790                for (int i = 0; i < views.size(); i++) {
13791                        Table viewModel = (Table) modelManager.getViewModel(views.get(i));
13792                        if (!tableIds.contains(viewModel.getId())) {
13793                                appendViewModel(dataflow, viewModel);
13794                                tableIds.add(viewModel.getId());
13795                        }
13796                }
13797
13798                List<TTable> tables = modelManager.getBaseTables();
13799                for (int i = 0; i < tables.size(); i++) {
13800                        Object model = modelManager.getModel(tables.get(i));
13801                        if (model instanceof Table) {
13802                                Table tableModel = (Table) model;
13803                                if (tableModel.isView()) {
13804                                        if (!tableIds.contains(tableModel.getId())) {
13805                                                appendViewModel(dataflow, tableModel);
13806                                                tableIds.add(tableModel.getId());
13807                                        }
13808                                }
13809                        }
13810                }
13811
13812                List<Table> tableNames = modelManager.getTablesByName();
13813                for (int i = 0; i < tableNames.size(); i++) {
13814                        Table tableModel = tableNames.get(i);
13815                        if (tableModel.isView()) {
13816                                if (!tableIds.contains(tableModel.getId())) {
13817                                        appendViewModel(dataflow, tableModel);
13818                                        tableIds.add(tableModel.getId());
13819                                }
13820                        }
13821                }
13822        }
13823
13824        private void appendViewModel(dataflow dataflow, Table viewModel) {
13825                table viewElement = new table();
13826                viewElement.setId(String.valueOf(viewModel.getId()));
13827                if (!SQLUtil.isEmpty(viewModel.getDatabase())) {
13828                        viewElement.setDatabase(viewModel.getDatabase());
13829                }
13830                if (!SQLUtil.isEmpty(viewModel.getSchema())) {
13831                        viewElement.setSchema(viewModel.getSchema());
13832                }
13833                viewElement.setServer(viewModel.getServer());
13834                viewElement.setName(viewModel.getName());
13835                viewElement.setType("view");
13836                viewElement.setStarStmt(viewModel.getStarStmt());
13837
13838                if(viewModel.isFromDDL()){
13839                        viewElement.setFromDDL(String.valueOf(viewModel.isFromDDL()));
13840                }
13841
13842        if(option.isTraceTablePosition()){
13843            for (Pair<Pair3<Long, Long, String>, Pair3<Long, Long, String>> position:viewModel.getPositions()){
13844                viewElement.setCoordinate(convertCoordinate(position.first)+","+convertCoordinate(position.second));
13845            }
13846        }
13847        else {
13848            viewElement.setCoordinate(convertCoordinate(viewModel.getStartPosition()) + ","
13849                    + convertCoordinate(viewModel.getEndPosition()));
13850        }
13851
13852                if (viewModel.getProcesses() != null) {
13853                        List<String> processIds = new ArrayList<String>();
13854                        for (Process process : viewModel.getProcesses()) {
13855                                processIds.add(String.valueOf(process.getId()));
13856                        }
13857                        viewElement.setProcessIds(processIds);
13858                }
13859                dataflow.getViews().add(viewElement);
13860
13861                List<TableColumn> columns = viewModel.getColumns();
13862
13863                if (containStarColumn(columns)) {
13864                        for (TableColumn column : columns) {
13865                                if (column.getName().endsWith("*")) {
13866                                        for (TableColumn starElement : columns) {
13867                                                if (starElement == column) {
13868                                                        continue;
13869                                                }
13870                                                TObjectName columnObject = starElement.getColumnObject();
13871                                                column.bindStarLinkColumn(columnObject);
13872                                        }
13873                                        if (viewModel.isCreateTable()) {
13874                                                column.setShowStar(false);
13875                                        }
13876                                }
13877                        }
13878                }
13879
13880                for (int j = 0; j < columns.size(); j++) {
13881                        TableColumn columnModel = (TableColumn) columns.get(j);
13882                        if (!columnModel.isPseduo() && columnModel.hasStarLinkColumn()) {
13883                                List<String> starLinkColumnList = columnModel.getStarLinkColumnNames();
13884                                for (int k = 0; k < starLinkColumnList.size(); k++) {
13885                                        column columnElement = new column();
13886                                        columnElement.setId(String.valueOf(columnModel.getId()) + "_" + k);
13887                                        String columnName = starLinkColumnList.get(k);
13888                                        if (containStarColumn(columns, columnName)) {
13889                                                continue;
13890                                        }
13891                                        columnElement.setName(columnName);
13892                                        if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
13893                                                columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
13894                                                                + convertCoordinate(columnModel.getEndPosition()));
13895                                        }
13896                                        viewElement.getColumns().add(columnElement);
13897                                }
13898
13899                                if (columnModel.isShowStar()) {
13900                                        column columnElement = new column();
13901                                        columnElement.setId(String.valueOf(columnModel.getId()));
13902                                        columnElement.setName(columnModel.getName());
13903                                        if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
13904                                                columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
13905                                                                + convertCoordinate(columnModel.getEndPosition()));
13906                                        }
13907                                        viewElement.getColumns().add(columnElement);
13908                                }
13909
13910                        } else {
13911                                column columnElement = new column();
13912                                columnElement.setId(String.valueOf(columnModel.getId()));
13913                                columnElement.setName(columnModel.getName());
13914                                if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
13915                                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
13916                                                        + convertCoordinate(columnModel.getEndPosition()));
13917                                }
13918                                if(columnModel.isPseduo()) {
13919                                        columnElement.setSource("system");
13920                                }
13921                                viewElement.getColumns().add(columnElement);
13922                        }
13923                }
13924
13925                TableRelationRows relationRows = viewModel.getRelationRows();
13926                if (relationRows.hasRelation()) {
13927                        column relationRowsElement = new column();
13928                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
13929                        relationRowsElement.setName(relationRows.getName());
13930                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
13931                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
13932                                                + convertCoordinate(relationRows.getEndPosition()));
13933                        }
13934                        relationRowsElement.setSource("system");
13935                        viewElement.getColumns().add(relationRowsElement);
13936                }
13937        }
13938
13939        private void appendStreamModel(dataflow dataflow, Table streamModel) {
13940                table streamElement = new table();
13941                streamElement.setId(String.valueOf(streamModel.getId()));
13942                if (!SQLUtil.isEmpty(streamModel.getDatabase())) {
13943                        streamElement.setDatabase(streamModel.getDatabase());
13944                }
13945                if (!SQLUtil.isEmpty(streamModel.getSchema())) {
13946                        streamElement.setSchema(streamModel.getSchema());
13947                }
13948                streamElement.setServer(streamModel.getServer());
13949                streamElement.setName(streamModel.getName());
13950                streamElement.setType("stream");
13951                if (streamModel.getFileType() != null) {
13952                        streamElement.setFileType(SQLUtil.trimColumnStringQuote(streamModel.getFileType()));
13953                }
13954
13955                if (streamModel.getStartPosition() != null && streamModel.getEndPosition() != null) {
13956                        streamElement.setCoordinate(convertCoordinate(streamModel.getStartPosition()) + ","
13957                                        + convertCoordinate(streamModel.getEndPosition()));
13958                }
13959
13960                if (streamModel.getProcesses() != null) {
13961                        List<String> processIds = new ArrayList<String>();
13962                        for (Process process : streamModel.getProcesses()) {
13963                                processIds.add(String.valueOf(process.getId()));
13964                        }
13965                        streamElement.setProcessIds(processIds);
13966                }
13967                dataflow.getStreams().add(streamElement);
13968
13969                List<TableColumn> columns = streamModel.getColumns();
13970
13971                for (int j = 0; j < columns.size(); j++) {
13972                        TableColumn columnModel = (TableColumn) columns.get(j);
13973                        column columnElement = new column();
13974                        columnElement.setId(String.valueOf(columnModel.getId()));
13975                        columnElement.setName(SQLUtil.trimColumnStringQuote(columnModel.getName()));
13976                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
13977                                        + convertCoordinate(columnModel.getEndPosition()));
13978                        streamElement.getColumns().add(columnElement);
13979                }
13980
13981                TableRelationRows relationRows = streamModel.getRelationRows();
13982                if (relationRows.hasRelation()) {
13983                        column relationRowsElement = new column();
13984                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
13985                        relationRowsElement.setName(relationRows.getName());
13986                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
13987                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
13988                                                + convertCoordinate(relationRows.getEndPosition()));
13989                        }
13990                        relationRowsElement.setSource("system");
13991                        streamElement.getColumns().add(relationRowsElement);
13992                }
13993        }
13994
13995        private void appendStageModel(dataflow dataflow, Table stageModel) {
13996                table stageElement = new table();
13997                stageElement.setId(String.valueOf(stageModel.getId()));
13998                if (!SQLUtil.isEmpty(stageModel.getDatabase())) {
13999                        stageElement.setDatabase(stageModel.getDatabase());
14000                }
14001                if (!SQLUtil.isEmpty(stageModel.getSchema())) {
14002                        stageElement.setSchema(stageModel.getSchema());
14003                }
14004                stageElement.setServer(stageModel.getServer());
14005                stageElement.setName(stageModel.getName());
14006                stageElement.setType("stage");
14007                stageElement.setLocation(stageModel.getLocation());
14008                if (stageModel.getFileType() != null) {
14009                        stageElement.setFileType(SQLUtil.trimColumnStringQuote(stageModel.getFileType()));
14010                }
14011
14012                if (stageModel.getStartPosition() != null && stageModel.getEndPosition() != null) {
14013                        stageElement.setCoordinate(convertCoordinate(stageModel.getStartPosition()) + ","
14014                                        + convertCoordinate(stageModel.getEndPosition()));
14015                }
14016
14017                if (stageModel.getProcesses() != null) {
14018                        List<String> processIds = new ArrayList<String>();
14019                        for (Process process : stageModel.getProcesses()) {
14020                                processIds.add(String.valueOf(process.getId()));
14021                        }
14022                        stageElement.setProcessIds(processIds);
14023                }
14024                dataflow.getStages().add(stageElement);
14025
14026                List<TableColumn> columns = stageModel.getColumns();
14027
14028                for (int j = 0; j < columns.size(); j++) {
14029                        TableColumn columnModel = (TableColumn) columns.get(j);
14030                        column columnElement = new column();
14031                        columnElement.setId(String.valueOf(columnModel.getId()));
14032                        columnElement.setName(SQLUtil.trimColumnStringQuote(columnModel.getName()));
14033                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
14034                                        + convertCoordinate(columnModel.getEndPosition()));
14035                        stageElement.getColumns().add(columnElement);
14036                }
14037
14038                TableRelationRows relationRows = stageModel.getRelationRows();
14039                if (relationRows.hasRelation()) {
14040                        column relationRowsElement = new column();
14041                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
14042                        relationRowsElement.setName(relationRows.getName());
14043                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
14044                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
14045                                                + convertCoordinate(relationRows.getEndPosition()));
14046                        }
14047                        relationRowsElement.setSource("system");
14048                        stageElement.getColumns().add(relationRowsElement);
14049                }
14050        }
14051
14052        private void appendSequenceModel(dataflow dataflow, Table sequenceModel) {
14053                table sequenceElement = new table();
14054                sequenceElement.setId(String.valueOf(sequenceModel.getId()));
14055                if (!SQLUtil.isEmpty(sequenceModel.getDatabase())) {
14056                        sequenceElement.setDatabase(sequenceModel.getDatabase());
14057                }
14058                if (!SQLUtil.isEmpty(sequenceModel.getSchema())) {
14059                        sequenceElement.setSchema(sequenceModel.getSchema());
14060                }
14061                sequenceElement.setServer(sequenceModel.getServer());
14062                sequenceElement.setName(sequenceModel.getName());
14063                sequenceElement.setType("sequence");
14064                sequenceElement.setLocation(sequenceModel.getLocation());
14065                if (sequenceModel.getFileType() != null) {
14066                        sequenceElement.setFileType(SQLUtil.trimColumnStringQuote(sequenceModel.getFileType()));
14067                }
14068
14069                if (sequenceModel.getStartPosition() != null && sequenceModel.getEndPosition() != null) {
14070                        sequenceElement.setCoordinate(convertCoordinate(sequenceModel.getStartPosition()) + ","
14071                                        + convertCoordinate(sequenceModel.getEndPosition()));
14072                }
14073
14074                if (sequenceModel.getProcesses() != null) {
14075                        List<String> processIds = new ArrayList<String>();
14076                        for (Process process : sequenceModel.getProcesses()) {
14077                                processIds.add(String.valueOf(process.getId()));
14078                        }
14079                        sequenceElement.setProcessIds(processIds);
14080                }
14081                dataflow.getSequences().add(sequenceElement);
14082
14083                List<TableColumn> columns = sequenceModel.getColumns();
14084
14085                for (int j = 0; j < columns.size(); j++) {
14086                        TableColumn columnModel = (TableColumn) columns.get(j);
14087                        column columnElement = new column();
14088                        columnElement.setId(String.valueOf(columnModel.getId()));
14089                        columnElement.setName(SQLUtil.trimColumnStringQuote(columnModel.getName()));
14090                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
14091                                        + convertCoordinate(columnModel.getEndPosition()));
14092                        sequenceElement.getColumns().add(columnElement);
14093                }
14094
14095                TableRelationRows relationRows = sequenceModel.getRelationRows();
14096                if (relationRows.hasRelation()) {
14097                        column relationRowsElement = new column();
14098                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
14099                        relationRowsElement.setName(relationRows.getName());
14100                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
14101                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
14102                                                + convertCoordinate(relationRows.getEndPosition()));
14103                        }
14104                        relationRowsElement.setSource("system");
14105                        sequenceElement.getColumns().add(relationRowsElement);
14106                }
14107        }
14108
14109        private void appendDataSourceModel(dataflow dataflow, Table datasourceModel) {
14110                table datasourceElement = new table();
14111                datasourceElement.setId(String.valueOf(datasourceModel.getId()));
14112                if (!SQLUtil.isEmpty(datasourceModel.getDatabase())) {
14113                        datasourceElement.setDatabase(datasourceModel.getDatabase());
14114                }
14115                if (!SQLUtil.isEmpty(datasourceModel.getSchema())) {
14116                        datasourceElement.setSchema(datasourceModel.getSchema());
14117                }
14118                datasourceElement.setServer(datasourceModel.getServer());
14119                datasourceElement.setName(datasourceModel.getName());
14120                datasourceElement.setType("datasource");
14121                datasourceElement.setLocation(datasourceModel.getLocation());
14122                if (datasourceModel.getFileType() != null) {
14123                        datasourceElement.setFileType(SQLUtil.trimColumnStringQuote(datasourceModel.getFileType()));
14124                }
14125
14126                if (datasourceModel.getStartPosition() != null && datasourceModel.getEndPosition() != null) {
14127                        datasourceElement.setCoordinate(convertCoordinate(datasourceModel.getStartPosition()) + ","
14128                                        + convertCoordinate(datasourceModel.getEndPosition()));
14129                }
14130
14131                if (datasourceModel.getProcesses() != null) {
14132                        List<String> processIds = new ArrayList<String>();
14133                        for (Process process : datasourceModel.getProcesses()) {
14134                                processIds.add(String.valueOf(process.getId()));
14135                        }
14136                        datasourceElement.setProcessIds(processIds);
14137                }
14138                dataflow.getDatasources().add(datasourceElement);
14139
14140                List<TableColumn> columns = datasourceModel.getColumns();
14141
14142                for (int j = 0; j < columns.size(); j++) {
14143                        TableColumn columnModel = (TableColumn) columns.get(j);
14144                        column columnElement = new column();
14145                        columnElement.setId(String.valueOf(columnModel.getId()));
14146                        columnElement.setName(SQLUtil.trimColumnStringQuote(columnModel.getName()));
14147                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
14148                                        + convertCoordinate(columnModel.getEndPosition()));
14149                        datasourceElement.getColumns().add(columnElement);
14150                }
14151
14152                TableRelationRows relationRows = datasourceModel.getRelationRows();
14153                if (relationRows.hasRelation()) {
14154                        column relationRowsElement = new column();
14155                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
14156                        relationRowsElement.setName(relationRows.getName());
14157                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
14158                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
14159                                                + convertCoordinate(relationRows.getEndPosition()));
14160                        }
14161                        relationRowsElement.setSource("system");
14162                        datasourceElement.getColumns().add(relationRowsElement);
14163                }
14164        }
14165
14166        private void appendDatabaseModel(dataflow dataflow, Table databaseModel) {
14167                table databaseElement = new table();
14168                databaseElement.setId(String.valueOf(databaseModel.getId()));
14169                if (!SQLUtil.isEmpty(databaseModel.getDatabase())) {
14170                        databaseElement.setDatabase(databaseModel.getDatabase());
14171                }
14172                if (!SQLUtil.isEmpty(databaseModel.getSchema())) {
14173                        databaseElement.setSchema(databaseModel.getSchema());
14174                }
14175                databaseElement.setServer(databaseModel.getServer());
14176                databaseElement.setName(databaseModel.getName());
14177                databaseElement.setType("database");
14178                if (databaseModel.getFileType() != null) {
14179                        databaseElement.setFileType(SQLUtil.trimColumnStringQuote(databaseModel.getFileType()));
14180                }
14181
14182                if (databaseModel.getStartPosition() != null && databaseModel.getEndPosition() != null) {
14183                        databaseElement.setCoordinate(convertCoordinate(databaseModel.getStartPosition()) + ","
14184                                        + convertCoordinate(databaseModel.getEndPosition()));
14185                }
14186
14187                if (databaseModel.getProcesses() != null) {
14188                        List<String> processIds = new ArrayList<String>();
14189                        for (Process process : databaseModel.getProcesses()) {
14190                                processIds.add(String.valueOf(process.getId()));
14191                        }
14192                        databaseElement.setProcessIds(processIds);
14193                }
14194                dataflow.getDatabases().add(databaseElement);
14195
14196                List<TableColumn> columns = databaseModel.getColumns();
14197
14198                for (int j = 0; j < columns.size(); j++) {
14199                        TableColumn columnModel = (TableColumn) columns.get(j);
14200                        column columnElement = new column();
14201                        columnElement.setId(String.valueOf(columnModel.getId()));
14202                        columnElement.setName(SQLUtil.trimColumnStringQuote(columnModel.getName()));
14203                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
14204                                        + convertCoordinate(columnModel.getEndPosition()));
14205                        databaseElement.getColumns().add(columnElement);
14206                }
14207
14208                TableRelationRows relationRows = databaseModel.getRelationRows();
14209                if (relationRows.hasRelation()) {
14210                        column relationRowsElement = new column();
14211                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
14212                        relationRowsElement.setName(relationRows.getName());
14213                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
14214                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
14215                                                + convertCoordinate(relationRows.getEndPosition()));
14216                        }
14217                        relationRowsElement.setSource("system");
14218                        databaseElement.getColumns().add(relationRowsElement);
14219                }
14220        }
14221
14222        private void appendSchemaModel(dataflow dataflow, Table schemaModel) {
14223                table schemaElement = new table();
14224                schemaElement.setId(String.valueOf(schemaModel.getId()));
14225                if (!SQLUtil.isEmpty(schemaModel.getDatabase())) {
14226                        schemaElement.setDatabase(schemaModel.getDatabase());
14227                }
14228                if (!SQLUtil.isEmpty(schemaModel.getSchema())) {
14229                        schemaElement.setSchema(schemaModel.getSchema());
14230                }
14231                schemaElement.setServer(schemaModel.getServer());
14232                schemaElement.setName(schemaModel.getName());
14233                schemaElement.setType("schema");
14234                if (schemaModel.getFileType() != null) {
14235                        schemaElement.setFileType(SQLUtil.trimColumnStringQuote(schemaModel.getFileType()));
14236                }
14237
14238                if (schemaModel.getStartPosition() != null && schemaModel.getEndPosition() != null) {
14239                        schemaElement.setCoordinate(convertCoordinate(schemaModel.getStartPosition()) + ","
14240                                        + convertCoordinate(schemaModel.getEndPosition()));
14241                }
14242
14243                if (schemaModel.getProcesses() != null) {
14244                        List<String> processIds = new ArrayList<String>();
14245                        for (Process process : schemaModel.getProcesses()) {
14246                                processIds.add(String.valueOf(process.getId()));
14247                        }
14248                        schemaElement.setProcessIds(processIds);
14249                }
14250                dataflow.getSchemas().add(schemaElement);
14251
14252                List<TableColumn> columns = schemaModel.getColumns();
14253
14254                for (int j = 0; j < columns.size(); j++) {
14255                        TableColumn columnModel = (TableColumn) columns.get(j);
14256                        column columnElement = new column();
14257                        columnElement.setId(String.valueOf(columnModel.getId()));
14258                        columnElement.setName(SQLUtil.trimColumnStringQuote(columnModel.getName()));
14259                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
14260                                        + convertCoordinate(columnModel.getEndPosition()));
14261                        schemaElement.getColumns().add(columnElement);
14262                }
14263
14264                TableRelationRows relationRows = schemaModel.getRelationRows();
14265                if (relationRows.hasRelation()) {
14266                        column relationRowsElement = new column();
14267                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
14268                        relationRowsElement.setName(relationRows.getName());
14269                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
14270                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
14271                                                + convertCoordinate(relationRows.getEndPosition()));
14272                        }
14273                        relationRowsElement.setSource("system");
14274                        schemaElement.getColumns().add(relationRowsElement);
14275                }
14276        }
14277
14278        private void appendPathModel(dataflow dataflow, Table pathModel) {
14279                table pathElement = new table();
14280                pathElement.setId(String.valueOf(pathModel.getId()));
14281                if (!SQLUtil.isEmpty(pathModel.getDatabase())) {
14282                        pathElement.setDatabase(pathModel.getDatabase());
14283                }
14284                if (!SQLUtil.isEmpty(pathModel.getSchema())) {
14285                        pathElement.setSchema(pathModel.getSchema());
14286                }
14287                pathElement.setServer(pathModel.getServer());
14288                pathElement.setName(pathModel.getName());
14289                pathElement.setType("path");
14290                if (pathModel.getFileFormat() != null) {
14291                        pathElement.setFileFormat(SQLUtil.trimColumnStringQuote(pathModel.getFileFormat()));
14292                }
14293
14294                if (pathModel.getStartPosition() != null && pathModel.getEndPosition() != null) {
14295                        pathElement.setCoordinate(convertCoordinate(pathModel.getStartPosition()) + ","
14296                                        + convertCoordinate(pathModel.getEndPosition()));
14297                }
14298
14299                if (pathModel.getProcesses() != null) {
14300                        List<String> processIds = new ArrayList<String>();
14301                        for (Process process : pathModel.getProcesses()) {
14302                                processIds.add(String.valueOf(process.getId()));
14303                        }
14304                        pathElement.setProcessIds(processIds);
14305                }
14306
14307                pathElement.setUri(pathModel.getName());
14308
14309                dataflow.getPaths().add(pathElement);
14310
14311                List<TableColumn> columns = pathModel.getColumns();
14312
14313                for (int j = 0; j < columns.size(); j++) {
14314                        TableColumn columnModel = (TableColumn) columns.get(j);
14315                        column columnElement = new column();
14316                        columnElement.setId(String.valueOf(columnModel.getId()));
14317                        columnElement.setName(SQLUtil.trimColumnStringQuote(columnModel.getName()));
14318                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
14319                                        + convertCoordinate(columnModel.getEndPosition()));
14320                        pathElement.getColumns().add(columnElement);
14321                }
14322
14323                TableRelationRows relationRows = pathModel.getRelationRows();
14324                if (relationRows.hasRelation()) {
14325                        column relationRowsElement = new column();
14326                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
14327                        relationRowsElement.setName(relationRows.getName());
14328                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
14329                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
14330                                                + convertCoordinate(relationRows.getEndPosition()));
14331                        }
14332                        relationRowsElement.setSource("system");
14333                        pathElement.getColumns().add(relationRowsElement);
14334                }
14335        }
14336
14337        private void appendVariableModel(dataflow dataflow, Table variableModel) {
14338                table variableElement = new table();
14339                variableElement.setId(String.valueOf(variableModel.getId()));
14340                if (!SQLUtil.isEmpty(variableModel.getDatabase())) {
14341                        variableElement.setDatabase(variableModel.getDatabase());
14342                }
14343                if (!SQLUtil.isEmpty(variableModel.getSchema())) {
14344                        variableElement.setSchema(variableModel.getSchema());
14345                }
14346                variableElement.setServer(variableModel.getServer());
14347                variableElement.setName(variableModel.getName());
14348                variableElement.setType("variable");
14349                variableElement.setParent(variableModel.getParent());
14350                if (variableModel.getSubType() != null) {
14351                        variableElement.setSubType(variableModel.getSubType().name());
14352                }
14353
14354                if (variableModel.getStartPosition() != null && variableModel.getEndPosition() != null) {
14355                        variableElement.setCoordinate(convertCoordinate(variableModel.getStartPosition()) + ","
14356                                        + convertCoordinate(variableModel.getEndPosition()));
14357                }
14358                dataflow.getVariables().add(variableElement);
14359
14360                List<TableColumn> columns = variableModel.getColumns();
14361
14362                if (containStarColumn(columns)) {
14363                        for (TableColumn column : columns) {
14364                                if (column.getName().endsWith("*")) {
14365                                        for (TableColumn starElement : columns) {
14366                                                if (starElement == column) {
14367                                                        continue;
14368                                                }
14369                                                TObjectName columnObject = starElement.getColumnObject();
14370                                                column.bindStarLinkColumn(columnObject);
14371                                        }
14372//                                      column.setShowStar(false);
14373                                }
14374                        }
14375                }
14376
14377                for (int j = 0; j < columns.size(); j++) {
14378                        TableColumn columnModel = columns.get(j);
14379                        if (columnModel.hasStarLinkColumn()) {
14380                                List<String> starLinkColumnList = columnModel.getStarLinkColumnNames();
14381                                for (int k = 0; k < starLinkColumnList.size(); k++) {
14382                                        column columnElement = new column();
14383                                        columnElement.setId(columnModel.getId() + "_" + k);
14384                                        String columnName = starLinkColumnList.get(k);
14385                                        if (containStarColumn(columns, columnName)) {
14386                                                continue;
14387                                        }
14388                                        columnElement.setName(columnName);
14389                                        if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
14390                                                columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
14391                                                                + convertCoordinate(columnModel.getEndPosition()));
14392                                        }
14393                                        variableElement.getColumns().add(columnElement);
14394                                }
14395
14396                                if (columnModel.isShowStar()) {
14397                                        column columnElement = new column();
14398                                        columnElement.setId(String.valueOf(columnModel.getId()));
14399                                        columnElement.setName(columnModel.getName());
14400                                        if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
14401                                                columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
14402                                                                + convertCoordinate(columnModel.getEndPosition()));
14403                                        }
14404                                        variableElement.getColumns().add(columnElement);
14405                                }
14406
14407                        } else {
14408                                column columnElement = new column();
14409                                columnElement.setId(String.valueOf(columnModel.getId()));
14410                                columnElement.setName(columnModel.getName());
14411                                if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
14412                                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
14413                                                        + convertCoordinate(columnModel.getEndPosition()));
14414                                }
14415                                variableElement.getColumns().add(columnElement);
14416                        }
14417                }
14418
14419                TableRelationRows relationRows = variableModel.getRelationRows();
14420                if (relationRows.hasRelation()) {
14421                        column relationRowsElement = new column();
14422                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
14423                        relationRowsElement.setName(relationRows.getName());
14424                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
14425                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
14426                                                + convertCoordinate(relationRows.getEndPosition()));
14427                        }
14428                        relationRowsElement.setSource("system");
14429                        variableElement.getColumns().add(relationRowsElement);
14430                }
14431        }
14432
14433        private void appendCursorModel(dataflow dataflow, Table cursorModel) {
14434                table cursorElement = new table();
14435                cursorElement.setId(String.valueOf(cursorModel.getId()));
14436                if (!SQLUtil.isEmpty(cursorModel.getDatabase())) {
14437                        cursorElement.setDatabase(cursorModel.getDatabase());
14438                }
14439                if (!SQLUtil.isEmpty(cursorModel.getSchema())) {
14440                        cursorElement.setSchema(cursorModel.getSchema());
14441                }
14442                cursorElement.setServer(cursorModel.getServer());
14443                cursorElement.setName(cursorModel.getName());
14444                cursorElement.setType("variable");
14445                if (cursorElement.getSubType() != null) {
14446                        cursorElement.setSubType(cursorModel.getSubType().name());
14447                }
14448
14449                if (cursorModel.getStartPosition() != null && cursorModel.getEndPosition() != null) {
14450                        cursorElement.setCoordinate(convertCoordinate(cursorModel.getStartPosition()) + ","
14451                                        + convertCoordinate(cursorModel.getEndPosition()));
14452                }
14453                dataflow.getVariables().add(cursorElement);
14454
14455                List<TableColumn> columns = cursorModel.getColumns();
14456
14457                if (containStarColumn(columns)) {
14458                        for (TableColumn column : columns) {
14459                                if (column.getName().endsWith("*")) {
14460                                        for (TableColumn starElement : columns) {
14461                                                if (starElement == column) {
14462                                                        continue;
14463                                                }
14464                                                TObjectName columnObject = starElement.getColumnObject();
14465                                                column.bindStarLinkColumn(columnObject);
14466                                        }
14467                                        column.setShowStar(false);
14468                                }
14469                        }
14470                }
14471
14472                for (int j = 0; j < columns.size(); j++) {
14473                        TableColumn columnModel = (TableColumn) columns.get(j);
14474                        if (columnModel.hasStarLinkColumn()) {
14475                                List<String> starLinkColumnList = columnModel.getStarLinkColumnNames();
14476                                for (int k = 0; k < starLinkColumnList.size(); k++) {
14477                                        column columnElement = new column();
14478                                        columnElement.setId(String.valueOf(columnModel.getId()) + "_" + k);
14479                                        String columnName = starLinkColumnList.get(k);
14480                                        if (containStarColumn(columns, columnName)) {
14481                                                continue;
14482                                        }
14483                                        columnElement.setName(columnName);
14484                                        if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
14485                                                columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
14486                                                                + convertCoordinate(columnModel.getEndPosition()));
14487                                        }
14488                                        cursorElement.getColumns().add(columnElement);
14489                                }
14490
14491                                if (columnModel.isShowStar()) {
14492                                        column columnElement = new column();
14493                                        columnElement.setId(String.valueOf(columnModel.getId()));
14494                                        columnElement.setName(columnModel.getName());
14495                                        if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
14496                                                columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
14497                                                                + convertCoordinate(columnModel.getEndPosition()));
14498                                        }
14499                                        cursorElement.getColumns().add(columnElement);
14500                                }
14501
14502                        } else {
14503                                column columnElement = new column();
14504                                columnElement.setId(String.valueOf(columnModel.getId()));
14505                                columnElement.setName(columnModel.getName());
14506                                if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
14507                                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
14508                                                        + convertCoordinate(columnModel.getEndPosition()));
14509                                }
14510                                cursorElement.getColumns().add(columnElement);
14511                        }
14512                }
14513
14514                TableRelationRows relationRows = cursorModel.getRelationRows();
14515                if (relationRows.hasRelation()) {
14516                        column relationRowsElement = new column();
14517                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
14518                        relationRowsElement.setName(relationRows.getName());
14519                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
14520                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
14521                                                + convertCoordinate(relationRows.getEndPosition()));
14522                        }
14523                        relationRowsElement.setSource("system");
14524                        cursorElement.getColumns().add(relationRowsElement);
14525                }
14526        }
14527
14528        private void appendErrors(dataflow dataflow) {
14529                List<ErrorInfo> errorInfos = this.getErrorMessages();
14530                if (metadataErrors != null) {
14531                        for (int i = 0; i < metadataErrors.size(); i++) {
14532                                errorInfos.add(i, metadataErrors.get(i));
14533                        }
14534                }
14535
14536                for (int i = 0; i < errorInfos.size(); ++i) {
14537                        ErrorInfo errorInfo = errorInfos.get(i);
14538                        error error = new error();
14539                        if (!SQLUtil.isEmpty(errorInfo.getErrorMessage())) {
14540                                error.setErrorMessage(errorInfo.getErrorMessage());
14541                        }
14542                        if (!SQLUtil.isEmpty(errorInfo.getErrorType())) {
14543                                error.setErrorType(errorInfo.getErrorType());
14544                        }
14545                        if (errorInfo.getStartPosition() != null && errorInfo.getEndPosition() != null) {
14546                                error.setCoordinate(convertCoordinate(errorInfo.getStartPosition()) + ","
14547                                                + convertCoordinate(errorInfo.getEndPosition()));
14548                        }
14549                        if (!SQLUtil.isEmpty(errorInfo.getFileName())) {
14550                                error.setFile(errorInfo.getFileName());
14551                        }
14552                        if (errorInfo.getOriginStartPosition() != null && errorInfo.getOriginEndPosition() != null) {
14553                                error.setOriginCoordinate(errorInfo.getOriginStartPosition() + "," + errorInfo.getOriginEndPosition());
14554                        }
14555                        dataflow.getErrors().add(error);
14556                }
14557        }
14558
14559        private void appendOraclePackages(dataflow dataflow) {
14560                List<OraclePackage> packages = this.modelManager.getOraclePackageModels();
14561
14562                for (int i = 0; i < packages.size(); ++i) {
14563                        OraclePackage model = packages.get(i);
14564                        oraclePackage oraclePackage = new oraclePackage();
14565                        oraclePackage.setId(String.valueOf(model.getId()));
14566                        if (!SQLUtil.isEmpty(model.getDatabase())) {
14567                                oraclePackage.setDatabase(model.getDatabase());
14568                        }
14569                        if (!SQLUtil.isEmpty(model.getSchema())) {
14570                                oraclePackage.setSchema(model.getSchema());
14571                        }
14572                        oraclePackage.setServer(model.getServer());
14573                        oraclePackage.setName(model.getName());
14574                        if (model.getType() != null) {
14575                                oraclePackage.setType(model.getType().name().replace("sst", ""));
14576                        }
14577                        if (model.getStartPosition() != null && model.getEndPosition() != null) {
14578                                oraclePackage.setCoordinate(
14579                                                convertCoordinate(model.getStartPosition()) + "," + convertCoordinate(model.getEndPosition()));
14580                        }
14581
14582                        dataflow.getPackages().add(oraclePackage);
14583
14584                        List<Argument> arguments = model.getArguments();
14585
14586                        for (int j = 0; j < arguments.size(); ++j) {
14587                                Argument argumentModel = (Argument) arguments.get(j);
14588                                argument argumentElement = new argument();
14589                                argumentElement.setId(String.valueOf(argumentModel.getId()));
14590                                argumentElement.setName(argumentModel.getName());
14591                                if (argumentModel.getStartPosition() != null && argumentModel.getEndPosition() != null) {
14592                                        argumentElement.setCoordinate(convertCoordinate(argumentModel.getStartPosition()) + ","
14593                                                        + convertCoordinate(argumentModel.getEndPosition()));
14594                                }
14595
14596                                argumentElement.setDatatype(argumentModel.getDataType().getDataTypeName());
14597                                argumentElement.setInout(argumentModel.getMode().name());
14598                                oraclePackage.getArguments().add(argumentElement);
14599                        }
14600
14601                        for (int j = 0; j < model.getProcedures().size(); j++) {
14602                                Procedure procedureModel = model.getProcedures().get(j);
14603                                procedure procedure = new procedure();
14604                                procedure.setId(String.valueOf(procedureModel.getId()));
14605                                if (!SQLUtil.isEmpty(procedureModel.getDatabase())) {
14606                                        procedure.setDatabase(procedureModel.getDatabase());
14607                                }
14608                                if (!SQLUtil.isEmpty(procedureModel.getSchema())) {
14609                                        procedure.setSchema(procedureModel.getSchema());
14610                                }
14611                                procedure.setServer(procedureModel.getServer());
14612                                procedure.setName(procedureModel.getName());
14613                                if (procedureModel.getType() != null) {
14614                                        procedure.setType(procedureModel.getType().name().replace("sst", ""));
14615                                }
14616                                if (procedureModel.getStartPosition() != null && procedureModel.getEndPosition() != null) {
14617                                        procedure.setCoordinate(convertCoordinate(procedureModel.getStartPosition()) + ","
14618                                                        + convertCoordinate(procedureModel.getEndPosition()));
14619                                }
14620
14621                                oraclePackage.getProcedures().add(procedure);
14622
14623                                List<Argument> procedureArguments = procedureModel.getArguments();
14624
14625                                for (int k = 0; k < procedureArguments.size(); ++k) {
14626                                        Argument argumentModel = (Argument) procedureArguments.get(k);
14627                                        argument argumentElement = new argument();
14628                                        argumentElement.setId(String.valueOf(argumentModel.getId()));
14629                                        argumentElement.setName(argumentModel.getName());
14630                                        if (argumentModel.getStartPosition() != null && argumentModel.getEndPosition() != null) {
14631                                                argumentElement.setCoordinate(convertCoordinate(argumentModel.getStartPosition()) + ","
14632                                                                + convertCoordinate(argumentModel.getEndPosition()));
14633                                        }
14634
14635                                        argumentElement.setDatatype(argumentModel.getDataType().getDataTypeName());
14636                                        argumentElement.setInout(argumentModel.getMode().name());
14637                                        procedure.getArguments().add(argumentElement);
14638                                }
14639                        }
14640                }
14641        }
14642
14643        private void appendProcedures(dataflow dataflow) {
14644                List<Procedure> procedures = this.modelManager.getProcedureModels();
14645
14646                for (int i = 0; i < procedures.size(); ++i) {
14647                        Procedure model = procedures.get(i);
14648                        if (model.getParentPackage() != null) {
14649                                continue;
14650                        }
14651                        procedure procedure = new procedure();
14652                        procedure.setId(String.valueOf(model.getId()));
14653                        if (!SQLUtil.isEmpty(model.getDatabase())) {
14654                                procedure.setDatabase(model.getDatabase());
14655                        }
14656                        if (!SQLUtil.isEmpty(model.getSchema())) {
14657                                procedure.setSchema(model.getSchema());
14658                        }
14659                        procedure.setServer(model.getServer());
14660                        procedure.setName(model.getName());
14661                        if (model.getType() != null) {
14662                                procedure.setType(model.getType().name().replace("sst", ""));
14663                        }
14664                        if (model.getStartPosition() != null && model.getEndPosition() != null) {
14665                                procedure.setCoordinate(
14666                                                convertCoordinate(model.getStartPosition()) + "," + convertCoordinate(model.getEndPosition()));
14667                        }
14668
14669                        dataflow.getProcedures().add(procedure);
14670
14671                        List<Argument> arguments = model.getArguments();
14672
14673                        for (int j = 0; j < arguments.size(); ++j) {
14674                                Argument argumentModel = (Argument) arguments.get(j);
14675                                argument argumentElement = new argument();
14676                                argumentElement.setId(String.valueOf(argumentModel.getId()));
14677                                argumentElement.setName(argumentModel.getName());
14678                                if (argumentModel.getStartPosition() != null && argumentModel.getEndPosition() != null) {
14679                                        argumentElement.setCoordinate(convertCoordinate(argumentModel.getStartPosition()) + ","
14680                                                        + convertCoordinate(argumentModel.getEndPosition()));
14681                                }
14682
14683                                argumentElement.setDatatype(argumentModel.getDataType().getDataTypeName());
14684                                argumentElement.setInout(argumentModel.getMode().name());
14685                                procedure.getArguments().add(argumentElement);
14686                        }
14687                }
14688        }
14689
14690        private void appendProcesses(dataflow dataflow) {
14691                List<Process> processes = this.modelManager.getProcessModels();
14692
14693                for (int i = 0; i < processes.size(); ++i) {
14694                        Process model = processes.get(i);
14695                        process process = new process();
14696                        process.setId(String.valueOf(model.getId()));
14697                        if (!SQLUtil.isEmpty(model.getDatabase())) {
14698                                process.setDatabase(model.getDatabase());
14699                        }
14700                        if (!SQLUtil.isEmpty(model.getSchema())) {
14701                                process.setSchema(model.getSchema());
14702                        }
14703                        process.setServer(model.getServer());
14704                        process.setName(getProcessName(model));
14705                        if (!SQLUtil.isEmpty(model.getProcedureName())) {
14706                                process.setProcedureName(model.getProcedureName());
14707                        }
14708                        if (model.getProcedureId() != null) {
14709                                process.setProcedureId(String.valueOf(model.getProcedureId()));
14710                        }
14711                        if (!SQLUtil.isEmpty(model.getQueryHashId())) {
14712                                process.setQueryHashId(model.getQueryHashId());
14713                        }
14714                        if (model.getGspObject() != null) {
14715                                process.setType(model.getGspObject().sqlstatementtype.name());
14716                        }
14717                        if (model.getStartPosition() != null && model.getEndPosition() != null) {
14718                                process.setCoordinate(
14719                                                convertCoordinate(model.getStartPosition()) + "," + convertCoordinate(model.getEndPosition()));
14720                        }
14721                        if (model.getTransforms() != null && !model.getTransforms().isEmpty()) {
14722                                for (Transform transformItem : model.getTransforms()) {
14723                                        process.addTransform(transformItem);
14724                                }
14725                        }
14726                        dataflow.getProcesses().add(process);
14727                }
14728        }
14729
14730        private void appendTables(dataflow dataflow) {
14731                List<TTable> tables = modelManager.getBaseTables();
14732                Map<String, table> tableMap = new HashMap<String, table>();
14733                Set<Long> tableModelIds = new HashSet<Long>();
14734                for (int i = 0; i < tables.size(); i++) {
14735                        Object model = modelManager.getModel(tables.get(i));
14736                        if (model instanceof Table) {
14737                                Table tableModel = (Table) model;
14738                                if(tableModelIds.contains(tableModel.getId())) {
14739                                        continue;
14740                                }
14741                                else {
14742                                        tableModelIds.add(tableModel.getId());
14743                                }
14744                                if (tableModel.isView()) {
14745                                        continue;
14746                                }
14747                                if (tableModel.isStage()) {
14748                                        appendStageModel(dataflow, tableModel);
14749                                        continue;
14750                                }
14751                                if (tableModel.isSequence()) {
14752                                        appendSequenceModel(dataflow, tableModel);
14753                                        continue;
14754                                }
14755                                if (tableModel.isDataSource()) {
14756                                        appendDataSourceModel(dataflow, tableModel);
14757                                        continue;
14758                                }
14759                                if (tableModel.isDatabase()) {
14760                                        appendDatabaseModel(dataflow, tableModel);
14761                                        continue;
14762                                }
14763                                if (tableModel.isSchema()) {
14764                                        appendSchemaModel(dataflow, tableModel);
14765                                        continue;
14766                                }
14767                                if (tableModel.isStream()) {
14768                                        appendStreamModel(dataflow, tableModel);
14769                                        continue;
14770                                }
14771                                if (tableModel.isPath()) {
14772                                        appendPathModel(dataflow, tableModel);
14773                                        continue;
14774                                }
14775                                if (tableModel.isVariable() && !tableModel.isCursor()) {
14776                                        appendVariableModel(dataflow, tableModel);
14777                                        continue;
14778                                }
14779                                if (tableModel.isCursor()) {
14780                                        appendCursorModel(dataflow, tableModel);
14781                                        continue;
14782                                }
14783                                if (tableModel.isConstant()) {
14784                                        appendConstantModel(dataflow, tableModel);
14785                                        continue;
14786                                }
14787                                if (!tableIds.contains(tableModel.getId())) {
14788                                        appendTableModel(dataflow, tableModel, tableMap);
14789                                        tableIds.add(tableModel.getId());
14790                                }
14791                        } else if (model instanceof QueryTable) {
14792                                QueryTable queryTable = (QueryTable) model;
14793                                if (!tableIds.contains(queryTable.getId())) {
14794                                        appendResultSet(dataflow, queryTable);
14795                                        tableIds.add(queryTable.getId());
14796                                }
14797                        }
14798                }
14799
14800                List<Table> tableNames = modelManager.getTablesByName();
14801                tableNames.addAll(modelManager.getDropTables());
14802                
14803                for (int i = 0; i < tableNames.size(); i++) {
14804                        Table tableModel = tableNames.get(i);
14805                        if(tableModelIds.contains(tableModel.getId())) {
14806                                continue;
14807                        }
14808                        else {
14809                                tableModelIds.add(tableModel.getId());
14810                        }
14811                        if (tableModel.isView()) {
14812                                continue;
14813                        }
14814                        if (tableModel.isDatabase()) {
14815                                appendDatabaseModel(dataflow, tableModel);
14816                                continue;
14817                        }
14818                        if (tableModel.isSchema()) {
14819                                appendSchemaModel(dataflow, tableModel);
14820                                continue;
14821                        }
14822                        if (tableModel.isStage()) {
14823                                appendStageModel(dataflow, tableModel);
14824                                continue;
14825                        }
14826                        if (tableModel.isSequence()) {
14827                                appendSequenceModel(dataflow, tableModel);
14828                                continue;
14829                        }
14830                        if (tableModel.isDataSource()) {
14831                                appendDataSourceModel(dataflow, tableModel);
14832                                continue;
14833                        }
14834                        if (tableModel.isStream()) {
14835                                appendStreamModel(dataflow, tableModel);
14836                                continue;
14837                        }
14838                        if (tableModel.isPath()) {
14839                                appendPathModel(dataflow, tableModel);
14840                                continue;
14841                        }
14842                        if (tableModel.isVariable()) {
14843                                appendVariableModel(dataflow, tableModel);
14844                                continue;
14845                        }
14846                        if (tableModel.isCursor()) {
14847                                appendCursorModel(dataflow, tableModel);
14848                                continue;
14849                        }
14850                        if (tableModel.isConstant()) {
14851                                appendConstantModel(dataflow, tableModel);
14852                                continue;
14853                        }
14854                        if (!tableIds.contains(tableModel.getId())) {
14855                                appendTableModel(dataflow, tableModel, tableMap);
14856                                tableIds.add(tableModel.getId());
14857                        }
14858                }
14859        }
14860
14861        private void appendConstantModel(dataflow dataflow, Table tableModel) {
14862                table constantElement = new table();
14863                constantElement.setId(String.valueOf(tableModel.getId()));
14864                if (!SQLUtil.isEmpty(tableModel.getDatabase())) {
14865                        constantElement.setDatabase(tableModel.getDatabase());
14866                }
14867                if (!SQLUtil.isEmpty(tableModel.getSchema())) {
14868                        constantElement.setSchema(tableModel.getSchema());
14869                }
14870                constantElement.setServer(tableModel.getServer());
14871                constantElement.setName(getConstantName(tableModel));
14872                constantElement.setType("constantTable");
14873
14874                if (tableModel.getStartPosition() != null && tableModel.getEndPosition() != null) {
14875                        constantElement.setCoordinate(convertCoordinate(tableModel.getStartPosition()) + ","
14876                                        + convertCoordinate(tableModel.getEndPosition()));
14877                }
14878                dataflow.getTables().add(constantElement);
14879
14880                List<TableColumn> columns = tableModel.getColumns();
14881                for (int j = 0; j < columns.size(); j++) {
14882                        TableColumn columnModel = (TableColumn) columns.get(j);
14883                        column columnElement = new column();
14884                        columnElement.setId(String.valueOf(columnModel.getId()));
14885                        columnElement.setName(columnModel.getName());
14886                        if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
14887                                columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
14888                                                + convertCoordinate(columnModel.getEndPosition()));
14889                        }
14890                        constantElement.getColumns().add(columnElement);
14891                }
14892        }
14893
14894        private void appendTableModel(dataflow dataflow, Table tableModel, Map<String, table> tableMap) {
14895                if(tableModel.getSubType() == SubType.unnest) {}
14896                table tableElement = new table();
14897                tableElement.setId(String.valueOf(tableModel.getId()));
14898                if (!SQLUtil.isEmpty(tableModel.getDatabase())) {
14899                        tableElement.setDatabase(tableModel.getDatabase());
14900                }
14901                if (!SQLUtil.isEmpty(tableModel.getSchema())) {
14902                        tableElement.setSchema(tableModel.getSchema());
14903                }
14904                tableElement.setServer(tableModel.getServer());
14905                tableElement.setName(tableModel.getName());
14906                tableElement.setDisplayName(tableModel.getDisplayName());
14907                tableElement.setStarStmt(tableModel.getStarStmt());
14908                if(tableModel.isFromDDL()) {
14909                        tableElement.setFromDDL(String.valueOf(tableModel.isFromDDL()));
14910                }
14911
14912                if (tableModel.isPseudo()) {
14913                        tableElement.setType("pseudoTable");
14914                } else {
14915                        tableElement.setType("table");
14916                }
14917
14918                if (tableModel.getSubType() != null) {
14919                        if (tableModel.getSubType() == SubType.unnest) {
14920                                tableElement.setType(SubType.unnest.name());
14921                        }
14922                        else {
14923                                tableElement.setSubType(tableModel.getSubType().name());
14924                        }
14925                }
14926                if (tableModel.getParent() != null) {
14927                        tableElement.setParent(tableModel.getParent());
14928                }
14929                if (tableModel.getAlias() != null && tableModel.getAlias().trim().length() > 0) {
14930                        tableElement.setAlias(tableModel.getAlias());
14931                }
14932                if (tableModel.getStartPosition() != null && tableModel.getEndPosition() != null) {
14933            if(option.isTraceTablePosition()){
14934                for (Pair<Pair3<Long, Long, String>, Pair3<Long, Long, String>> position:tableModel.getPositions()){
14935                    tableElement.appendCoordinate(convertCoordinate(position.first)+","+convertCoordinate(position.second));
14936                }
14937            }
14938            else {
14939                tableElement.setCoordinate(convertCoordinate(tableModel.getStartPosition()) + ","
14940                    + convertCoordinate(tableModel.getEndPosition()));
14941            }
14942                }
14943                if (tableModel.getProcesses() != null) {
14944                        List<String> processIds = new ArrayList<String>();
14945                        for (Process process : tableModel.getProcesses()) {
14946                                processIds.add(String.valueOf(process.getId()));
14947                        }
14948                        tableElement.setProcessIds(processIds);
14949                }
14950
14951                table oldTableElement = null;
14952                String tableFullName = DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getQualifiedTableName(tableElement));
14953                
14954                if(tableMap.containsKey(tableFullName)) {
14955                        oldTableElement = tableMap.get(tableFullName);
14956                }
14957                else {
14958                        tableMap.put(tableFullName, tableElement);
14959                }
14960                
14961                if (tableModel.getSubType() == SubType.unnest) {
14962                        dataflow.getResultsets().add(tableElement);
14963                } else {
14964                        dataflow.getTables().add(tableElement);
14965                }
14966
14967                List<TableColumn> columns = tableModel.getColumns();
14968
14969                if (containStarColumn(columns)) {
14970                        for (TableColumn column : columns) {
14971                                if (column.getName().endsWith("*")) {
14972                                        for (TableColumn starElement : columns) {
14973                                                if (starElement == column) {
14974                                                        continue;
14975                                                }
14976                                                if (starElement.isNotBindStarLinkColumn()) {
14977                                                        continue;
14978                                                }
14979                                                TObjectName columnObject = starElement.getColumnObject();
14980                                                column.bindStarLinkColumn(columnObject);
14981                                        }
14982                                        if (tableModel.isCreateTable() && column.isExpandStar()) {
14983                                                column.setShowStar(false);
14984                                        }
14985                                }
14986                        }
14987                }
14988
14989                for (int j = 0; j < columns.size(); j++) {
14990                        TableColumn columnModel = columns.get(j);
14991
14992                        if(oldTableElement != null){
14993                                if(!CollectionUtil.isEmpty(oldTableElement.getColumns())){
14994                                        for(column oldColumnElement: oldTableElement.getColumns()){
14995                                                if(oldColumnElement.getName().equalsIgnoreCase(columnModel.getName())){
14996                                                        if(columnModel.getDataType() == null && oldColumnElement.getDataType() != null){
14997                                                                columnModel.setDataType(oldColumnElement.getDataType());
14998                                                        }
14999                                                        if(columnModel.getPrimaryKey() == null && oldColumnElement.isPrimaryKey() != null){
15000                                                                columnModel.setPrimaryKey(oldColumnElement.isPrimaryKey());
15001                                                        }
15002                                                        if(columnModel.getIndexKey() == null && oldColumnElement.isIndexKey() != null){
15003                                                                columnModel.setIndexKey(oldColumnElement.isIndexKey());
15004                                                        }
15005                                                        if(columnModel.getUnqiueKey() == null && oldColumnElement.isUnqiueKey() != null){
15006                                                                columnModel.setUnqiueKey(oldColumnElement.isUnqiueKey());
15007                                                        }
15008                                                        if(columnModel.getForeignKey() == null && oldColumnElement.isForeignKey() != null){
15009                                                                columnModel.setForeignKey(oldColumnElement.isForeignKey());
15010                                                        }
15011
15012                                                        if(oldColumnElement.getDataType() == null && columnModel.getDataType() != null){
15013                                                                oldColumnElement.setDataType(columnModel.getDataType());
15014                                                        }
15015                                                        if(oldColumnElement.isPrimaryKey() == null && columnModel.getPrimaryKey() != null){
15016                                                                oldColumnElement.setPrimaryKey(columnModel.getPrimaryKey());
15017                                                        }
15018                                                        if(oldColumnElement.isIndexKey() == null && columnModel.getIndexKey() != null){
15019                                                                oldColumnElement.setIndexKey(columnModel.getIndexKey());
15020                                                        }
15021                                                        if(oldColumnElement.isUnqiueKey() == null && columnModel.getUnqiueKey() != null){
15022                                                                oldColumnElement.setUnqiueKey(columnModel.getUnqiueKey());
15023                                                        }
15024                                                        if(oldColumnElement.isForeignKey() == null && columnModel.getForeignKey() != null){
15025                                                                oldColumnElement.setForeignKey(columnModel.getForeignKey());
15026                                                        }
15027                                                        break;
15028                                                }
15029                                        }
15030                                }
15031
15032                        }
15033
15034                        if (!columnModel.isPseduo() && columnModel.hasStarLinkColumn() && !columnModel.isVariant()) {
15035                                List<String> starLinkColumnList = columnModel.getStarLinkColumnNames();
15036                                for (int k = 0; k < starLinkColumnList.size(); k++) {
15037                                        column columnElement = new column();
15038                                        columnElement.setId(String.valueOf(columnModel.getId()) + "_" + k);
15039                                        String columnName = starLinkColumnList.get(k);
15040                                        if (containStarColumn(columns, columnName)) {
15041                                                continue;
15042                                        }
15043                                        columnElement.setName(columnName);
15044                                        if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
15045                                                columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
15046                                                                + convertCoordinate(columnModel.getEndPosition()));
15047                                        }
15048                                        if (columnModel.getForeignKey()) {
15049                                                columnElement.setForeignKey(columnModel.getForeignKey());
15050                                        }
15051                                        if (columnModel.getIndexKey()) {
15052                                                columnElement.setIndexKey(columnModel.getIndexKey());
15053                                        }
15054                                        if (columnModel.getPrimaryKey()) {
15055                                                columnElement.setPrimaryKey(columnModel.getPrimaryKey());
15056                                        }
15057                                        if (columnModel.getUnqiueKey()) {
15058                                                columnElement.setUnqiueKey(columnModel.getUnqiueKey());
15059                                        }
15060                                        if (columnModel.getDataType() != null) {
15061                                                columnElement.setDataType(columnModel.getDataType());
15062                                        }
15063                                        tableElement.getColumns().add(columnElement);
15064                                }
15065                                if (columnModel.isShowStar()) {
15066                                        column columnElement = new column();
15067                                        columnElement.setId(String.valueOf(columnModel.getId()));
15068                                        columnElement.setName(columnModel.getName());
15069                                        if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
15070                                                columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
15071                                                                + convertCoordinate(columnModel.getEndPosition()));
15072                                        }
15073                                        if (columnModel.getForeignKey()) {
15074                                                columnElement.setForeignKey(columnModel.getForeignKey());
15075                                        }
15076                                        if (columnModel.getIndexKey()) {
15077                                                columnElement.setIndexKey(columnModel.getIndexKey());
15078                                        }
15079                                        if (columnModel.getPrimaryKey()) {
15080                                                columnElement.setPrimaryKey(columnModel.getPrimaryKey());
15081                                        }
15082                                        if (columnModel.getUnqiueKey()) {
15083                                                columnElement.setUnqiueKey(columnModel.getUnqiueKey());
15084                                        }
15085                                        if (columnModel.getDataType() != null) {
15086                                                columnElement.setDataType(columnModel.getDataType());
15087                                        }
15088                                        tableElement.getColumns().add(columnElement);
15089                                }
15090                        } else {
15091                                column columnElement = new column();
15092                                columnElement.setId(String.valueOf(columnModel.getId()));
15093                                columnElement.setName(columnModel.getName());
15094                                columnElement.setDisplayName(columnModel.getDisplayName());
15095                                if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
15096                                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
15097                                                        + convertCoordinate(columnModel.getEndPosition()));
15098                                }
15099                                if (columnModel.isPseduo()) {
15100                                        columnElement.setSource("system");
15101                                }
15102                                if (columnModel.getForeignKey()) {
15103                                        columnElement.setForeignKey(columnModel.getForeignKey());
15104                                }
15105                                if (columnModel.getIndexKey()) {
15106                                        columnElement.setIndexKey(columnModel.getIndexKey());
15107                                }
15108                                if (columnModel.getPrimaryKey()) {
15109                                        columnElement.setPrimaryKey(columnModel.getPrimaryKey());
15110                                }
15111                                if (columnModel.getUnqiueKey()) {
15112                                        columnElement.setUnqiueKey(columnModel.getUnqiueKey());
15113                                }
15114                                if (columnModel.getDataType() != null) {
15115                                        columnElement.setDataType(columnModel.getDataType());
15116                                }
15117                                tableElement.getColumns().add(columnElement);
15118                        }
15119                }
15120
15121                TableRelationRows relationRows = tableModel.getRelationRows();
15122                if (relationRows.hasRelation()) {
15123                        column relationRowsElement = new column();
15124                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
15125                        relationRowsElement.setName(relationRows.getName());
15126                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
15127                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
15128                                                + convertCoordinate(relationRows.getEndPosition()));
15129                        }
15130                        relationRowsElement.setSource("system");
15131                        tableElement.getColumns().add(relationRowsElement);
15132                }
15133        }
15134
15135        private boolean containStarColumn(List<TableColumn> columns, String qualifiedColumnName) {
15136                for (TableColumn tableColumn : columns) {
15137                        if (DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()).equals(qualifiedColumnName)) {
15138                                return true;
15139                        }
15140                }
15141                return false;
15142        }
15143
15144        private TableColumn searchTableColumn(List<TableColumn> columns, String qualifiedColumnName) {
15145                for (TableColumn tableColumn : columns) {
15146                        if (DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()).equals(qualifiedColumnName)) {
15147                                return tableColumn;
15148                        }
15149                }
15150                return null;
15151        }
15152
15153        private boolean containStarColumn(Collection<RelationshipElement<?>> elements, String qualifiedColumnName) {
15154                for (RelationshipElement element : elements) {
15155                        if (element.getElement() instanceof TableColumn) {
15156                                if (DlineageUtil.getIdentifierNormalColumnName(((TableColumn) element.getElement()).getName())
15157                                                .equals(qualifiedColumnName)) {
15158                                        return true;
15159                                }
15160                        } else if (element.getElement() instanceof ResultColumn) {
15161                                if (DlineageUtil.getIdentifierNormalColumnName(((ResultColumn) element.getElement()).getName())
15162                                                .equals(qualifiedColumnName)) {
15163                                        return true;
15164                                }
15165                        }
15166                }
15167                return false;
15168        }
15169
15170        /**
15171         * Returns true if there is a non-star source element matching the column name
15172         * whose parent (table/resultset) does NOT also contribute a star source in
15173         * the same relationship. This identifies "definitive" explicit sources like
15174         * subquery columns (e.g., "coalesce(...) as col_a") vs incidental references
15175         * (e.g., "aTab.id" where aTab.* is also a source).
15176         */
15177        private boolean hasDefinitiveNonStarSource(Collection<RelationshipElement<?>> elements, String qualifiedColumnName) {
15178                // First, collect parent IDs of all star (*) sources
15179                Set<Long> starSourceParentIds = new HashSet<Long>();
15180                for (RelationshipElement<?> element : elements) {
15181                        if (element.getElement() instanceof TableColumn) {
15182                                TableColumn tc = (TableColumn) element.getElement();
15183                                if ("*".equals(tc.getName()) && tc.getTable() != null) {
15184                                        starSourceParentIds.add(tc.getTable().getId());
15185                                }
15186                        } else if (element.getElement() instanceof ResultColumn) {
15187                                ResultColumn rc = (ResultColumn) element.getElement();
15188                                if ("*".equals(rc.getName()) && rc.getResultSet() != null) {
15189                                        starSourceParentIds.add(rc.getResultSet().getId());
15190                                }
15191                        }
15192                }
15193
15194                // Then check if any non-star source matching the column name has a parent
15195                // that does NOT also have a star source
15196                for (RelationshipElement<?> element : elements) {
15197                        if (element.getElement() instanceof TableColumn) {
15198                                TableColumn tc = (TableColumn) element.getElement();
15199                                if (!"*".equals(tc.getName())
15200                                                && DlineageUtil.getIdentifierNormalColumnName(tc.getName()).equals(qualifiedColumnName)) {
15201                                        if (tc.getTable() != null && !starSourceParentIds.contains(tc.getTable().getId())) {
15202                                                return true;
15203                                        }
15204                                }
15205                        } else if (element.getElement() instanceof ResultColumn) {
15206                                ResultColumn rc = (ResultColumn) element.getElement();
15207                                if (!"*".equals(rc.getName())
15208                                                && DlineageUtil.getIdentifierNormalColumnName(rc.getName()).equals(qualifiedColumnName)) {
15209                                        if (rc.getResultSet() != null && !starSourceParentIds.contains(rc.getResultSet().getId())) {
15210                                                return true;
15211                                        }
15212                                }
15213                        }
15214                }
15215                return false;
15216        }
15217
15218        private void analyzeSelectStmt(TSelectSqlStatement stmt) {
15219                if (!accessedSubqueries.contains(stmt)) {
15220                        accessedSubqueries.add(stmt);
15221                } else {
15222                        if (modelManager.getModel(stmt) != null) {
15223                                return;
15224                        }
15225                }
15226
15227                if (stmt.getParentStmt() == null && stmt.getIntoClause() == null && stmt.getIntoTableClause() == null) {
15228                        if(option.isIgnoreTopSelect() && (option.isIgnoreRecordSet() || option.isSimpleOutput())){
15229                                if(option.getAnalyzeMode() == null || option.getAnalyzeMode() == AnalyzeMode.dataflow){
15230                                        return;
15231                                }
15232                        }
15233                }
15234
15235                if (stmt.getSetOperatorType() != ESetOperatorType.none) {
15236
15237                        // Iteratively analyze all descendant UNION branches before processing this node.
15238                        // Uses iterative post-order traversal to preserve the original left-right-self
15239                        // processing order, avoiding StackOverflow with deeply nested UNION trees.
15240                        {
15241                                Deque<TSelectSqlStatement> stack1 = new ArrayDeque<>();
15242                                List<TSelectSqlStatement> postOrder = new ArrayList<>();
15243                                stack1.push(stmt);
15244                                while (!stack1.isEmpty()) {
15245                                        TSelectSqlStatement node = stack1.pop();
15246                                        postOrder.add(node);
15247                                        if (node.getSetOperatorType() != ESetOperatorType.none) {
15248                                                // Push left first, then right, so that after reversal left comes first
15249                                                if (node.getLeftStmt() != null) stack1.push(node.getLeftStmt());
15250                                                if (node.getRightStmt() != null) stack1.push(node.getRightStmt());
15251                                        }
15252                                }
15253                                Collections.reverse(postOrder);
15254                                // Process all descendants in post-order, skip stmt itself (processed below)
15255                                for (int pi = 0; pi < postOrder.size() - 1; pi++) {
15256                                        TSelectSqlStatement node = postOrder.get(pi);
15257                                        if (!accessedStatements.contains(node)) {
15258                                                accessedStatements.add(node);
15259                                                analyzeSelectStmt(node);
15260                                        }
15261                                }
15262                        }
15263
15264                        stmtStack.push(stmt);
15265                        SelectSetResultSet resultSet = modelFactory.createSelectSetResultSet(stmt);
15266
15267                        ResultSet leftResultSetModel = (ResultSet) modelManager.getModel(stmt.getLeftStmt());
15268                        if (leftResultSetModel != null && leftResultSetModel != resultSet
15269                                        && !leftResultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
15270                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
15271                                impactRelation.setEffectType(EffectType.select);
15272                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
15273                                                leftResultSetModel.getRelationRows()));
15274                                impactRelation.setTarget(
15275                                                new RelationRowsRelationshipElement<ResultSetRelationRows>(resultSet.getRelationRows()));
15276                        }
15277                        
15278                        ResultSet rightResultSetModel = (ResultSet) modelManager.getModel(stmt.getRightStmt());
15279                        if (rightResultSetModel != null && rightResultSetModel != resultSet
15280                                        && !rightResultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
15281                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
15282                                impactRelation.setEffectType(EffectType.select);
15283                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
15284                                                rightResultSetModel.getRelationRows()));
15285                                impactRelation.setTarget(
15286                                                new RelationRowsRelationshipElement<ResultSetRelationRows>(resultSet.getRelationRows()));
15287                        }
15288                        
15289                        if ((leftResultSetModel != null && leftResultSetModel.isDetermined())
15290                                        || (rightResultSetModel != null && rightResultSetModel.isDetermined())) {
15291                                resultSet.setDetermined(true);
15292                        }
15293                        
15294                        if (resultSet.getColumns() == null || resultSet.getColumns().isEmpty()) {
15295                                if (getResultColumnList(stmt.getLeftStmt()) != null) {
15296                                        createSelectSetResultColumns(resultSet, stmt.getLeftStmt());
15297                                } else if (getResultColumnList(stmt.getRightStmt()) != null) {
15298                                        createSelectSetResultColumns(resultSet, stmt.getRightStmt());
15299                                }
15300                        }
15301
15302                        List<ResultColumn> columns = resultSet.getColumns();
15303                        for (int i = 0; i < columns.size(); i++) {
15304                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
15305                                relation.setEffectType(EffectType.select);
15306                                relation.setTarget(new ResultColumnRelationshipElement(columns.get(i)));
15307
15308                                if (!stmt.getLeftStmt().isCombinedQuery()) {
15309                                        ResultSet sourceResultSet = (ResultSet) modelManager
15310                                                        .getModel(stmt.getLeftStmt().getResultColumnList());
15311                                        if (sourceResultSet!=null && sourceResultSet.getColumns().size() > i) {
15312                                                if (columns.get(i).getName().endsWith("*")) {
15313                                                        for (ResultColumn column : sourceResultSet.getColumns()) {
15314                                                                relation.addSource(new ResultColumnRelationshipElement(column));
15315                                                        }
15316                                                } else {
15317                                                        relation.addSource(
15318                                                                        new ResultColumnRelationshipElement(sourceResultSet.getColumns().get(i)));
15319                                                }
15320                                        }
15321                                } else {
15322                                        ResultSet sourceResultSet = (ResultSet) modelManager.getModel(stmt.getLeftStmt());
15323                                        if (sourceResultSet != null && sourceResultSet.getColumns().size() > i) {
15324                                                if (columns.get(i).getName().endsWith("*")) {
15325                                                        for (ResultColumn column : sourceResultSet.getColumns()) {
15326                                                                relation.addSource(new ResultColumnRelationshipElement(column));
15327                                                        }
15328                                                } else {
15329                                                        relation.addSource(
15330                                                                        new ResultColumnRelationshipElement(sourceResultSet.getColumns().get(i)));
15331                                                }
15332                                        }
15333                                }
15334
15335                                if (!stmt.getRightStmt().isCombinedQuery()) {
15336                                        ResultSet sourceResultSet = (ResultSet) modelManager
15337                                                        .getModel(stmt.getRightStmt().getResultColumnList());
15338                                        if (sourceResultSet != null && sourceResultSet.getColumns().size() > i) {
15339                                                if (columns.get(i).getName().endsWith("*")) {
15340                                                        for (ResultColumn column : sourceResultSet.getColumns()) {
15341                                                                relation.addSource(new ResultColumnRelationshipElement(column));
15342                                                        }
15343                                                } else {
15344                                                        relation.addSource(
15345                                                                        new ResultColumnRelationshipElement(sourceResultSet.getColumns().get(i)));
15346                                                }
15347                                        } else if (sourceResultSet != null) {
15348                                                for (ResultColumn column : sourceResultSet.getColumns()) {
15349                                                        if (column.hasStarLinkColumn()) {
15350                                                                relation.addSource(new ResultColumnRelationshipElement(column));
15351                                                        }
15352                                                }
15353                                        }
15354                                } else {
15355                                        ResultSet sourceResultSet = (ResultSet) modelManager.getModel(stmt.getRightStmt());
15356                                        if (sourceResultSet != null && sourceResultSet.getColumns().size() > i) {
15357                                                relation.addSource(new ResultColumnRelationshipElement(sourceResultSet.getColumns().get(i)));
15358                                        } else if (sourceResultSet != null) {
15359                                                for (ResultColumn column : sourceResultSet.getColumns()) {
15360                                                        if (column.hasStarLinkColumn()) {
15361                                                                relation.addSource(new ResultColumnRelationshipElement(column));
15362                                                        }
15363                                                }
15364                                        }
15365                                }
15366                        }
15367                        
15368                        analyzeSelectIntoClause(stmt);
15369                        
15370                        stmtStack.pop();
15371                } else {
15372
15373                        // handle hive stmt, issue_id I3SGZB
15374                        if (stmt.getHiveBodyList() != null && stmt.getHiveBodyList().size() > 0) {
15375                                stmtStack.push(stmt);
15376                                hiveFromTables = stmt.tables;
15377                                if (hiveFromTables != null) {
15378                                        for (int i = 0; i < hiveFromTables.size(); i++) {
15379                                                modelFactory.createTable(hiveFromTables.getTable(i));
15380                                        }
15381                                }
15382                                for (int i = 0; i < stmt.getHiveBodyList().size(); i++) {
15383                                        analyzeCustomSqlStmt(stmt.getHiveBodyList().get(i));
15384                                }
15385                                stmtStack.pop();
15386                                return;
15387                        }
15388                        
15389                        if (stmt.getTransformClause() != null) {
15390                                analyzeHiveTransformClause(stmt, stmt.getTransformClause());
15391                                return;
15392                        }
15393
15394                        if (stmt.getResultColumnList() == null) {
15395                                return;
15396                        }
15397
15398                        stmtStack.push(stmt);
15399
15400                        TTableList fromTables = stmt.tables;
15401                        if ((fromTables == null || fromTables.size() == 0)
15402                                        && (hiveFromTables != null && hiveFromTables.size() > 0)) {
15403                                fromTables = hiveFromTables;
15404                        }
15405
15406                        for (int i = 0; i < fromTables.size(); i++) {
15407                                TTable table = fromTables.getTable(i);
15408                                if (table.getLateralViewList() != null && !table.getLateralViewList().isEmpty()) {
15409                                        analyzeTableSubquery(table);
15410                                        analyzeLateralView(stmt, table, table.getLateralViewList());
15411                                        stmtStack.pop();
15412                                        return;
15413                                }
15414                                if (table.getUnnestClause() != null && option.getVendor() == EDbVendor.dbvpresto) {
15415                                        analyzeTableSubquery(table);
15416                                        analyzePrestoUnnest(stmt, table);
15417                                        stmtStack.pop();
15418                                        return;
15419                                }
15420                                if (table.getUnnestClause() != null && option.getVendor() == EDbVendor.dbvbigquery) {
15421                                        analyzeBigQueryUnnest(stmt, table);
15422                                }
15423                        }
15424                        
15425                        //用来获取 pivotedTable 对应的columns
15426                        TPivotedTable pivotedTable = null; 
15427                        if (stmt.getJoins() != null && stmt.getJoins().size() > 0) {
15428                                for (TJoin join : stmt.getJoins()) {
15429                                        if (join.getTable() != null && join.getTable().getPivotedTable() != null) {
15430                                                if (isUnPivotedTable(join.getTable().getPivotedTable())) {
15431                                                        analyzeUnPivotedTable(stmt, join.getTable().getPivotedTable());
15432                                                        pivotedTable = join.getTable().getPivotedTable();
15433                                                } else {
15434                                                        analyzePivotedTable(stmt, join.getTable().getPivotedTable());
15435                                                        pivotedTable = join.getTable().getPivotedTable();
15436                                                }
15437                                        }
15438                                }
15439                        }
15440
15441                        for (int i = 0; i < fromTables.size(); i++) {
15442                                TTable table = fromTables.getTable(i);
15443                                // Handle TABLE(func()) which has tableType=tableExpr and funcCall=null
15444                                if (table.getTableType() == ETableSource.tableExpr && table.getFuncCall() == null && pipelinedAnalyzer != null) {
15445                                        try {
15446                                                pipelinedAnalyzer.tryStitchTableExpr(table);
15447                                        } catch (Exception e) {
15448                                                // Don't let pipelined stitching failure break main flow
15449                                        }
15450                                }
15451
15452                                if (table.getFuncCall() != null || (table.getTableExpr()!=null && table.getTableExpr().getFunctionCall()!=null)) {
15453                                        TFunctionCall functionCall = table.getFuncCall();
15454                                        if (functionCall == null) {
15455                                                functionCall = table.getTableExpr().getFunctionCall();
15456                                        }
15457                                        Procedure callee = modelManager.getProcedureByName(
15458                                                        DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
15459                                        if (callee == null && procedureDDLMap.containsKey(DlineageUtil.getFunctionNameWithArgNum(functionCall))) {
15460                                                analyzeCustomSqlStmt(procedureDDLMap.get(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
15461                                                callee = modelManager.getProcedureByName(
15462                                                                DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
15463                                        }
15464
15465                                        if(callee!=null) {
15466                                                if (callee.getArguments() != null) {
15467                                                        for (int j = 0; j < callee.getArguments().size(); j++) {
15468                                                                Argument argument = callee.getArguments().get(j);
15469                                                                Variable variable = modelFactory.createVariable(callee, argument.getName(), false);
15470                                                                if(variable!=null) {
15471                                                                        if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
15472                                                                                Transform transform = new Transform();
15473                                                                                transform.setType(Transform.FUNCTION);
15474                                                                                transform.setCode(functionCall);
15475                                                                                variable.getColumns().get(0).setTransform(transform);
15476                                                                        }
15477                                                                        Process process = modelFactory.createProcess(functionCall);
15478                                                                        variable.addProcess(process);
15479                                                                        analyzeFunctionArgumentsDataFlowRelation(variable.getColumns().get(0), functionCall, j, process);
15480                                                                }
15481                                                        }
15482                                                }
15483                                                Set<Object> functionTableModelObjs = modelManager.getFunctionTable(DlineageUtil
15484                                                                .getIdentifierNormalTableName(functionCall.getFunctionName().toString()));
15485                                                if (functionTableModelObjs != null) {
15486                                                        modelManager.bindModel(table, functionTableModelObjs.iterator().next());
15487                                                }
15488                                                // Try pipelined function stitching
15489                                                if (pipelinedAnalyzer != null) {
15490                                                        try {
15491                                                                pipelinedAnalyzer.tryStitchCallSite(table, functionCall);
15492                                                        } catch (Exception e) {
15493                                                                // Don't let pipelined stitching failure break main flow
15494                                                        }
15495                                                }
15496                                                continue;
15497                                        } else {
15498                                                Set<Object> functionTableModelObjs = modelManager.getFunctionTable(DlineageUtil
15499                                                                .getIdentifierNormalTableName(functionCall.getFunctionName().toString()));
15500                                                if (functionTableModelObjs == null) {
15501//                                                      Procedure procedure = modelManager.getProcedureByName(DlineageUtil
15502//                                                                      .getIdentifierNormalTableName(functionCall.getFunctionName().toString()));
15503//                                                      if (procedure != null) {
15504                                                                createFunction(functionCall);
15505                                                                continue;
15506//                                                      }
15507                                                }
15508                                                if (functionTableModelObjs!=null && functionTableModelObjs.iterator().next() instanceof Table) {
15509                                                        Table functionTableModel = (Table) functionTableModelObjs.iterator().next();
15510                                                        if (functionTableModel.getColumns() != null) {
15511                                                                Table functionTable = modelFactory.createTableFromCreateDDL(table, true);
15512                                                                if (functionTable == functionTableModel)
15513                                                                        continue;
15514                                                                for (int j = 0; j < functionTableModel.getColumns().size(); j++) {
15515                                                                        TableColumn column = modelFactory.createTableColumn(functionTable,
15516                                                                                        functionTableModel.getColumns().get(j).getColumnObject(), true);
15517                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
15518                                                                        relation.setEffectType(EffectType.select);
15519                                                                        relation.setTarget(new TableColumnRelationshipElement(column));
15520                                                                        relation.addSource(
15521                                                                                        new TableColumnRelationshipElement(functionTableModel.getColumns().get(j)));
15522                                                                }
15523                                                        }
15524                                                } else if (functionTableModelObjs!=null && functionTableModelObjs.iterator().next() instanceof ResultSet) {
15525                                                        ResultSet functionTableModel = (ResultSet) functionTableModelObjs.iterator().next();
15526                                                        if (functionTableModel.getColumns() != null) {
15527                                                                Table functionTable = modelFactory.createTableFromCreateDDL(table, true);
15528                                                                for (int j = 0; j < functionTableModel.getColumns().size(); j++) {
15529                                                                        TParseTreeNode columnObj = functionTableModel.getColumns().get(j).getColumnObject();
15530                                                                        if (columnObj instanceof TObjectName) {
15531                                                                                TableColumn column = modelFactory.createTableColumn(functionTable,
15532                                                                                                (TObjectName) columnObj, true);
15533                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
15534                                                                                relation.setEffectType(EffectType.select);
15535                                                                                relation.setTarget(new TableColumnRelationshipElement(column));
15536                                                                                relation.addSource(new ResultColumnRelationshipElement(
15537                                                                                                functionTableModel.getColumns().get(j)));
15538                                                                        } else if (columnObj instanceof TResultColumn) {
15539                                                                                TableColumn column = modelFactory.createTableColumn(functionTable,
15540                                                                                                (TResultColumn) columnObj);
15541                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
15542                                                                                relation.setEffectType(EffectType.select);
15543                                                                                relation.setTarget(new TableColumnRelationshipElement(column));
15544                                                                                relation.addSource(new ResultColumnRelationshipElement(
15545                                                                                                functionTableModel.getColumns().get(j)));
15546                                                                        }
15547                                                                }
15548                                                        }
15549                                                }
15550                                                // Try pipelined function stitching for unresolved function calls
15551                                                if (pipelinedAnalyzer != null) {
15552                                                        try {
15553                                                                pipelinedAnalyzer.tryStitchCallSite(table, functionCall);
15554                                                        } catch (Exception e) {
15555                                                                // Don't let pipelined stitching failure break main flow
15556                                                        }
15557                                                }
15558                                        }
15559                                }
15560
15561                                if (table.getPartitionExtensionClause() != null
15562                                                && table.getPartitionExtensionClause().getKeyValues() != null) {
15563                                        TExpressionList values = table.getPartitionExtensionClause().getKeyValues();
15564                                        Table tableModel = modelFactory.createTable(table);
15565                                        for (TExpression value : values) {
15566                                                if (value.getExpressionType() == EExpressionType.simple_constant_t) {
15567                                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
15568                                                        impactRelation.setEffectType(EffectType.select);
15569                                                        Table constantTable = modelFactory.createConstantsTable(stmtStack.peek());
15570                                                        TableColumn constantColumn = modelFactory.createTableColumn(constantTable, value.getConstantOperand());
15571                                                        impactRelation.addSource(new TableColumnRelationshipElement(constantColumn));
15572                                                        impactRelation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(
15573                                                                        tableModel.getRelationRows()));
15574                                                }
15575                                        }
15576                                }
15577
15578                                if (table.getSubquery() != null) {
15579                                        QueryTable queryTable = modelFactory.createQueryTable(table);
15580                                        TSelectSqlStatement subquery = table.getSubquery();
15581                                        analyzeSelectStmt(subquery);
15582
15583                                        ResultSet resultSetModel = (ResultSet) modelManager.getModel(subquery);
15584                                        if (resultSetModel != null && resultSetModel.isDetermined()) {
15585                                                queryTable.setDetermined(true);
15586                                        }
15587
15588                                        if (resultSetModel != null && resultSetModel != queryTable
15589                                                        && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
15590                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
15591                                                impactRelation.setEffectType(EffectType.select);
15592                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
15593                                                                resultSetModel.getRelationRows()));
15594                                                impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
15595                                                                queryTable.getRelationRows()));
15596                                        }
15597
15598                                        if (resultSetModel != null && resultSetModel != queryTable
15599                                                        && queryTable.getTableObject().getAliasClause() != null
15600                                                        && queryTable.getTableObject().getAliasClause().getColumns() != null) {
15601                                                for (int j = 0; j < queryTable.getColumns().size()
15602                                                                && j < resultSetModel.getColumns().size(); j++) {
15603                                                        ResultColumn sourceColumn = resultSetModel.getColumns().get(j);
15604                                                        ResultColumn targetColumn = queryTable.getColumns().get(j);
15605
15606                                                        DataFlowRelationship queryRalation = modelFactory.createDataFlowRelation();
15607                                                        queryRalation.setEffectType(EffectType.select);
15608                                                        queryRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
15609                                                        queryRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
15610                                                }
15611                                        } else if (subquery.getSetOperatorType() != ESetOperatorType.none) {
15612                                                SelectSetResultSet selectSetResultSetModel = (SelectSetResultSet) modelManager
15613                                                                .getModel(subquery);
15614                                                for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) {
15615                                                        ResultColumn sourceColumn = selectSetResultSetModel.getColumns().get(j);
15616                                                        ResultColumn targetColumn = modelFactory.createSelectSetResultColumn(queryTable,
15617                                                                        sourceColumn);
15618                                                        for (TObjectName starLinkColumn : sourceColumn.getStarLinkColumnList()) {
15619                                                                targetColumn.bindStarLinkColumn(starLinkColumn);
15620                                                        }
15621                                                        DataFlowRelationship selectSetRalation = modelFactory.createDataFlowRelation();
15622                                                        selectSetRalation.setEffectType(EffectType.select);
15623                                                        selectSetRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
15624                                                        selectSetRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
15625                                                }
15626                                        }
15627                                } else if (table.getOutputMerge() != null) {
15628                                        QueryTable queryTable = modelFactory.createQueryTable(table);
15629                                        TMergeSqlStatement subquery = table.getOutputMerge();
15630                                        analyzeMergeStmt(subquery);
15631
15632                                        for (TResultColumn column : subquery.getOutputClause().getSelectItemList()) {
15633                                                modelFactory.createResultColumn(queryTable, column);
15634                                                analyzeResultColumn(column, EffectType.select);
15635                                        }
15636
15637                                        ResultSet resultSetModel = (ResultSet) modelManager
15638                                                        .getModel(subquery.getOutputClause().getSelectItemList());
15639
15640                                        if (resultSetModel != null && resultSetModel != queryTable
15641                                                        && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
15642                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
15643                                                impactRelation.setEffectType(EffectType.select);
15644                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
15645                                                                resultSetModel.getRelationRows()));
15646                                                impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
15647                                                                queryTable.getRelationRows()));
15648                                        }
15649
15650                                        if (resultSetModel != null && resultSetModel != queryTable
15651                                                        && queryTable.getTableObject().getAliasClause() != null
15652                                                        && queryTable.getTableObject().getAliasClause().getColumns() != null) {
15653                                                for (int j = 0; j < queryTable.getColumns().size()
15654                                                                && j < resultSetModel.getColumns().size(); j++) {
15655                                                        ResultColumn sourceColumn = resultSetModel.getColumns().get(j);
15656                                                        ResultColumn targetColumn = queryTable.getColumns().get(j);
15657
15658                                                        DataFlowRelationship queryRalation = modelFactory.createDataFlowRelation();
15659                                                        queryRalation.setEffectType(EffectType.select);
15660                                                        queryRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
15661                                                        queryRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
15662                                                }
15663                                        }
15664                                } else if (table.getTableExpr() != null && table.getTableExpr().getSubQuery() != null) {
15665                                        QueryTable queryTable = modelFactory.createQueryTable(table);
15666                                        TSelectSqlStatement subquery = table.getTableExpr().getSubQuery();
15667                                        analyzeSelectStmt(subquery);
15668
15669                                        ResultSet resultSetModel = (ResultSet) modelManager.getModel(subquery);
15670
15671                                        if (resultSetModel != null && resultSetModel != queryTable
15672                                                        && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
15673                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
15674                                                impactRelation.setEffectType(EffectType.select);
15675                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
15676                                                                resultSetModel.getRelationRows()));
15677                                                impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
15678                                                                queryTable.getRelationRows()));
15679                                        }
15680
15681                                        if (resultSetModel != null && resultSetModel != queryTable
15682                                                        && queryTable.getTableObject().getAliasClause() != null) {
15683                                                for (int j = 0; j < resultSetModel.getColumns().size(); j++) {
15684                                                        ResultColumn sourceColumn = resultSetModel.getColumns().get(j);
15685                                                        ResultColumn targetColumn = modelFactory.createSelectSetResultColumn(queryTable,
15686                                                                        sourceColumn);
15687
15688                                                        DataFlowRelationship queryRalation = modelFactory.createDataFlowRelation();
15689                                                        queryRalation.setEffectType(EffectType.select);
15690                                                        queryRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
15691                                                        queryRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
15692                                                }
15693                                        } else if (subquery.getSetOperatorType() != ESetOperatorType.none) {
15694                                                SelectSetResultSet selectSetResultSetModel = (SelectSetResultSet) modelManager
15695                                                                .getModel(subquery);
15696                                                for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) {
15697                                                        ResultColumn sourceColumn = selectSetResultSetModel.getColumns().get(j);
15698                                                        ResultColumn targetColumn = modelFactory.createSelectSetResultColumn(queryTable,
15699                                                                        sourceColumn);
15700                                                        for (TObjectName starLinkColumn : sourceColumn.getStarLinkColumnList()) {
15701                                                                targetColumn.bindStarLinkColumn(starLinkColumn);
15702                                                        }
15703                                                        DataFlowRelationship selectSetRalation = modelFactory.createDataFlowRelation();
15704                                                        selectSetRalation.setEffectType(EffectType.select);
15705                                                        selectSetRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
15706                                                        selectSetRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
15707                                                }
15708                                        }
15709                                } else if (table.getCTE() != null) {
15710                                        QueryTable queryTable = modelFactory.createQueryTable(table);
15711
15712                                        TObjectNameList cteColumns = table.getCTE().getColumnList();
15713                                        if (cteColumns != null) {
15714                                                for (int j = 0; j < cteColumns.size(); j++) {
15715                                                        modelFactory.createResultColumn(queryTable, cteColumns.getObjectName(j));
15716                                                }
15717                                                queryTable.setDetermined(true);
15718                                        }
15719                                        TSelectSqlStatement subquery = table.getCTE().getSubquery();
15720                                        if (subquery != null && !stmtStack.contains(subquery) && subquery.getResultColumnList() != null) {
15721                                                analyzeSelectStmt(subquery);
15722
15723                                                ResultSet resultSetModel = (ResultSet) modelManager.getModel(subquery);
15724                                                if (resultSetModel != null && resultSetModel != queryTable
15725                                                                && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
15726                                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
15727                                                        impactRelation.setEffectType(EffectType.select);
15728                                                        impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
15729                                                                        resultSetModel.getRelationRows()));
15730                                                        impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
15731                                                                        queryTable.getRelationRows()));
15732                                                }
15733
15734                                                if (subquery.getSetOperatorType() != ESetOperatorType.none) {
15735                                                        SelectSetResultSet selectSetResultSetModel = (SelectSetResultSet) modelManager
15736                                                                        .getModel(subquery);
15737                                                        int x = 0;
15738                                                        for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) {
15739                                                                ResultColumn sourceColumn = selectSetResultSetModel.getColumns().get(j);
15740                                                                ResultColumn targetColumn = null;
15741                                                                if (cteColumns != null) {
15742                                                                        if (queryTable.getColumns().size() <= j) {
15743                                                                                for (int k = 0; k < queryTable.getColumns().size(); k++) {
15744                                                                                        if (queryTable.getColumns().get(k).getName().equals(sourceColumn.getName())
15745                                                                                                        || queryTable.getColumns().get(k).getName()
15746                                                                                                                        .equals(sourceColumn.getAlias())) {
15747                                                                                                targetColumn = queryTable.getColumns().get(k);
15748                                                                                        }
15749                                                                                }
15750                                                                        } else {
15751                                                                                if (x < j) {
15752                                                                                        x = j;
15753                                                                                }
15754                                                                                targetColumn = queryTable.getColumns().get(x);
15755                                                                                x++;
15756                                                                                if (resultSetModel.getColumns().get(j).getName().contains("*")
15757                                                                                                && x < cteColumns.size()) {
15758                                                                                        j--;
15759                                                                                }
15760                                                                        }
15761                                                                } else {
15762                                                                        targetColumn = modelFactory.createSelectSetResultColumn(queryTable, sourceColumn);
15763                                                                }
15764                                                                for (TObjectName starLinkColumn : sourceColumn.getStarLinkColumnList()) {
15765                                                                        targetColumn.bindStarLinkColumn(starLinkColumn);
15766                                                                }
15767                                                                DataFlowRelationship selectSetRalation = modelFactory.createDataFlowRelation();
15768                                                                selectSetRalation.setEffectType(EffectType.select);
15769                                                                selectSetRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
15770                                                                selectSetRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
15771                                                        }
15772                                                        if (!queryTable.isDetermined()) {
15773                                                                queryTable.setDetermined(selectSetResultSetModel.isDetermined());
15774                                                        }
15775                                                } else {
15776                                                        int x = 0;
15777                                                        for (int j = 0; j < resultSetModel.getColumns().size(); j++) {
15778                                                                ResultColumn sourceColumn = resultSetModel.getColumns().get(j);
15779                                                                ResultColumn targetColumn = null;
15780                                                                if (cteColumns != null) {
15781                                                                        if (queryTable.getColumns().size() <= j) {
15782                                                                                for (int k = 0; k < queryTable.getColumns().size(); k++) {
15783                                                                                        if (queryTable.getColumns().get(k).getName().equals(sourceColumn.getName())
15784                                                                                                        || queryTable.getColumns().get(k).getName()
15785                                                                                                                        .equals(sourceColumn.getAlias())) {
15786                                                                                                targetColumn = queryTable.getColumns().get(k);
15787                                                                                        }
15788                                                                                }
15789                                                                        } else {
15790                                                                                if (x < j) {
15791                                                                                        x = j;
15792                                                                                }
15793                                                                                targetColumn = queryTable.getColumns().get(x);
15794                                                                                x++;
15795                                                                                if (resultSetModel.getColumns().get(j).getName().contains("*")
15796                                                                                                && x < cteColumns.size()) {
15797                                                                                        j--;
15798                                                                                }
15799                                                                        }
15800                                                                } else {
15801                                                                        targetColumn = modelFactory.createSelectSetResultColumn(queryTable, sourceColumn);
15802                                                                }
15803                                                                for (TObjectName starLinkColumn : sourceColumn.getStarLinkColumnList()) {
15804                                                                        targetColumn.bindStarLinkColumn(starLinkColumn);
15805                                                                }
15806                                                                DataFlowRelationship selectSetRalation = modelFactory.createDataFlowRelation();
15807                                                                selectSetRalation.setEffectType(EffectType.select);
15808                                                                selectSetRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
15809                                                                selectSetRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
15810                                                        }
15811                                                        if (!queryTable.isDetermined()) {
15812                                                                queryTable.setDetermined(resultSetModel.isDetermined());
15813                                                        }
15814                                                }
15815                                        } else if (table.getCTE().getUpdateStmt() != null) {
15816                                                analyzeCustomSqlStmt(table.getCTE().getUpdateStmt());
15817                                        } else if (table.getCTE().getInsertStmt() != null) {
15818                                                analyzeCustomSqlStmt(table.getCTE().getInsertStmt());
15819                                        } else if (table.getCTE().getDeleteStmt() != null) {
15820                                                analyzeCustomSqlStmt(table.getCTE().getDeleteStmt());
15821                                        }
15822                                } else if (table.getTableType().name().startsWith("open")) {
15823                                        continue;
15824                                } else if (table.getTableType() == ETableSource.jsonTable) {
15825                                        Table functionTable = modelFactory.createJsonTable(table);
15826                                        TJsonTable jsonTable = table.getJsonTable();
15827                                        TColumnDefinitionList definitions = jsonTable.getColumnDefinitions();
15828                                        if (definitions != null) {
15829                                                for (int j = 0; j < definitions.size(); j++) {
15830                            TColumnDefinitionList nestDefinitions = definitions.getColumn(j).getNestedTableColumns();
15831                                                        if(nestDefinitions!=null) {
15832                                                                for(int k=0;k<nestDefinitions.size();k++){
15833                                                                        TableColumn column = modelFactory.createTableColumn(functionTable,
15834                                            nestDefinitions.getColumn(k).getColumnName(), true);
15835                                    if (nestDefinitions.getColumn(k).getColumnPath() != null) {
15836                                        column.setDisplayName(
15837                                                column.getName() + ":" + nestDefinitions.getColumn(k).getColumnPath());
15838                                    }
15839                                                                }
15840                                                        }
15841                                                        else {
15842                                TableColumn column = modelFactory.createTableColumn(functionTable,
15843                                        definitions.getColumn(j).getColumnName(), true);
15844                                if (definitions.getColumn(j).getColumnPath() != null) {
15845                                    column.setDisplayName(
15846                                            column.getName() + ":" + definitions.getColumn(j).getColumnPath());
15847                                }
15848                            }
15849                                                }
15850                                        } else {
15851                                                TObjectName keyColumn = new TObjectName();
15852                                                keyColumn.setString("key");
15853                                                modelFactory.createJsonTableColumn(functionTable, keyColumn);
15854                                                TObjectName valueColumn = new TObjectName();
15855                                                valueColumn.setString("value");
15856                                                modelFactory.createJsonTableColumn(functionTable, valueColumn);
15857                                                TObjectName typeColumn = new TObjectName();
15858                                                typeColumn.setString("type");
15859                                                modelFactory.createJsonTableColumn(functionTable, typeColumn);
15860                                        }
15861
15862                                        functionTable.setCreateTable(true);
15863                                        functionTable.setSubType(SubType.function);
15864                                        modelManager.bindCreateModel(table, functionTable);
15865
15866                                        if (jsonTable.getJsonExpression() == null) {
15867                                                ErrorInfo errorInfo = new ErrorInfo();
15868                                                errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
15869                                                errorInfo.setErrorMessage("Can't handle the json table: " + jsonTable.toString());
15870                                                errorInfo.setStartPosition(new Pair3<Long, Long, String>(jsonTable.getStartToken().lineNo,
15871                                                                jsonTable.getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
15872                                                errorInfo.setEndPosition(new Pair3<Long, Long, String>(jsonTable.getEndToken().lineNo,
15873                                                                jsonTable.getEndToken().columnNo + jsonTable.getEndToken().getAstext().length(),
15874                                                                ModelBindingManager.getGlobalHash()));
15875                                                errorInfo.fillInfo(this);
15876                                                errorInfos.add(errorInfo);
15877                                        } else {
15878                                                String jsonName = jsonTable.getJsonExpression().toString();
15879                                                if (!jsonName.startsWith("@")) {
15880                                                        columnsInExpr visitor = new columnsInExpr();
15881                                                        jsonTable.getJsonExpression().inOrderTraverse(visitor);
15882                                                        List<TObjectName> objectNames = visitor.getObjectNames();
15883                                                        List<TParseTreeNode> functions = visitor.getFunctions();
15884                                                        List<TParseTreeNode> constants = visitor.getConstants();
15885                                                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
15886
15887                                                        for (int j = 0; j < functionTable.getColumns().size(); j++) {
15888                                                                TableColumn tableColumn = functionTable.getColumns().get(j);
15889                                                                if (functions != null && !functions.isEmpty()) {
15890                                                                        analyzeFunctionDataFlowRelation(tableColumn, functions, EffectType.function);
15891                                                                }
15892                                                                if (subquerys != null && !subquerys.isEmpty()) {
15893                                                                        analyzeSubqueryDataFlowRelation(tableColumn, subquerys, EffectType.select);
15894                                                                }
15895                                                                if (objectNames != null && !objectNames.isEmpty()) {
15896                                                                        analyzeDataFlowRelation(tableColumn, objectNames, EffectType.select, functions);
15897                                                                }
15898                                                                if (constants != null && !constants.isEmpty()) {
15899                                                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select,
15900                                                                                        functions);
15901                                                                }
15902                                                        }
15903                                                } else {
15904                                                        TStatementList stmts = stmt.getGsqlparser().getSqlstatements();
15905                                                        for (int j = 0; j < stmts.size(); j++) {
15906                                                                TCustomSqlStatement item = stmts.get(j);
15907                                                                if (item instanceof TMssqlDeclare) {
15908                                                                        if (analyzeMssqlJsonDeclare((TMssqlDeclare) item, jsonName, functionTable)) {
15909                                                                                break;
15910                                                                        }
15911                                                                }
15912                                                        }
15913                                                }
15914                                        }
15915                                } else if (table.getTableType() == ETableSource.xmltable) {
15916                                        Table functionTable = modelFactory.createXmlTable(table);
15917                                        TXmlTable xmlTable = table.getXmlTable();
15918                                        TXmlTableParameter param = xmlTable != null ? xmlTable.getArg() : null;
15919
15920                                        // Output columns from COLUMNS clause
15921                                        if (param != null && param.getXmlTableColumns() != null) {
15922                                                TColumnDefinitionList defs = param.getXmlTableColumns();
15923                                                for (int j = 0; j < defs.size(); j++) {
15924                                                        TColumnDefinition def = defs.getColumn(j);
15925                                                        TableColumn column = modelFactory.createTableColumn(
15926                                                                        functionTable, def.getColumnName(), true);
15927                                                        if (def.getXmlTableColumnPath() != null) {
15928                                                                column.setDisplayName(
15929                                                                                column.getName() + ":" + def.getXmlTableColumnPath());
15930                                                        }
15931                                                }
15932                                        }
15933                                        functionTable.setCreateTable(true);
15934                                        functionTable.setSubType(SubType.function);
15935                                        modelManager.bindCreateModel(table, functionTable);
15936
15937                                        // Source columns from PASSING (v1: many-to-many)
15938                                        TResultColumnList passing = null;
15939                                        if (param != null && param.getXmlPassingClause() != null) {
15940                                                passing = param.getXmlPassingClause().getPassingList();
15941                                        }
15942                                        if (passing != null && passing.size() > 0) {
15943                                                columnsInExpr visitor = new columnsInExpr();
15944                                                for (int p = 0; p < passing.size(); p++) {
15945                                                        TExpression e = passing.getResultColumn(p).getExpr();
15946                                                        if (e != null) e.inOrderTraverse(visitor);
15947                                                }
15948                                                List<TObjectName> objectNames = visitor.getObjectNames();
15949                                                List<TParseTreeNode> functions = visitor.getFunctions();
15950                                                List<TParseTreeNode> constants = visitor.getConstants();
15951                                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
15952
15953                                                for (int j = 0; j < functionTable.getColumns().size(); j++) {
15954                                                        TableColumn tc = functionTable.getColumns().get(j);
15955                                                        if (functions != null && !functions.isEmpty())
15956                                                                analyzeFunctionDataFlowRelation(tc, functions, EffectType.function);
15957                                                        if (subquerys != null && !subquerys.isEmpty())
15958                                                                analyzeSubqueryDataFlowRelation(tc, subquerys, EffectType.select);
15959                                                        if (objectNames != null && !objectNames.isEmpty())
15960                                                                analyzeDataFlowRelation(tc, objectNames, EffectType.select, functions);
15961                                                        if (constants != null && !constants.isEmpty())
15962                                                                analyzeConstantDataFlowRelation(tc, constants, EffectType.select, functions);
15963                                                }
15964                                        } else {
15965                                                ErrorInfo errorInfo = new ErrorInfo();
15966                                                errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
15967                                                errorInfo.setErrorMessage("Can't analyze XMLTABLE: missing PASSING clause.");
15968                                                errorInfos.add(errorInfo);
15969                                        }
15970                                } else if (getTableLinkedColumns(table) != null && getTableLinkedColumns(table).size() > 0) {
15971                                        if (table.getTableType() == ETableSource.rowList && table.getRowList() != null
15972                                                        && table.getRowList().size() > 0) {
15973                                                Table tableModel = modelFactory.createTable(table);
15974                                                for (int j = 0; j < table.getRowList().size(); j++) {
15975                                                        TMultiTarget rowList = table.getRowList().getMultiTarget(j);
15976                                                        for (int k = 0; k < rowList.getColumnList().size(); k++) {
15977                                                                TResultColumn column = rowList.getColumnList().getResultColumn(k);
15978                                                                if (column.getFieldAttr() == null)
15979                                                                        continue;
15980                                                                TableColumn tableColumn = modelFactory.createTableColumn(tableModel,
15981                                                                                column.getFieldAttr(), true);
15982
15983                                                                columnsInExpr visitor = new columnsInExpr();
15984                                                                column.getExpr().inOrderTraverse(visitor);
15985                                                                List<TObjectName> objectNames = visitor.getObjectNames();
15986                                                                List<TParseTreeNode> functions = visitor.getFunctions();
15987                                                                List<TParseTreeNode> constants = visitor.getConstants();
15988                                                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
15989
15990                                                                if (functions != null && !functions.isEmpty()) {
15991                                                                        analyzeFunctionDataFlowRelation(tableColumn, functions, EffectType.function);
15992                                                                }
15993                                                                if (subquerys != null && !subquerys.isEmpty()) {
15994                                                                        analyzeSubqueryDataFlowRelation(tableColumn, subquerys, EffectType.select);
15995                                                                }
15996                                                                if (objectNames != null && !objectNames.isEmpty()) {
15997                                                                        analyzeDataFlowRelation(tableColumn, objectNames, EffectType.select, functions);
15998                                                                }
15999                                                                if (constants != null && !constants.isEmpty()) {
16000                                                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select,
16001                                                                                        functions);
16002                                                                }
16003                                                        }
16004                                                }
16005                                        } else {
16006                                                if(table.getTableType() == ETableSource.pivoted_table) {
16007                                                        continue;
16008                                                }
16009                                                if (table.getTableType() == ETableSource.rowList) {
16010                                                        List<TResultColumnList> rowList = table.getValueClause().getRows();
16011                                                        
16012                                                        QueryTable tableModel = modelFactory.createQueryTable(table);
16013                                                        TAliasClause aliasClause = table.getAliasClause();
16014                                                        if (aliasClause != null && aliasClause.getColumns() != null) {
16015                                                                for (TObjectName column : aliasClause.getColumns()) {
16016                                                                        modelFactory.createResultColumn(tableModel, column, true);
16017                                                                }       
16018                                                        } else {        
16019                                                                int columnCount = rowList.get(0).size();
16020                                                                for (int j = 1; j <= columnCount; j++) {
16021                                                                        TObjectName columnName = new TObjectName();
16022                                                                        columnName.setString("column" + j);
16023                                                                        modelFactory.createResultColumn(tableModel, columnName, true);
16024                                                                }
16025                                                        }
16026                                                        tableModel.setDetermined(true);
16027                                                        
16028                                                        for (TResultColumnList resultColumnList : rowList) {
16029                                                                for (int j = 0; j < resultColumnList.size(); j++) {
16030                                                                        TResultColumn resultColumn = resultColumnList.getResultColumn(j);
16031                                                                        analyzeValueColumn(tableModel.getColumns().get(j), resultColumn, EffectType.select);
16032                                                                }
16033                                                        }
16034                                                        
16035                                                } else {
16036                                                        Table tableModel = modelFactory.createTable(table);
16037                                                        for (int j = 0; j < getTableLinkedColumns(table).size(); j++) {
16038                                                                TObjectName object = getTableLinkedColumns(table).getObjectName(j);
16039
16040                                                                if (object.getDbObjectType() == EDbObjectType.variable) {
16041                                                                        continue;
16042                                                                }
16043
16044                                                                if (object.getColumnNameOnly().startsWith("@")
16045                                                                                && (option.getVendor() == EDbVendor.dbvmssql
16046                                                                                                || option.getVendor() == EDbVendor.dbvazuresql)) {
16047                                                                        continue;
16048                                                                }
16049
16050                                                                if (object.getColumnNameOnly().startsWith(":")
16051                                                                                && (option.getVendor() == EDbVendor.dbvhana
16052                                                                                                || option.getVendor() == EDbVendor.dbvteradata)) {
16053                                                                        continue;
16054                                                                }
16055
16056                                                                if (isBuiltInFunctionName(object) && isFromFunction(object)) {
16057                                                                        continue;
16058                                                                }
16059
16060                                                                if (!"*".equals(getColumnName(object))) {
16061                                                                        if (isStructColumn(object)) {
16062
16063                                                                        } else {
16064                                                                                if (pivotedTable != null) {
16065                                                                                        ResultColumn resultColumn = getPivotedTableColumn(pivotedTable, object);
16066                                                                                        if (resultColumn != null) {
16067                                                                                                continue;
16068                                                                                        }
16069                                                                                }
16070                                                                                TableColumn tableColumn = modelFactory.createTableColumn(tableModel, object,
16071                                                                                                false);
16072                                                                                if(tableColumn == null) {
16073                                                                                        continue;
16074                                                                                }
16075                                                                                if (table.getUnnestClause() != null
16076                                                                                                && table.getUnnestClause().getArrayExpr() != null) {
16077                                                                                        columnsInExpr visitor = new columnsInExpr();
16078                                                                                        table.getUnnestClause().getArrayExpr().inOrderTraverse(visitor);
16079
16080                                                                                        List<TObjectName> objectNames = visitor.getObjectNames();
16081                                                                                        List<TParseTreeNode> functions = visitor.getFunctions();
16082
16083                                                                                        if (functions != null && !functions.isEmpty()) {
16084                                                                                                analyzeFunctionDataFlowRelation(tableColumn, functions,
16085                                                                                                                EffectType.select);
16086
16087                                                                                        }
16088
16089                                                                                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
16090                                                                                        if (subquerys != null && !subquerys.isEmpty()) {
16091                                                                                                analyzeSubqueryDataFlowRelation(tableColumn, subquerys,
16092                                                                                                                EffectType.select);
16093                                                                                        }
16094
16095                                                                                        analyzeDataFlowRelation(tableColumn, objectNames, EffectType.select,
16096                                                                                                        functions);
16097
16098                                                                                        List<TParseTreeNode> constants = visitor.getConstants();
16099                                                                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select,
16100                                                                                                        functions);
16101                                                                                }
16102                                                                        }
16103                                                                } else {
16104                                                                        boolean flag = false;
16105                                                                        for (TObjectName column : getTableLinkedColumns(table)) {
16106                                                                                if ("*".equals(getColumnName(column))) {
16107                                                                                        continue;
16108                                                                                }
16109                                                                                if (column.getLocation() != ESqlClause.where
16110                                                                                                && column.getLocation() != ESqlClause.joinCondition) {
16111                                                                                        flag = true;
16112                                                                                }
16113                                                                        }
16114                                                                        if (!flag) {
16115                                                                                TableColumn tableColumn = modelFactory.createTableColumn(tableModel, object,
16116                                                                                                false);
16117                                                                                if (tableColumn != null && !tableModel.hasSQLEnv()) {
16118                                                                                        tableColumn.setShowStar(true);
16119                                                                                }
16120                                                                        }
16121                                                                }
16122
16123                                                        }
16124                                                }
16125                                        }
16126                                }
16127                                else {
16128                                        modelFactory.createTable(table);
16129                                }
16130                        }
16131
16132                        if (pivotedTable!=null) {
16133                                for (TJoin join : stmt.getJoins()) {
16134                                        if (join.getTable() != null && join.getTable().getPivotedTable() != null) {
16135                                                if (isUnPivotedTable(join.getTable().getPivotedTable())) {
16136                                                        analyzeUnPivotedTable(stmt, join.getTable().getPivotedTable());
16137                                                } else {
16138                                                        analyzePivotedTable(stmt, join.getTable().getPivotedTable());
16139                                                }
16140                                                stmtStack.pop();
16141                                                return;
16142                                        }
16143                                }
16144                        }
16145
16146                        if (!stmt.isCombinedQuery()) {
16147                                Object queryModel = modelManager.getModel(stmt.getResultColumnList());
16148
16149                                if (queryModel == null) {
16150                                        TSelectSqlStatement parentStmt = getParentSetSelectStmt(stmt);
16151                                        if (isTopResultSet(stmt) || parentStmt == null) {
16152                                                ResultSet resultSetModel = modelFactory.createResultSet(stmt,
16153                                                                isTopResultSet(stmt) && isShowTopSelectResultSet() && stmt.getIntoClause() == null);
16154
16155                                                createPseudoImpactRelation(stmt, resultSetModel, EffectType.select);
16156
16157                                                boolean isDetermined = true;
16158                                                Map<String, AtomicInteger> keyMap = new HashMap<String, AtomicInteger>();
16159                                                Set<String> columnNames = new HashSet<String>();
16160                                                for (int i = 0; i < stmt.getResultColumnList().size(); i++) {
16161                                                        TResultColumn column = stmt.getResultColumnList().getResultColumn(i);
16162
16163                                                        if (column.getExpr().getComparisonType() == EComparisonType.equals
16164                                                                        && column.getExpr().getLeftOperand().getObjectOperand() != null) {
16165                                                                TObjectName columnObject = column.getExpr().getLeftOperand().getObjectOperand();
16166
16167                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel,
16168                                                                                columnObject);
16169                                                                if (columnObject.getDbObjectType() == EDbObjectType.variable) {
16170                                                                        Table variable = modelManager
16171                                                                                        .getTableByName(DlineageUtil.getTableFullName(columnObject.toString()));
16172                                                                        if (variable != null) {
16173                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16174                                                                                relation.setEffectType(EffectType.select);
16175                                                                                TableColumn columnModel = variable.getColumns().get(0);
16176                                                                                relation.setTarget(new TableColumnRelationshipElement(columnModel));
16177                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
16178                                                                        } else {
16179                                                                                variable = modelFactory.createVariable(columnObject);
16180                                                                                variable.setCreateTable(true);
16181                                                                                variable.setSubType(SubType.record);
16182                                                                                TObjectName variableProperties = new TObjectName();
16183                                                                                variableProperties.setString("*");
16184                                                                                TableColumn variableProperty = modelFactory.createTableColumn(variable,
16185                                                                                                variableProperties, true);
16186                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16187                                                                                relation.setEffectType(EffectType.select);
16188                                                                                relation.setTarget(new TableColumnRelationshipElement(variableProperty));
16189                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
16190                                                                        }
16191                                                                }
16192
16193                                                                columnsInExpr visitor = new columnsInExpr();
16194                                                                column.getExpr().getRightOperand().inOrderTraverse(visitor);
16195
16196                                                                List<TObjectName> objectNames = visitor.getObjectNames();
16197                                                                List<TParseTreeNode> functions = visitor.getFunctions();
16198
16199                                                                if (functions != null && !functions.isEmpty()) {
16200                                                                        analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.select);
16201
16202                                                                }
16203
16204                                                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
16205                                                                if (subquerys != null && !subquerys.isEmpty()) {
16206                                                                        analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.select);
16207                                                                }
16208
16209                                                                analyzeDataFlowRelation(resultColumn, objectNames, EffectType.select, functions);
16210
16211                                                                List<TParseTreeNode> constants = visitor.getConstants();
16212                                                                analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.select, functions);
16213                                                        } else {
16214                                                                if (column.getFieldAttr() != null && isStructColumn(column.getFieldAttr())) {
16215                                                                        Table table = modelFactory.createTable(column.getFieldAttr().getSourceTable());
16216                                                                        for (int j = 0; j < table.getColumns().size(); j++) {
16217                                                                                TObjectName columnName = new TObjectName();
16218                                                                                if (table.getColumns().get(j).getName().equals(table.getAlias())) {
16219                                                                                        if (!SQLUtil.isEmpty(column.getColumnAlias())) {
16220                                                                                                columnName.setString(column.getColumnAlias());
16221                                                                                        } else {
16222                                                                                                columnName.setString(table.getColumns().get(j).getName());
16223                                                                                        }
16224                                                                                } else {
16225                                                                                        columnName.setString(
16226                                                                                                        table.getAlias() + "." + table.getColumns().get(j).getName());
16227                                                                                }
16228                                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel,
16229                                                                                                columnName);
16230                                                                                DataFlowRelationship relationship = modelFactory.createDataFlowRelation();
16231                                                                                relationship.setTarget(new ResultColumnRelationshipElement(resultColumn));
16232                                                                                relationship.addSource(
16233                                                                                                new TableColumnRelationshipElement(table.getColumns().get(j)));
16234                                                                        }
16235                                                                } else if (column.getExpr().getFunctionCall() != null && column.getExpr().getFunctionCall().getFunctionType() == EFunctionType.struct_t) {
16236                                                                        Function function = (Function) createFunction(column.getExpr().getFunctionCall());
16237                                                                        String functionName = getResultSetName(function);
16238                                                                        for (int j = 0; j < function.getColumns().size(); j++) {
16239                                                                                TObjectName columnName = new TObjectName();
16240                                                                                if (column.getAliasClause() != null) {
16241                                                                                        columnName.setString(column.getAliasClause() + "."
16242                                                                                                        + function.getColumns().get(j).getName());
16243                                                                                }
16244                                                                                else {
16245                                                                                        columnName.setString(functionName + "."
16246                                                                                                        + function.getColumns().get(j).getName());
16247                                                                                }
16248                                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel,
16249                                                                                                columnName);
16250                                                                                resultColumn.setStruct(true);
16251                                                                                DataFlowRelationship relationship = modelFactory.createDataFlowRelation();
16252                                                                                relationship.setTarget(new ResultColumnRelationshipElement(resultColumn));
16253                                                                                relationship.addSource(
16254                                                                                                new ResultColumnRelationshipElement(function.getColumns().get(j)));
16255                                                                        }
16256                                                                } else if (column.getExpr().getFunctionCall() != null && column.getExpr().getFunctionCall().getFunctionType() == EFunctionType.array_t) {
16257                                                                        Function function = (Function) createFunction(column.getExpr().getFunctionCall());
16258                                                                        String functionName = getResultSetName(function);
16259                                                                        if (function.getColumns().size() == 1) {
16260                                                                                // Scalar result inside ARRAY() - use just alias as column name
16261                                                                                TObjectName columnName = new TObjectName();
16262                                                                                if (column.getAliasClause() != null) {
16263                                                                                        columnName.setString(column.getAliasClause().toString());
16264                                                                                } else {
16265                                                                                        columnName.setString(functionName);
16266                                                                                }
16267                                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel,
16268                                                                                                columnName);
16269                                                                                DataFlowRelationship relationship = modelFactory.createDataFlowRelation();
16270                                                                                relationship.setTarget(new ResultColumnRelationshipElement(resultColumn));
16271                                                                                relationship.addSource(
16272                                                                                                new ResultColumnRelationshipElement(function.getColumns().get(0)));
16273                                                                        } else {
16274                                                                                for (int j = 0; j < function.getColumns().size(); j++) {
16275                                                                                        TObjectName columnName = new TObjectName();
16276                                                                                        if (column.getAliasClause() != null) {
16277                                                                                                columnName.setString(column.getAliasClause() + "."
16278                                                                                                                + getColumnNameOnly(function.getColumns().get(j).getName()));
16279                                                                                        }
16280                                                                                        else {
16281                                                                                                columnName.setString(functionName + "."
16282                                                                                                                + getColumnNameOnly(function.getColumns().get(j).getName()));
16283                                                                                        }
16284                                                                                        ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel,
16285                                                                                                        columnName);
16286                                                                                        resultColumn.setStruct(true);
16287                                                                                        DataFlowRelationship relationship = modelFactory.createDataFlowRelation();
16288                                                                                        relationship.setTarget(new ResultColumnRelationshipElement(resultColumn));
16289                                                                                        relationship.addSource(
16290                                                                                                        new ResultColumnRelationshipElement(function.getColumns().get(j)));
16291                                                                                }
16292                                                                        }
16293                                                                } else if (column.getExpr().getExprList() != null && column.getExpr().getExpressionType() == EExpressionType.array_t) {
16294                                                                        if(column.getExpr().getObjectOperand()!=null) {
16295                                                                                TObjectName exprColumnName = column.getExpr().getObjectOperand();
16296                                                                                Table table = modelFactory.createTable(exprColumnName.getSourceTable());
16297                                                                                for (int z = 0; z < table.getColumns().size(); z++) {
16298                                                                                        TableColumn tableColumn = table.getColumns().get(z);
16299                                                                                        if(getColumnName(exprColumnName.toString()).equals(getColumnName(tableColumn.getName()))) {
16300                                                                                                TObjectName columnName = new TObjectName();
16301                                                                                                if (column.getAliasClause() != null) {
16302                                                                                                        columnName.setString(column.getAliasClause().toString());
16303                                                                                                } else {
16304                                                                                                        columnName
16305                                                                                                                        .setString(getColumnNameOnly(columnName.toString()));
16306                                                                                                }
16307                                                                                                ResultColumn resultColumn = modelFactory
16308                                                                                                                .createResultColumn(resultSetModel, columnName);
16309                                                                                                resultColumn.setStruct(true);
16310                                                                                                DataFlowRelationship relationship = modelFactory
16311                                                                                                                .createDataFlowRelation();
16312                                                                                                relationship.setTarget(
16313                                                                                                                new ResultColumnRelationshipElement(resultColumn));
16314                                                                                                relationship.addSource(new TableColumnRelationshipElement(
16315                                                                                                                tableColumn));
16316                                                                                        }
16317                                                                                }
16318                                                                        }
16319                                                                        else {
16320                                                                                for (int j = 0; j < column.getExpr().getExprList().size(); j++) {
16321                                                                                        TExpression expression = column.getExpr().getExprList().getExpression(j);
16322                                                                                        if(expression.getExpressionType() == EExpressionType.function_t) {
16323                                                                                                Function function = (Function)createFunction(expression.getFunctionCall());
16324                                                                                                String functionName = getResultSetName(function);
16325                                                                                                if (function != null && function.getColumns() != null) {
16326                                                                                                        if (function.getColumns().size() == 1) {
16327                                                                                                                // Scalar function inside ARRAY[] (e.g., ARRAY[TO_JSON_STRING(col)])
16328                                                                                                                // Use just the alias as column name, not alias.functionName
16329                                                                                                                TObjectName columnName = new TObjectName();
16330                                                                                                                if (column.getAliasClause() != null) {
16331                                                                                                                        columnName.setString(column.getAliasClause().toString());
16332                                                                                                                } else {
16333                                                                                                                        columnName.setString(functionName);
16334                                                                                                                }
16335                                                                                                                ResultColumn resultColumn = modelFactory
16336                                                                                                                                .createResultColumn(resultSetModel, columnName);
16337                                                                                                                DataFlowRelationship relationship = modelFactory
16338                                                                                                                                .createDataFlowRelation();
16339                                                                                                                relationship.setTarget(
16340                                                                                                                                new ResultColumnRelationshipElement(resultColumn));
16341                                                                                                                relationship.addSource(new ResultColumnRelationshipElement(
16342                                                                                                                                function.getColumns().get(0)));
16343                                                                                                        } else {
16344                                                                                                                // Multi-column (struct-like) function - use dotted naming
16345                                                                                                                for (int x = 0; x < function.getColumns().size(); x++) {
16346                                                                                                                        TObjectName columnName = new TObjectName();
16347                                                                                                                        if (column.getAliasClause() != null) {
16348                                                                                                                                columnName.setString(column.getAliasClause() + "."
16349                                                                                                                                                + getColumnNameOnly(
16350                                                                                                                                                function.getColumns().get(x).getName()));
16351                                                                                                                        } else {
16352                                                                                                                                columnName.setString(
16353                                                                                                                                                functionName + "." + getColumnNameOnly(
16354                                                                                                                                                                function.getColumns().get(x).getName()));
16355                                                                                                                        }
16356                                                                                                                        ResultColumn resultColumn = modelFactory
16357                                                                                                                                        .createResultColumn(resultSetModel, columnName);
16358                                                                                                                        resultColumn.setStruct(true);
16359                                                                                                                        DataFlowRelationship relationship = modelFactory
16360                                                                                                                                        .createDataFlowRelation();
16361                                                                                                                        relationship.setTarget(
16362                                                                                                                                        new ResultColumnRelationshipElement(resultColumn));
16363                                                                                                                        relationship.addSource(new ResultColumnRelationshipElement(
16364                                                                                                                                        function.getColumns().get(x)));
16365                                                                                                                }
16366                                                                                                        }
16367                                                                                                }
16368                                                                                        }
16369                                                                                        else if(expression.getExpressionType() == EExpressionType.simple_object_name_t) {
16370                                                                                                TObjectName exprColumnName = expression.getObjectOperand();
16371                                                                                                Table table = modelFactory.createTable(exprColumnName.getSourceTable());
16372                                                                                                for (int z = 0; z < table.getColumns().size(); z++) {
16373                                                                                                        TableColumn tableColumn = table.getColumns().get(z);
16374                                                                                                        if(getColumnName(exprColumnName.toString()).equals(getColumnName(tableColumn.getName()))) {
16375                                                                                                                TObjectName columnName = new TObjectName();
16376                                                                                                                if (column.getAliasClause() != null) {
16377                                                                                                                        columnName.setString(column.getAliasClause().toString());
16378                                                                                                                } else {
16379                                                                                                                        columnName
16380                                                                                                                                        .setString(getColumnNameOnly(columnName.toString()));
16381                                                                                                                }
16382                                                                                                                ResultColumn resultColumn = modelFactory
16383                                                                                                                                .createResultColumn(resultSetModel, columnName);
16384                                                                                                                resultColumn.setStruct(true);
16385                                                                                                                DataFlowRelationship relationship = modelFactory
16386                                                                                                                                .createDataFlowRelation();
16387                                                                                                                relationship.setTarget(
16388                                                                                                                                new ResultColumnRelationshipElement(resultColumn));
16389                                                                                                                relationship.addSource(new TableColumnRelationshipElement(
16390                                                                                                                                tableColumn));
16391                                                                                                        }
16392                                                                                                }
16393                                                                                        }
16394                                                                                }
16395                                                                        }
16396                                                                } else {
16397                                                                        if ("*".equals(column.getColumnNameOnly())) {
16398                                                                                Map<String, Pair<String, TExpression>> replaceAsIdentifierMap = new HashMap<String, Pair<String, TExpression>>();
16399                                                                                Map<String, TObjectName> replaceColumnMap = new HashMap<String, TObjectName>();
16400                                                                                if(column.getReplaceExprAsIdentifiers()!=null && column.getReplaceExprAsIdentifiers().size()>0) {
16401                                                                                        for(TReplaceExprAsIdentifier replace: column.getReplaceExprAsIdentifiers()) {
16402                                                                                                replaceAsIdentifierMap.put(replace.getIdentifier().toString(), new Pair<String, TExpression>(column.getExpr().getExceptReplaceClause().toString(), replace.getExpr()));
16403                                                                                                replaceColumnMap.put(replace.getIdentifier().toString(), replace.getIdentifier());
16404                                                                                        }
16405                                                                                }
16406                                                                                
16407                                                                                TObjectName columnObject = column.getFieldAttr();
16408                                                                                List<TTable> sourceTables = columnObject.getSourceTableList();
16409                                                                                if (sourceTables != null && !sourceTables.isEmpty()) {
16410                                                                                        boolean[] determine = new boolean[sourceTables.size()];
16411                                                                                        for (int k = 0; k < sourceTables.size(); k++) {
16412                                                                                                TTable sourceTable = sourceTables.get(k);
16413                                                                                                Object tableModel = modelManager.getModel(sourceTable);
16414                                                                                                if (tableModel instanceof Table && ((Table) tableModel).isCreateTable()) {
16415                                                                                                        Table table = (Table) tableModel;
16416                                                                                                        for (int j = 0; j < table.getColumns().size(); j++) {
16417                                                                                                                TableColumn tableColumn = table.getColumns().get(j);
16418                                                                                                                if (column.getExceptColumnList() != null) {
16419                                                                                                                        boolean except = false;
16420                                                                                                                        for (TObjectName objectName : column.getExceptColumnList()) {
16421                                                                                                                                if(getColumnName(objectName.toString()).equals(getColumnName(tableColumn.getName()))) {
16422                                                                                                                                        except = true;
16423                                                                                                                                        break;
16424                                                                                                                                }
16425                                                                                                                        }
16426                                                                                                                        if (!except && tableColumn.isStruct()) {
16427                                                                                                                                List<String> names = SQLUtil
16428                                                                                                                                                .parseNames(tableColumn.getName());
16429                                                                                                                                for (String name : names) {
16430                                                                                                                                        for (TObjectName objectName : column
16431                                                                                                                                                        .getExceptColumnList()) {
16432                                                                                                                                                if (getColumnName(objectName.toString())
16433                                                                                                                                                                .equals(getColumnName(name))) {
16434                                                                                                                                                        except = true;
16435                                                                                                                                                        break;
16436                                                                                                                                                }
16437                                                                                                                                        }
16438                                                                                                                                        if (except) {
16439                                                                                                                                                break;
16440                                                                                                                                        }
16441                                                                                                                                }
16442                                                                                                                        }
16443                                                                                                                        if(except){
16444                                                                                                                                continue;
16445                                                                                                                        }
16446                                                                                                                }
16447                                                                                                                
16448                                                                                                                if (replaceAsIdentifierMap.containsKey(tableColumn.getName())) {
16449                                                                                                                        Pair<String, TExpression> expr = replaceAsIdentifierMap.get(tableColumn.getName());
16450                                                                                                                        ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel, replaceColumnMap.get(tableColumn.getName()));
16451                                                                                                                        Transform transform = new Transform();
16452                                                                                                                        transform.setType(Transform.EXPRESSION);
16453                                                                                                                        TObjectName expression = new TObjectName();
16454                                                                                                                        expression.setString(expr.first);
16455                                                                                                                transform.setCode(expression);
16456                                                                                                                        resultColumn.setTransform(transform);
16457                                                                                                                        analyzeResultColumnExpressionRelation(resultColumn, expr.second);
16458                                                                                                                }
16459                                                                                                                else {
16460                                                                                                                        String columnName = DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName());
16461                                                                                                                        //bigquery
16462                                                                                                                        boolean exist = false;
16463                                                                                                                        if (!columnName.matches("(?i)f\\d+_")) {
16464                                                                                                                                if (!keyMap.containsKey(columnName)) {
16465                                                                                                                                        keyMap.put(columnName, new AtomicInteger(0));
16466                                                                                                                                } else {
16467                                                                                                                                        while (columnNames.contains(columnName)) {
16468                                                                                                                                                if(columnName.indexOf("*") == -1 && stmt.toString().matches("(?is).*using\\s*\\(\\s*"+columnName+"\\s*\\).*")) {
16469                                                                                                                                                        exist = true;
16470                                                                                                                                                        break;
16471                                                                                                                                                } else if (keyMap.containsKey(columnName)) {
16472                                                                                                                                                        int index = keyMap.get(columnName)
16473                                                                                                                                                                        .incrementAndGet();
16474                                                                                                                                                        columnName = columnName + index;
16475                                                                                                                                                }
16476                                                                                                                                        }
16477                                                                                                                                }
16478                                                                                                                                columnNames.add(columnName);
16479                                                                                                                        }
16480                                                                                                                        if (exist) {
16481                                                                                                                                String targetColumn = columnName;
16482                                                                                                                                DataFlowRelationship relation = modelFactory
16483                                                                                                                                                .createDataFlowRelation();
16484                                                                                                                                relation.setEffectType(EffectType.select);
16485                                                                                                                                relation.setTarget(new ResultColumnRelationshipElement(
16486                                                                                                                                                resultSetModel.getColumns().stream()
16487                                                                                                                                                                .filter(t -> t.getName()
16488                                                                                                                                                                                .equalsIgnoreCase(targetColumn))
16489                                                                                                                                                                .findFirst().get()));
16490                                                                                                                                relation.addSource(new TableColumnRelationshipElement(
16491                                                                                                                                                tableColumn));
16492                                                                                                                                continue;
16493                                                                                                                        }
16494                                                                                                                        if (DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName())
16495                                                                                                                                        .equalsIgnoreCase(DlineageUtil.getIdentifierNormalColumnName(columnName))) {
16496                                                                                                                                columnName = tableColumn.getName();
16497                                                                                                                        }
16498                                                                                                                        ResultColumn resultColumn = modelFactory.createStarResultColumn(resultSetModel, column, columnName);
16499                                                                                                                        resultColumn.setStruct(tableColumn.isStruct());
16500                                                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16501                                                                                                                        relation.setEffectType(EffectType.select);
16502                                                                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
16503                                                                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
16504                                                                                                                }
16505                                                                                                        }
16506                                                                                                        determine[k] = true;
16507                                                                                                        continue;
16508                                                                                                }
16509                                                                                                else if (tableModel instanceof ResultSet && ((ResultSet) tableModel).isDetermined()) {
16510                                                                                                        ResultSet table = (ResultSet) tableModel;
16511                                                                                                        for (int j = 0; j < table.getColumns().size(); j++) {
16512                                                                                                                ResultColumn tableColumn = table.getColumns().get(j);
16513                                                                                                                if (column.getExceptColumnList() != null) {
16514                                                                                                                        boolean except = false;
16515                                                                                                                        for (TObjectName objectName : column.getExceptColumnList()) {
16516                                                                                                                                if(getColumnName(objectName.toString()).equals(getColumnName(tableColumn.getName()))) {
16517                                                                                                                                        except = true;
16518                                                                                                                                        break;
16519                                                                                                                                }
16520                                                                                                                        }
16521                                                                                                                        if (!except && tableColumn.isStruct()) {
16522                                                                                                                                List<String> names = SQLUtil
16523                                                                                                                                                .parseNames(tableColumn.getName());
16524                                                                                                                                for (String name : names) {
16525                                                                                                                                        for (TObjectName objectName : column
16526                                                                                                                                                        .getExceptColumnList()) {
16527                                                                                                                                                if (getColumnName(objectName.toString())
16528                                                                                                                                                                .equals(getColumnName(name))) {
16529                                                                                                                                                        except = true;
16530                                                                                                                                                        break;
16531                                                                                                                                                }
16532                                                                                                                                        }
16533                                                                                                                                        if (except) {
16534                                                                                                                                                break;
16535                                                                                                                                        }
16536                                                                                                                                }
16537                                                                                                                        }
16538                                                                                                                        if(except){
16539                                                                                                                                continue;
16540                                                                                                                        }
16541                                                                                                                }
16542                                                                                                                if (replaceAsIdentifierMap.containsKey(tableColumn.getName())) {
16543                                                                                                                        Pair<String, TExpression> expr = replaceAsIdentifierMap.get(tableColumn.getName());
16544                                                                                                                        ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel, replaceColumnMap.get(tableColumn.getName()));
16545                                                                                                                        Transform transform = new Transform();
16546                                                                                                                        transform.setType(Transform.EXPRESSION);
16547                                                                                                                        TObjectName expression = new TObjectName();
16548                                                                                                                        expression.setString(expr.first);
16549                                                                                                                transform.setCode(expression);
16550                                                                                                                        resultColumn.setTransform(transform);
16551                                                                                                                        analyzeResultColumnExpressionRelation(resultColumn, expr.second);
16552                                                                                                                }
16553                                                                                                                else if(tableColumn.getRefColumnName()!=null) {
16554                                                                                                                        ResultColumn resultColumn = modelFactory.createStarResultColumn(resultSetModel, column, tableColumn.getRefColumnName());
16555                                                                                                                        if(tableColumn.isStruct()) {
16556                                                                                                                                resultColumn.setStruct(true);
16557                                                                                                                        }
16558                                                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16559                                                                                                                        relation.setEffectType(EffectType.select);
16560                                                                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
16561                                                                                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn));
16562                                                                                                                }
16563                                                                                                                else {
16564                                                                                                                        String columnName = DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName());
16565                                                                                                                        //bigquery
16566                                                                                                                        if (!columnName.matches("(?i)f\\d+_")) {
16567                                                                                                                                if (!keyMap.containsKey(columnName)) {
16568                                                                                                                                        keyMap.put(columnName, new AtomicInteger(0));
16569                                                                                                                                } else {
16570                                                                                                                                        while (columnNames.contains(columnName)) {
16571                                                                                                                                                int index = keyMap.get(columnName)
16572                                                                                                                                                                .incrementAndGet();
16573                                                                                                                                                columnName = columnName + index;
16574                                                                                                                                        }
16575                                                                                                                                }
16576                                                                                                                                columnNames.add(columnName);
16577                                                                                                                        }
16578                                                                                                                        if (DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName())
16579                                                                                                                                        .equalsIgnoreCase(DlineageUtil.getIdentifierNormalColumnName(columnName))) {
16580                                                                                                                                columnName = tableColumn.getName();
16581                                                                                                                        }
16582                                                                                                                        if (modelManager.getModel(column) instanceof ResultColumn) {
16583                                                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16584                                                                                                                                relation.setEffectType(EffectType.select);
16585                                                                                                                                relation.setTarget(new ResultColumnRelationshipElement((ResultColumn)modelManager.getModel(column)));
16586                                                                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
16587                                                                                                                        }
16588                                                                                                                        else {
16589                                                                                                                                ResultColumn resultColumn = modelFactory.createStarResultColumn(resultSetModel, column, columnName);
16590                                                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16591                                                                                                                                relation.setEffectType(EffectType.select);
16592                                                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
16593                                                                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
16594                                                                                                                        }
16595                                                                                                                }
16596                                                                                                        }
16597                                                                                                        determine[k] = true;
16598                                                                                                        continue;
16599                                                                                                }
16600                                                                                                else {
16601                                                                                                        ResultColumn resultColumn = modelFactory
16602                                                                                                                        .createResultColumn(resultSetModel, column);
16603                                                                                                        if (tableModel instanceof Function) {
16604                                                                                                
16605                                                                                                        } else {
16606                                                                                                                TObjectName[] columns = modelManager
16607                                                                                                                                .getTableColumns(sourceTable);
16608                                                                                                                for (int j = 0; j < columns.length; j++) {
16609                                                                                                                        TObjectName columnName = columns[j];
16610                                                                                                                        if (columnName == null
16611                                                                                                                                        || "*".equals(getColumnName(columnName))) {
16612                                                                                                                                continue;
16613                                                                                                                        }
16614                                                                                                                        if (isStructColumn(columnName)) {
16615                                                                                                                                continue;
16616                                                                                                                        }
16617
16618                                                                                                                        resultColumn.bindStarLinkColumn(columnName);
16619                                                                                                                        if (column.getExceptColumnList() != null) {
16620                                                                                                                                for (TObjectName objectName : column
16621                                                                                                                                                .getExceptColumnList()) {
16622                                                                                                                                        resultColumn.unbindStarLinkColumn(objectName);
16623                                                                                                                                }
16624                                                                                                                        }
16625                                                                                                                }
16626                                                                                                                if (tableModel instanceof ResultSet) {
16627                                                                                                                        ResultSet queryTable = (ResultSet) tableModel;
16628                                                                                                                        if (!containStarColumn(queryTable)) {
16629                                                                                                                                resultColumn.setShowStar(false);
16630                                                                                                                        }
16631                                                                                                                }
16632                                                                                                                if (tableModel instanceof Table) {
16633                                                                                                                        Table table = (Table) tableModel;
16634                                                                                                                        if (table.isCreateTable()) {
16635                                                                                                                                resultColumn.setShowStar(false);
16636                                                                                                                        }
16637                                                                                                                }
16638                                                                                                        }
16639                                                                                                }
16640                                                                                        }
16641                                                                                        if(!Arrays.toString(determine).contains("false")) {
16642                                                                                                continue;
16643                                                                                        }
16644                                                                                } else {
16645                                                                                        ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel, column);
16646                                                                                        TTableList tables = stmt.getTables();
16647                                                                                        for (int k = 0; k < tables.size(); k++) {
16648                                                                                                TTable table = tables.getTable(k);
16649                                                                                                TObjectName[] columns = modelManager.getTableColumns(table);
16650                                                                                                for (int j = 0; j < columns.length; j++) {
16651                                                                                                        TObjectName columnName = columns[j];
16652                                                                                                        if (columnName == null) {
16653                                                                                                                continue;
16654                                                                                                        }
16655                                                                                                        if ("*".equals(getColumnName(columnName))) {
16656                                                                                                                if (modelManager.getModel(table) instanceof Table) {
16657                                                                                                                        Table tableModel = (Table) modelManager.getModel(table);
16658                                                                                                                        if (tableModel != null
16659                                                                                                                                        && !tableModel.getColumns().isEmpty()) {
16660                                                                                                                                for (TableColumn item : tableModel.getColumns()) {
16661                                                                                                                                        resultColumn
16662                                                                                                                                                        .bindStarLinkColumn(item.getColumnObject());
16663                                                                                                                                        if (table.getSubquery() == null
16664                                                                                                                                                        && table.getCTE() == null
16665                                                                                                                                                        && !tableModel.isCreateTable()) {
16666                                                                                                                                                resultColumn.setShowStar(true);
16667                                                                                                                                        }
16668                                                                                                                                }
16669                                                                                                                        }
16670                                                                                                                } else if (modelManager.getModel(table) instanceof QueryTable) {
16671                                                                                                                        QueryTable tableModel = (QueryTable) modelManager
16672                                                                                                                                        .getModel(table);
16673                                                                                                                        if (tableModel != null
16674                                                                                                                                        && !tableModel.getColumns().isEmpty()) {
16675                                                                                                                                for (ResultColumn item : tableModel.getColumns()) {
16676                                                                                                                                        if (item.hasStarLinkColumn()) {
16677                                                                                                                                                for (TObjectName starLinkColumn : item
16678                                                                                                                                                                .getStarLinkColumnList()) {
16679                                                                                                                                                        resultColumn
16680                                                                                                                                                                        .bindStarLinkColumn(starLinkColumn);
16681                                                                                                                                                }
16682                                                                                                                                        } else if (item
16683                                                                                                                                                        .getColumnObject() instanceof TObjectName) {
16684                                                                                                                                                resultColumn.bindStarLinkColumn(
16685                                                                                                                                                                (TObjectName) item.getColumnObject());
16686                                                                                                                                        } else if (item
16687                                                                                                                                                        .getColumnObject() instanceof TResultColumn) {
16688                                                                                                                                                TResultColumn queryTableColumn = (TResultColumn) item
16689                                                                                                                                                                .getColumnObject();
16690                                                                                                                                                TObjectName tableColumnObject = queryTableColumn
16691                                                                                                                                                                .getFieldAttr();
16692                                                                                                                                                if (tableColumnObject != null) {
16693                                                                                                                                                        resultColumn.bindStarLinkColumn(
16694                                                                                                                                                                        tableColumnObject);
16695                                                                                                                                                } else if (queryTableColumn
16696                                                                                                                                                                .getAliasClause() != null && !item.isStruct()) {
16697                                                                                                                                                        resultColumn.bindStarLinkColumn(
16698                                                                                                                                                                        queryTableColumn.getAliasClause()
16699                                                                                                                                                                                        .getAliasName());
16700                                                                                                                                                }
16701                                                                                                                                        }
16702                                                                                                                                }
16703                                                                                                                        }
16704                                                                                                                }
16705                                                                                                                continue;
16706                                                                                                        }
16707                                                                                                        resultColumn.bindStarLinkColumn(columnName);
16708                                                                                                }
16709                                                                                        }
16710                                                                                }
16711                                                                                isDetermined = false;
16712                                                                        }
16713                                                                        else {
16714                                                                                if(column.getAliasClause()!=null && column.getAliasClause().getColumns()!=null) {
16715                                                                                        for(TObjectName aliasColumn: column.getAliasClause().getColumns()) {
16716                                                                                                modelFactory.createResultColumn(resultSetModel, aliasColumn);
16717                                                                                        }
16718                                                                                }
16719                                                                                else {
16720                                                                                        modelFactory.createResultColumn(resultSetModel, column);
16721                                                                                }
16722                                                                        }
16723                                                                        analyzeResultColumn(column, EffectType.select);
16724                                                                }
16725                                                        }
16726                                                }
16727                                                if (isDetermined) {
16728                                                        resultSetModel.setDetermined(isDetermined);
16729                                                }
16730                                        }
16731
16732                                        TSelectSqlStatement parent = getParentSetSelectStmt(stmt);
16733                                        if (parent != null && parent.getSetOperatorType() != ESetOperatorType.none) {
16734                                                ResultSet resultSetModel = modelFactory.createResultSet(stmt, false);
16735                                                if(queryModel == null) {
16736                                                        queryModel = resultSetModel;
16737                                                } 
16738
16739                                                createPseudoImpactRelation(stmt, resultSetModel, EffectType.select);
16740
16741                                                boolean isDetermined = true;
16742                                                for (int i = 0; i < stmt.getResultColumnList().size(); i++) {
16743                                                        TResultColumn column = stmt.getResultColumnList().getResultColumn(i);
16744                                                        if ("*".equals(column.getColumnNameOnly())) {
16745                                                                
16746                                                                Map<String, Pair<String, TExpression>> replaceAsIdentifierMap = new HashMap<String, Pair<String, TExpression>>();
16747                                                                Map<String, TObjectName> replaceColumnMap = new HashMap<String, TObjectName>();
16748                                                                if(column.getReplaceExprAsIdentifiers()!=null && column.getReplaceExprAsIdentifiers().size()>0) {
16749                                                                        for(TReplaceExprAsIdentifier replace: column.getReplaceExprAsIdentifiers()) {
16750                                                                                replaceAsIdentifierMap.put(replace.getIdentifier().toString(), new Pair<String, TExpression>(column.getExpr().getExceptReplaceClause().toString(), replace.getExpr()));
16751                                                                                replaceColumnMap.put(replace.getIdentifier().toString(), replace.getIdentifier());
16752                                                                        }
16753                                                                }
16754                                                                
16755                                                                TObjectName columnObject = column.getFieldAttr();
16756                                                                TTable sourceTable = columnObject.getSourceTable();
16757                                                                if (sourceTable != null) {
16758                                                                        Object tableModel = modelManager.getModel(sourceTable);
16759                                                                        if (tableModel instanceof Table && ((Table) tableModel).isCreateTable()) {
16760                                                                                Table table = (Table) tableModel;
16761                                                                                for (int j = 0; j < table.getColumns().size(); j++) {
16762                                                                                        TableColumn tableColumn = table.getColumns().get(j);
16763                                                                                        if (column.getExceptColumnList() != null) {
16764                                                                                                boolean except = false;
16765                                                                                                for (TObjectName objectName : column.getExceptColumnList()) {
16766                                                                                                        if (getColumnName(objectName.toString())
16767                                                                                                                        .equals(getColumnName(tableColumn.getName()))) {
16768                                                                                                                except = true;
16769                                                                                                                break;
16770                                                                                                        }
16771                                                                                                }
16772                                                                                                if (!except && tableColumn.isStruct()) {
16773                                                                                                        List<String> names = SQLUtil
16774                                                                                                                        .parseNames(tableColumn.getName());
16775                                                                                                        for (String name : names) {
16776                                                                                                                for (TObjectName objectName : column
16777                                                                                                                                .getExceptColumnList()) {
16778                                                                                                                        if (getColumnName(objectName.toString())
16779                                                                                                                                        .equals(getColumnName(name))) {
16780                                                                                                                                except = true;
16781                                                                                                                                break;
16782                                                                                                                        }
16783                                                                                                                }
16784                                                                                                                if (except) {
16785                                                                                                                        break;
16786                                                                                                                }
16787                                                                                                        }
16788                                                                                                }
16789                                                                                                if (except) {
16790                                                                                                        continue;
16791                                                                                                }
16792                                                                                        }
16793                                                                                        
16794                                                                                        if (replaceAsIdentifierMap.containsKey(tableColumn.getName())) {
16795                                                                                                Pair<String, TExpression> expr = replaceAsIdentifierMap.get(tableColumn.getName());
16796                                                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel, replaceColumnMap.get(tableColumn.getName()));
16797                                                                                                Transform transform = new Transform();
16798                                                                                                transform.setType(Transform.EXPRESSION);
16799                                                                                                TObjectName expression = new TObjectName();
16800                                                                                                expression.setString(expr.first);
16801                                                                                        transform.setCode(expression);
16802                                                                                                resultColumn.setTransform(transform);
16803                                                                                                analyzeResultColumnExpressionRelation(resultColumn, expr.second);
16804                                                                                        }
16805                                                                                        else {
16806                                                                                                ResultColumn resultColumn = modelFactory.createStarResultColumn(
16807                                                                                                                resultSetModel, column, tableColumn.getName());
16808                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16809                                                                                                relation.setEffectType(EffectType.select);
16810                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
16811                                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
16812                                                                                        }
16813                                                                                }
16814                                                                                continue;
16815                                                                        } else if (tableModel instanceof ResultSet
16816                                                                                        && ((ResultSet) tableModel).isDetermined()) {
16817                                                                                ResultSet table = (ResultSet) tableModel;
16818                                                                                for (int j = 0; j < table.getColumns().size(); j++) {
16819                                                                                        ResultColumn tableColumn = table.getColumns().get(j);
16820                                                                                        if (column.getExceptColumnList() != null) {
16821                                                                                                boolean except = false;
16822                                                                                                for (TObjectName objectName : column.getExceptColumnList()) {
16823                                                                                                        if (getColumnName(objectName.toString())
16824                                                                                                                        .equals(getColumnName(tableColumn.getName()))) {
16825                                                                                                                except = true;
16826                                                                                                                break;
16827                                                                                                        }
16828                                                                                                }
16829                                                                                                if (!except && tableColumn.isStruct()) {
16830                                                                                                        List<String> names = SQLUtil
16831                                                                                                                        .parseNames(tableColumn.getName());
16832                                                                                                        for (String name : names) {
16833                                                                                                                for (TObjectName objectName : column
16834                                                                                                                                .getExceptColumnList()) {
16835                                                                                                                        if (getColumnName(objectName.toString())
16836                                                                                                                                        .equals(getColumnName(name))) {
16837                                                                                                                                except = true;
16838                                                                                                                                break;
16839                                                                                                                        }
16840                                                                                                                }
16841                                                                                                                if (except) {
16842                                                                                                                        break;
16843                                                                                                                }
16844                                                                                                        }
16845                                                                                                }
16846                                                                                                if (except) {
16847                                                                                                        continue;
16848                                                                                                }
16849                                                                                        }
16850                                                                                        
16851                                                                                        if (replaceAsIdentifierMap.containsKey(tableColumn.getName())) {
16852                                                                                                Pair<String, TExpression> expr = replaceAsIdentifierMap.get(tableColumn.getName());
16853                                                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel, replaceColumnMap.get(tableColumn.getName()));
16854                                                                                                Transform transform = new Transform();
16855                                                                                                transform.setType(Transform.EXPRESSION);
16856                                                                                                TObjectName expression = new TObjectName();
16857                                                                                                expression.setString(expr.first);
16858                                                                                        transform.setCode(expression);
16859                                                                                                resultColumn.setTransform(transform);
16860                                                                                                analyzeResultColumnExpressionRelation(resultColumn, expr.second);
16861                                                                                        }
16862                                                                                        else if (tableColumn.getRefColumnName() != null) {
16863                                                                                                ResultColumn resultColumn = modelFactory.createStarResultColumn(
16864                                                                                                                (ResultSet)queryModel, column, tableColumn.getRefColumnName());
16865                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16866                                                                                                relation.setEffectType(EffectType.select);
16867                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
16868                                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
16869                                                                                        } else {
16870                                                                                                ResultColumn resultColumn = modelFactory.createStarResultColumn(
16871                                                                                                                resultSetModel, column, tableColumn.getName());
16872                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16873                                                                                                relation.setEffectType(EffectType.select);
16874                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
16875                                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
16876                                                                                        }
16877                                                                                }
16878                                                                                continue;
16879                                                                        }
16880                                                                        else {
16881                                                                                isDetermined = false;
16882                                                                        }
16883                                                                }
16884                                                                
16885                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel, column);
16886                                                                
16887                                                                if (columnObject.getTableToken() != null && sourceTable != null) {
16888                                                                        TObjectName[] columns = modelManager.getTableColumns(sourceTable);
16889                                                                        for (int j = 0; j < columns.length; j++) {
16890                                                                                TObjectName columnName = columns[j];
16891                                                                                if (columnName == null || "*".equals(getColumnName(columnName))) {
16892                                                                                        continue;
16893                                                                                }
16894                                                                                resultColumn.bindStarLinkColumn(columnName);
16895                                                                        }
16896
16897                                                                        if (modelManager.getModel(sourceTable) instanceof Table) {
16898                                                                                Table tableModel = (Table) modelManager.getModel(sourceTable);
16899                                                                                if (tableModel != null && !tableModel.getColumns().isEmpty()) {
16900                                                                                        for (TableColumn item : tableModel.getColumns()) {
16901                                                                                                if ("*".equals(getColumnName(item.getColumnObject()))) {
16902                                                                                                        continue;
16903                                                                                                }
16904                                                                                                resultColumn.bindStarLinkColumn(item.getColumnObject());
16905                                                                                        }
16906                                                                                }
16907                                                                        } else if (modelManager.getModel(sourceTable) instanceof QueryTable) {
16908                                                                                QueryTable tableModel = (QueryTable) modelManager.getModel(sourceTable);
16909                                                                                if (tableModel != null && !tableModel.getColumns().isEmpty()) {
16910                                                                                        for (ResultColumn item : tableModel.getColumns()) {
16911                                                                                                if (item.hasStarLinkColumn()) {
16912                                                                                                        for (TObjectName starLinkColumn : item.getStarLinkColumnList()) {
16913                                                                                                                if ("*".equals(getColumnName(starLinkColumn))) {
16914                                                                                                                        continue;
16915                                                                                                                }
16916                                                                                                                resultColumn.bindStarLinkColumn(starLinkColumn);
16917                                                                                                        }
16918                                                                                                } else if (item.getColumnObject() instanceof TObjectName) {
16919                                                                                                        TObjectName starLinkColumn = (TObjectName) item.getColumnObject();
16920                                                                                                        if ("*".equals(getColumnName(starLinkColumn))) {
16921                                                                                                                continue;
16922                                                                                                        }
16923                                                                                                        resultColumn.bindStarLinkColumn(starLinkColumn);
16924                                                                                                }
16925                                                                                        }
16926                                                                                }
16927                                                                        }
16928
16929                                                                } else {
16930                                                                        TTableList tables = stmt.getTables();
16931                                                                        for (int k = 0; k < tables.size(); k++) {
16932                                                                                TTable table = tables.getTable(k);
16933                                                                                TObjectName[] columns = modelManager.getTableColumns(table);
16934                                                                                for (int j = 0; j < columns.length; j++) {
16935                                                                                        TObjectName columnName = columns[j];
16936                                                                                        if (columnName == null) {
16937                                                                                                continue;
16938                                                                                        }
16939                                                                                        if ("*".equals(getColumnName(columnName))) {
16940                                                                                                if (modelManager.getModel(table) instanceof Table) {
16941                                                                                                        Table tableModel = (Table) modelManager.getModel(table);
16942                                                                                                        if (tableModel != null && !tableModel.getColumns().isEmpty()) {
16943                                                                                                                for (TableColumn item : tableModel.getColumns()) {
16944                                                                                                                        resultColumn.bindStarLinkColumn(item.getColumnObject());
16945                                                                                                                }
16946                                                                                                        }
16947                                                                                                } else if (modelManager.getModel(table) instanceof QueryTable) {
16948                                                                                                        QueryTable tableModel = (QueryTable) modelManager.getModel(table);
16949                                                                                                        if (tableModel != null && !tableModel.getColumns().isEmpty()) {
16950                                                                                                                for (ResultColumn item : tableModel.getColumns()) {
16951                                                                                                                        if (item.hasStarLinkColumn()) {
16952                                                                                                                                for (TObjectName starLinkColumn : item
16953                                                                                                                                                .getStarLinkColumnList()) {
16954                                                                                                                                        resultColumn.bindStarLinkColumn(starLinkColumn);
16955                                                                                                                                }
16956                                                                                                                        } else if (item.getColumnObject() instanceof TObjectName) {
16957                                                                                                                                resultColumn.bindStarLinkColumn(
16958                                                                                                                                                (TObjectName) item.getColumnObject());
16959                                                                                                                        } else if (item
16960                                                                                                                                        .getColumnObject() instanceof TResultColumn) {
16961                                                                                                                                TResultColumn queryTableColumn = (TResultColumn) item
16962                                                                                                                                                .getColumnObject();
16963                                                                                                                                TObjectName tableColumnObject = queryTableColumn
16964                                                                                                                                                .getFieldAttr();
16965                                                                                                                                if (tableColumnObject != null) {
16966                                                                                                                                        resultColumn.bindStarLinkColumn(tableColumnObject);
16967                                                                                                                                } else if (queryTableColumn.getAliasClause() != null) {
16968                                                                                                                                        resultColumn.bindStarLinkColumn(queryTableColumn
16969                                                                                                                                                        .getAliasClause().getAliasName());
16970                                                                                                                                }
16971                                                                                                                        }
16972                                                                                                                }
16973                                                                                                        }
16974                                                                                                }
16975
16976                                                                                                continue;
16977                                                                                        }
16978                                                                                        resultColumn.bindStarLinkColumn(columnName);
16979                                                                                }
16980                                                                        }
16981                                                                }
16982                                                        }
16983                                                        else {
16984                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel, column);
16985                                                        }
16986                                                        analyzeResultColumn(column, EffectType.select);
16987
16988                                                }
16989                                                
16990                                                resultSetModel.setDetermined(isDetermined);
16991                                        }
16992                                } else {
16993                                        for (int i = 0; i < stmt.getResultColumnList().size(); i++) {
16994                                                TResultColumn column = stmt.getResultColumnList().getResultColumn(i);
16995
16996                                                if (!(queryModel instanceof ResultSet)) {
16997                                                        continue;
16998                                                }
16999                                                
17000                                                ResultSet resultSetModel = (ResultSet)queryModel;
17001                                                        
17002                                                if ("*".equals(column.getColumnNameOnly())) {
17003                                                        TObjectName columnObject = column.getFieldAttr();
17004                                                        TTable sourceTable = columnObject.getSourceTable();
17005                                                        if (column.toString().indexOf(".") == -1 && stmt.getTables().size() > 1) {
17006                                                                sourceTable = null;
17007                                                        }
17008                                                        if (sourceTable != null) {
17009                                                                {
17010                                                                        Object tableModel = modelManager.getModel(sourceTable);
17011                                                                        if (tableModel instanceof Table && ((Table) tableModel).isCreateTable()) {
17012                                                                                Table table = (Table) tableModel;
17013                                                                                for (int j = 0; j < table.getColumns().size(); j++) {
17014                                                                                        TableColumn tableColumn = table.getColumns().get(j);
17015                                                                                        if (column.getExceptColumnList() != null) {
17016                                                                                                boolean except = false;
17017                                                                                                for (TObjectName objectName : column.getExceptColumnList()) {
17018                                                                                                        if (getColumnName(objectName.toString())
17019                                                                                                                        .equals(getColumnName(tableColumn.getName()))) {
17020                                                                                                                except = true;
17021                                                                                                                break;
17022                                                                                                        }
17023                                                                                                }
17024                                                                                                if (!except && tableColumn.isStruct()) {
17025                                                                                                        List<String> names = SQLUtil
17026                                                                                                                        .parseNames(tableColumn.getName());
17027                                                                                                        for (String name : names) {
17028                                                                                                                for (TObjectName objectName : column
17029                                                                                                                                .getExceptColumnList()) {
17030                                                                                                                        if (getColumnName(objectName.toString())
17031                                                                                                                                        .equals(getColumnName(name))) {
17032                                                                                                                                except = true;
17033                                                                                                                                break;
17034                                                                                                                        }
17035                                                                                                                }
17036                                                                                                                if (except) {
17037                                                                                                                        break;
17038                                                                                                                }
17039                                                                                                        }
17040                                                                                                }
17041                                                                                                if (except) {
17042                                                                                                        continue;
17043                                                                                                }
17044                                                                                        }
17045                                                                                        ResultColumn resultColumn = modelFactory.createStarResultColumn(
17046                                                                                                        resultSetModel, column, tableColumn.getName());
17047                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17048                                                                                        relation.setEffectType(EffectType.select);
17049                                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
17050                                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
17051                                                                                }
17052                                                                                if(stmt.getResultColumnList().size() == 1) {
17053                                                                                        resultSetModel.setDetermined(true);
17054                                                                                }
17055                                                                                else {
17056                                                                                        int starCount = 0;
17057                                                                                        for (int j = 0; j < stmt.getResultColumnList().size(); j++) {
17058                                                                                                if (stmt.getResultColumnList().getResultColumn(j).getColumnNameOnly()
17059                                                                                                                .endsWith("*")) {
17060                                                                                                        starCount += 1;
17061                                                                                                }
17062                                                                                        }
17063                                                                                        if (starCount <= 1) {
17064                                                                                                resultSetModel.setDetermined(true);
17065                                                                                        }
17066                                                                                }
17067                                                                                continue;
17068                                                                        } else if (tableModel instanceof ResultSet
17069                                                                                        && ((ResultSet) tableModel).isDetermined()) {
17070                                                                                ResultSet table = (ResultSet) tableModel;
17071                                                                                for (int j = 0; j < table.getColumns().size(); j++) {
17072                                                                                        ResultColumn tableColumn = table.getColumns().get(j);
17073                                                                                        if (column.getExceptColumnList() != null) {
17074                                                                                                boolean except = false;
17075                                                                                                for (TObjectName objectName : column.getExceptColumnList()) {
17076                                                                                                        if (getColumnName(objectName.toString())
17077                                                                                                                        .equals(getColumnName(tableColumn.getName()))) {
17078                                                                                                                except = true;
17079                                                                                                                break;
17080                                                                                                        }
17081                                                                                                }
17082                                                                                                if (!except && tableColumn.isStruct()) {
17083                                                                                                        List<String> names = SQLUtil
17084                                                                                                                        .parseNames(tableColumn.getName());
17085                                                                                                        for (String name : names) {
17086                                                                                                                for (TObjectName objectName : column
17087                                                                                                                                .getExceptColumnList()) {
17088                                                                                                                        if (getColumnName(objectName.toString())
17089                                                                                                                                        .equals(getColumnName(name))) {
17090                                                                                                                                except = true;
17091                                                                                                                                break;
17092                                                                                                                        }
17093                                                                                                                }
17094                                                                                                                if (except) {
17095                                                                                                                        break;
17096                                                                                                                }
17097                                                                                                        }
17098                                                                                                }
17099                                                                                                if (except) {
17100                                                                                                        continue;
17101                                                                                                }
17102                                                                                        }
17103                                                                                        if (tableColumn.getRefColumnName() != null) {
17104                                                                                                ResultColumn resultColumn = modelFactory.createStarResultColumn(
17105                                                                                                                (ResultSet)queryModel, column, tableColumn.getRefColumnName());
17106                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17107                                                                                                relation.setEffectType(EffectType.select);
17108                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
17109                                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
17110                                                                                        } else {
17111                                                                                                ResultColumn resultColumn = modelFactory.createStarResultColumn(
17112                                                                                                                resultSetModel, column, tableColumn.getName());
17113                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17114                                                                                                relation.setEffectType(EffectType.select);
17115                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
17116                                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
17117                                                                                        }
17118                                                                                }
17119                                                                                if(stmt.getResultColumnList().size() == 1) {
17120                                                                                        resultSetModel.setDetermined(true);
17121                                                                                }
17122                                                                                else {
17123                                                                                        int starCount = 0;
17124                                                                                        for (int j = 0; j < stmt.getResultColumnList().size(); j++) {
17125                                                                                                if (stmt.getResultColumnList().getResultColumn(j).getColumnNameOnly()
17126                                                                                                                .endsWith("*")) {
17127                                                                                                        starCount += 1;
17128                                                                                                }
17129                                                                                        }
17130                                                                                        if (starCount <= 1) {
17131                                                                                                resultSetModel.setDetermined(true);
17132                                                                                        }
17133                                                                                }
17134                                                                                continue;
17135                                                                        }
17136                                                                }
17137                                                                
17138                                                                ResultColumn resultColumn  = modelFactory.createResultColumn(resultSetModel, column);   
17139                                                                if (modelManager.getModel(sourceTable) instanceof Table) {
17140                                                                        Table tableModel = (Table) modelManager.getModel(sourceTable);
17141                                                                        if (tableModel != null) {
17142                                                                                modelFactory.createTableColumn(tableModel, columnObject, false);
17143                                                                        }
17144                                                                        TObjectName[] columns = modelManager.getTableColumns(sourceTable);
17145                                                                        for (int j = 0; j < columns.length; j++) {
17146                                                                                TObjectName columnName = columns[j];
17147                                                                                if (columnName == null || "*".equals(getColumnName(columnName))) {
17148                                                                                        continue;
17149                                                                                }
17150                                                                                resultColumn.bindStarLinkColumn(columnName);
17151                                                                        }
17152
17153                                                                        if (tableModel.getColumns() != null) {
17154                                                                                for (int j = 0; j < tableModel.getColumns().size(); j++) {
17155                                                                                        TableColumn tableColumn = tableModel.getColumns().get(j);
17156                                                                                        TObjectName columnName = tableColumn.getColumnObject();
17157                                                                                        if (columnName == null || "*".equals(getColumnName(columnName))) {
17158                                                                                                continue;
17159                                                                                        }
17160                                                                                        resultColumn.bindStarLinkColumn(columnName);
17161                                                                                }
17162                                                                        }
17163                                                                } else if (modelManager.getModel(sourceTable) instanceof QueryTable) {
17164                                                                        QueryTable tableModel = (QueryTable) modelManager.getModel(sourceTable);
17165                                                                        if (tableModel != null && !tableModel.getColumns().isEmpty()) {
17166                                                                                for (ResultColumn item : tableModel.getColumns()) {
17167                                                                                        if (item.hasStarLinkColumn()) {
17168                                                                                                for (TObjectName starLinkColumn : item.getStarLinkColumnList()) {
17169                                                                                                        resultColumn.bindStarLinkColumn(starLinkColumn);
17170                                                                                                }
17171                                                                                        } else if (item.getColumnObject() instanceof TObjectName) {
17172                                                                                                resultColumn.bindStarLinkColumn((TObjectName) item.getColumnObject());
17173                                                                                        } else if (item.getColumnObject() instanceof TResultColumn) {
17174                                                                                                TResultColumn queryTableColumn = (TResultColumn) item.getColumnObject();
17175                                                                                                TObjectName tableColumnObject = queryTableColumn.getFieldAttr();
17176                                                                                                if (tableColumnObject != null) {
17177                                                                                                        resultColumn.bindStarLinkColumn(tableColumnObject);
17178                                                                                                } else if (queryTableColumn.getAliasClause() != null) {
17179                                                                                                        resultColumn.bindStarLinkColumn(
17180                                                                                                                        queryTableColumn.getAliasClause().getAliasName());
17181                                                                                                }
17182                                                                                        }
17183                                                                                }
17184                                                                        }
17185                                                                }
17186                                                        } else {
17187                                                                ResultColumn resultColumn  = modelFactory.createResultColumn(resultSetModel, column);   
17188                                                                TTableList tables = stmt.getTables();
17189                                                                for (int k = 0; k < tables.size(); k++) {
17190                                                                        TTable table = tables.getTable(k);
17191                                                                        TObjectName[] columns = modelManager.getTableColumns(table);
17192                                                                        for (int j = 0; j < columns.length; j++) {
17193                                                                                TObjectName columnName = columns[j];
17194                                                                                if (columnName == null) {
17195                                                                                        continue;
17196                                                                                }
17197                                                                                if ("*".equals(getColumnName(columnName))) {
17198                                                                                        if (modelManager.getModel(table) instanceof Table) {
17199                                                                                                Table tableModel = (Table) modelManager.getModel(table);
17200                                                                                                if (tableModel != null) {
17201                                                                                                        modelFactory.createTableColumn(tableModel, columnName, false);
17202                                                                                                }
17203                                                                                                if (tableModel != null && !tableModel.getColumns().isEmpty()) {
17204                                                                                                        for (int z = 0; z < tableModel.getColumns().size(); z++) {
17205                                                                                                                resultColumn.bindStarLinkColumn(
17206                                                                                                                                tableModel.getColumns().get(z).getColumnObject());
17207                                                                                                        }
17208                                                                                                }
17209                                                                                        } else if (modelManager.getModel(table) instanceof QueryTable) {
17210                                                                                                QueryTable tableModel = (QueryTable) modelManager.getModel(table);
17211                                                                                                if (tableModel != null && !tableModel.getColumns().isEmpty()) {
17212                                                                                                        for (ResultColumn item : tableModel.getColumns()) {
17213                                                                                                                if (item.hasStarLinkColumn()) {
17214                                                                                                                        for (TObjectName starLinkColumn : item
17215                                                                                                                                        .getStarLinkColumnList()) {
17216                                                                                                                                resultColumn.bindStarLinkColumn(starLinkColumn);
17217                                                                                                                        }
17218                                                                                                                } else if (item.getColumnObject() instanceof TObjectName) {
17219                                                                                                                        resultColumn.bindStarLinkColumn(
17220                                                                                                                                        (TObjectName) item.getColumnObject());
17221                                                                                                                } else if (item.getColumnObject() instanceof TResultColumn) {
17222                                                                                                                        TResultColumn queryTableColumn = (TResultColumn) item
17223                                                                                                                                        .getColumnObject();
17224                                                                                                                        TObjectName tableColumnObject = queryTableColumn
17225                                                                                                                                        .getFieldAttr();
17226                                                                                                                        if (tableColumnObject != null) {
17227                                                                                                                                resultColumn.bindStarLinkColumn(tableColumnObject);
17228                                                                                                                        } else if (queryTableColumn.getAliasClause() != null) {
17229                                                                                                                                resultColumn.bindStarLinkColumn(queryTableColumn
17230                                                                                                                                                .getAliasClause().getAliasName());
17231                                                                                                                        }
17232                                                                                                                }
17233                                                                                                        }
17234                                                                                                }
17235                                                                                        }
17236                                                                                        continue;
17237                                                                                }
17238                                                                                resultColumn.bindStarLinkColumn(columnName);
17239                                                                        }
17240                                                                }
17241                                                        }
17242                                                }
17243                                                else {
17244                                                        ResultColumn resultColumn  = modelFactory.createResultColumn(resultSetModel, column);   
17245                                                }
17246                                                
17247                                                analyzeResultColumn(column, EffectType.select);
17248
17249                                        }
17250                                        
17251                                        if (queryModel instanceof ResultSet) {
17252                                                boolean isDetermined = true;
17253                                                ResultSet resultSet = (ResultSet) queryModel;
17254                                                for (ResultColumn column : resultSet.getColumns()) {
17255                                                        if (column.getName().endsWith("*")) {
17256                                                                isDetermined = false;
17257                                                                break;
17258                                                        }
17259                                                }
17260                                                if (isDetermined) {
17261                                                        resultSet.setDetermined(isDetermined);
17262                                                }
17263                                        }
17264                                }
17265                        }
17266
17267                
17268                        analyzeSelectIntoClause(stmt);
17269                
17270
17271                        if (stmt.getJoins() != null && stmt.getJoins().size() > 0) {
17272                                for (int i = 0; i < stmt.getJoins().size(); i++) {
17273                                        TJoin join = stmt.getJoins().getJoin(i);
17274                                        ResultSet topResultSet = (ResultSet) modelManager.getModel(stmt);
17275                                        if (join.getJoinItems() != null && join.getJoinItems().size() > 0) {
17276                                                for (int k = 0; k < join.getJoinItems().size(); k++) {
17277                                                        TTable table = join.getJoinItems().getJoinItem(k).getTable();
17278                                                        if (table != null && table.getSubquery() != null) {
17279                                                                
17280                                                                ResultSet joinResultSet = (ResultSet) modelManager.getModel(table.getSubquery());
17281                                                                for (int x = 0; x < joinResultSet.getColumns().size(); x++) {
17282                                                                        ResultColumn sourceColumn = joinResultSet.getColumns().get(x);
17283                                                                        ResultColumn resultColumn = matchResultColumn(topResultSet.getColumns(),
17284                                                                                        sourceColumn);
17285                                                                        if (resultColumn != null
17286                                                                                        && resultColumn.getColumnObject() instanceof TResultColumn) {
17287                                                                                TResultColumn column = (TResultColumn) resultColumn.getColumnObject();
17288                                                                                if (column.getAliasClause() == null && column.getFieldAttr() != null) {
17289                                                                                        TObjectName resultObject = column.getFieldAttr();
17290                                                                                        if (resultObject.getSourceTable() == null
17291                                                                                                        || resultObject.getSourceTable().equals(table)) {
17292                                                                                                DataFlowRelationship combinedQueryRelation = modelFactory
17293                                                                                                                .createDataFlowRelation();
17294                                                                                                combinedQueryRelation.setEffectType(EffectType.select);
17295                                                                                                combinedQueryRelation
17296                                                                                                                .setTarget(new ResultColumnRelationshipElement(resultColumn));
17297                                                                                                combinedQueryRelation
17298                                                                                                                .addSource(new ResultColumnRelationshipElement(sourceColumn));
17299                                                                                        }
17300                                                                                }
17301                                                                        }
17302                                                                }
17303                                                        }
17304                                                        
17305                                                        if(join.getJoinItems().getJoinItem(k).getJoin()!=null) {
17306                                                                analyzeJoin(join.getJoinItems().getJoinItem(k).getJoin(), EffectType.select);
17307                                                        }
17308                                                }
17309                                        }
17310                                        analyzeJoin(join, EffectType.select);
17311                                }
17312                        }
17313
17314                        if (stmt.getWhereClause() != null) {
17315                                TExpression expr = stmt.getWhereClause().getCondition();
17316                                if (expr != null) {
17317                                        analyzeFilterCondition(null, expr, null, JoinClauseType.where, EffectType.select);
17318                                }
17319                        }
17320
17321                        stmtStack.pop();
17322                }
17323        }
17324
17325        protected TObjectNameList getTableLinkedColumns(TTable table) {
17326                if(structObjectMap.containsKey(table)) {
17327                        return structObjectMap.get(table);
17328                }
17329                return table.getLinkedColumns();
17330        }
17331
17332        protected boolean isTopResultSet(TSelectSqlStatement stmt) {
17333                TCustomSqlStatement parent = stmt.getParentStmt();
17334                if (parent == null)
17335                        return true;
17336                if (parent instanceof TMssqlReturn) {
17337                        return true;
17338                }
17339                if (parent instanceof TReturnStmt) {
17340                        return true;
17341                }
17342                if (parent instanceof TCommonBlock) {
17343                        TCommonBlock block = (TCommonBlock) parent;
17344                        if (block.getStatements() != null) {
17345                                for (int i = 0; i < block.getStatements().size(); i++) {
17346                                        TCustomSqlStatement child = block.getStatements().get(i);
17347                                        if(stmt == child) {
17348                                                return true;
17349                                        }
17350                                }
17351                        }
17352                }
17353                if (parent instanceof TMssqlBlock) {
17354                        TMssqlBlock block = (TMssqlBlock) parent;
17355                        if (block.getStatements() != null) {
17356                                for (int i = 0; i < block.getStatements().size(); i++) {
17357                                        TCustomSqlStatement child = block.getStatements().get(i);
17358                                        if(stmt == child) {
17359                                                return true;
17360                                        }
17361                                }
17362                        }
17363                }
17364                if (parent instanceof TStoredProcedureSqlStatement) {
17365                        TStoredProcedureSqlStatement block = (TStoredProcedureSqlStatement) parent;
17366                        if (block.getStatements() != null) {
17367                                for (int i = 0; i < block.getStatements().size(); i++) {
17368                                        TCustomSqlStatement child = block.getStatements().get(i);
17369                                        if(child == stmt) {
17370                                                return true;
17371                                        }
17372                                        if (child instanceof TReturnStmt) {
17373                                                TReturnStmt returnStmt = (TReturnStmt) child;
17374                                                if (returnStmt.getStatements() != null) {
17375                                                        for (int j = 0; j < returnStmt.getStatements().size(); j++) {
17376                                                                TCustomSqlStatement child1 = returnStmt.getStatements().get(j);
17377                                                                if(child1 == stmt) {
17378                                                                        return true;
17379                                                                }
17380                                                        }
17381                                                }
17382                                        }
17383                                        if (child instanceof TMssqlReturn) {
17384                                                TMssqlReturn returnStmt = (TMssqlReturn) child;
17385                                                if (returnStmt.getStatements() != null) {
17386                                                        for (int j = 0; j < returnStmt.getStatements().size(); j++) {
17387                                                                TCustomSqlStatement child1 = returnStmt.getStatements().get(j);
17388                                                                if(child1 == stmt) {
17389                                                                        return true;
17390                                                                }
17391                                                        }
17392                                                }
17393                                        }
17394                                }
17395                        }
17396                }
17397                return false;
17398        }
17399
17400        protected void analyzeTableSubquery(TTable table) {
17401                if(table.getSubquery()!=null) {
17402                        QueryTable queryTable = modelFactory.createQueryTable(table);
17403                        TSelectSqlStatement subquery = table.getSubquery();
17404                        analyzeSelectStmt(subquery);
17405
17406                        ResultSet resultSetModel = (ResultSet) modelManager.getModel(subquery);
17407
17408                        if (resultSetModel != null && resultSetModel != queryTable
17409                                        && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
17410                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
17411                                impactRelation.setEffectType(EffectType.select);
17412                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
17413                                                resultSetModel.getRelationRows()));
17414                                impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
17415                                                queryTable.getRelationRows()));
17416                        }
17417
17418                        if (resultSetModel != null && resultSetModel != queryTable
17419                                        && queryTable.getTableObject().getAliasClause() != null
17420                                        && queryTable.getTableObject().getAliasClause().getColumns() != null) {
17421                                for (int j = 0; j < queryTable.getColumns().size()
17422                                                && j < resultSetModel.getColumns().size(); j++) {
17423                                        ResultColumn sourceColumn = resultSetModel.getColumns().get(j);
17424                                        ResultColumn targetColumn = queryTable.getColumns().get(j);
17425
17426                                        DataFlowRelationship queryRalation = modelFactory.createDataFlowRelation();
17427                                        queryRalation.setEffectType(EffectType.select);
17428                                        queryRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
17429                                        queryRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
17430                                }
17431                        } else if (subquery.getSetOperatorType() != ESetOperatorType.none) {
17432                                SelectSetResultSet selectSetResultSetModel = (SelectSetResultSet) modelManager
17433                                                .getModel(subquery);
17434                                for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) {
17435                                        ResultColumn sourceColumn = selectSetResultSetModel.getColumns().get(j);
17436                                        ResultColumn targetColumn = modelFactory.createSelectSetResultColumn(queryTable,
17437                                                        sourceColumn);
17438                                        for (TObjectName starLinkColumn : sourceColumn.getStarLinkColumnList()) {
17439                                                targetColumn.bindStarLinkColumn(starLinkColumn);
17440                                        }
17441                                        DataFlowRelationship selectSetRalation = modelFactory.createDataFlowRelation();
17442                                        selectSetRalation.setEffectType(EffectType.select);
17443                                        selectSetRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
17444                                        selectSetRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
17445                                }
17446                        }
17447                }
17448        }
17449
17450        private ResultColumn getPivotedTableColumn(TPivotedTable pivotedTable, TObjectName columnName) {
17451                List<TPivotClause> pivotClauses = new ArrayList<TPivotClause>();
17452                if (pivotedTable.getPivotClause() != null) {
17453                        pivotClauses.add(pivotedTable.getPivotClause());
17454                }
17455                if (pivotedTable.getPivotClauseList() != null) {
17456                        for (int i = 0; i < pivotedTable.getPivotClauseList().size(); i++) {
17457                                pivotClauses.add(pivotedTable.getPivotClauseList().getElement(i));
17458                        }
17459                }
17460                for (TPivotClause clause : pivotClauses) {
17461                        Object model = modelManager.getModel(clause);
17462                        if (model instanceof PivotedTable) {
17463                                PivotedTable pivotedTableModel = (PivotedTable) model;
17464                                if (pivotedTableModel.getColumns() != null) {
17465                                        for (ResultColumn column : pivotedTableModel.getColumns()) {
17466                                                if (DlineageUtil.compareColumnIdentifier(getColumnName(columnName),
17467                                                                getColumnName(SQLUtil.trimColumnStringQuote(column.getName())))) {
17468                                                        return column;
17469                                                }
17470                                        }
17471                                }
17472                        }
17473                }
17474                return null;
17475        }
17476
17477        private void analyzeHiveTransformClause(TSelectSqlStatement stmt, THiveTransformClause transformClause) {
17478                Table mapSourceTable = null;
17479                QueryTable mapQueryTable = null;
17480                if(stmt.getTables()!=null) {
17481                        for(int i=0;i<stmt.getTables().size();i++) {
17482                                TTable table = stmt.getTables().getTable(i);
17483                                if (table.getSubquery() != null) {
17484                                        if (transformClause.getTransformType() == ETransformType.ettReduce) {
17485                                                mapQueryTable = modelFactory.createQueryTable(table);
17486                                        }
17487                                        analyzeSelectStmt(table.getSubquery());
17488                                }
17489                                else {
17490                                        mapSourceTable = modelFactory.createTable(table);
17491                                }
17492                        }
17493                }
17494                
17495                if (transformClause.getTransformType() == ETransformType.ettReduce) {
17496                        modelFactory.createResultSet(stmt, false);
17497                }
17498                
17499                List<TableColumn> mapTableColumns = new ArrayList<TableColumn>();
17500                List<ResultColumn> mapResultSetColumns = new ArrayList<ResultColumn>();
17501                List<ResultColumn> redueResultSetColumns = new ArrayList<ResultColumn>();
17502                
17503                if(transformClause.getExpressionList()!=null) {
17504                        for(TExpression expression: transformClause.getExpressionList()) {
17505                                if(expression.getObjectOperand()!=null) {
17506                                        if (transformClause.getTransformType() == ETransformType.ettMap || transformClause.getTransformType() == ETransformType.ettSelect) {
17507                                                if (mapSourceTable != null) {
17508                                                        TableColumn tableColumn = modelFactory.createTableColumn(mapSourceTable,
17509                                                                        expression.getObjectOperand(), false);
17510                                                        if (tableColumn != null) {
17511                                                                mapTableColumns.add(tableColumn);
17512                                                        }
17513                                                }
17514                                        }
17515                                        else if (transformClause.getTransformType() == ETransformType.ettReduce) {
17516                                                if (mapQueryTable != null) {
17517                                                        ResultColumn resultColumn = modelFactory.createResultColumn(mapQueryTable,
17518                                                                        expression.getObjectOperand(), false);
17519                                                        if (resultColumn != null) {
17520                                                                mapResultSetColumns.add(resultColumn);
17521                                                        }
17522                                                }
17523                                        }
17524                                }
17525                        }
17526                }
17527                
17528                if (transformClause.getAliasClause() != null) {
17529                        Object model = modelManager.getModel(stmt);
17530                        if (model instanceof ResultSet) {
17531                                ResultSet result = (ResultSet) model;
17532                                if (result!=null && transformClause.getAliasClause().getColumns() != null) {
17533                                        for (TObjectName column : transformClause.getAliasClause().getColumns()) {
17534                                                ResultColumn resultColumn = modelFactory.createResultColumn(result, column);
17535                                                if (resultColumn != null) {
17536                                                        if (transformClause.getTransformType() == ETransformType.ettMap 
17537                                                                        || transformClause.getTransformType() == ETransformType.ettSelect) {
17538                                                                mapResultSetColumns.add(resultColumn);
17539                                                        }
17540                                                        else if (transformClause.getTransformType() == ETransformType.ettReduce) {
17541                                                                redueResultSetColumns.add(resultColumn);
17542                                                        }
17543                                                }
17544                                        }
17545                                }
17546                        }
17547                }
17548                
17549                if (transformClause.getTransformType() == ETransformType.ettMap 
17550                                || transformClause.getTransformType() == ETransformType.ettSelect) {
17551                        if (!mapTableColumns.isEmpty() && !mapResultSetColumns.isEmpty()) {
17552                                for (ResultColumn resultColumn : mapResultSetColumns) {
17553                                        for (TableColumn tableColumn : mapTableColumns) {
17554                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17555                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
17556                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
17557                                                relation.setEffectType(EffectType.select);
17558                                        }
17559                                }
17560                        }
17561                }
17562                else if (transformClause.getTransformType() == ETransformType.ettReduce) {
17563                        if (!redueResultSetColumns.isEmpty() && !mapResultSetColumns.isEmpty()) {
17564                                for (ResultColumn reduceResultColumn : redueResultSetColumns) {
17565                                        for (ResultColumn mapResultColumn : mapResultSetColumns) {
17566                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17567                                                relation.setTarget(new ResultColumnRelationshipElement(reduceResultColumn));
17568                                                relation.addSource(new ResultColumnRelationshipElement(mapResultColumn));
17569                                                relation.setEffectType(EffectType.select);
17570                                        }
17571                                }
17572                        }
17573                }
17574        }
17575
17576        protected boolean isStructColumn(TObjectName columnName) {
17577                return columnName.getSourceTable() != null && columnName.getSourceTable().getAliasClause() != null
17578                                && columnName.getSourceTable().getUnnestClause() != null
17579                                && DlineageUtil.compareColumnIdentifier(getColumnName(columnName),
17580                                                getColumnName(columnName.getSourceTable().getAliasClause().getAliasName()));
17581        }
17582
17583        private TObjectName getObjectName(ResultColumn column) {
17584                if (column.getColumnObject() instanceof TResultColumn) {
17585                        TResultColumn resultColumn = (TResultColumn) column.getColumnObject();
17586                        if (resultColumn.getAliasClause() != null && resultColumn.getAliasClause().getAliasName() != null) {
17587                                return resultColumn.getAliasClause().getAliasName();
17588                        }
17589                        if (resultColumn.getFieldAttr() != null) {
17590                                return resultColumn.getFieldAttr();
17591                        }
17592                        if (resultColumn.getExpr() != null
17593                                        && resultColumn.getExpr().getExpressionType() == EExpressionType.simple_object_name_t) {
17594                                return resultColumn.getExpr().getObjectOperand();
17595                        }
17596                } else if (column.getColumnObject() instanceof TObjectName) {
17597                        return (TObjectName) column.getColumnObject();
17598                }
17599                return null;
17600        }
17601
17602        private boolean isShowTopSelectResultSet() {
17603                if (option.isSimpleOutput() && !option.isSimpleShowTopSelectResultSet())
17604                        return false;
17605                return true;
17606        }
17607
17608        private void analyzeSelectIntoClause(TSelectSqlStatement stmt) {
17609                if (stmt.getParentStmt() instanceof TSelectSqlStatement) {
17610                        return;
17611                }
17612                
17613                TableColumn oracleIntoTableColumn = null;
17614                
17615                TIntoClause intoClause = stmt.getIntoClause();
17616                
17617                TSelectSqlStatement leftStmt = DlineageUtil.getLeftStmt(stmt);
17618                
17619                if (intoClause == null && leftStmt != null) {
17620                        intoClause = leftStmt.getIntoClause();
17621                }
17622                
17623                if (intoClause != null) {
17624                        List<TObjectName> tableNames = new ArrayList<TObjectName>();
17625                        if (intoClause.getExprList() != null) {
17626                                for (int j = 0; j < intoClause.getExprList().size(); j++) {
17627                                        TObjectName tableName = intoClause.getExprList().getExpression(j).getObjectOperand();
17628                                        if (tableName != null) {
17629                                                if (tableName.toString().startsWith(":") && option.getVendor() == EDbVendor.dbvoracle
17630                                                                && tableName.getDbObjectType() == EDbObjectType.column) {
17631                                                        TObjectName tableAlias = new TObjectName();
17632                                                        tableAlias.setString(tableName.getTableString());
17633                                                        tableNames.add(tableAlias);
17634                                                        TTable sourceTable = tableName.getSourceTable();
17635                                                        Table sourceTableModel = modelFactory.createTable(sourceTable, tableAlias);
17636                                                        oracleIntoTableColumn = modelFactory.createTableColumn(sourceTableModel, tableName, true);
17637                                                } else {
17638                                                        if (tableName != null) {
17639                                                                tableNames.add(tableName);
17640                                                        }
17641                                                }
17642                                        }
17643                                        else if(intoClause.getExprList().getExpression(j).getFunctionCall()!=null) {
17644                                                TObjectName variableName = intoClause.getExprList().getExpression(j).getFunctionCall().getFunctionName();
17645                                                tableNames.add(variableName);
17646                                                Variable variable = modelFactory.createVariable(variableName);
17647                                                variable.setSubType(SubType.record);
17648                                                TObjectName variableProperties = new TObjectName();
17649                                                variableProperties.setString("*");
17650                                                modelFactory.createTableColumn(variable, variableProperties, true);
17651                                        }
17652                                }
17653                        } else if (intoClause.getVariableList() != null) {
17654                                for (int j = 0; j < intoClause.getVariableList().size(); j++) {
17655                                        TObjectName tableName = intoClause.getVariableList().getObjectName(j);
17656                                        if (tableName != null) {
17657                                                tableNames.add(tableName);
17658                                        }
17659                                }
17660                        } else if (intoClause.getIntoName() != null) {
17661                                tableNames.add(intoClause.getIntoName());
17662                        }
17663
17664                        ResultSet queryModel = (ResultSet) modelManager.getModel(stmt.getResultColumnList());
17665                        if (stmt.getSetOperatorType() != ESetOperatorType.none) {
17666                                queryModel = (ResultSet) modelManager.getModel(stmt);
17667                        }
17668                        for (int j = 0; j < tableNames.size(); j++) {
17669                                TObjectName tableName = tableNames.get(j);
17670                                if (tableName.getColumnNameOnly().startsWith("@")
17671                                                && (option.getVendor() == EDbVendor.dbvmssql || option.getVendor() == EDbVendor.dbvazuresql)) {
17672                                        continue;
17673                                }
17674
17675                                if (tableName.getColumnNameOnly().startsWith(":")
17676                                                && (option.getVendor() == EDbVendor.dbvhana || option.getVendor() == EDbVendor.dbvteradata)) {
17677                                        continue;
17678                                }
17679
17680                                Table tableModel;
17681                                TableColumn variableColumn = null;
17682
17683                                if (tableName.getDbObjectType() == EDbObjectType.variable) {
17684                                        if (tableName.toString().indexOf(".") != -1) {
17685                                                List<String> splits = SQLUtil.parseNames(tableName.toString());
17686                                                tableModel = modelFactory.createVariable(splits.get(splits.size() - 2));
17687                                        } else {
17688                                                tableModel = modelFactory.createVariable(tableName);
17689                                        }
17690                                        if (tableModel.getSubType() == null) {
17691                                                tableModel.setSubType(SubType.record);
17692                                        }
17693                                        if(tableModel.getColumns() == null || tableModel.getColumns().isEmpty()) {
17694                                                TObjectName variableProperties = new TObjectName();
17695                                                variableProperties.setString("*");
17696                                                variableColumn = modelFactory.createTableColumn(tableModel, variableProperties, true);
17697                                        }
17698                                } else {
17699                                        tableModel = modelFactory.createTableByName(tableName);
17700                                }
17701                                
17702                                if (queryModel instanceof ResultSet && (stmt.getWhereClause() != null || hasJoin(stmt))) {
17703                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
17704                                        impactRelation.setEffectType(EffectType.insert);
17705                                        impactRelation.addSource(
17706                                                        new RelationRowsRelationshipElement<ResultSetRelationRows>(((ResultSet)queryModel).getRelationRows()));
17707                                        impactRelation.setTarget(
17708                                                        new RelationRowsRelationshipElement<TableRelationRows>(tableModel.getRelationRows()));
17709                                }
17710                                
17711                                Process process = modelFactory.createProcess(stmt);
17712                                tableModel.addProcess(process);
17713                                
17714                                if (stmt.getSetOperatorType() != ESetOperatorType.none) {
17715                                        for (ResultColumn resultColumn : queryModel.getColumns()) {
17716                                                TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
17717                                                                resultColumn.getName());
17718
17719                                                if (DlineageUtil.isTempTable(tableModel, option.getVendor()) && sqlenv != null
17720                                                                && tableModel.getDatabase() != null && tableModel.getSchema() != null) {
17721                                                        TSQLSchema schema = sqlenv
17722                                                                        .getSQLSchema(tableModel.getDatabase() + "." + tableModel.getSchema(), true);
17723                                                        if (schema != null) {
17724                                                                TSQLTable tempTable = schema
17725                                                                                .createTable(DlineageUtil.getSimpleTableName(tableModel.getName()));
17726                                                                tempTable.addColumn(tableColumn.getName());
17727                                                        }
17728                                                }
17729
17730                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17731                                                relation.setEffectType(EffectType.insert);
17732                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
17733                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
17734                                                relation.setProcess(process);
17735                                        }
17736                                        
17737                                        tableModel.setDetermined(queryModel.isDetermined());
17738                                        if(queryModel.isDetermined() && DlineageUtil.isTempTable(tableModel, option.getVendor())) {
17739                                                tableModel.setCreateTable(true, false);
17740                                        }
17741                                        return;
17742                                }
17743
17744                                boolean isDetermined = true;
17745                                for (int i = 0; i < stmt.getResultColumnList().size(); i++) {
17746                                        if (tableNames.size() > 1 && tableName.getDbObjectType() == EDbObjectType.variable) {
17747                                                if (i != j) {
17748                                                        continue;
17749                                                }
17750                                        }
17751                                        TResultColumn column = stmt.getResultColumnList().getResultColumn(i);
17752
17753                                        if ("*".equals(column.getColumnNameOnly()) && column.getFieldAttr() != null
17754                                                        && column.getFieldAttr().getSourceTable() != null) {
17755                                                Object model = modelManager.getModel(column);
17756                                                if(model instanceof LinkedHashMap) {
17757                                                        LinkedHashMap<String, ResultColumn> columns = (LinkedHashMap<String, ResultColumn>)model;
17758                                                        for(String key: columns.keySet()) {
17759                                                                ResultColumn sourceColumn = columns.get(key);
17760                                                                TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
17761                                                                                sourceColumn.getName());
17762                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17763                                                                relation.setEffectType(EffectType.insert);
17764                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
17765                                                                relation.addSource(new ResultColumnRelationshipElement(sourceColumn));
17766                                                                relation.setProcess(process);
17767                                                        }
17768                                                }
17769                                                else if(model instanceof ResultColumn) {
17770                                                        isDetermined = false;
17771                                                        ResultColumn resultColumn = (ResultColumn) model;
17772                                                        List<TObjectName> columns = resultColumn.getStarLinkColumnList();
17773                                                        if (columns.size() > 0) {
17774                                                                for (int k = 0; k < columns.size(); k++) {
17775        
17776                                                                        TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
17777                                                                                        columns.get(k));
17778        
17779                                                                        if (DlineageUtil.isTempTable(tableModel, option.getVendor()) && sqlenv != null
17780                                                                                        && tableModel.getDatabase() != null && tableModel.getSchema() != null) {
17781                                                                                TSQLSchema schema = sqlenv.getSQLSchema(
17782                                                                                                tableModel.getDatabase() + "." + tableModel.getSchema(), true);
17783                                                                                if (schema != null) {
17784                                                                                        TSQLTable tempTable = schema.createTable(DlineageUtil.getSimpleTableName(tableModel.getName()));
17785                                                                                        tempTable.addColumn(tableColumn.getName());
17786                                                                                }
17787                                                                        }
17788        
17789                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17790                                                                        relation.setEffectType(EffectType.insert);
17791                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
17792                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
17793                                                                        relation.setProcess(process);
17794                                                                }
17795                                                                if (resultColumn.isShowStar()) {
17796                                                                        TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
17797                                                                                        column.getFieldAttr());
17798                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17799                                                                        relation.setEffectType(EffectType.insert);
17800                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
17801                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
17802                                                                        relation.setProcess(process);
17803                                                                }
17804                                                        } else {
17805                                                                TObjectName columnObject = column.getFieldAttr();
17806                                                                if (column.getAliasClause() != null) {
17807                                                                        columnObject = column.getAliasClause().getAliasName();
17808                                                                }
17809                                                                if (columnObject != null) {
17810                                                                        TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
17811                                                                                        columnObject);
17812                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17813                                                                        relation.setEffectType(EffectType.insert);
17814                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
17815                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
17816                                                                        relation.setProcess(process);
17817                                                                } else if (!SQLUtil.isEmpty(column.getColumnAlias())) {
17818                                                                        TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
17819                                                                                        column.getAliasClause().getAliasName());
17820                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17821                                                                        relation.setEffectType(EffectType.insert);
17822                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
17823                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
17824                                                                        relation.setProcess(process);
17825                                                                }
17826                                                        }
17827                                                }
17828                                        } else {
17829                                                ResultColumn resultColumn = null;
17830
17831                                                if (queryModel instanceof QueryTable) {
17832                                                        resultColumn = (ResultColumn) modelManager.getModel(column);
17833                                                } else if (queryModel instanceof ResultSet) {
17834                                                        resultColumn = (ResultColumn) modelManager.getModel(column);
17835                                                } else {
17836                                                        continue;
17837                                                }
17838
17839                                                if (resultColumn == null && column.getAliasClause() != null) {
17840                                                        resultColumn = (ResultColumn) modelManager.getModel(column.getAliasClause().getAliasName());
17841                                                }
17842
17843                                                if (resultColumn != null) {
17844                                                        TObjectName columnObject = column.getFieldAttr();
17845                                                        if (column.getAliasClause() != null) {
17846                                                                columnObject = column.getAliasClause().getAliasName();
17847                                                        }
17848                                                        TableColumn tableColumn = null;
17849                                                        if (columnObject != null) {
17850                                                                if (tableModel.isVariable()) {
17851                                                                        if (variableColumn != null) {
17852                                                                                tableColumn = variableColumn;
17853                                                                        } else {
17854                                                                                tableColumn = tableModel.getColumns().get(0);
17855                                                                        }
17856                                                                } 
17857                                                                else if (oracleIntoTableColumn != null) {
17858                                                                        tableColumn = oracleIntoTableColumn;
17859                                                                }
17860                                                                else {
17861                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel, columnObject);
17862                                                                        if (containStarColumn(queryModel)) {
17863                                                                                tableColumn.notBindStarLinkColumn(true);
17864                                                                        }
17865                                                                }
17866                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17867                                                                relation.setEffectType(EffectType.insert);
17868                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
17869                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
17870                                                                relation.setProcess(process);
17871                                                        } else if (!SQLUtil.isEmpty(column.getColumnAlias())) {
17872                                                                if (tableModel.isVariable()) {
17873                                                                        if (variableColumn != null) {
17874                                                                                tableColumn = variableColumn;
17875                                                                        } else {
17876                                                                                tableColumn = tableModel.getColumns().get(0);
17877                                                                        }
17878                                                                } else {
17879                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
17880                                                                                        column.getAliasClause().getAliasName());
17881                                                                }
17882                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17883                                                                relation.setEffectType(EffectType.insert);
17884                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
17885                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
17886                                                                relation.setProcess(process);
17887                                                        } else {
17888                                                                if (tableModel.isVariable()) {
17889                                                                        if (variableColumn != null) {
17890                                                                                tableColumn = variableColumn;
17891                                                                        } else {
17892                                                                                tableColumn = tableModel.getColumns().get(0);
17893                                                                        }
17894                                                                }
17895                                                                if (tableColumn != null) {
17896                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17897                                                                        relation.setEffectType(EffectType.insert);
17898                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
17899                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
17900                                                                        relation.setProcess(process);
17901                                                                }
17902                                                        }
17903                                                }
17904                                        }
17905                                }
17906                                tableModel.setDetermined(isDetermined);
17907                                if(isDetermined && DlineageUtil.isTempTable(tableModel, option.getVendor())) {
17908                                        tableModel.setCreateTable(true, false);
17909                                }
17910                        }
17911                }
17912        }
17913
17914        private boolean isUnPivotedTable(TPivotedTable pivotedTable) {
17915                if (pivotedTable.getPivotClauseList() != null && pivotedTable.getPivotClauseList().size() > 0) {
17916                        return pivotedTable.getPivotClauseList().getElement(0).getType() == TPivotClause.unpivot;
17917                } else {
17918                        TPivotClause pivotClause = pivotedTable.getPivotClause();
17919                        return pivotClause.getType() == TPivotClause.unpivot;
17920                }
17921        }
17922
17923        private void analyzeUnPivotedTable(TSelectSqlStatement stmt, TPivotedTable pivotedTable) {
17924                List<Object> tables = new ArrayList<Object>();
17925                Set<Object> pivotedColumns = new HashSet<Object>();
17926                TTable fromTable = pivotedTable.getTableSource();
17927                Object table = modelManager.getModel(fromTable);
17928                List<TPivotClause> pivotClauses = new ArrayList<TPivotClause>();
17929                if (pivotedTable.getPivotClauseList() != null && pivotedTable.getPivotClauseList().size() > 0) {
17930                        for (int i = 0; i < pivotedTable.getPivotClauseList().size(); i++) {
17931                                pivotClauses.add(pivotedTable.getPivotClauseList().getElement(i));
17932                        }
17933                } else {
17934                        TPivotClause pivotClause = pivotedTable.getPivotClause();
17935                        pivotClauses.add(pivotClause);
17936                }
17937
17938                for (int y = 0; y < pivotClauses.size(); y++) {
17939                        TPivotClause pivotClause = pivotClauses.get(y);
17940                        PivotedTable pivotTable = modelFactory.createPivotdTable(pivotClause);
17941                        pivotTable.setUnpivoted(true);
17942
17943                        if (pivotClause.getValueColumnList() != null) {
17944                                for (int j = 0; j < pivotClause.getValueColumnList().size(); j++) {
17945                                        modelFactory.createResultColumn(pivotTable, pivotClause.getValueColumnList().getObjectName(j));
17946                                }
17947                        }
17948                        if (pivotClause.getPivotColumnList() != null) {
17949                                for (int j = 0; j < pivotClause.getPivotColumnList().size(); j++) {
17950                                        modelFactory.createResultColumn(pivotTable, pivotClause.getPivotColumnList().getObjectName(j));
17951                                }
17952                        }
17953                        if (pivotClause.getUnpivotInClause()!=null && pivotClause.getUnpivotInClause().getItems() != null) {
17954                                for (int j = 0; j < pivotClause.getUnpivotInClause().getItems().size(); j++) {
17955                                        TObjectName columnName = pivotClause.getUnpivotInClause().getItems().getElement(j).getColumn();
17956                                        if (columnName != null) {
17957                                                if (table instanceof QueryTable) {
17958                                                        for (ResultColumn tableColumn : ((QueryTable) table).getColumns()) {
17959                                                                if (getColumnName(columnName)
17960                                                                                .equals(DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
17961                                                                        for (ResultColumn resultColumn : pivotTable.getColumns()) {
17962                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17963                                                                                relation.setEffectType(EffectType.select);
17964                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
17965                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
17966                                                                                pivotedColumns.add(tableColumn);
17967                                                                        }
17968                                                                        break;
17969                                                                }
17970                                                        }
17971                                                } else if (table instanceof Table) {
17972                                                        for (TableColumn tableColumn : ((Table) table).getColumns()) {
17973                                                                if (getColumnName(columnName)
17974                                                                                .equals(DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
17975                                                                        for (ResultColumn resultColumn : pivotTable.getColumns()) {
17976                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17977                                                                                relation.setEffectType(EffectType.select);
17978                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
17979                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
17980                                                                                pivotedColumns.add(tableColumn);
17981                                                                        }
17982                                                                        break;
17983                                                                }
17984                                                        }
17985                                                }
17986                                        } else {
17987                                                TObjectNameList columnNames = pivotClause.getUnpivotInClause().getItems().getElement(j)
17988                                                                .getColumnList();
17989                                                for (TObjectName columnName1 : columnNames) {
17990                                                        if (table instanceof QueryTable) {
17991                                                                for (ResultColumn tableColumn : ((QueryTable) table).getColumns()) {
17992                                                                        if (getColumnName(columnName1).equals(
17993                                                                                        DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
17994                                                                                for (ResultColumn resultColumn : pivotTable.getColumns()) {
17995                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17996                                                                                        relation.setEffectType(EffectType.select);
17997                                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
17998                                                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn));
17999                                                                                        pivotedColumns.add(tableColumn);
18000                                                                                }
18001                                                                                break;
18002                                                                        }
18003                                                                }
18004                                                        } else if (table instanceof Table) {
18005                                                                for (TableColumn tableColumn : ((Table) table).getColumns()) {
18006                                                                        if (getColumnName(columnName1).equals(
18007                                                                                        DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
18008                                                                                for (ResultColumn resultColumn : pivotTable.getColumns()) {
18009                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18010                                                                                        relation.setEffectType(EffectType.select);
18011                                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
18012                                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
18013                                                                                        pivotedColumns.add(tableColumn);
18014                                                                                }
18015                                                                                break;
18016                                                                        }
18017                                                                }
18018                                                        }
18019                                                }
18020                                        }
18021                                }
18022                        }
18023                        tables.add(pivotTable);
18024                        tables.add(table);
18025                }
18026
18027                ResultSet resultSet = modelFactory.createResultSet(stmt,
18028                                isTopResultSet(stmt) && isShowTopSelectResultSet());
18029                TResultColumnList columnList = stmt.getResultColumnList();
18030                for (int i = 0; i < columnList.size(); i++) {
18031                        TResultColumn column = columnList.getResultColumn(i);
18032                        ResultColumn resultColumn = modelFactory.createAndBindingSelectSetResultColumn(resultSet, column, i);
18033                        if (resultColumn.getColumnObject() instanceof TResultColumn) {
18034                                TResultColumn columnObject = (TResultColumn) resultColumn.getColumnObject();
18035                                if (columnObject.getFieldAttr() != null) {
18036                                        if ("*".equals(getColumnName(columnObject.getFieldAttr()))) {
18037                                                resultColumn.setShowStar(false);
18038                                                int index = 0;
18039                                                for (int k = 0; k < tables.size(); k++) {
18040                                                        Object tableItem = tables.get(k);
18041                                                        if (tableItem instanceof ResultSet) {
18042                                                                for (int x = 0; x < ((ResultSet) tableItem).getColumns().size(); x++) {
18043                                                                        ResultColumn tableColumn = ((ResultSet) tableItem).getColumns().get(x);
18044                                                                        if (pivotedColumns.contains(tableColumn)) {
18045                                                                                continue;
18046                                                                        }
18047                                                                        if (tableColumn.getColumnObject() instanceof TObjectName) {
18048                                                                                resultColumn.bindStarLinkColumn((TObjectName) tableColumn.getColumnObject());
18049                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18050                                                                                relation.setEffectType(EffectType.select);
18051                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn, (TObjectName) tableColumn.getColumnObject()));
18052                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
18053                                                                        } else if (tableColumn.getColumnObject() instanceof TResultColumn) {
18054                                                                                if (((TResultColumn) tableColumn.getColumnObject()).getFieldAttr() != null) {
18055                                                                                        if (tableColumn.hasStarLinkColumn()) {
18056                                                                                                for (int z = 0; z < tableColumn.getStarLinkColumnList().size(); z++) {
18057                                                                                                        ResultColumn resultColumn1 = modelFactory.createResultColumn(
18058                                                                                                                        (ResultSet) tableItem,
18059                                                                                                                        tableColumn.getStarLinkColumnList().get(z));
18060                                                                                                        DataFlowRelationship relation = modelFactory
18061                                                                                                                        .createDataFlowRelation();
18062                                                                                                        relation.setEffectType(EffectType.select);
18063                                                                                                        relation.setTarget(
18064                                                                                                                        new ResultColumnRelationshipElement(resultColumn));
18065                                                                                                        relation.addSource(
18066                                                                                                                        new ResultColumnRelationshipElement(resultColumn1));
18067                                                                                                        tableColumn.getStarLinkColumns().remove(
18068                                                                                                                        getColumnName(tableColumn.getStarLinkColumnList().get(z)));
18069                                                                                                        z--;
18070                                                                                                }
18071                                                                                        } else {
18072                                                                                                resultColumn.bindStarLinkColumn(
18073                                                                                                                ((TResultColumn) tableColumn.getColumnObject()).getFieldAttr());
18074                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18075                                                                                                relation.setEffectType(EffectType.select);
18076                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn, ((TResultColumn) tableColumn.getColumnObject()).getFieldAttr()));
18077                                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
18078                                                                                        }
18079                                                                                } else if (((TResultColumn) tableColumn.getColumnObject()).getExpr() != null) {
18080                                                                                        TExpression expr = ((TResultColumn) tableColumn.getColumnObject())
18081                                                                                                        .getExpr();
18082                                                                                        if (expr.getExpressionType() == EExpressionType.simple_constant_t) {
18083                                                                                                TObjectName columnName = new TObjectName();
18084                                                                                                columnName.setString(expr.toString());
18085                                                                                                resultColumn.bindStarLinkColumn(columnName);
18086                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18087                                                                                                relation.setEffectType(EffectType.select);
18088                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn, columnName));
18089                                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
18090                                                                                        }
18091                                                                                }
18092                                                                        }
18093                                                                }
18094                                                        } else if (tableItem instanceof Table) {
18095                                                                for (TableColumn tableColumn : ((Table) tableItem).getColumns()) {
18096                                                                        if (pivotedColumns.contains(tableColumn)) {
18097                                                                                continue;
18098                                                                        }
18099                                                                        resultColumn.bindStarLinkColumn((TObjectName) tableColumn.getColumnObject(), index);
18100                                                                        index++;
18101                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18102                                                                        relation.setEffectType(EffectType.select);
18103                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn, (TObjectName) tableColumn.getColumnObject()));
18104                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
18105                                                                }
18106                                                        }
18107                                                }
18108                                        } else {
18109                                                for (int k = 0; k < tables.size(); k++) {
18110                                                        Object tableItem = tables.get(k);
18111                                                        if (tableItem instanceof ResultSet) {
18112                                                                for (ResultColumn tableColumn : ((ResultSet) tableItem).getColumns()) {
18113                                                                        if (getColumnName(columnObject.getFieldAttr()).equals(
18114                                                                                        DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
18115                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18116                                                                                relation.setEffectType(EffectType.select);
18117                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
18118                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
18119                                                                                break;
18120                                                                        }
18121                                                                }
18122                                                        } else if (tableItem instanceof Table) {
18123                                                                for (TableColumn tableColumn : ((Table) tableItem).getColumns()) {
18124                                                                        if (getColumnName(columnObject.getFieldAttr()).equals(
18125                                                                                        DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
18126                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18127                                                                                relation.setEffectType(EffectType.select);
18128                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
18129                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
18130                                                                                break;
18131                                                                        }
18132                                                                }
18133                                                        }
18134                                                }
18135                                        }
18136                                } else if (columnObject.getExpr() != null) {
18137                                        analyzeResultColumn(column, EffectType.select);
18138                                }
18139                        }
18140                }
18141        }
18142
18143        private void analyzePivotedTable(TSelectSqlStatement stmt, TPivotedTable pivotedTable) {
18144                List<Object> tables = new ArrayList<Object>();
18145                Set<Object> pivotedColumns = new HashSet<Object>();
18146                TTable fromTable = pivotedTable.getTableSource();
18147                Object table = modelManager.getModel(fromTable);
18148                if(table == null && fromTable.getSubquery()!=null) {
18149                        table = modelFactory.createQueryTable(fromTable);
18150                        TSelectSqlStatement subquery = fromTable.getSubquery();
18151                        analyzeSelectStmt(subquery);
18152                }
18153                List<TPivotClause> pivotClauses = new ArrayList<TPivotClause>();
18154                if (pivotedTable.getPivotClauseList() != null && pivotedTable.getPivotClauseList().size() > 0) {
18155                        for (int i = 0; i < pivotedTable.getPivotClauseList().size(); i++) {
18156                                pivotClauses.add(pivotedTable.getPivotClauseList().getElement(i));
18157                        }
18158                } else {
18159                        TPivotClause pivotClause = pivotedTable.getPivotClause();
18160                        pivotClauses.add(pivotClause);
18161                }
18162
18163                for (int y = 0; y < pivotClauses.size(); y++) {
18164                        TPivotClause pivotClause = pivotClauses.get(y);
18165                        List<TFunctionCall> functionCalls = new ArrayList<TFunctionCall>();
18166                        if (pivotClause.getAggregation_function() != null || pivotClause.getAggregation_function_list() != null) {
18167                                if (pivotClause.getAggregation_function() != null) {
18168                                        functionCalls.add((TFunctionCall) pivotClause.getAggregation_function());
18169                                } else if (pivotClause.getAggregation_function_list() != null) {
18170                                        for (int i = 0; i < pivotClause.getAggregation_function_list().size(); i++) {
18171                                                functionCalls.add((TFunctionCall) pivotClause.getAggregation_function_list().getResultColumn(i)
18172                                                                .getExpr().getFunctionCall());
18173                                        }
18174                                }
18175
18176                                if (functionCalls.isEmpty()) {
18177                                        return;
18178                                }
18179
18180                                if (pivotClause.getPivotColumnList() == null) {
18181                                        return;
18182                                }
18183
18184                                if (pivotClause.getPivotInClause() == null) {
18185                                        return;
18186                                }
18187
18188                                for (int x = 0; x < functionCalls.size(); x++) {
18189                                        TFunctionCall functionCall = functionCalls.get(x);
18190                                        Function function = modelFactory.createFunction(functionCall);
18191                                        ResultColumn column = modelFactory.createFunctionResultColumn(function,
18192                                                        ((TFunctionCall) functionCall).getFunctionName());
18193
18194                                        List<TExpression> expressions = new ArrayList<TExpression>();
18195                                        getFunctionExpressions(expressions, new ArrayList<TExpression>(), functionCall);
18196
18197                                        for (int j = 0; j < expressions.size(); j++) {
18198                                                columnsInExpr visitor = new columnsInExpr();
18199                                                expressions.get(j).inOrderTraverse(visitor);
18200                                                List<TObjectName> objectNames = visitor.getObjectNames();
18201                                                if (objectNames == null) {
18202                                                        continue;
18203                                                }
18204                                                for (TObjectName columnName : objectNames) {
18205                                                        if (table instanceof QueryTable) {
18206                                                                for (int i = 0; i < ((QueryTable) table).getColumns().size(); i++) {
18207                                                                        boolean find = false;
18208                                                                        ResultColumn tableColumn = ((QueryTable) table).getColumns().get(i);
18209                                                                        if (tableColumn.hasStarLinkColumn()) {
18210                                                                                for (int k = 0; k < tableColumn.getStarLinkColumnList().size(); k++) {
18211                                                                                        TObjectName objectName = tableColumn.getStarLinkColumnList().get(k);
18212                                                                                        if (getColumnName(columnName).equals(getColumnName(objectName))) {
18213                                                                                                ResultColumn resultColumn = modelFactory
18214                                                                                                                .createResultColumn((QueryTable) table, objectName);
18215                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18216                                                                                                relation.setEffectType(EffectType.select);
18217                                                                                                relation.setTarget(new ResultColumnRelationshipElement(column));
18218                                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
18219                                                                                                pivotedColumns.add(resultColumn);
18220                                                                                                tableColumn.getStarLinkColumns()
18221                                                                                                                .remove(DlineageUtil.getColumnName(objectName));
18222                                                                                                find = true;
18223                                                                                                break;
18224                                                                                        }
18225                                                                                }
18226                                                                        } else if (getColumnName(columnName).equals(
18227                                                                                        DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
18228                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18229                                                                                relation.setEffectType(EffectType.select);
18230                                                                                relation.setTarget(new ResultColumnRelationshipElement(column));
18231                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
18232                                                                                pivotedColumns.add(tableColumn);
18233                                                                                find = true;
18234                                                                                break;
18235                                                                        } 
18236                                                                        
18237                                                                        if (!find && tableColumn.getName().endsWith("*")) {
18238                                                                                QueryTable queryTable = (QueryTable)table;
18239                                                                                ResultColumn resultColumn = modelFactory.createResultColumn(queryTable, columnName);
18240                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18241                                                                                relation.setEffectType(EffectType.select);
18242                                                                                relation.setTarget(new ResultColumnRelationshipElement(column));
18243                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
18244                                                                                tableColumn.bindStarLinkColumn(columnName);
18245                                                                        }
18246                                                                }
18247                                                        } else if (table instanceof Table) {
18248                                                                for (TableColumn tableColumn : ((Table) table).getColumns()) {
18249                                                                        if (getColumnName(columnName).equals(
18250                                                                                        DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
18251                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18252                                                                                relation.setEffectType(EffectType.select);
18253                                                                                relation.setTarget(new ResultColumnRelationshipElement(column));
18254                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
18255                                                                                pivotedColumns.add(tableColumn);
18256                                                                                break;
18257                                                                        }
18258                                                                }
18259                                                        }
18260                                                }
18261                                        }
18262
18263                                        PivotedTable pivotTable = modelFactory.createPivotdTable(pivotClause);
18264                                        pivotTable.setUnpivoted(false);
18265
18266                                        if (pivotClause.getPivotInClause().getItems() != null) {
18267                                                for (int j = 0; j < pivotClause.getPivotInClause().getItems().size(); j++) {
18268                                                        ResultColumn resultColumn = null;
18269                                                        if (pivotClause.getAggregation_function_list() != null
18270                                                                        && pivotClause.getAggregation_function_list().size() > 1) {
18271                                                                TResultColumn functionColumn = pivotClause.getAggregation_function_list()
18272                                                                                .getResultColumn(x);
18273                                                                TObjectName tableColumn = new TObjectName();
18274                                                                if (option.getVendor() == EDbVendor.dbvbigquery) {
18275                                                                        tableColumn.setString(getResultColumnString(functionColumn) + "_"
18276                                                                                        + SQLUtil.trimColumnStringQuote(getResultColumnString(
18277                                                                                                        pivotClause.getPivotInClause().getItems().getResultColumn(j))));
18278                                                                }
18279                                                                else {
18280                                                                        tableColumn.setString(SQLUtil
18281                                                                                        .trimColumnStringQuote(getResultColumnString(
18282                                                                                                        pivotClause.getPivotInClause().getItems().getResultColumn(j)))
18283                                                                                        + "_" + getResultColumnString(functionColumn));
18284                                                                }
18285                                                                resultColumn = modelFactory.createResultColumn(pivotTable, tableColumn);
18286                                                        } else {
18287                                                                resultColumn = modelFactory.createSelectSetResultColumn(pivotTable,
18288                                                                                pivotClause.getPivotInClause().getItems().getResultColumn(j), j);
18289                                                        }
18290                                                        {
18291                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18292                                                                relation.setEffectType(EffectType.select);
18293                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
18294                                                                relation.addSource(new ResultColumnRelationshipElement(column));
18295                                                        }
18296                                                        {
18297                                                                for (TObjectName columnName : pivotClause.getPivotColumnList()) {
18298                                                                        if (table instanceof QueryTable) {
18299                                                                                for (int i = 0; i < ((QueryTable) table).getColumns().size(); i++) {
18300                                                                                        ResultColumn tableColumn = ((QueryTable) table).getColumns().get(i);
18301                                                                                        if (tableColumn.hasStarLinkColumn()) {
18302                                                                                                for (int k = 0; k < tableColumn.getStarLinkColumnList().size(); k++) {
18303                                                                                                        TObjectName objectName = tableColumn.getStarLinkColumnList().get(k);
18304                                                                                                        if (getColumnName(columnName).equals(getColumnName(objectName))) {
18305                                                                                                                ResultColumn resultColumn1 = modelFactory
18306                                                                                                                                .createResultColumn((QueryTable) table, objectName);
18307                                                                                                                DataFlowRelationship relation = modelFactory
18308                                                                                                                                .createDataFlowRelation();
18309                                                                                                                relation.setEffectType(EffectType.select);
18310                                                                                                                relation.setTarget(new ResultColumnRelationshipElement(column));
18311                                                                                                                relation.addSource(
18312                                                                                                                                new ResultColumnRelationshipElement(resultColumn1));
18313                                                                                                                pivotedColumns.add(resultColumn1);
18314                                                                                                                tableColumn.getStarLinkColumns()
18315                                                                                                                                .remove(DlineageUtil.getColumnName(objectName));
18316                                                                                                                break;
18317                                                                                                        }
18318                                                                                                }
18319                                                                                        } else if (getColumnName(columnName).equals(DlineageUtil
18320                                                                                                        .getIdentifierNormalColumnName(tableColumn.getName()))) {
18321                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18322                                                                                                relation.setEffectType(EffectType.select);
18323                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
18324                                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
18325                                                                                                pivotedColumns.add(tableColumn);
18326                                                                                                break;
18327                                                                                        }
18328                                                                                }
18329                                                                        } else if (table instanceof Table) {
18330                                                                                for (TableColumn tableColumn : ((Table) table).getColumns()) {
18331                                                                                        if (getColumnName(columnName).equals(DlineageUtil
18332                                                                                                        .getIdentifierNormalColumnName(tableColumn.getName()))) {
18333                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18334                                                                                                relation.setEffectType(EffectType.select);
18335                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
18336                                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
18337                                                                                                pivotedColumns.add(tableColumn);
18338                                                                                                break;
18339                                                                                        }
18340                                                                                }
18341                                                                        }
18342                                                                }
18343                                                        }
18344                                                }
18345                                        } else if (pivotClause.getPivotInClause().getSubQuery() != null) {
18346                                                TSelectSqlStatement subquery = pivotClause.getPivotInClause().getSubQuery();
18347                                                analyzeSelectStmt(subquery);
18348                                                ResultSet selectSetResultSetModel = (ResultSet) modelManager.getModel(subquery);
18349                                                for (int j = 0; j < subquery.getResultColumnList().size(); j++) {
18350                                                        ResultColumn resultColumn = modelFactory.createSelectSetResultColumn(pivotTable,
18351                                                                        subquery.getResultColumnList().getResultColumn(j), j);
18352                                                        {
18353                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18354                                                                relation.setEffectType(EffectType.select);
18355                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
18356                                                                relation.addSource(new ResultColumnRelationshipElement(column));
18357                                                                relation.addSource(new ResultColumnRelationshipElement(
18358                                                                                selectSetResultSetModel.getColumns().get(j)));
18359                                                        }
18360                                                        {
18361                                                                for (TObjectName columnName : pivotClause.getPivotColumnList()) {
18362                                                                        if (table instanceof QueryTable) {
18363                                                                                for (ResultColumn tableColumn : ((QueryTable) table).getColumns()) {
18364                                                                                        if (getColumnName(columnName).equals(DlineageUtil
18365                                                                                                        .getIdentifierNormalColumnName(tableColumn.getName()))) {
18366                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18367                                                                                                relation.setEffectType(EffectType.select);
18368                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
18369                                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
18370                                                                                                pivotedColumns.add(tableColumn);
18371                                                                                                break;
18372                                                                                        }
18373                                                                                }
18374                                                                        } else if (table instanceof Table) {
18375                                                                                for (TableColumn tableColumn : ((Table) table).getColumns()) {
18376                                                                                        if (getColumnName(columnName).equals(DlineageUtil
18377                                                                                                        .getIdentifierNormalColumnName(tableColumn.getName()))) {
18378                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18379                                                                                                relation.setEffectType(EffectType.select);
18380                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
18381                                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
18382                                                                                                pivotedColumns.add(tableColumn);
18383                                                                                                break;
18384                                                                                        }
18385                                                                                }
18386                                                                        }
18387                                                                }
18388                                                        }
18389                                                }
18390                                        }
18391                                        tables.add(pivotTable);
18392                                        tables.add(table);
18393                                }
18394                        }
18395                }
18396
18397                TPivotClause pivotClause = pivotClauses.get(0);
18398                boolean hasAlias = pivotClause.getAliasClause() != null && pivotClause.getAliasClause().getColumns() != null
18399                                && pivotClause.getAliasClause().getColumns().size() > 0;
18400                if (hasAlias) {
18401                        Alias alias = modelFactory.createAlias(pivotClause.getAliasClause());
18402                        List<TObjectName> aliasColumns = new ArrayList<TObjectName>();
18403                        int index = 0;
18404                        for (int k = 0; k < tables.size(); k++) {
18405                                Object tableItem = tables.get(k);
18406                                if (tableItem instanceof ResultSet) {
18407                                        for (ResultColumn tableColumn : ((ResultSet) tableItem).getColumns()) {
18408                                                if (pivotedColumns.contains(tableColumn)) {
18409                                                        continue;
18410                                                }
18411                                                if (tableColumn.getColumnObject() instanceof TObjectName) {
18412                                                        aliasColumns.add((TObjectName) tableColumn.getColumnObject());
18413                                                } else if (tableColumn.getColumnObject() instanceof TResultColumn) {
18414                                                        if (((TResultColumn) tableColumn.getColumnObject()).getFieldAttr() != null) {
18415                                                                aliasColumns.add(((TResultColumn) tableColumn.getColumnObject()).getFieldAttr());
18416                                                        } else {
18417                                                                TExpression expr = ((TResultColumn) tableColumn.getColumnObject()).getExpr();
18418                                                                if (expr.getExpressionType() == EExpressionType.simple_constant_t) {
18419                                                                        TObjectName columnName = new TObjectName();
18420                                                                        columnName.setString(expr.toString());
18421                                                                        aliasColumns.add(columnName);
18422                                                                }
18423                                                        }
18424                                                }
18425                                        }
18426                                } else if (tableItem instanceof Table) {
18427                                        for (TableColumn tableColumn : ((Table) tableItem).getColumns()) {
18428                                                if (pivotedColumns.contains(tableColumn)) {
18429                                                        continue;
18430                                                }
18431                                                aliasColumns.add(index, (TObjectName) tableColumn.getColumnObject());
18432                                                index++;
18433                                        }
18434                                }
18435                        }
18436
18437                        IndexedLinkedHashMap<String, ResultColumn> aliasColumnMap = new IndexedLinkedHashMap<String, ResultColumn>();
18438                        int diffCount = pivotClause.getAliasClause().getColumns().size() -  aliasColumns.size();
18439                        for (int k = 0; k < pivotClause.getAliasClause().getColumns().size(); k++) {
18440                                if (pivotClause.getAliasClause().getColumns().size() > aliasColumns.size()) {
18441                                        if (k < diffCount) {
18442                                                continue;
18443                                        }
18444                                        ResultColumn resultColumn = modelFactory.createResultColumn(alias,
18445                                                        pivotClause.getAliasClause().getColumns().getObjectName(k));
18446                                        if ((k - diffCount) < aliasColumns.size()) {
18447                                                aliasColumnMap.put(aliasColumns.get(k - diffCount).toString(), resultColumn);
18448                                        }
18449                                } else {
18450                                        ResultColumn resultColumn = modelFactory.createResultColumn(alias,
18451                                                        pivotClause.getAliasClause().getColumns().getObjectName(k));
18452                                        if (k < aliasColumns.size()) {
18453                                                aliasColumnMap.put(aliasColumns.get(k).toString(), resultColumn);
18454                                        }
18455                                }
18456                        }
18457
18458                        for (int k = 0; k < tables.size(); k++) {
18459                                Object tableItem = tables.get(k);
18460                                if (tableItem instanceof ResultSet) {
18461                                        int resultColumnSize = ((ResultSet) tableItem).getColumns().size();
18462                                        for (int x = 0; x < resultColumnSize; x++) {
18463                                                ResultColumn tableColumn = ((ResultSet) tableItem).getColumns().get(x);
18464                                                if (pivotedColumns.contains(tableColumn)) {
18465                                                        continue;
18466                                                }
18467                                                if (tableColumn.getColumnObject() instanceof TObjectName) {
18468                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18469                                                        relation.setEffectType(EffectType.select);
18470                                                        relation.setTarget(new ResultColumnRelationshipElement(
18471                                                                        aliasColumnMap.get(tableColumn.getColumnObject().toString())));
18472                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn));
18473                                                } else if (tableColumn.getColumnObject() instanceof TResultColumn) {
18474                                                        if (((TResultColumn) tableColumn.getColumnObject()).getFieldAttr() != null) {
18475                                                                ResultColumn targetColumn = aliasColumnMap.get(
18476                                                                                ((TResultColumn) tableColumn.getColumnObject()).getFieldAttr().toString());
18477                                                                if(targetColumn == null) {
18478                                                                        continue;
18479                                                                }
18480                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18481                                                                relation.setEffectType(EffectType.select);
18482                                                                relation.setTarget(new ResultColumnRelationshipElement(targetColumn));
18483                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
18484                                                        } else if (((TResultColumn) tableColumn.getColumnObject()).getExpr() != null) {
18485                                                                TExpression expr = ((TResultColumn) tableColumn.getColumnObject()).getExpr();
18486                                                                if (expr.getExpressionType() == EExpressionType.simple_constant_t) {
18487                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18488                                                                        relation.setEffectType(EffectType.select);
18489                                                                        ResultColumn targetColumn = (ResultColumn) aliasColumnMap
18490                                                                                        .getValueAtIndex(aliasColumnMap.size() - resultColumnSize + x);
18491                                                                        relation.setTarget(new ResultColumnRelationshipElement(targetColumn));
18492                                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn));
18493                                                                }
18494                                                        }
18495                                                }
18496                                        }
18497                                } else if (tableItem instanceof Table) {
18498                                        for (TableColumn tableColumn : ((Table) tableItem).getColumns()) {
18499                                                if (pivotedColumns.contains(tableColumn)) {
18500                                                        continue;
18501                                                }
18502                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18503                                                relation.setEffectType(EffectType.select);
18504                                                relation.setTarget(new ResultColumnRelationshipElement(
18505                                                                aliasColumnMap.get(tableColumn.getColumnObject().toString())));
18506                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
18507                                        }
18508                                }
18509                        }
18510
18511                        ResultSet resultSet = modelFactory.createResultSet(stmt,
18512                                        isTopResultSet(stmt) && isShowTopSelectResultSet());
18513                        TResultColumnList columnList = stmt.getResultColumnList();
18514                        for (int i = 0; i < columnList.size(); i++) {
18515                                TResultColumn column = columnList.getResultColumn(i);
18516                                ResultColumn resultColumn = modelFactory.createAndBindingSelectSetResultColumn(resultSet, column, i);
18517                                if (resultColumn.getColumnObject() instanceof TResultColumn) {
18518                                        TResultColumn columnObject = (TResultColumn) resultColumn.getColumnObject();
18519                                        if (columnObject.getFieldAttr() != null) {
18520                                                if ("*".equals(getColumnName(columnObject.getFieldAttr()))) {
18521                                                        resultColumn.setShowStar(false);
18522                                                        for (ResultColumn tableColumn : ((ResultSet) alias).getColumns()) {
18523                                                                resultColumn.bindStarLinkColumn((TObjectName) tableColumn.getColumnObject());
18524                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18525                                                                relation.setEffectType(EffectType.select);
18526                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn, (TObjectName) tableColumn.getColumnObject()));
18527                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
18528                                                        }
18529                                                } else {
18530                                                        for (ResultColumn tableColumn : ((ResultSet) alias).getColumns()) {
18531                                                                if (getColumnName(columnObject.getFieldAttr())
18532                                                                                .equals(DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
18533                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18534                                                                        relation.setEffectType(EffectType.select);
18535                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
18536                                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn));
18537                                                                        break;
18538                                                                }
18539                                                        }
18540                                                }
18541                                        } else if (columnObject.getExpr() != null && columnObject.getExpr()
18542                                                        .getExpressionType() == EExpressionType.sqlserver_proprietary_column_alias_t) {
18543                                                for (ResultColumn tableColumn : ((ResultSet) alias).getColumns()) {
18544                                                        if (getColumnName(columnObject.getExpr().getRightOperand().getObjectOperand())
18545                                                                        .equals(DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
18546                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18547                                                                relation.setEffectType(EffectType.select);
18548                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
18549                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
18550                                                                break;
18551                                                        }
18552                                                }
18553                                        } else if (columnObject.getExpr() != null && columnObject.getExpr()
18554                                                        .getExpressionType() == EExpressionType.function_t) {
18555                                                Function function = (Function) createFunction(columnObject.getExpr().getFunctionCall());
18556                                                for (ResultColumn arg : function.getColumns()) {
18557                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18558                                                        relation.setEffectType(EffectType.select);
18559                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
18560                                                        relation.addSource(new ResultColumnRelationshipElement(arg));
18561                                                }
18562                                        }
18563                                }
18564                        }
18565                } else {
18566                        ResultSet resultSet = modelFactory.createResultSet(stmt, isTopResultSet(stmt));
18567                        TResultColumnList columnList = stmt.getResultColumnList();
18568                        for (int i = 0; i < columnList.size(); i++) {
18569                                TResultColumn column = columnList.getResultColumn(i);
18570                                ResultColumn resultColumn = modelFactory.createAndBindingSelectSetResultColumn(resultSet, column, i);
18571                                if (resultColumn.getColumnObject() instanceof TResultColumn) {
18572                                        boolean fromFunction = false;
18573                                        TResultColumn columnObject = (TResultColumn) resultColumn.getColumnObject();
18574                                        TObjectName resultColumnFieldAttr = columnObject.getFieldAttr();
18575                                        List<TObjectName> resultColumnNames = new ArrayList<TObjectName>();
18576                                        if (resultColumnFieldAttr != null) {
18577                                                resultColumnNames.add(resultColumnFieldAttr);
18578                                        } else if (columnObject.getExpr() != null
18579                                                        && column.getExpr().getExpressionType() == EExpressionType.function_t) {
18580                                                extractFunctionObjectNames(column.getExpr().getFunctionCall(), resultColumnNames);
18581                                                fromFunction = true;
18582                                        }
18583                                        
18584                                        if (!resultColumnNames.isEmpty()) {
18585                                                for (TObjectName resultColumnName : resultColumnNames) {
18586                                                        if ("*".equals(getColumnName(resultColumnName))) {
18587                                                                resultColumn.setShowStar(false);
18588                                                                int index = 0;
18589                                                                for (int k = 0; k < tables.size(); k++) {
18590                                                                        Object tableItem = tables.get(k);
18591                                                                        if (tableItem instanceof ResultSet && !(tableItem instanceof QueryTable)) {
18592                                                                                for (int x = 0; x < ((ResultSet) tableItem).getColumns().size(); x++) {
18593                                                                                        ResultColumn tableColumn = ((ResultSet) tableItem).getColumns().get(x);
18594                                                                                        if (pivotedColumns.contains(tableColumn)) {
18595                                                                                                continue;
18596                                                                                        }
18597                                                                                        if (tableColumn.getColumnObject() instanceof TObjectName) {
18598                                                                                                resultColumn.bindStarLinkColumn(
18599                                                                                                                (TObjectName) tableColumn.getColumnObject());
18600                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18601                                                                                                relation.setEffectType(EffectType.select);
18602                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn, (TObjectName) tableColumn.getColumnObject()));
18603                                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
18604                                                                                        } else if (tableColumn.getColumnObject() instanceof TResultColumn) {
18605                                                                                                if (((TResultColumn) tableColumn.getColumnObject())
18606                                                                                                                .getFieldAttr() != null) {
18607                                                                                                        if (tableColumn.hasStarLinkColumn()) {
18608                                                                                                                for (int z = 0; z < tableColumn.getStarLinkColumnList()
18609                                                                                                                                .size(); z++) {
18610                                                                                                                        ResultColumn resultColumn1 = modelFactory
18611                                                                                                                                        .createResultColumn((ResultSet) tableItem,
18612                                                                                                                                                        tableColumn.getStarLinkColumnList().get(z));
18613                                                                                                                        DataFlowRelationship relation = modelFactory
18614                                                                                                                                        .createDataFlowRelation();
18615                                                                                                                        relation.setEffectType(EffectType.select);
18616                                                                                                                        relation.setTarget(
18617                                                                                                                                        new ResultColumnRelationshipElement(resultColumn));
18618                                                                                                                        relation.addSource(
18619                                                                                                                                        new ResultColumnRelationshipElement(resultColumn1));
18620                                                                                                                        tableColumn.getStarLinkColumns().remove(getColumnName(
18621                                                                                                                                        tableColumn.getStarLinkColumnList().get(z)));
18622                                                                                                                        z--;
18623                                                                                                                }
18624                                                                                                        } else {
18625                                                                                                                resultColumn.bindStarLinkColumn(
18626                                                                                                                                ((TResultColumn) tableColumn.getColumnObject())
18627                                                                                                                                                .getFieldAttr());
18628                                                                                                                DataFlowRelationship relation = modelFactory
18629                                                                                                                                .createDataFlowRelation();
18630                                                                                                                relation.setEffectType(EffectType.select);
18631                                                                                                                relation.setTarget(
18632                                                                                                                                new ResultColumnRelationshipElement(resultColumn, ((TResultColumn) tableColumn.getColumnObject())
18633                                                                                                                                                .getFieldAttr()));
18634                                                                                                                relation.addSource(
18635                                                                                                                                new ResultColumnRelationshipElement(tableColumn));
18636                                                                                                        }
18637                                                                                                } else if (((TResultColumn) tableColumn.getColumnObject())
18638                                                                                                                .getExpr() != null) {
18639                                                                                                        TExpression expr = ((TResultColumn) tableColumn.getColumnObject())
18640                                                                                                                        .getExpr();
18641                                                                                                        if (expr.getExpressionType() == EExpressionType.simple_constant_t) {
18642                                                                                                                TObjectName columnName = new TObjectName();
18643                                                                                                                columnName.setString(expr.toString());
18644                                                                                                                resultColumn.bindStarLinkColumn(columnName);
18645                                                                                                                DataFlowRelationship relation = modelFactory
18646                                                                                                                                .createDataFlowRelation();
18647                                                                                                                relation.setEffectType(EffectType.select);
18648                                                                                                                relation.setTarget(
18649                                                                                                                                new ResultColumnRelationshipElement(resultColumn, columnName));
18650                                                                                                                relation.addSource(
18651                                                                                                                                new ResultColumnRelationshipElement(tableColumn));
18652                                                                                                        }
18653                                                                                                }
18654                                                                                        }
18655                                                                                }
18656                                                                        } else if (tableItem instanceof Table) {
18657                                                                                for (TableColumn tableColumn : ((Table) tableItem).getColumns()) {
18658                                                                                        if (pivotedColumns.contains(tableColumn)) {
18659                                                                                                continue;
18660                                                                                        }
18661                                                                                        resultColumn.bindStarLinkColumn((TObjectName) tableColumn.getColumnObject(),
18662                                                                                                        index);
18663                                                                                        index++;
18664                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18665                                                                                        relation.setEffectType(EffectType.select);
18666                                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn, (TObjectName) tableColumn.getColumnObject()));
18667                                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
18668                                                                                }
18669                                                                        } else if (tableItem instanceof QueryTable) {
18670                                                                                for (ResultColumn tableColumn : ((QueryTable) tableItem).getColumns()) {
18671                                                                                        if (pivotedColumns.contains(tableColumn)) {
18672                                                                                                continue;
18673                                                                                        }
18674                                                                                        TObjectName column1 = new TObjectName();
18675                                                                                        column1.setString(tableColumn.getName());
18676                                                                                        resultColumn.bindStarLinkColumn(column1, index);
18677                                                                                        index++;
18678                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18679                                                                                        relation.setEffectType(EffectType.select);
18680                                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn, column1));
18681                                                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn), false);
18682                                                                                }
18683                                                                        }
18684                                                                }
18685                                                        } else {
18686                                                                ResultColumn pivotedTableColumn = getPivotedTableColumn(pivotedTable, resultColumnName);
18687                                                                        if (pivotedTableColumn != null) {
18688                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18689                                                                                relation.setEffectType(EffectType.select);
18690                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
18691                                                                                relation.addSource(new ResultColumnRelationshipElement(pivotedTableColumn));
18692                                                                        } else {
18693                                                                                for (int k = 0; k < tables.size(); k++) {
18694                                                                                        Object tableItem = tables.get(k);
18695                                                                                        if (tableItem instanceof ResultSet) {
18696                                                                                                for (ResultColumn tableColumn : ((ResultSet) tableItem).getColumns()) {
18697                                                                                                        if (DlineageUtil
18698                                                                                                                        .getIdentifierNormalColumnName(tableColumn.getName()).equals(getColumnName(resultColumnName))) {
18699                                                                                                                if (fromFunction) {
18700                                                                                                                        Function function = (Function)createPivotedFunction(column.getExpr().getFunctionCall(), tableColumn);
18701                                                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18702                                                                                                                        relation.setEffectType(EffectType.select);
18703                                                                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
18704                                                                                                                        if (function.getColumns() != null && !function.getColumns().isEmpty()) {
18705                                                                                                                                for (ResultColumn functionColumn : function.getColumns()) {
18706                                                                                                                                        relation.addSource(new ResultColumnRelationshipElement(functionColumn));
18707                                                                                                                                }
18708                                                                                                                        }
18709                                                                                                                } else {
18710                                                                                                                        DataFlowRelationship relation = modelFactory
18711                                                                                                                                        .createDataFlowRelation();
18712                                                                                                                        relation.setEffectType(EffectType.select);
18713                                                                                                                        relation.setTarget(
18714                                                                                                                                        new ResultColumnRelationshipElement(resultColumn));
18715                                                                                                                        relation.addSource(
18716                                                                                                                                        new ResultColumnRelationshipElement(tableColumn));
18717                                                                                                                }
18718                                                                                                                break;
18719                                                                                                        }
18720                                                                                                }
18721                                                                                        } else if (tableItem instanceof Table) {
18722                                                                                                for (TableColumn tableColumn : ((Table) tableItem).getColumns()) {
18723                                                                                                        if (getColumnName(resultColumnName).equals(DlineageUtil
18724                                                                                                                        .getIdentifierNormalColumnName(tableColumn.getName()))) {
18725                                                                                                                DataFlowRelationship relation = modelFactory
18726                                                                                                                                .createDataFlowRelation();
18727                                                                                                                relation.setEffectType(EffectType.select);
18728                                                                                                                relation.setTarget(
18729                                                                                                                                new ResultColumnRelationshipElement(resultColumn));
18730                                                                                                                relation.addSource(
18731                                                                                                                                new TableColumnRelationshipElement(tableColumn));
18732                                                                                                                break;
18733                                                                                                        }
18734                                                                                                }
18735                                                                                        }
18736                                                                                }
18737                                                                        }
18738                                                                }
18739                                                }
18740                                        } else if (columnObject.getExpr() != null && columnObject.getExpr()
18741                                                        .getExpressionType() == EExpressionType.sqlserver_proprietary_column_alias_t) {
18742                                                for (int k = 0; k < tables.size(); k++) {
18743                                                        Object tableItem = tables.get(k);
18744                                                        if (tableItem instanceof ResultSet) {
18745                                                                for (ResultColumn tableColumn : ((ResultSet) tableItem).getColumns()) {
18746                                                                        if (columnObject.getExpr().getRightOperand().getObjectOperand() != null
18747                                                                                        && getColumnName(
18748                                                                                                        columnObject.getExpr().getRightOperand().getObjectOperand())
18749                                                                                                                        .equals(DlineageUtil.getIdentifierNormalColumnName(
18750                                                                                                                                        tableColumn.getName()))) {
18751                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18752                                                                                relation.setEffectType(EffectType.select);
18753                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
18754                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
18755                                                                                break;
18756                                                                        } else if (columnObject.getExpr().getRightOperand()
18757                                                                                        .getExpressionType() == EExpressionType.function_t) {
18758                                                                                List<TExpression> expressions = new ArrayList<TExpression>();
18759                                                                                getFunctionExpressions(expressions, new ArrayList<TExpression>(),
18760                                                                                                columnObject.getExpr().getRightOperand().getFunctionCall());
18761                                                                                for (int j = 0; j < expressions.size(); j++) {
18762                                                                                        columnsInExpr visitor = new columnsInExpr();
18763                                                                                        expressions.get(j).inOrderTraverse(visitor);
18764                                                                                        List<TObjectName> objectNames = visitor.getObjectNames();
18765                                                                                        if (objectNames == null) {
18766                                                                                                continue;
18767                                                                                        }
18768                                                                                        for (TObjectName columnName : objectNames) {
18769                                                                                                if (getColumnName(columnName).equals(DlineageUtil
18770                                                                                                                .getIdentifierNormalColumnName(tableColumn.getName()))) {
18771                                                                                                        DataFlowRelationship relation = modelFactory
18772                                                                                                                        .createDataFlowRelation();
18773                                                                                                        relation.setEffectType(EffectType.select);
18774                                                                                                        relation.setTarget(
18775                                                                                                                        new ResultColumnRelationshipElement(resultColumn));
18776                                                                                                        relation.addSource(
18777                                                                                                                        new ResultColumnRelationshipElement(tableColumn));
18778                                                                                                        break;
18779                                                                                                }
18780                                                                                        }
18781                                                                                }
18782                                                                        }
18783                                                                }
18784                                                        } else if (tableItem instanceof Table) {
18785                                                                for (TableColumn tableColumn : ((Table) tableItem).getColumns()) {
18786                                                                        if (getColumnName(columnObject.getExpr().getRightOperand().getObjectOperand())
18787                                                                                        .equals(DlineageUtil
18788                                                                                                        .getIdentifierNormalColumnName(tableColumn.getName()))) {
18789                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18790                                                                                relation.setEffectType(EffectType.select);
18791                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
18792                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
18793                                                                                break;
18794                                                                        }
18795                                                                }
18796                                                        }
18797                                                }
18798                                        }
18799                                }
18800                        }
18801                }
18802
18803                analyzeSelectIntoClause(stmt);
18804        }
18805
18806        private Function createPivotedFunction(TFunctionCall functionCall, ResultColumn sourceColumn) {
18807                Function function = modelFactory.createFunction((TFunctionCall) functionCall);
18808                ResultColumn column = modelFactory.createFunctionResultColumn(function,
18809                                ((TFunctionCall) functionCall).getFunctionName());
18810                if ("COUNT".equalsIgnoreCase(((TFunctionCall) functionCall).getFunctionName().toString())) {
18811                        // @see https://e.gitee.com/gudusoft/issues/list?issue=I40NUP
18812                        // COUNT特殊处理，不和参数关联
18813                        if (option.isShowCountTableColumn()) {
18814                                analyzePivotedFunctionArgumentsDataFlowRelation(column, functionCall, sourceColumn);
18815                        }
18816                } else {
18817                        analyzePivotedFunctionArgumentsDataFlowRelation(column, functionCall, sourceColumn);
18818                        Set<Object> functionTableModelObjs = modelManager.getFunctionTable(getIdentifiedFunctionName(function));
18819                        if (functionTableModelObjs!=null && functionTableModelObjs.iterator().next() instanceof ResultSet) {
18820                                ResultSet functionTableModel = (ResultSet) functionTableModelObjs.iterator().next();
18821                                if (functionTableModel.getColumns() != null) {
18822                                        for (int j = 0; j < functionTableModel.getColumns().size(); j++) {
18823                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18824                                                        relation.setEffectType(EffectType.select);
18825                                                        relation.setTarget(new ResultColumnRelationshipElement(column));
18826                                                        relation.addSource(new ResultColumnRelationshipElement(
18827                                                                        functionTableModel.getColumns().get(j)));
18828                                        }
18829                                }
18830                        }
18831                }
18832                return function;
18833        }
18834
18835        private void analyzePivotedFunctionArgumentsDataFlowRelation(ResultColumn column, TFunctionCall functionCall,
18836                        ResultColumn sourceColumn) {
18837                List<TExpression> directExpressions = new ArrayList<TExpression>();
18838                List<TExpression> indirectExpressions = new ArrayList<TExpression>();
18839
18840                getFunctionExpressions(directExpressions, indirectExpressions, functionCall);
18841
18842                for (int j = 0; j < directExpressions.size(); j++) {
18843                        columnsInExpr visitor = new columnsInExpr();
18844                        directExpressions.get(j).inOrderTraverse(visitor);
18845
18846                        List<TObjectName> objectNames = visitor.getObjectNames();
18847                        List<TParseTreeNode> constants = visitor.getConstants();
18848
18849                        if (objectNames != null) {
18850                                for (TObjectName name : objectNames) {
18851                                        if (DlineageUtil.compareColumnIdentifier(name.toString(), sourceColumn.getName())) {
18852                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18853                                                relation.setEffectType(EffectType.select);
18854                                                relation.setTarget(new ResultColumnRelationshipElement(column));
18855                                                relation.addSource(new ResultColumnRelationshipElement(sourceColumn));
18856                                        }
18857                                }
18858                        }
18859                        if (constants != null) {
18860                                for (TParseTreeNode name : constants) {
18861                                        if (name.toString().equals(sourceColumn.getName())) {
18862                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18863                                                relation.setEffectType(EffectType.select);
18864                                                relation.setTarget(new ResultColumnRelationshipElement(column));
18865                                                relation.addSource(new ResultColumnRelationshipElement(sourceColumn));
18866                                        }
18867                                }
18868                        }
18869                }
18870        }
18871        
18872        private void extractFunctionObjectNames(TFunctionCall functionCall, List<TObjectName> resultColumnNames) {
18873                List<TExpression> directExpressions = new ArrayList<TExpression>();
18874                List<TExpression> indirectExpressions = new ArrayList<TExpression>();
18875
18876                getFunctionExpressions(directExpressions, indirectExpressions, functionCall);
18877
18878                for (int j = 0; j < directExpressions.size(); j++) {
18879                        columnsInExpr visitor = new columnsInExpr();
18880                        directExpressions.get(j).inOrderTraverse(visitor);
18881
18882                        List<TObjectName> objectNames = visitor.getObjectNames();
18883                        List<TParseTreeNode> functions = visitor.getFunctions();
18884                        List<TParseTreeNode> constants = visitor.getConstants(); 
18885
18886                        if (objectNames != null) {
18887                                resultColumnNames.addAll(objectNames);
18888                        }
18889                        
18890                        if (constants != null) {
18891                                for(TParseTreeNode item: constants) {
18892                                        if(item instanceof TConstant && ((TConstant) item).getLiteralType().getText().equals(ELiteralType.string_et.getText())) {
18893                                                TObjectName object = new TObjectName();
18894                                                object.setString(item.toString());
18895                                                resultColumnNames.add(object);
18896                                        }
18897                                }
18898                        }
18899
18900                        if (functions != null && !functions.isEmpty()) {
18901                                for (TParseTreeNode function : functions) {
18902                                        if (function instanceof TFunctionCall) {
18903                                                extractFunctionObjectNames((TFunctionCall) function, resultColumnNames);
18904                                        }
18905                                }
18906                        }
18907                }
18908        }
18909
18910        private String getResultColumnString(TResultColumn resultColumn) {
18911                if (resultColumn.getAliasClause() != null) {
18912                        return resultColumn.getAliasClause().toString();
18913                }
18914                return resultColumn.toString();
18915        }
18916
18917        private void analyzeBigQueryUnnest(TSelectSqlStatement stmt, TTable table) {
18918                Table unnestTable = modelFactory.createTableFromCreateDDL(table, false, getTempTableName(table));
18919                unnestTable.setSubType(SubType.unnest);
18920                TUnnestClause clause = table.getUnnestClause();
18921                TExpression arrayExpr = clause.getArrayExpr();
18922                if (arrayExpr == null){
18923                        if (clause.getColumns() != null) {
18924                                for (TObjectName column : clause.getColumns()) {
18925                                        if (clause.getDerivedColumnList() != null) {
18926                                                unnestTable.setCreateTable(true);
18927                                                for (int i = 0; i < clause.getDerivedColumnList().size(); i++) {
18928                                                        TObjectName columnName = new TObjectName();
18929                                                        columnName.setString(column.getColumnNameOnly() + "."
18930                                                                        + clause.getDerivedColumnList().getObjectName(i).getColumnNameOnly());
18931                                                        TableColumn tableColumn = modelFactory.createTableColumn(unnestTable, columnName, true);
18932                                                        List<TObjectName> columns = new ArrayList<TObjectName>();
18933                                                        columns.add(column);
18934                                                        analyzeDataFlowRelation(tableColumn, columns, EffectType.select, null);
18935                                                }
18936                                        }
18937                                        else {
18938                                                unnestTable.setCreateTable(true);
18939                                                boolean find = false;
18940                                                if (column.getSourceTable() != null && modelManager.getModel(column.getSourceTable()) instanceof Table) {
18941                                                        Table sourceTable = (Table)modelManager.getModel(column.getSourceTable());
18942                                                        if(sourceTable!=null) {
18943                                                                for(TableColumn tableColumn: sourceTable.getColumns()) {
18944                                                                        if(tableColumn.isStruct()) {
18945                                                                                List<String> names = SQLUtil.parseNames(tableColumn.getName());
18946                                                                                if (names.get(0).equalsIgnoreCase(column.getColumnNameOnly())) {
18947                                                                                        TObjectName columnName = new TObjectName();
18948                                                                                        columnName.setString(tableColumn.getName());
18949                                                                                        TableColumn unnestTableColumn = modelFactory.createTableColumn(unnestTable,
18950                                                                                                        columnName, true);
18951                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18952                                                                                        relation.setEffectType(EffectType.select);
18953                                                                                        relation.setTarget(new TableColumnRelationshipElement(unnestTableColumn));
18954                                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
18955                                                                                        find = true;
18956                                                                                }
18957                                                                        }
18958                                                                }
18959                                                        }
18960                                                }
18961                                                if (!find) {
18962                                                        TObjectName colName = column;
18963                                                        if (table.getAliasClause() != null && table.getAliasClause().getAliasName() != null) {
18964                                                                colName = table.getAliasClause().getAliasName();
18965                                                        }
18966                                                        TableColumn tableColumn = modelFactory.createTableColumn(unnestTable, colName, true);
18967                                                        List<TObjectName> columns = new ArrayList<TObjectName>();
18968                                                        columns.add(column);
18969                                                        analyzeDataFlowRelation(tableColumn, columns, EffectType.select, null);
18970                                                }
18971                                        }
18972                                }
18973                        }
18974                        return;
18975                }
18976                List<TExpression> expressions = new ArrayList<TExpression>();
18977                TExpressionList values = arrayExpr.getExprList();
18978                if (values == null) {
18979                        expressions.add(arrayExpr);
18980                }
18981                else {
18982                        for(TExpression value: values) {
18983                                expressions.add(value);
18984                        }
18985                }
18986                for (TExpression value : expressions) {
18987                        unnestTable.setCreateTable(true);
18988                        if (value.getExpressionType() == EExpressionType.simple_object_name_t) {
18989                                if (table.getAliasClause() != null) {
18990                                        TableColumn tableColumn = modelFactory.createTableColumn(unnestTable,
18991                                                        table.getAliasClause().getAliasName(), true);
18992                                        columnsInExpr visitor = new columnsInExpr();
18993                                        value.inOrderTraverse(visitor);
18994                                        List<TObjectName> columns = visitor.getObjectNames();
18995                                        analyzeDataFlowRelation(tableColumn, columns, EffectType.select, null);
18996                                } else {
18997                                        TResultColumnList resultColumnList = stmt.getResultColumnList();
18998                                        for (int i = 0; i < resultColumnList.size(); i++) {
18999                                                TObjectName firstColumn = new TObjectName();
19000                                                firstColumn.setString(resultColumnList.getResultColumn(0).getColumnNameOnly());
19001                                                TableColumn tableColumn = modelFactory.createTableColumn(unnestTable, firstColumn, true);
19002                                                columnsInExpr visitor = new columnsInExpr();
19003                                                value.inOrderTraverse(visitor);
19004                                                List<TObjectName> columns = visitor.getObjectNames();
19005                                                analyzeDataFlowRelation(tableColumn, columns, EffectType.select, null);
19006                                        }
19007                                }
19008                        } else if (value.getExpressionType() == EExpressionType.function_t) {
19009                                if (table.getAliasClause() != null) {
19010                                        TableColumn tableColumn = modelFactory.createTableColumn(unnestTable,
19011                                                        table.getAliasClause().getAliasName(), true);
19012                                        columnsInExpr visitor = new columnsInExpr();
19013                                        value.inOrderTraverse(visitor);
19014                                        List<TParseTreeNode> functions = visitor.getFunctions();
19015                                        analyzeFunctionDataFlowRelation(tableColumn, functions, EffectType.select, null);
19016                                } else {
19017                                        TResultColumnList resultColumnList = stmt.getResultColumnList();
19018                                        for (int i = 0; i < resultColumnList.size(); i++) {
19019                                                TObjectName firstColumn = new TObjectName();
19020                                                firstColumn.setString(resultColumnList.getResultColumn(0).getColumnNameOnly());
19021                                                TableColumn tableColumn = modelFactory.createTableColumn(unnestTable, firstColumn, true);
19022                                                columnsInExpr visitor = new columnsInExpr();
19023                                                value.inOrderTraverse(visitor);
19024                                                List<TObjectName> columns = visitor.getObjectNames();
19025                                                analyzeDataFlowRelation(tableColumn, columns, EffectType.select, null);
19026                                        }
19027                                }
19028                        } else if (value.getExpressionType() == EExpressionType.simple_constant_t) {
19029                                if (table.getAliasClause() != null) {
19030                                        TableColumn tableColumn = modelFactory.createTableColumn(unnestTable,
19031                                                        table.getAliasClause().getAliasName(), true);
19032                                        columnsInExpr visitor = new columnsInExpr();
19033                                        value.inOrderTraverse(visitor);
19034                                        List<TParseTreeNode> constants = visitor.getConstants();
19035                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select, null);
19036                                } else {
19037                                        TObjectName firstColumn = new TObjectName();
19038                                        firstColumn.setString("f0_");
19039                                        TableColumn tableColumn = modelFactory.createTableColumn(unnestTable, firstColumn, true);
19040                                        columnsInExpr visitor = new columnsInExpr();
19041                                        value.inOrderTraverse(visitor);
19042                                        List<TParseTreeNode> constants = visitor.getConstants();
19043                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select, null);
19044                                }
19045                        } else if (value.getExpressionType() == EExpressionType.list_t) {
19046                                if (arrayExpr.getTypeName() != null && arrayExpr.getTypeName().getColumnDefList() != null) {
19047                                        for (int i = 0; i < arrayExpr.getTypeName().getColumnDefList().size(); i++) {
19048                                                TColumnDefinition column = arrayExpr.getTypeName().getColumnDefList().getColumn(i);
19049                                                if (column != null && column.getColumnName() != null) {
19050                                                        TableColumn tableColumn = modelFactory.createTableColumn(unnestTable,
19051                                                                        column.getColumnName(), true);
19052                                                        columnsInExpr visitor = new columnsInExpr();
19053                                                        value.getExprList().getExpression(i).inOrderTraverse(visitor);
19054                                                        List<TParseTreeNode> constants = visitor.getConstants();
19055                                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select, null);
19056                                                }
19057                                        }
19058                                } else {
19059                                        for (int i = 0; i < value.getExprList().size(); i++) {
19060                                                TObjectName firstColumn = new TObjectName();
19061                                                firstColumn.setString("f" + i + "_");
19062                                                TableColumn tableColumn = modelFactory.createTableColumn(unnestTable, firstColumn, true);
19063                                                columnsInExpr visitor = new columnsInExpr();
19064                                                value.getExprList().getExpression(i).inOrderTraverse(visitor);
19065                                                List<TParseTreeNode> constants = visitor.getConstants();
19066                                                analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select, null);
19067                                        }
19068                                }
19069                        } else if (value.getExpressionType() == EExpressionType.subquery_t) {
19070                                analyzeSelectStmt(value.getSubQuery());
19071                                ResultSet resultSet = (ResultSet) modelManager.getModel(value.getSubQuery());
19072                                if (resultSet != null) {
19073                                        for (int i = 0; i < resultSet.getColumns().size(); i++) {
19074                                                TObjectName columnName =  new TObjectName();
19075                                                if(resultSet.getColumns().get(i).getAlias()!=null) {
19076                                                        columnName.setString(resultSet.getColumns().get(i).getAlias());
19077                                                }
19078                                                else {
19079                                                        columnName.setString(getColumnName(resultSet.getColumns().get(i).getName()));
19080                                                }
19081                                                TableColumn tableColumn = modelFactory.createTableColumn(unnestTable, columnName, true);
19082                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19083                                                relation.setEffectType(EffectType.select);
19084                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
19085                                                relation.addSource(
19086                                                                new ResultColumnRelationshipElement(resultSet.getColumns().get(i)));
19087                                        }
19088                                }
19089                        }
19090                }
19091        }
19092
19093        private void analyzePrestoUnnest(TSelectSqlStatement stmt, TTable table) {
19094                List<Table> tables = new ArrayList<Table>();
19095                TTable targetTable = stmt.getTables().getTable(0);
19096                Table stmtTable = modelFactory.createTable(targetTable);
19097                Object sourceTable = null;
19098                if (targetTable.getSubquery() != null) {
19099                        analyzeSelectStmt(targetTable.getSubquery());
19100                        if (targetTable.getSubquery().isCombinedQuery()) {
19101                                ResultSet sourceResultSet = (ResultSet) modelManager.getModel(targetTable.getSubquery());
19102                                sourceTable = sourceResultSet;
19103                        } else if (targetTable.getSubquery().getResultColumnList() != null) {
19104                                ResultSet sourceResultSet = (ResultSet) modelManager
19105                                                .getModel(targetTable.getSubquery().getResultColumnList());
19106                                sourceTable = sourceResultSet;
19107                        } else if (targetTable.getSubquery().getValueClause() != null) {
19108                                List<TResultColumnList> rowList = targetTable.getSubquery().getValueClause().getRows();
19109                                if (rowList != null && rowList.size() > 0) {
19110                                        Table valuesTable = modelFactory.createTableByName("Values-Table", true);
19111                                        int columnCount = rowList.get(0).size();
19112                                        for (int j = 1; j <= columnCount; j++) {
19113                                                TObjectName columnName = new TObjectName();
19114                                                TResultColumn columnObject = rowList.get(0).getResultColumn(j - 1);
19115                                                if (columnObject.getExpr().getExpressionType() == EExpressionType.typecast_t) {
19116                                                        columnName.setString(columnObject.getExpr().getLeftOperand().toString());
19117                                                } else {
19118                                                        columnName.setString(columnObject.getExpr().toString());
19119                                                }
19120                                                modelFactory.createTableColumn(valuesTable, columnName, true);
19121                                        }
19122                                        valuesTable.setCreateTable(true);
19123                                        valuesTable.setSubType(SubType.values_table);
19124                                        sourceTable = valuesTable;
19125                                }
19126                        }
19127                }
19128                tables.add(stmtTable);
19129                Table unnestTable = modelFactory.createTable(table);
19130                unnestTable.setSubType(SubType.unnest);
19131                tables.add(unnestTable);
19132                if (table.getAliasClause() != null && table.getAliasClause().getColumns() != null
19133                                && table.getUnnestClause().getColumns() != null) {
19134                        int unnestTableSize = table.getUnnestClause().getColumns().size();
19135                        for (int i = 0; i < table.getAliasClause().getColumns().size(); i++) {
19136                                TableColumn sourceColumn = null;
19137                                if (unnestTableSize > i) {
19138                                        sourceColumn = modelFactory.createTableColumn(stmtTable,
19139                                                        table.getUnnestClause().getColumns().getObjectName(i), true);
19140                                        if (sourceTable instanceof Table) {
19141                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19142                                                relation.setEffectType(EffectType.select);
19143                                                relation.setTarget(new TableColumnRelationshipElement(sourceColumn));
19144                                                relation.addSource(
19145                                                                new TableColumnRelationshipElement(((Table) sourceTable).getColumns().get(i)));
19146                                        } else if (sourceTable instanceof ResultSet) {
19147                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19148                                                relation.setEffectType(EffectType.select);
19149                                                relation.setTarget(new TableColumnRelationshipElement(sourceColumn));
19150                                                relation.addSource(
19151                                                                new ResultColumnRelationshipElement(((ResultSet) sourceTable).getColumns().get(i)));
19152                                        }
19153                                } else {
19154                                        sourceColumn = modelFactory.createTableColumn(stmtTable,
19155                                                        table.getUnnestClause().getColumns().getObjectName(unnestTableSize - 1), true);
19156                                }
19157                                TableColumn targetColumn = modelFactory.createTableColumn(unnestTable,
19158                                                table.getAliasClause().getColumns().getObjectName(i), true);
19159                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19160                                relation.setEffectType(EffectType.select);
19161                                relation.setTarget(new TableColumnRelationshipElement(targetColumn));
19162                                relation.addSource(new TableColumnRelationshipElement(sourceColumn));
19163                        }
19164                }
19165
19166                ResultSet resultSet = modelFactory.createResultSet(stmt,
19167                                isTopResultSet(stmt) && isShowTopSelectResultSet());
19168                TResultColumnList columnList = stmt.getResultColumnList();
19169                for (int i = 0; i < columnList.size(); i++) {
19170                        TResultColumn column = columnList.getResultColumn(i);
19171                        ResultColumn resultColumn = modelFactory.createSelectSetResultColumn(resultSet, column, i);
19172                        if (resultColumn.getColumnObject() instanceof TResultColumn) {
19173                                TResultColumn columnObject = (TResultColumn) resultColumn.getColumnObject();
19174                                if (columnObject.getFieldAttr() != null) {
19175                                        if ("*".equals(getColumnName(columnObject.getFieldAttr()))) {
19176                                                for (int k = 0; k < tables.size(); k++) {
19177                                                        Table tableItem = tables.get(k);
19178                                                        for (TableColumn tableColumn : tableItem.getColumns()) {
19179                                                                resultColumn.bindStarLinkColumn(tableColumn.getColumnObject());
19180                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19181                                                                relation.setEffectType(EffectType.select);
19182                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn, tableColumn.getColumnObject()));
19183                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
19184                                                        }
19185                                                }
19186                                        } else {
19187                                                boolean match = false;
19188                                                for (int k = 0; k < tables.size(); k++) {
19189                                                        Table tableItem = tables.get(k);
19190                                                        for (TableColumn tableColumn : tableItem.getColumns()) {
19191                                                                if (getColumnName(columnObject.getFieldAttr())
19192                                                                                .equals(DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
19193                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19194                                                                        relation.setEffectType(EffectType.select);
19195                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
19196                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
19197                                                                        match = true;
19198                                                                }
19199                                                        }
19200                                                }
19201                                                if (!match) {
19202                                                        TableColumn tableColumn = modelFactory.createTableColumn(stmtTable,
19203                                                                        columnObject.getFieldAttr(), false);
19204                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19205                                                        relation.setEffectType(EffectType.select);
19206                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
19207                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
19208                                                }
19209                                        }
19210                                }
19211                        }
19212                }
19213        }
19214
19215        private void analyzeLateralView(TSelectSqlStatement stmt, TTable table, ArrayList<TLateralView> lateralViews) {
19216                List<Object> tables = new ArrayList<Object>();
19217                Object stmtTable = null;
19218                if(table.getSubquery()!=null) {
19219                        stmtTable = modelFactory.createQueryTable(table);
19220                        tables.add(stmtTable);
19221                }
19222                else {
19223                        stmtTable = modelFactory.createTable(table);
19224                        tables.add(stmtTable);
19225                }
19226                for (int i = 0; i < lateralViews.size(); i++) {
19227                        TLateralView lateralView = lateralViews.get(i);
19228                        TFunctionCall functionCall = lateralView.getUdtf();
19229                        List<TExpression> expressions = new ArrayList<TExpression>();
19230                        if (functionCall == null) {
19231                                continue;
19232                        }
19233                        Function function = modelFactory.createFunction(functionCall);
19234                        ResultColumn column = modelFactory.createFunctionResultColumn(function,
19235                                        ((TFunctionCall) functionCall).getFunctionName());
19236
19237                        Table lateralTable = null;
19238                        if (lateralView.getTableAlias() != null) {
19239                                lateralTable = modelFactory.createTableByName(lateralView.getTableAlias().getAliasName(), true);
19240                        } else {
19241                                lateralTable = modelFactory.createTableByName(functionCall.toString(), true);
19242                        }
19243
19244                        for (int j = 0; j < lateralView.getColumnAliasList().size(); j++) {
19245                                TObjectName viewColumn = lateralView.getColumnAliasList().getObjectName(j);
19246                                TableColumn tableColumn = modelFactory.createTableColumn(lateralTable, viewColumn, true);
19247
19248                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19249                                relation.setEffectType(EffectType.select);
19250                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
19251                                relation.addSource(new ResultColumnRelationshipElement(column));
19252                        }
19253
19254                        getFunctionExpressions(expressions, new ArrayList<TExpression>(), functionCall);
19255                        for (int j = 0; j < expressions.size(); j++) {
19256                                columnsInExpr visitor = new columnsInExpr();
19257                                expressions.get(j).inOrderTraverse(visitor);
19258                                List<TObjectName> objectNames = visitor.getObjectNames();
19259                                if (objectNames == null) {
19260                                        continue;
19261                                }
19262                                for (TObjectName columnName : objectNames) {
19263                                        boolean match = false;
19264                                        for (int k = 0; k < tables.size(); k++) {
19265                                                Object item = tables.get(k);
19266                                                if(item instanceof Table) {
19267                                                        Table tableItem = (Table)item;
19268                                                        for (TableColumn tableColumn : tableItem.getColumns()) {
19269                                                                if (getColumnName(columnName)
19270                                                                                .equals(DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
19271                                                                        match = true;
19272                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19273                                                                        relation.setEffectType(EffectType.select);
19274                                                                        relation.setTarget(new ResultColumnRelationshipElement(column));
19275                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
19276                                                                }
19277                                                        }
19278                                                }
19279                                                else {
19280                                                        ResultSet tableItem = (ResultSet)item;
19281                                                        for (ResultColumn tableColumn : tableItem.getColumns()) {
19282                                                                if (getColumnName(columnName)
19283                                                                                .equals(DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
19284                                                                        match = true;
19285                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19286                                                                        relation.setEffectType(EffectType.select);
19287                                                                        relation.setTarget(new ResultColumnRelationshipElement(column));
19288                                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn));
19289                                                                }
19290                                                        }
19291                                                }
19292                                        }
19293                                        if (!match) {
19294                                                if(stmtTable instanceof Table) {
19295                                                        TableColumn tableColumn = modelFactory.createTableColumn((Table)stmtTable, columnName, false);
19296                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19297                                                        relation.setEffectType(EffectType.select);
19298                                                        relation.setTarget(new ResultColumnRelationshipElement(column));
19299                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
19300                                                }
19301                                                else {
19302                                                        ResultColumn tableColumn = modelFactory.createResultColumn((ResultSet)stmtTable, columnName, false);
19303                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19304                                                        relation.setEffectType(EffectType.select);
19305                                                        relation.setTarget(new ResultColumnRelationshipElement(column));
19306                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn));
19307                                                }
19308                                        }
19309                                }
19310                                List<TParseTreeNode> constants = visitor.getConstants();
19311                                if (!constants.isEmpty()) {
19312                                        if (option.isShowConstantTable()) {
19313                                                Table constantTable = modelFactory.createConstantsTable(stmtStack.peek());
19314                                                for (TParseTreeNode constant : constants) {
19315                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19316                                                        relation.setEffectType(EffectType.select);
19317                                                        relation.setTarget(new ResultColumnRelationshipElement(column));
19318                                                        if (constant instanceof TConstant) {
19319                                                                TableColumn constantColumn = modelFactory.createTableColumn(constantTable,
19320                                                                                (TConstant) constant);
19321                                                                relation.addSource(new ConstantRelationshipElement(constantColumn));
19322                                                        } else if (constant instanceof TObjectName) {
19323                                                                TableColumn constantColumn = modelFactory.createTableColumn(constantTable,
19324                                                                                (TObjectName) constant, false);
19325                                                                relation.addSource(new ConstantRelationshipElement(constantColumn));
19326                                                        }
19327                                                }
19328                                        }
19329                                }
19330                                
19331                                List<TParseTreeNode> functions = visitor.getFunctions();
19332                                if (functions != null && !functions.isEmpty()) {
19333                                        analyzeFunctionDataFlowRelation(column, functions, EffectType.function);
19334                                }
19335                        }
19336                        tables.add(lateralTable);
19337                }
19338
19339                ResultSet resultSet = modelFactory.createResultSet(stmt,
19340                                isTopResultSet(stmt) && isShowTopSelectResultSet());
19341                TResultColumnList columnList = stmt.getResultColumnList();
19342                for (int i = 0; i < columnList.size(); i++) {
19343                        TResultColumn column = columnList.getResultColumn(i);
19344                        ResultColumn resultColumn = modelFactory.createSelectSetResultColumn(resultSet, column, i);
19345                        if (resultColumn.getColumnObject() instanceof TResultColumn) {
19346                                TResultColumn columnObject = (TResultColumn) resultColumn.getColumnObject();
19347                                if (columnObject.getFieldAttr() != null) {
19348                                        if ("*".equals(getColumnName(columnObject.getFieldAttr()))) {
19349                                                for (int k = 0; k < tables.size(); k++) {
19350                                                        Object item = tables.get(k);
19351                                                        if(item instanceof Table) {
19352                                                                Table tableItem = (Table)item;
19353                                                                for (TableColumn tableColumn : tableItem.getColumns()) {
19354                                                                        resultColumn.bindStarLinkColumn(tableColumn.getColumnObject());
19355                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19356                                                                        relation.setEffectType(EffectType.select);
19357                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn, tableColumn.getColumnObject()));
19358                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
19359                                                                }
19360                                                        }
19361                                                        else {
19362                                                                ResultSet tableItem = (ResultSet)item;
19363                                                                for (ResultColumn tableColumn : tableItem.getColumns()) {
19364                                                                        TObjectName linkColumn = new TObjectName();
19365                                                                        linkColumn.setString(tableColumn.getName());
19366                                                                        resultColumn.bindStarLinkColumn(linkColumn);
19367                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19368                                                                        relation.setEffectType(EffectType.select);
19369                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn, linkColumn));
19370                                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn));
19371                                                                }
19372                                                        }
19373                                                }
19374                                        } else {
19375                                                boolean match = false;
19376                                                for (int k = 0; k < tables.size(); k++) {
19377                                                        Object item = tables.get(k);
19378                                                        if(item instanceof Table) {
19379                                                                Table tableItem = (Table)item;
19380                                                                for (TableColumn tableColumn : tableItem.getColumns()) {
19381                                                                        if (getColumnName(columnObject.getFieldAttr())
19382                                                                                        .equals(DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
19383                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19384                                                                                relation.setEffectType(EffectType.select);
19385                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
19386                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
19387                                                                                match = true;
19388                                                                        }
19389                                                                }
19390                                                        }
19391                                                        else {
19392                                                                ResultSet tableItem = (ResultSet)item;
19393                                                                for (ResultColumn tableColumn : tableItem.getColumns()) {
19394                                                                        if (getColumnName(columnObject.getFieldAttr())
19395                                                                                        .equals(DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
19396                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19397                                                                                relation.setEffectType(EffectType.select);
19398                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
19399                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
19400                                                                                match = true;
19401                                                                        }
19402                                                                }
19403                                                        }
19404                                                }
19405                                                if (!match) {
19406                                                        if(stmtTable instanceof Table) {
19407                                                                TableColumn tableColumn = modelFactory.createTableColumn((Table)stmtTable,
19408                                                                                columnObject.getFieldAttr(), false);
19409                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19410                                                                relation.setEffectType(EffectType.select);
19411                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
19412                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
19413                                                        }
19414                                                        else {
19415                                                                ResultColumn tableColumn = modelFactory.createResultColumn((ResultSet)stmtTable,
19416                                                                                columnObject.getFieldAttr(), false);
19417                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19418                                                                relation.setEffectType(EffectType.select);
19419                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
19420                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
19421                                                        }
19422                                                }
19423                                        }
19424                                }
19425                                else if (columnObject.getExpr() != null
19426                                                && columnObject.getExpr().getExpressionType() == EExpressionType.function_t) {
19427                                        analyzeResultColumn(column, EffectType.select);
19428                                        for (int k = 0; k < tables.size(); k++) {
19429                                                Object item = tables.get(k);
19430                                                if (item instanceof Table) {
19431                                                        Table tableItem = (Table) item;
19432                                                        for (TableColumn tableColumn : tableItem.getColumns()) {
19433                                                                if (getColumnName(resultColumn.getName()).equals(
19434                                                                                DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
19435                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19436                                                                        relation.setEffectType(EffectType.select);
19437                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
19438                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
19439                                                                }
19440                                                        }
19441                                                } else {
19442                                                        ResultSet tableItem = (ResultSet) item;
19443                                                        for (ResultColumn tableColumn : tableItem.getColumns()) {
19444                                                                if (getColumnName(resultColumn.getName()).equals(
19445                                                                                DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
19446                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19447                                                                        relation.setEffectType(EffectType.select);
19448                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
19449                                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn));
19450                                                                }
19451                                                        }
19452                                                }
19453                                        }
19454                                }
19455                        }
19456                }
19457        }
19458
19459        private boolean isFromFunction(TObjectName object) {
19460
19461                Stack<TParseTreeNode> nodes = object.getStartToken().getNodesStartFromThisToken();
19462                if (nodes != null) {
19463                        for (int i = 0; i < nodes.size(); i++) {
19464                                if (nodes.get(i) instanceof TFunctionCall) {
19465                                        return true;
19466                                }
19467                        }
19468                }
19469                return false;
19470        }
19471
19472        private TResultColumnList getResultColumnList(TSelectSqlStatement stmt) {
19473                // Iterative DFS (left-first) to find the first non-combined query's result column list.
19474                // Avoids StackOverflow with deeply nested UNION trees.
19475                Deque<TSelectSqlStatement> stack = new ArrayDeque<>();
19476                stack.push(stmt);
19477                while (!stack.isEmpty()) {
19478                        TSelectSqlStatement current = stack.pop();
19479                        if (current.isCombinedQuery()) {
19480                                // Push right first so left is processed first (stack is LIFO)
19481                                if (current.getRightStmt() != null) stack.push(current.getRightStmt());
19482                                if (current.getLeftStmt() != null) stack.push(current.getLeftStmt());
19483                        } else {
19484                                if (current.getResultColumnList() != null) {
19485                                        return current.getResultColumnList();
19486                                }
19487                        }
19488                }
19489                return null;
19490        }
19491
19492        private void createPseudoImpactRelation(TCustomSqlStatement stmt, ResultSet resultSetModel, EffectType effectType) {
19493                if (stmt.getTables() != null) {
19494                        for (int i = 0; i < stmt.getTables().size(); i++) {
19495                                TTable table = stmt.getTables().getTable(i);
19496                                if (modelManager.getModel(table) instanceof ResultSet) {
19497                                        ResultSet tableModel = (ResultSet) modelManager.getModel(table);
19498                                        if (tableModel != resultSetModel && !tableModel.getRelationRows().getHoldRelations().isEmpty()) {
19499                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
19500                                                impactRelation.setEffectType(effectType);
19501                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
19502                                                                tableModel.getRelationRows()));
19503                                                impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
19504                                                                resultSetModel.getRelationRows()));
19505                                        }
19506                                } else if (modelManager.getModel(table) instanceof Table) {
19507                                        Table tableModel = (Table) modelManager.getModel(table);
19508                                        if (!tableModel.getRelationRows().getHoldRelations().isEmpty()) {
19509                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
19510                                                impactRelation.setEffectType(effectType);
19511                                                impactRelation.addSource(
19512                                                                new RelationRowsRelationshipElement<TableRelationRows>(tableModel.getRelationRows()));
19513                                                impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
19514                                                                resultSetModel.getRelationRows()));
19515                                        }
19516                                }
19517                        }
19518                }
19519        }
19520
19521        private void analyzeFunctionDataFlowRelation(Object gspObject, List<TParseTreeNode> functions,
19522                        EffectType effectType) {
19523                for (int i = 0; i < functions.size(); i++) {
19524                        TParseTreeNode functionCall = functions.get(i);
19525                        if (functionCall instanceof TFunctionCall) {
19526                                String functionName = DlineageUtil.getIdentifierNormalTableName(
19527                                                DlineageUtil.getFunctionNameWithArgNum((TFunctionCall) functionCall));
19528                                Procedure procedure = modelManager.getProcedureByName(functionName);
19529                                if (procedure != null) {
19530                                        String procedureParent = getProcedureParentName(stmtStack.peek());
19531                                        if (procedureParent != null) {
19532                                                Procedure caller = modelManager
19533                                                                .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
19534                                                if (caller != null) {
19535                                                        CallRelationship callRelation = modelFactory.createCallRelation();
19536                                                        callRelation.setCallObject(functionCall);
19537                                                        callRelation.setTarget(new ProcedureRelationshipElement(caller));
19538                                                        callRelation.addSource(new ProcedureRelationshipElement(procedure));
19539                                                        if (isBuiltInFunctionName(((TFunctionCall)functionCall).getFunctionName())||isKeyword(((TFunctionCall)functionCall).getFunctionName())) {
19540                                                                callRelation.setBuiltIn(true);
19541                                                        }
19542                                                }
19543                                        }
19544                                        if (procedure.getProcedureObject() instanceof TCreateFunctionStmt) {
19545                                                TCreateFunctionStmt createFunction = (TCreateFunctionStmt)procedure.getProcedureObject();
19546                                                TTypeName dataType = createFunction.getReturnDataType();
19547                                                if (dataType!=null && dataType.getTypeOfList() != null && dataType.getTypeOfList().getColumnDefList() != null) {
19548                                                        Object modelObject = modelManager.getModel(gspObject);
19549                                                        if(modelObject instanceof ResultColumn) {
19550                                                                ResultColumn resultColumn = (ResultColumn)modelObject;
19551                                                                ResultSet resultSet = resultColumn.getResultSet();
19552                                                                for (int j = 0; j < dataType.getTypeOfList().getColumnDefList().size(); j++) {
19553                                                                        TObjectName columnName = new TObjectName();
19554                                                                        if( dataType.getDataType() == EDataType.array_t) {
19555//                                                                              columnName.setString(resultColumn.getName() + ".array."
19556//                                                                                              + dataType.getTypeOfList().getColumnDefList().getColumn(j)
19557//                                                                                                              .getColumnName().getColumnNameOnly());
19558                                                                                columnName.setString(resultColumn.getName() + "."
19559                                                                                                + dataType.getTypeOfList().getColumnDefList().getColumn(j)
19560                                                                                                .getColumnName().getColumnNameOnly());
19561                                                                        }
19562                                                                        else {
19563                                                                                columnName.setString(resultColumn.getName() + "."
19564                                                                                                + dataType.getTypeOfList().getColumnDefList().getColumn(j)
19565                                                                                                                .getColumnName().getColumnNameOnly());
19566                                                                        }
19567                                                                        ResultColumn sturctColumn = modelFactory.createResultColumn(resultSet, columnName,
19568                                                                                        true);
19569                                                                        sturctColumn.setStruct(true);
19570                                                                        Function sourceFunction = (Function)createFunction(functionCall);
19571                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19572                                                                        relation.setEffectType(effectType);
19573                                                                        relation.setTarget(new ResultColumnRelationshipElement(sturctColumn));
19574                                                                        if (sourceFunction.getColumns() != null && !sourceFunction.getColumns().isEmpty()) {
19575                                                                                for (ResultColumn column : sourceFunction.getColumns()) {
19576                                                                                        relation.addSource(new ResultColumnRelationshipElement(column));
19577                                                                                }
19578                                                                        }
19579                                                                }
19580                                                                resultSet.getColumns().remove(resultColumn);
19581                                                        }
19582                                                        return;
19583                                                }
19584                                        }
19585                                }
19586                        }
19587                        
19588                        if(gspObject instanceof TResultColumn) {
19589                                TResultColumn resultColumn = (TResultColumn)gspObject;
19590                                if(resultColumn.getAliasClause()!=null && resultColumn.getAliasClause().getColumns()!=null) {
19591                                        for(TObjectName columnName: resultColumn.getAliasClause().getColumns()) {
19592                                                analyzeFunctionDataFlowRelation(columnName, Arrays.asList(functionCall), effectType, null);
19593                                        }
19594                                        return;
19595                                }
19596                        }
19597                        analyzeFunctionDataFlowRelation(gspObject, Arrays.asList(functionCall), effectType, null);
19598                }
19599        }
19600
19601        private void analyzeFunctionDataFlowRelation(Object gspObject, List<TParseTreeNode> functions,
19602                        EffectType effectType, Process process) {
19603
19604                Object modelObject = modelManager.getModel(gspObject);
19605                if (modelObject == null) {
19606                        if (gspObject instanceof ResultColumn || gspObject instanceof TableColumn) {
19607                                modelObject = gspObject;
19608                        }
19609                }
19610
19611                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19612                relation.setEffectType(effectType);
19613                relation.setProcess(process);
19614
19615                if (modelObject instanceof ResultColumn) {
19616                        relation.setTarget(new ResultColumnRelationshipElement((ResultColumn) modelObject));
19617
19618                } else if (modelObject instanceof TableColumn) {
19619                        relation.setTarget(new TableColumnRelationshipElement((TableColumn) modelObject));
19620
19621                } else {
19622                        throw new UnsupportedOperationException();
19623                }
19624
19625                for (int i = 0; i < functions.size(); i++) {
19626                        TParseTreeNode functionCall = functions.get(i);
19627
19628                        if (functionCall instanceof TFunctionCall) {
19629                                TFunctionCall call = (TFunctionCall) functionCall;
19630                                Procedure callee = modelManager.getProcedureByName(
19631                                                DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(call)));
19632                                if (callee == null && procedureDDLMap.containsKey(DlineageUtil.getFunctionNameWithArgNum(call))) {
19633                                        analyzeCustomSqlStmt(procedureDDLMap.get(DlineageUtil.getFunctionNameWithArgNum(call)));
19634                                        callee = modelManager.getProcedureByName(
19635                                                        DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(call)));
19636                                }
19637
19638                                if (callee != null) {
19639                                        String procedureParent = getProcedureParentName(stmtStack.peek());
19640                                        if (procedureParent != null) {
19641                                                Procedure caller = modelManager
19642                                                                .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
19643                                                if (caller != null) {
19644                                                        CallRelationship callRelation = modelFactory.createCallRelation();
19645                                                        callRelation.setCallObject(functionCall);
19646                                                        callRelation.setTarget(new ProcedureRelationshipElement(caller));
19647                                                        callRelation.addSource(new ProcedureRelationshipElement(callee));
19648                                                        if (isBuiltInFunctionName(call.getFunctionName()) || isKeyword(call.getFunctionName())) {
19649                                                                callRelation.setBuiltIn(true);
19650                                                        }
19651                                                }
19652                                        }
19653                                        if (callee.getArguments() != null) {
19654                                                for (int j = 0; j < callee.getArguments().size(); j++) {
19655                                                        Argument argument = callee.getArguments().get(j);
19656                                                        Variable variable = modelFactory.createVariable(callee, argument.getName(), false);
19657                                                        if(variable!=null) {
19658                                                                if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
19659                                                                        Transform transform = new Transform();
19660                                                                        transform.setType(Transform.FUNCTION);
19661                                                                        transform.setCode(call);
19662                                                                        variable.getColumns().get(0).setTransform(transform);
19663                                                                }
19664                                                                Process callProcess = modelFactory.createProcess(call);
19665                                                                variable.addProcess(callProcess);
19666                                                                analyzeFunctionArgumentsDataFlowRelation(variable.getColumns().get(0), call, j, callProcess);
19667                                                        }
19668                                                }
19669                                        }
19670                                        Set<Object> functionTableModelObjs = modelManager.getFunctionTable(DlineageUtil
19671                                                        .getIdentifierNormalTableName(call.getFunctionName().toString()));
19672                                        if (functionTableModelObjs != null) {
19673                                                modelManager.bindModel(call, functionTableModelObjs.iterator().next());
19674                                                for (Object functionTableModelObj : functionTableModelObjs) {
19675                                                        if (functionTableModelObj instanceof ResultSet) {
19676                                                                ResultSet resultSet = (ResultSet) functionTableModelObj;
19677                                                                for (ResultColumn column : resultSet.getColumns()) {
19678                                                                        relation.addSource(new ResultColumnRelationshipElement(column));
19679                                                                }
19680                                                        }
19681                                                }
19682                                        }
19683                                        continue;
19684                                }
19685                        }
19686
19687
19688                        Object functionModel = createFunction(functionCall);
19689                        if (functionModel instanceof Function) {
19690                                Function sourceFunction = (Function)functionModel;
19691                                if (sourceFunction.getColumns() != null && !sourceFunction.getColumns().isEmpty()) {
19692                                        for (ResultColumn column : sourceFunction.getColumns()) {
19693                                                relation.addSource(new ResultColumnRelationshipElement(column));
19694                                        }
19695                                }
19696                                else if (functionCall instanceof TFunctionCall) {
19697                                        relation.addSource(new ResultColumnRelationshipElement((FunctionResultColumn) modelManager
19698                                                        .getModel(((TFunctionCall) functionCall).getFunctionName())));
19699                                } else if (functionCall instanceof TCaseExpression) {
19700                                        relation.addSource(new ResultColumnRelationshipElement((FunctionResultColumn) modelManager
19701                                                        .getModel(((TCaseExpression) functionCall).getWhenClauseItemList())));
19702                                }
19703                                
19704                                if (sourceFunction != null && !sourceFunction.getRelationRows().getHoldRelations().isEmpty()) {
19705                                        boolean find = false;
19706                                        if (modelObject instanceof ResultColumn) {
19707                                                ResultSetRelationRows targetRelationRows = ((ResultColumn) modelObject).getResultSet().getRelationRows();
19708                                                if(targetRelationRows.hasRelation()) {
19709                                                        for(Relationship relationship: targetRelationRows.getHoldRelations()) {
19710                                                                if(relationship.getSources().contains(sourceFunction.getRelationRows())) {
19711                                                                        find = true;
19712                                                                        break;
19713                                                                }
19714                                                        }
19715                                                }
19716                                        }
19717                                        else if (modelObject instanceof TableColumn) {
19718                                                TableRelationRows targetRelationRows = ((TableColumn) modelObject).getTable().getRelationRows();
19719                                                if(targetRelationRows.hasRelation()) {
19720                                                        for(Relationship relationship: targetRelationRows.getHoldRelations()) {
19721                                                                if(relationship.getSources().contains(sourceFunction.getRelationRows())) {
19722                                                                        find = true;
19723                                                                        break;
19724                                                                }
19725                                                        }
19726                                                }
19727                                        }
19728                                        
19729                                        if (!find) {
19730                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
19731                                                impactRelation.setEffectType(EffectType.select);
19732                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
19733                                                                sourceFunction.getRelationRows()));
19734                                                if (modelObject instanceof ResultColumn) {
19735                                                        impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
19736                                                                        ((ResultColumn) modelObject).getResultSet().getRelationRows()));
19737                                                } else if (modelObject instanceof TableColumn) {
19738                                                        impactRelation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(
19739                                                                        ((TableColumn) modelObject).getTable().getRelationRows()));
19740                                                }
19741                                        }
19742                                }
19743                        } else if (functionModel instanceof Table) {
19744                                TFunctionCall call = (TFunctionCall) functionCall;
19745                                String functionName = call.getFunctionName().toString();
19746                                boolean flag = false;
19747                                if (functionName.indexOf(".") != -1) {
19748                                        String columnName = functionName.substring(functionName.indexOf(".") + 1);
19749                                        for (TableColumn tableColumn : ((Table) functionModel).getColumns()) {
19750                                                if (getColumnName(tableColumn.getName()).equalsIgnoreCase(columnName)) {
19751                                                        TableColumnRelationshipElement element = new TableColumnRelationshipElement(tableColumn);
19752                                                        relation.addSource(element);
19753                                                        flag = true;
19754                                                        break;
19755                                                }
19756                                        }
19757                                }
19758
19759                                if (!flag) {
19760                                        TableColumn tableColumn = modelFactory.createTableColumn((Table) functionModel,
19761                                                        ((TFunctionCall) functionCall));
19762                                        TableColumnRelationshipElement element = new TableColumnRelationshipElement(tableColumn);
19763                                        relation.addSource(element);
19764                                }
19765                        }
19766                }
19767
19768        }
19769
19770        private void analyzeSubqueryDataFlowRelation(Object gspObject, List<TSelectSqlStatement> subquerys,
19771                        EffectType effectType) {
19772                analyzeSubqueryDataFlowRelation(gspObject, subquerys, effectType, null);
19773        }
19774
19775        private void analyzeSubqueryDataFlowRelation(Object gspObject, List<TSelectSqlStatement> subquerys,
19776                        EffectType effectType, Process process) {
19777
19778                Object modelObject = modelManager.getModel(gspObject);
19779                if (modelObject == null) {
19780                        if (gspObject instanceof ResultColumn || gspObject instanceof TableColumn) {
19781                                modelObject = gspObject;
19782                        }
19783                }
19784
19785                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19786                relation.setEffectType(effectType);
19787                relation.setProcess(process);
19788
19789                if (modelObject instanceof ResultColumn) {
19790                        relation.setTarget(new ResultColumnRelationshipElement((ResultColumn) modelObject));
19791
19792                } else if (modelObject instanceof TableColumn) {
19793                        relation.setTarget(new TableColumnRelationshipElement((TableColumn) modelObject));
19794
19795                } else {
19796                        throw new UnsupportedOperationException();
19797                }
19798
19799                for (int i = 0; i < subquerys.size(); i++) {
19800                        TSelectSqlStatement subquery = subquerys.get(i);
19801                        ResultSet resultSetModel = (ResultSet) modelManager.getModel(subquery);
19802                        if (resultSetModel != null && resultSetModel.getColumns() != null) {
19803                                for (ResultColumn column : resultSetModel.getColumns()) {
19804                                        relation.addSource(new ResultColumnRelationshipElement(column));
19805                                }
19806                        }
19807                        
19808                        if (resultSetModel != null && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
19809                                boolean find = false;
19810                                if (modelObject instanceof ResultColumn) {
19811                                        ResultSetRelationRows targetRelationRows = ((ResultColumn) modelObject).getResultSet().getRelationRows();
19812                                        if(targetRelationRows.hasRelation()) {
19813                                                for(Relationship relationship: targetRelationRows.getHoldRelations()) {
19814                                                        if(relationship.getSources().contains(resultSetModel.getRelationRows())) {
19815                                                                find = true;
19816                                                                break;
19817                                                        }
19818                                                }
19819                                        }
19820                                }
19821                                else if (modelObject instanceof TableColumn) {
19822                                        TableRelationRows targetRelationRows = ((TableColumn) modelObject).getTable().getRelationRows();
19823                                        if(targetRelationRows.hasRelation()) {
19824                                                for(Relationship relationship: targetRelationRows.getHoldRelations()) {
19825                                                        if(relationship.getSources().contains(resultSetModel.getRelationRows())) {
19826                                                                find = true;
19827                                                                break;
19828                                                        }
19829                                                }
19830                                        }
19831                                }
19832                                
19833                                if (!find) {
19834                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
19835                                        impactRelation.setEffectType(EffectType.select);
19836                                        impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
19837                                                        resultSetModel.getRelationRows()));
19838                                        if (modelObject instanceof ResultColumn) {
19839                                                impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
19840                                                                ((ResultColumn) modelObject).getResultSet().getRelationRows()));
19841                                        } else if (modelObject instanceof TableColumn) {
19842                                                impactRelation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(
19843                                                                ((TableColumn) modelObject).getTable().getRelationRows()));
19844                                        }
19845                                }
19846                        }
19847                }
19848
19849        }
19850
19851        private Object createFunction(TParseTreeNode functionCall) {
19852                if (functionCall instanceof TFunctionCall) {
19853                        TFunctionCall functionObj = (TFunctionCall) functionCall;
19854
19855                        // Generic structured-dataflow dispatch. When a vendor adapter (e.g.
19856                        // Spark from_json+explode) returns a descriptor, model the function
19857                        // as a structured generator: per-field result columns linked to
19858                        // exact structured source paths such as nodes[*].key. When no
19859                        // adapter matches, fall through to the existing function logic
19860                        // unchanged.
19861                        StructuredAdapterContext sctx = new StructuredAdapterContext(option.getVendor());
19862                        StructuredDataflowDescriptor sdescriptor =
19863                                        StructuredDataflowRegistry.defaultRegistry().describe(functionObj, sctx);
19864                        if (sdescriptor != null) {
19865                                Function structuredFn = createStructuredDataflowFunction(sdescriptor);
19866                                if (structuredFn != null) {
19867                                        return structuredFn;
19868                                }
19869                        }
19870
19871                        if (!isBuiltInFunctionName(functionObj.getFunctionName())) {
19872                                TCustomSqlStatement stmt = stmtStack.peek();
19873                                String procedureParent = getProcedureParentName(stmt);
19874                                if (procedureParent != null) {
19875                                        Procedure procedureCallee = modelManager.getProcedureByName(DlineageUtil
19876                                                        .getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionObj)));
19877                                        if (procedureCallee != null) {
19878                                                if (procedureParent != null) {
19879                                                        Procedure caller = modelManager
19880                                                                        .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
19881                                                        if (caller != null) {
19882                                                                CallRelationship callRelation = modelFactory.createCallRelation();
19883                                                                callRelation.setCallObject(functionCall);
19884                                                                callRelation.setTarget(new ProcedureRelationshipElement(caller));
19885                                                                callRelation.addSource(new ProcedureRelationshipElement(procedureCallee));
19886                                                                if(isBuiltInFunctionName(functionObj.getFunctionName()) || isKeyword(functionObj.getFunctionName())){
19887                                                                        callRelation.setBuiltIn(true);
19888                                                                }
19889                                                        }
19890                                                }
19891                                                if (procedureCallee.getArguments() != null) {
19892                                                        for (int j = 0; j < procedureCallee.getArguments().size(); j++) {
19893                                                                Argument argument = procedureCallee.getArguments().get(j);
19894                                                                Variable variable = modelFactory.createVariable(procedureCallee, argument.getName(), false);
19895                                                                if(variable!=null) {
19896                                                                        if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
19897                                                                                Transform transform = new Transform();
19898                                                                                transform.setType(Transform.FUNCTION);
19899                                                                                transform.setCode(functionObj);
19900                                                                                variable.getColumns().get(0).setTransform(transform);
19901                                                                        }
19902                                                                        Process process = modelFactory.createProcess(functionObj);
19903                                                                        variable.addProcess(process);
19904                                                                        analyzeFunctionArgumentsDataFlowRelation(variable.getColumns().get(0), functionObj, j, process);
19905                                                                }
19906                                                        }
19907                                                }
19908                                        } else {
19909                                                TFunctionCall call = (TFunctionCall)functionCall;
19910                                                String functionName = call.getFunctionName().toString();
19911                                                if (functionName.indexOf(".") != -1) {
19912                                                        Table functionTable = modelManager
19913                                                                        .getTableByName(functionName.substring(0, functionName.indexOf(".")));
19914                                                        if (functionTable != null) {
19915                                                                String columnName = functionName.substring(functionName.indexOf(".") + 1);
19916                                                                for (TableColumn tableColumn : functionTable.getColumns()) {
19917                                                                        if (getColumnName(tableColumn.getName()).equalsIgnoreCase(columnName)) {
19918                                                                                return functionTable;
19919                                                                        }
19920                                                                }
19921                                                        }
19922                                                }
19923                                                Function function = modelFactory.createFunction(call);
19924                                                if (procedureParent != null) {
19925                                                        Procedure caller = modelManager
19926                                                                        .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
19927                                                        if (caller != null) {
19928                                                                CallRelationship callRelation = modelFactory.createCallRelation();
19929                                                                callRelation.setCallObject(functionCall);
19930                                                                callRelation.setTarget(new ProcedureRelationshipElement(caller));
19931                                                                callRelation.addSource(new FunctionRelationshipElement(function));
19932                                                                if(isBuiltInFunctionName(functionObj.getFunctionName()) || isKeyword(functionObj.getFunctionName())){
19933                                                                        callRelation.setBuiltIn(true);
19934                                                                }
19935                                                        }
19936                                                }
19937                                        }
19938                                }
19939                        } else if (isConstantFunction(functionObj.getFunctionName())
19940                                        && (functionObj.getArgs() == null || functionObj.getArgs().size() == 0)) {
19941                                if (option.isShowConstantTable()) {
19942                                        Table constantTable = modelFactory.createConstantsTable(stmtStack.peek());
19943                                        modelFactory.createTableColumn(constantTable, functionObj);
19944                                        return constantTable;
19945                                } else {
19946                                        return null;
19947                                }
19948                        }  
19949                        
19950                        if (functionObj.getFunctionType() == EFunctionType.struct_t) {
19951                                Function function = modelFactory.createFunction((TFunctionCall) functionCall);
19952                                if(functionObj instanceof TTableFunction) {
19953                                        TTableFunction tableFunction = (TTableFunction) functionObj;
19954                                        if (tableFunction.getFieldValues() != null) {
19955                                                for (int i = 0; i < tableFunction.getFieldValues().size(); i++) {
19956                                                        TResultColumn resultColumn = tableFunction.getFieldValues().getResultColumn(i);
19957                                                        if (resultColumn.getAliasClause() != null) {
19958                                                                ResultColumn column = modelFactory.createFunctionResultColumn(function,
19959                                                                                resultColumn.getAliasClause().getAliasName());
19960                                                                columnsInExpr visitor = new columnsInExpr();
19961                                                                resultColumn.getExpr().inOrderTraverse(visitor);
19962                                                                List<TObjectName> objectNames = visitor.getObjectNames();
19963                                                                List<TParseTreeNode> functions = visitor.getFunctions();
19964                                                                if (functions != null && !functions.isEmpty()) {
19965                                                                        analyzeFunctionDataFlowRelation(column, functions, EffectType.function);
19966                                                                }
19967                                                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
19968                                                                if (subquerys != null && !subquerys.isEmpty()) {
19969                                                                        analyzeSubqueryDataFlowRelation(column, subquerys, EffectType.function);
19970                                                                }
19971                                                                analyzeDataFlowRelation(column, objectNames, EffectType.function, functions);
19972                                                                List<TParseTreeNode> constants = visitor.getConstants();
19973                                                                analyzeConstantDataFlowRelation(column, constants, EffectType.function, functions);
19974                                                        } else if (resultColumn.getFieldAttr() != null) {
19975                                                                ResultColumn column = modelFactory.createFunctionResultColumn(function,
19976                                                                                resultColumn.getFieldAttr());
19977                                                                analyzeDataFlowRelation(column, Arrays.asList(resultColumn.getFieldAttr()), EffectType.function, null);
19978                                                        } else if (resultColumn.getExpr() != null) {
19979                                                                if (resultColumn.getExpr().getFunctionCall() != null) {
19980                                                                        Function resultColumnFunction = (Function) createFunction(
19981                                                                                        resultColumn.getExpr().getFunctionCall());
19982                                                                        String functionName = getResultSetName(function);
19983                                                                        for (int j = 0; j < resultColumnFunction.getColumns().size(); j++) {
19984                                                                                TObjectName columnName = new TObjectName();
19985                                                                                if (resultColumn.getAliasClause() != null) {
19986                                                                                        columnName.setString(resultColumn.getAliasClause() + "." + getColumnNameOnly(
19987                                                                                                        resultColumnFunction.getColumns().get(j).getName()));
19988                                                                                } else {
19989                                                                                        columnName.setString(functionName + "." + getColumnNameOnly(
19990                                                                                                        resultColumnFunction.getColumns().get(j).getName()));
19991                                                                                }
19992                                                                                ResultColumn functionResultColumn = modelFactory.createResultColumn(function,
19993                                                                                                columnName);
19994                                                                                DataFlowRelationship relationship = modelFactory.createDataFlowRelation();
19995                                                                                relationship.setTarget(new ResultColumnRelationshipElement(functionResultColumn));
19996                                                                                relationship.addSource(new ResultColumnRelationshipElement(
19997                                                                                                resultColumnFunction.getColumns().get(j)));
19998                                                                        }
19999                                                                }
20000                                                                else if (resultColumn.getExpr().getCaseExpression() != null) {
20001                                                                        function = modelFactory.createFunction(resultColumn.getExpr().getCaseExpression());
20002                                                                        ResultColumn column = modelFactory.createFunctionResultColumn(function,
20003                                                                                        ((TCaseExpression) resultColumn.getExpr().getCaseExpression()).getWhenClauseItemList());
20004                                                                        analyzeFunctionArgumentsDataFlowRelation(column, functionCall);
20005                                                                }
20006                                                        }
20007                                                }
20008                                                return function;
20009                                        }
20010                                }
20011                                else if(functionObj instanceof TFunctionCall) {
20012                                        
20013                                }
20014                        }
20015                        
20016                        // BigQuery array_agg(table_alias) row-reference expansion
20017                        if (functionObj.getFunctionType() == EFunctionType.array_agg_t
20018                                        && option.getVendor() == EDbVendor.dbvbigquery
20019                                        && functionObj.getArgs() != null
20020                                        && functionObj.getArgs().size() == 1) {
20021
20022                                TExpression arg0 = functionObj.getArgs().getExpression(0);
20023                                if (arg0 != null
20024                                                && arg0.getExpressionType() == EExpressionType.simple_object_name_t) {
20025
20026                                        TObjectName on = arg0.getObjectOperand();
20027                                        if (isArrayAggRowReference(on)) {
20028                                                Function function = (Function) modelManager.getModel(functionCall);
20029                                                if (function == null) {
20030                                                        function = modelFactory.createFunction((TFunctionCall) functionCall);
20031                                                }
20032
20033                                                // Only bind once (avoid duplicate processing when
20034                                                // createFunction is called from analyzeFunctionDataFlowRelation)
20035                                                if (function.getColumns() == null || function.getColumns().isEmpty()) {
20036                                                        TTable srcTable = on.getSourceTable();
20037                                                        String tableAlias = srcTable.getAliasName() != null
20038                                                                        ? srcTable.getAliasName().toString() : null;
20039                                                        
20040                                                        Table sourceTable = (Table) modelManager.getModel(srcTable);
20041                                                        sourceTable.removeColumn(tableAlias);
20042
20043                                                        // Collect inferred column names from ORDER BY (in function)
20044                                                        // and GROUP BY (in enclosing SELECT). Excludes the table alias.
20045                                                        List<TObjectName> inferredColumns = collectRowReferenceInferredColumns(
20046                                                                        functionObj, tableAlias);
20047
20048                                                        // Create a single * function result column carrying star-link
20049                                                        // metadata for the inferred columns and direct source edges
20050                                                        // for * and each inferred column.
20051                                                        createRowReferenceStarColumn(function, srcTable, inferredColumns);
20052                                                }
20053
20054                                                return function;
20055                                        }
20056                                }
20057                        }
20058
20059                        if (functionObj.getFunctionType() == EFunctionType.array_t || functionObj.getFunctionType() == EFunctionType.array_agg_t) {
20060                                Function function = modelFactory.createFunction((TFunctionCall) functionCall);
20061                                if(functionObj.getArgs()!=null) {
20062                                        if(functionObj.getArgs().getExpression(0).getSubQuery()!=null) {
20063                                                TSelectSqlStatement stmt = functionObj.getArgs().getExpression(0).getSubQuery();
20064                                                analyzeSelectStmt(stmt);
20065                                                ResultSet resultset = (ResultSet) modelManager.getModel(stmt);
20066                                                for (int i = 0; i < resultset.getColumns().size(); i++) {
20067                                                        ResultColumn sourceColumn = resultset.getColumns().get(i);
20068                                                        TObjectName columnName = new TObjectName();
20069                                                        columnName.setString(sourceColumn.getName());
20070                                                        ResultColumn resultColumn = modelFactory.createFunctionResultColumn(function,
20071                                                                        columnName);
20072                                                        DataFlowRelationship relationship = modelFactory.createDataFlowRelation();
20073                                                        relationship.setTarget(new ResultColumnRelationshipElement(resultColumn));
20074                                                        relationship.addSource(
20075                                                                        new ResultColumnRelationshipElement(sourceColumn));
20076                                                }
20077                                                return function;
20078                                        }
20079                                        else if (functionObj.getArgs().getExpression(0).getExpressionType() == EExpressionType.function_t) {
20080                                                Object functionTableModelObj = createFunction(functionObj.getArgs().getExpression(0).getFunctionCall());
20081                                                if (functionTableModelObj instanceof ResultSet) {
20082                                                        ResultSet resultset = (ResultSet) functionTableModelObj;
20083                                                        for (int i = 0; i < resultset.getColumns().size(); i++) {
20084                                                                ResultColumn sourceColumn = resultset.getColumns().get(i);
20085                                                                TObjectName columnName = new TObjectName();
20086                                                                columnName.setString(sourceColumn.getName());
20087                                                                ResultColumn resultColumn = modelFactory.createFunctionResultColumn(function, columnName);
20088                                                                DataFlowRelationship relationship = modelFactory.createDataFlowRelation();
20089                                                                relationship.setTarget(new ResultColumnRelationshipElement(resultColumn));
20090                                                                relationship.addSource(new ResultColumnRelationshipElement(sourceColumn));
20091                                                        }
20092                                                        return function;
20093                                                }
20094                                        }
20095                                }
20096                        }
20097                        
20098                        Function function = modelFactory.createFunction((TFunctionCall) functionCall);
20099                        ResultColumn column = modelFactory.createFunctionResultColumn(function,
20100                                        ((TFunctionCall) functionCall).getFunctionName());
20101                        if ("COUNT".equalsIgnoreCase(((TFunctionCall) functionCall).getFunctionName().toString())) {
20102                                // @see https://e.gitee.com/gudusoft/issues/list?issue=I40NUP
20103                                // COUNT特殊处理，不和参数关联
20104                                if (option.isShowCountTableColumn()) {
20105                                        analyzeFunctionArgumentsDataFlowRelation(column, functionCall);
20106                                }
20107                        } else {
20108                                boolean isCustomFunction = analyzeCustomFunctionCall((TFunctionCall)functionCall);
20109//                              if(!isCustomFunction) 
20110                                {
20111                                        analyzeFunctionArgumentsDataFlowRelation(column, functionCall);
20112                                }
20113                                Set<Object> functionTableModelObjs = modelManager.getFunctionTable(getIdentifiedFunctionName(function));
20114                                if(functionTableModelObjs!=null) {
20115                                        for(Object functionTableModelObj: functionTableModelObjs) {
20116                                                if (functionTableModelObj instanceof ResultSet) {
20117                                                        ResultSet functionTableModel = (ResultSet) functionTableModelObj;
20118                                                        if (functionTableModel.getColumns() != null) {
20119                                                                for (int j = 0; j < functionTableModel.getColumns().size(); j++) {
20120                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
20121                                                                        relation.setEffectType(EffectType.select);
20122                                                                        relation.setTarget(new ResultColumnRelationshipElement(column));
20123                                                                        relation.addSource(new ResultColumnRelationshipElement(
20124                                                                                        functionTableModel.getColumns().get(j)));
20125                                                                }
20126                                                        }
20127                                                }
20128                                        }
20129                                }
20130                        }
20131                        return function;
20132                } else if (functionCall instanceof TCaseExpression) {
20133                        Function function = modelFactory.createFunction((TCaseExpression) functionCall);
20134                        ResultColumn column = modelFactory.createFunctionResultColumn(function,
20135                                        ((TCaseExpression) functionCall).getWhenClauseItemList());
20136                        analyzeFunctionArgumentsDataFlowRelation(column, functionCall);
20137                        return function;
20138                } else if (functionCall instanceof TObjectName) {
20139                        Function function = modelFactory.createFunction((TObjectName) functionCall);
20140                        TObjectName columnName = new TObjectName();
20141                        columnName.setString(function.getFunctionName());
20142                        ResultColumn column = modelFactory.createResultColumn(function,
20143                                        columnName);
20144                        analyzeFunctionArgumentsDataFlowRelation(column, functionCall);
20145                        return function;
20146                }
20147                return null;
20148        }
20149
20150        protected String getIdentifiedFunctionName(Function function) {
20151                return DlineageUtil.getIdentifierNormalFunctionName(function.getFunctionName());
20152        }
20153        
20154        private boolean isConstantFunction(TObjectName functionName) {
20155                boolean result = CONSTANT_BUILTIN_FUNCTIONS.contains(functionName.toString().toUpperCase());
20156                if (result) {
20157                        return true;
20158                }
20159                return false;
20160        }
20161
20162        private void analyzeFunctionArgumentsDataFlowRelation(Object resultColumn, TParseTreeNode gspObject) {
20163                List<TExpression> directExpressions = new ArrayList<TExpression>();
20164                List<TExpression> indirectExpressions = new ArrayList<TExpression>();
20165                List<TExpression> conditionExpressions = new ArrayList<TExpression>();
20166                if (gspObject instanceof TFunctionCall) {
20167                        TFunctionCall functionCall = (TFunctionCall) gspObject;
20168                        getFunctionExpressions(directExpressions, indirectExpressions, functionCall);
20169                } else if (gspObject instanceof TCaseExpression) {
20170                        TCaseExpression expr = (TCaseExpression) gspObject;
20171                        TExpression inputExpr = expr.getInput_expr();
20172                        if (inputExpr != null) {
20173                                if(option.isShowCaseWhenAsDirect()){
20174                                        directExpressions.add(inputExpr);
20175                                }
20176                                else {
20177                                        conditionExpressions.add(inputExpr);
20178                                }
20179                        }
20180                        TExpression defaultExpr = expr.getElse_expr();
20181                        if (defaultExpr != null) {
20182                                directExpressions.add(defaultExpr);
20183                        }
20184                        TWhenClauseItemList list = expr.getWhenClauseItemList();
20185                        for (int i = 0; i < list.size(); i++) {
20186                                TWhenClauseItem element = list.getWhenClauseItem(i);
20187                                if(option.isShowCaseWhenAsDirect()){
20188                                        directExpressions.add(element.getComparison_expr());
20189                                }
20190                                else {
20191                                        conditionExpressions.add(element.getComparison_expr());
20192                                }
20193                                directExpressions.add(element.getReturn_expr());
20194                        }
20195                }
20196
20197                for (int j = 0; j < directExpressions.size(); j++) {
20198                        columnsInExpr visitor = new columnsInExpr();
20199                        directExpressions.get(j).inOrderTraverse(visitor);
20200
20201                        List<TObjectName> objectNames = visitor.getObjectNames();
20202                        List<TParseTreeNode> functions = visitor.getFunctions();
20203
20204                        if (functions != null && !functions.isEmpty()) {
20205                                analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function);
20206                        }
20207
20208                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
20209                        if (subquerys != null && !subquerys.isEmpty()) {
20210                                analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.function);
20211                        }
20212
20213                        analyzeDataFlowRelation(resultColumn, objectNames, EffectType.function, functions);
20214
20215                        List<TParseTreeNode> constants = visitor.getConstants();
20216                        analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.function, functions);
20217                }
20218
20219                conditionExpressions.addAll(indirectExpressions);
20220                for (int j = 0; j < conditionExpressions.size(); j++) {
20221                        analyzeFilterCondition(resultColumn, conditionExpressions.get(j), null, null, EffectType.function);
20222                }
20223        }
20224        
20225        private void analyzeFunctionArgumentsDataFlowRelation(Object resultColumn, TCallStatement callStatment, int argumentIndex, Process process) {
20226                List<TExpression> directExpressions = new ArrayList<TExpression>();
20227                List<TExpression> indirectExpressions = new ArrayList<TExpression>();
20228                List<TExpression> conditionExpressions = new ArrayList<TExpression>();
20229
20230                getFunctionExpressions(directExpressions, indirectExpressions, callStatment, argumentIndex);
20231
20232                for (int j = 0; j < directExpressions.size(); j++) {
20233                        columnsInExpr visitor = new columnsInExpr();
20234                        directExpressions.get(j).inOrderTraverse(visitor);
20235
20236                        List<TObjectName> objectNames = visitor.getObjectNames();
20237                        List<TParseTreeNode> functions = visitor.getFunctions();
20238
20239                        if (functions != null && !functions.isEmpty()) {
20240                                analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function);
20241                        }
20242
20243                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
20244                        if (subquerys != null && !subquerys.isEmpty()) {
20245                                analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.function);
20246                        }
20247
20248                        DataFlowRelationship relation = analyzeDataFlowRelation(resultColumn, objectNames, EffectType.function, functions);
20249                        if (relation != null) {
20250                                relation.setProcess(process);
20251                        }
20252
20253                        List<TParseTreeNode> constants = visitor.getConstants();
20254                        analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.function, functions);
20255                }
20256
20257                conditionExpressions.addAll(indirectExpressions);
20258                for (int j = 0; j < conditionExpressions.size(); j++) {
20259                        analyzeFilterCondition(resultColumn, conditionExpressions.get(j), null, null, EffectType.function);
20260                }
20261        }
20262
20263        private void analyzeFunctionArgumentsDataFlowRelation(Object resultColumn, TFunctionCall functionCall, int argumentIndex, Process process) {
20264                List<TExpression> directExpressions = new ArrayList<TExpression>();
20265                List<TExpression> indirectExpressions = new ArrayList<TExpression>();
20266                List<TExpression> conditionExpressions = new ArrayList<TExpression>();
20267
20268                getFunctionExpressions(directExpressions, indirectExpressions, functionCall, argumentIndex);
20269
20270                for (int j = 0; j < directExpressions.size(); j++) {
20271                        columnsInExpr visitor = new columnsInExpr();
20272                        directExpressions.get(j).inOrderTraverse(visitor);
20273
20274                        List<TObjectName> objectNames = visitor.getObjectNames();
20275                        List<TParseTreeNode> functions = visitor.getFunctions();
20276
20277                        if (functions != null && !functions.isEmpty()) {
20278                                analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function);
20279                        }
20280
20281                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
20282                        if (subquerys != null && !subquerys.isEmpty()) {
20283                                analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.function);
20284                        }
20285
20286                        DataFlowRelationship relation = analyzeDataFlowRelation(resultColumn, objectNames, EffectType.function, functions);
20287                        if (relation != null) {
20288                                relation.setProcess(process);
20289                        }
20290
20291                        List<TParseTreeNode> constants = visitor.getConstants();
20292                        analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.function, functions);
20293                }
20294
20295                conditionExpressions.addAll(indirectExpressions);
20296                for (int j = 0; j < conditionExpressions.size(); j++) {
20297                        analyzeFilterCondition(resultColumn, conditionExpressions.get(j), null, null, EffectType.function);
20298                }
20299        }
20300
20301        private void analyzeFunctionArgumentsDataFlowRelation(Object resultColumn, TMssqlExecute functionCall, String argumentName, int argumentIndex, Process process) {
20302                List<TExpression> directExpressions = new ArrayList<TExpression>();
20303                List<TExpression> indirectExpressions = new ArrayList<TExpression>();
20304                List<TExpression> conditionExpressions = new ArrayList<TExpression>();
20305
20306                getFunctionExpressions(directExpressions, indirectExpressions, functionCall, argumentName, argumentIndex);
20307
20308                for (int j = 0; j < directExpressions.size(); j++) {
20309                        columnsInExpr visitor = new columnsInExpr();
20310                        directExpressions.get(j).inOrderTraverse(visitor);
20311
20312                        List<TObjectName> objectNames = visitor.getObjectNames();
20313                        List<TParseTreeNode> functions = visitor.getFunctions();
20314
20315                        if (functions != null && !functions.isEmpty()) {
20316                                analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function);
20317                        }
20318
20319                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
20320                        if (subquerys != null && !subquerys.isEmpty()) {
20321                                analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.function);
20322                        }
20323
20324                        DataFlowRelationship relation = analyzeDataFlowRelation(resultColumn, objectNames, EffectType.function, functions);
20325                        if (relation != null) {
20326                                relation.setProcess(process);
20327                        }
20328
20329                        List<TParseTreeNode> constants = visitor.getConstants();
20330                        analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.function, functions);
20331                }
20332
20333                conditionExpressions.addAll(indirectExpressions);
20334                for (int j = 0; j < conditionExpressions.size(); j++) {
20335                        analyzeFilterCondition(resultColumn, conditionExpressions.get(j), null, null, EffectType.function);
20336                }
20337        }
20338
20339
20340        private static void addResultColumnListExpressions(
20341                        TResultColumnList list, List<TExpression> target) {
20342                if (list == null) return;
20343                for (int k = 0; k < list.size(); k++) {
20344                        TExpression e = list.getResultColumn(k).getExpr();
20345                        if (e != null) target.add(e);
20346                }
20347        }
20348
20349        private static void addPassingClauseExpressions(
20350                        TFunctionCall fc, List<TExpression> target) {
20351                TResultColumnList list = null;
20352                if (fc.getXmlPassingClause() != null
20353                                && fc.getXmlPassingClause().getPassingList() != null) {
20354                        list = fc.getXmlPassingClause().getPassingList();
20355                } else if (fc.getPassingClause() != null
20356                                && fc.getPassingClause().getPassingList() != null) {
20357                        list = fc.getPassingClause().getPassingList();
20358                }
20359                addResultColumnListExpressions(list, target);
20360        }
20361
20362        private void collectOracleXmlFunctionExpressions(
20363                        TFunctionCall fc,
20364                        List<TExpression> direct,
20365                        List<TExpression> indirect) {
20366
20367                if (option.getVendor() != EDbVendor.dbvoracle) return;
20368
20369                // XMLELEMENT: value exprs + XMLATTRIBUTES exprs (direct)
20370                addResultColumnListExpressions(fc.getXMLElementValueExprList(), direct);
20371                if (fc.getXMLAttributesClause() != null) {
20372                        addResultColumnListExpressions(
20373                                        fc.getXMLAttributesClause().getValueExprList(), direct);
20374                }
20375
20376                // XMLFOREST (direct)
20377                addResultColumnListExpressions(fc.getXMLForestValueList(), direct);
20378
20379                // EXTRACT(XML) — AST-shape disambiguation (V3)
20380                // XML form populates getXMLType_Instance(); scalar EXTRACT(YEAR FROM d) does not.
20381                if (fc.getXMLType_Instance() != null) {
20382                        direct.add(fc.getXMLType_Instance());
20383                }
20384
20385                String name = fc.getFunctionName() == null
20386                                ? "" : fc.getFunctionName().toString().toUpperCase();
20387
20388                // XMLEXISTS indirect (filter) from PASSING expressions
20389                if ("XMLEXISTS".equals(name)) {
20390                        addPassingClauseExpressions(fc, indirect);
20391                }
20392
20393                // XMLAGG / SYS_XMLAGG ORDER BY → indirect
20394                if (("XMLAGG".equals(name) || "SYS_XMLAGG".equals(name))
20395                                && fc.getSortClause() != null) {
20396                        TOrderByItemList orderByList = fc.getSortClause().getItems();
20397                        if (orderByList != null) {
20398                                for (int k = 0; k < orderByList.size(); k++) {
20399                                        TExpression e = orderByList.getOrderByItem(k).getSortKey();
20400                                        if (e != null) indirect.add(e);
20401                                }
20402                        }
20403                }
20404
20405                // DELIBERATELY NOT HARVESTED (metadata, not data):
20406                //   fc.getTypeExpression()       — target type for XMLCAST
20407                //   fc.getXMLElementNameExpr()   — element tag identifier
20408                //   XPath / XQuery literal strings that appear as function args
20409                //   datatype / format / style tokens
20410        }
20411
20412        private void collectArrayAggOrderByExpressions(
20413                        TFunctionCall fc,
20414                        List<TExpression> direct,
20415                        List<TExpression> indirect) {
20416                if (fc.getFunctionType() != EFunctionType.array_agg_t) return;
20417
20418                TOrderByItemList items = null;
20419                if (fc.getSortClause() != null) items = fc.getSortClause().getItems();
20420                if (items == null || items.size() == 0) {
20421                        items = fc.getOrderByList();
20422                }
20423                if (items == null) return;
20424
20425                // Resolve source tables for ORDER BY columns that the parser
20426                // doesn't link (they're inside a function call, outside the
20427                // resolver's normal scope)
20428                TTable fallbackTable = null;
20429                if (!stmtStack.isEmpty() && stmtStack.peek() instanceof TSelectSqlStatement) {
20430                        TTableList tables = ((TSelectSqlStatement) stmtStack.peek()).tables;
20431                        if (tables != null && tables.size() > 0) {
20432                                fallbackTable = tables.getTable(0);
20433                        }
20434                }
20435
20436                for (int k = 0; k < items.size(); k++) {
20437                        TExpression e = items.getOrderByItem(k).getSortKey();
20438                        if (e != null) {
20439                                if (fallbackTable != null
20440                                                && e.getExpressionType() == EExpressionType.simple_object_name_t
20441                                                && e.getObjectOperand() != null
20442                                                && e.getObjectOperand().getSourceTable() == null) {
20443                                        e.getObjectOperand().setSourceTable(fallbackTable);
20444                                }
20445                                direct.add(e);
20446                        }
20447                }
20448        }
20449
20450        private boolean isArrayAggRowReference(TObjectName on) {
20451                if (on == null) return false;
20452                TTable src = on.getSourceTable();
20453                if (src == null) return false;
20454
20455                String nm = DlineageUtil.getColumnName(on);
20456                if (nm == null || nm.isEmpty()) return false;
20457
20458                // Check if identifier text matches the table alias or table name
20459                boolean matchesAlias =
20460                        (src.getAliasName() != null
20461                                && src.getAliasName().toString().equalsIgnoreCase(nm))
20462                        || (src.getTableName() != null
20463                                && src.getTableName().toString().equalsIgnoreCase(nm));
20464                return matchesAlias;
20465        }
20466
20467        /**
20468         * Collects column names inferred from a row-reference array_agg:
20469         *  - ORDER BY keys inside the function call
20470         *  - GROUP BY keys in the enclosing SELECT
20471         * Excludes the table alias itself (so `original` in `array_agg(original)`
20472         * does not leak into the column list).
20473         */
20474        private List<TObjectName> collectRowReferenceInferredColumns(
20475                        TFunctionCall functionCall, String tableAlias) {
20476                List<TObjectName> columns = new ArrayList<TObjectName>();
20477
20478                TOrderByItemList orderItems = null;
20479                if (functionCall.getSortClause() != null) {
20480                        orderItems = functionCall.getSortClause().getItems();
20481                }
20482                if (orderItems == null || orderItems.size() == 0) {
20483                        orderItems = functionCall.getOrderByList();
20484                }
20485                if (orderItems != null) {
20486                        for (int k = 0; k < orderItems.size(); k++) {
20487                                TExpression sortKey = orderItems.getOrderByItem(k).getSortKey();
20488                                collectInferredColumn(sortKey, tableAlias, columns);
20489                        }
20490                }
20491
20492                if (!stmtStack.isEmpty() && stmtStack.peek() instanceof TSelectSqlStatement) {
20493                        TSelectSqlStatement select = (TSelectSqlStatement) stmtStack.peek();
20494                        if (select.getGroupByClause() != null) {
20495                                TGroupByItemList groupByList = select.getGroupByClause().getItems();
20496                                for (int k = 0; k < groupByList.size(); k++) {
20497                                        TExpression expr = groupByList.getGroupByItem(k).getExpr();
20498                                        collectInferredColumn(expr, tableAlias, columns);
20499                                }
20500                        }
20501                }
20502                return columns;
20503        }
20504
20505        private void collectInferredColumn(TExpression expr, String tableAlias,
20506                        List<TObjectName> columns) {
20507                if (expr == null) return;
20508                if (expr.getExpressionType() != EExpressionType.simple_object_name_t) return;
20509                if (expr.getObjectOperand() == null) return;
20510                String colName = DlineageUtil.getColumnName(expr.getObjectOperand());
20511                if (colName == null || colName.isEmpty()) return;
20512                if (tableAlias != null && tableAlias.equalsIgnoreCase(colName)) return;
20513                if(!columns.contains(expr.getObjectOperand())) {
20514                        columns.add(expr.getObjectOperand());
20515                }
20516        }
20517
20518        /**
20519         * Creates a `*` FunctionResultColumn plus one FunctionResultColumn per
20520         * inferred column. Each column gets its own source-edge back to
20521         * the source table (source.* -> function.*, source.id -> function.id, etc).
20522         *
20523         * Downstream, the `*` column collects all three edges into the inner
20524         * `unique` ResultColumn, and post-processing (rewriteArrayAggRowReferenceTargets)
20525         * splits the collapsed CTAS relation into one per column.
20526         */
20527        private void createRowReferenceStarColumn(Function function, TTable src,
20528                        List<TObjectName> inferredColumns) {
20529                Object tableModel = modelManager.getModel(src);
20530
20531                // Create function.* and bind source.* -> function.*
20532                TObjectName starName = new TObjectName();
20533                starName.setString("*");
20534                ResultColumn starColumn = modelFactory.createFunctionResultColumn(function, starName);
20535                starColumn.setStruct(true);
20536                starColumn.setShowStar(true);
20537
20538                if (tableModel instanceof Table) {
20539                        Table sourceTable = (Table) tableModel;
20540                        TableColumn sourceCol = findOrCreateTableColumn(sourceTable, "*");
20541                        
20542                        for (TObjectName colName : inferredColumns) {
20543                                TableColumn tableColumn = modelFactory.createTableColumn(sourceTable, colName, false);
20544                                if (tableColumn != null) {
20545                                        //order by and group by fdr relation
20546                                        AbstractRelationship relation = modelFactory.createImpactRelation();
20547                                        relation.setEffectType(EffectType.function);
20548                                        relation.setTarget(new ResultColumnRelationshipElement(starColumn));
20549                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
20550                                        
20551                                        // star column fdd relation
20552                                        relation = modelFactory.createDataFlowRelation();
20553                                        relation.setEffectType(EffectType.function);
20554                                        relation.setTarget(new ResultColumnRelationshipElement(starColumn));
20555                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
20556                                }
20557                        }
20558
20559                        
20560                        if (sourceCol != null) {
20561                                DataFlowRelationship rel = modelFactory.createDataFlowRelation();
20562                                rel.setEffectType(EffectType.function);
20563                                rel.setTarget(new ResultColumnRelationshipElement(starColumn));
20564                                rel.addSource(new TableColumnRelationshipElement(sourceCol));
20565                        }
20566                        
20567                } else if (tableModel instanceof ResultSet) {
20568                        ResultSet sourceRS = (ResultSet) tableModel;
20569                        ResultColumn sourceCol = findOrCreateResultColumn(sourceRS, "*");
20570                        for (TObjectName colName : inferredColumns) {
20571                                sourceCol.bindStarLinkColumn(colName);
20572                        }
20573                        
20574                        if (sourceCol != null) {
20575                                DataFlowRelationship rel = modelFactory.createDataFlowRelation();
20576                                rel.setEffectType(EffectType.function);
20577                                rel.setTarget(new ResultColumnRelationshipElement(starColumn));
20578                                rel.addSource(new ResultColumnRelationshipElement(sourceCol));
20579                        }
20580                }
20581        }
20582
20583        /**
20584         * Builds the Function lineage model for a structured-dataflow descriptor.
20585         * Creates per-field FunctionResultColumns and links each to an exact
20586         * structured source TableColumn (e.g. {@code nodes[*].key}). Returns the
20587         * Function on success; returns {@code null} when the source column cannot
20588         * be resolved so the caller falls back to existing function lineage.
20589         */
20590        private Function createStructuredDataflowFunction(StructuredDataflowDescriptor desc) {
20591                TParseTreeNode syntaxNode = desc.getSyntaxNode();
20592                if (!(syntaxNode instanceof TFunctionCall)) {
20593                        return null;
20594                }
20595                TFunctionCall fnCall = (TFunctionCall) syntaxNode;
20596
20597                Function function = null;
20598                Object existing = modelManager.getModel(fnCall);
20599                if (existing instanceof Function) {
20600                        function = (Function) existing;
20601                }
20602                boolean alreadyBuilt = function != null
20603                                && function.getColumns() != null
20604                                && !function.getColumns().isEmpty();
20605                if (alreadyBuilt) {
20606                        return function;
20607                }
20608                if (function == null) {
20609                        function = modelFactory.createFunction(fnCall);
20610                }
20611
20612                StructuredValueSource src = desc.getSource();
20613                TObjectName sourceColumn = src.getSourceColumn();
20614                TTable srcTable = sourceColumn != null ? sourceColumn.getSourceTable() : null;
20615                Object srcTableModel = srcTable != null ? modelManager.getModel(srcTable) : null;
20616                if (!(srcTableModel instanceof Table)) {
20617                        return null;
20618                }
20619                Table sourceTable = (Table) srcTableModel;
20620
20621                for (StructuredFieldBinding fb : desc.getFieldBindings()) {
20622                        String pathDisplay = fb.getSourcePath().toDisplayString();
20623                        TableColumn pathColumn = findOrCreateStructuredPathColumn(sourceTable, pathDisplay);
20624                        if (pathColumn == null) {
20625                                continue;
20626                        }
20627
20628                        TObjectName fieldName = new TObjectName();
20629                        fieldName.setString(fb.getOutputFieldName());
20630                        ResultColumn fnResult = modelFactory.createFunctionResultColumn(function, fieldName);
20631
20632                        DataFlowRelationship rel = modelFactory.createDataFlowRelation();
20633                        rel.setEffectType(EffectType.function);
20634                        rel.setTarget(new ResultColumnRelationshipElement(fnResult));
20635                        rel.addSource(new TableColumnRelationshipElement(pathColumn));
20636                }
20637                return function;
20638        }
20639
20640        /**
20641         * Resolve a struct-field-style outer reference ({@code structAlias.field}
20642         * or {@code relAlias.structAlias.field}) against in-scope subquery
20643         * QueryTables that hold a struct-bearing result column. When found,
20644         * append a relation source pointing at the function's field result
20645         * column so the exact source path (e.g. {@code nodes[*].key}) is
20646         * preserved through the outer projection. Returns {@code true} when
20647         * the reference was bound and the caller should skip orphan-column
20648         * fallback.
20649         */
20650        private boolean tryAppendStructuredFieldRelation(DataFlowRelationship relation,
20651                        TObjectName columnName, TTableList tableList) {
20652                if (relation == null || columnName == null || tableList == null) {
20653                        return false;
20654                }
20655                String fullText = columnName.toString();
20656                if (fullText == null) return false;
20657                int firstDot = fullText.indexOf('.');
20658                if (firstDot < 0) return false;
20659
20660                String structAlias;
20661                String fieldName;
20662                int lastDot = fullText.lastIndexOf('.');
20663                if (firstDot == lastDot) {
20664                        structAlias = fullText.substring(0, firstDot);
20665                        fieldName = fullText.substring(firstDot + 1);
20666                } else {
20667                        structAlias = fullText.substring(firstDot + 1, lastDot);
20668                        fieldName = fullText.substring(lastDot + 1);
20669                }
20670                if (structAlias == null || structAlias.isEmpty()) return false;
20671                if (fieldName == null || fieldName.isEmpty()) return false;
20672
20673                for (int i = 0; i < tableList.size(); i++) {
20674                        TTable tt = tableList.getTable(i);
20675                        if (tt == null) continue;
20676                        Object tm = modelManager.getModel(tt);
20677                        if (!(tm instanceof QueryTable)) continue;
20678                        QueryTable qt = (QueryTable) tm;
20679                        if (qt.getColumns() == null) continue;
20680                        for (ResultColumn rc : qt.getColumns()) {
20681                                if (rc == null || rc.getName() == null) continue;
20682                                if (!structAlias.equalsIgnoreCase(rc.getName())) continue;
20683                                ResultColumn fieldColumn = findStructuredFieldColumn(rc, fieldName);
20684                                if (fieldColumn != null) {
20685                                        relation.addSource(new ResultColumnRelationshipElement(fieldColumn));
20686                                        return true;
20687                                }
20688                        }
20689                }
20690                return false;
20691        }
20692
20693        /**
20694         * Given a QueryTable result column that aliases a structured generator
20695         * (e.g. {@code nodes1} backed by Spark {@code explode(from_json(...))}),
20696         * locate the FunctionResultColumn named {@code fieldName}. The Function
20697         * model is discovered by walking the existing data-flow relations whose
20698         * target is the alias column.
20699         */
20700        private ResultColumn findStructuredFieldColumn(ResultColumn aliasColumn, String fieldName) {
20701                if (aliasColumn == null || fieldName == null) return null;
20702                Relationship[] rels = modelManager.getRelations();
20703                if (rels == null) return null;
20704                String normalField = fieldName.trim();
20705                for (Relationship r : rels) {
20706                        if (!(r instanceof DataFlowRelationship)) continue;
20707                        DataFlowRelationship dfr = (DataFlowRelationship) r;
20708                        if (dfr.getTarget() == null) continue;
20709                        if (!(dfr.getTarget().getElement() == aliasColumn)) continue;
20710                        if (dfr.getSources() == null) continue;
20711                        for (RelationshipElement<?> se : dfr.getSources()) {
20712                                Object srcElement = se.getElement();
20713                                if (!(srcElement instanceof ResultColumn)) continue;
20714                                ResultColumn src = (ResultColumn) srcElement;
20715                                if (normalField.equalsIgnoreCase(src.getName())) {
20716                                        return src;
20717                                }
20718                        }
20719                }
20720                return null;
20721        }
20722
20723        /**
20724         * Find-or-create a TableColumn carrying a structured display path
20725         * (e.g. {@code nodes[*].key}) on the given source table. Unlike the
20726         * standard {@link #findOrCreateTableColumn(Table, String)}, this does
20727         * not treat the dotted/bracketed text as a schema-qualified name;
20728         * the entire path is used as the column display, so existing flat
20729         * columns named after the leaf field (e.g. {@code key}) do not
20730         * collide with structured-path columns.
20731         */
20732        private TableColumn findOrCreateStructuredPathColumn(Table table, String pathDisplay) {
20733                if (table == null || pathDisplay == null || pathDisplay.isEmpty()) return null;
20734                for (TableColumn tc : table.getColumns()) {
20735                        if (pathDisplay.equalsIgnoreCase(tc.getName())) {
20736                                return tc;
20737                        }
20738                }
20739                return new TableColumn(table, pathDisplay);
20740        }
20741
20742        private TableColumn findOrCreateTableColumn(Table table, String colName) {
20743                String normalName = DlineageUtil.getIdentifierNormalColumnName(colName);
20744                for (TableColumn tc : table.getColumns()) {
20745                        if ("*".equals(colName) && "*".equals(tc.getName())) return tc;
20746                        if (DlineageUtil.compareColumnIdentifier(tc.getName(), normalName)) return tc;
20747                }
20748                TObjectName obj = new TObjectName();
20749                obj.setString(colName);
20750                return modelFactory.createTableColumn(table, obj, "*".equals(colName));
20751        }
20752
20753        private ResultColumn findOrCreateResultColumn(ResultSet resultSet, String colName) {
20754                String normalName = DlineageUtil.getIdentifierNormalColumnName(colName);
20755                for (ResultColumn rc : resultSet.getColumns()) {
20756                        if ("*".equals(colName) && "*".equals(rc.getName())) return rc;
20757                        if (DlineageUtil.compareColumnIdentifier(rc.getName(), normalName)) return rc;
20758                }
20759                TObjectName obj = new TObjectName();
20760                obj.setString(colName);
20761                return modelFactory.createResultColumn(resultSet, obj);
20762        }
20763
20764        private void getFunctionExpressions(List<TExpression> directExpressions, List<TExpression> indirectExpressions,
20765                        TFunctionCall functionCall) {
20766                if (functionCall.getArgs() != null) {
20767                        for (int k = 0; k < functionCall.getArgs().size(); k++) {
20768                                TExpression expr = functionCall.getArgs().getExpression(k);
20769                                if(FunctionUtility.isDirectRelation(option.getVendor(), functionCall.getFunctionName().toString(),  functionCall.getArgs().size(), k)) {
20770                                        directExpressions.add(expr);
20771                                }
20772                                if(FunctionUtility.isIndirectRelation(option.getVendor(), functionCall.getFunctionName().toString(),  functionCall.getArgs().size(), k)) {
20773                                        if("DECODE".equalsIgnoreCase(functionCall.getFunctionName().toString()) && option.getVendor() == EDbVendor.dbvoracle) {
20774                                                if(option.isShowCaseWhenAsDirect()) {
20775                                                        directExpressions.add(expr);
20776                                                        continue;
20777                                                }
20778                                        }
20779                                        indirectExpressions.add(expr);
20780                                }
20781                        }
20782                }
20783                collectOracleXmlFunctionExpressions(functionCall, directExpressions, indirectExpressions);
20784                collectArrayAggOrderByExpressions(functionCall, directExpressions, indirectExpressions);
20785                if (functionCall.getTrimArgument() != null) {
20786                        TTrimArgument args = functionCall.getTrimArgument();
20787                        TExpression expr = args.getStringExpression();
20788                        if (expr != null) {
20789                                directExpressions.add(expr);
20790                        }
20791                        expr = args.getTrimCharacter();
20792                        if (expr != null) {
20793                                directExpressions.add(expr);
20794                        }
20795                }
20796
20797                if (functionCall.getAgainstExpr() != null) {
20798                        directExpressions.add(functionCall.getAgainstExpr());
20799                }
20800//              if (functionCall.getBetweenExpr() != null) {
20801//                      directExpressions.add(functionCall.getBetweenExpr());
20802//              }
20803                if (functionCall.getExpr1() != null) {
20804                        directExpressions.add(functionCall.getExpr1());
20805                }
20806                if (functionCall.getExpr2() != null) {
20807                        directExpressions.add(functionCall.getExpr2());
20808                }
20809                if (functionCall.getExpr3() != null) {
20810                        directExpressions.add(functionCall.getExpr3());
20811                }
20812                if (functionCall.getParameter() != null) {
20813                        directExpressions.add(functionCall.getParameter());
20814                }
20815                if (functionCall.getWindowDef() != null && functionCall.getWindowDef().getPartitionClause() != null) {
20816                        TExpressionList args = functionCall.getWindowDef().getPartitionClause().getExpressionList();
20817                        if (args != null) {
20818                                for (int k = 0; k < args.size(); k++) {
20819                                        TExpression expr = args.getExpression(k);
20820                                        if (expr != null) {
20821                                                indirectExpressions.add(expr);
20822                                        }
20823                                }
20824                        }
20825                }
20826                if (functionCall.getWindowDef() != null && functionCall.getWindowDef().getOrderBy() != null) {
20827                        TOrderByItemList orderByList = functionCall.getWindowDef().getOrderBy().getItems();
20828                        for (int i = 0; i < orderByList.size(); i++) {
20829                                TOrderByItem element = orderByList.getOrderByItem(i);
20830                                TExpression expression = element.getSortKey();
20831                                indirectExpressions.add(expression);
20832                        }
20833                }
20834                if (functionCall.getWithinGroup() != null && functionCall.getWithinGroup().getOrderBy() != null) {
20835                        TOrderByItemList orderByList = functionCall.getWithinGroup().getOrderBy().getItems();
20836                        for (int i = 0; i < orderByList.size(); i++) {
20837                                TOrderByItem element = orderByList.getOrderByItem(i);
20838                                TExpression expression = element.getSortKey();
20839                                indirectExpressions.add(expression);
20840                        }
20841                }
20842                if (functionCall.getCallTarget() != null) {
20843                        directExpressions.add(functionCall.getCallTarget().getExpr());
20844                }
20845                if (functionCall.getFieldValues() != null) {
20846                        for (int k = 0; k < functionCall.getFieldValues().size(); k++) {
20847                                TExpression expr = functionCall.getFieldValues().getResultColumn(k).getExpr();
20848                                directExpressions.add(expr);
20849                        }
20850                }
20851                if (functionCall instanceof TJsonObjectFunction) {
20852                        TJsonObjectFunction jsonObject = (TJsonObjectFunction)functionCall;
20853                        for (int k = 0; k < jsonObject.getKeyValues().size(); k++) {
20854                                TExpression expr = jsonObject.getKeyValues().get(k).getValue();
20855                                directExpressions.add(expr);
20856                        }
20857                }
20858                if (functionCall.getGroupConcatParam() != null) {
20859                        for (int k = 0; k < functionCall.getGroupConcatParam().getExprList().size(); k++) {
20860                                TExpression expr = functionCall.getGroupConcatParam().getExprList().getExpression(k);
20861                                directExpressions.add(expr);
20862                        }
20863                }
20864        }
20865
20866        private void getFunctionExpressions(List<TExpression> directExpressions, List<TExpression> indirectExpressions,
20867                                                                                TFunctionCall functionCall, int argumentIndex) {
20868                if (functionCall.getArgs() != null && argumentIndex < functionCall.getArgs().size()) {
20869                        TExpression expr = functionCall.getArgs().getExpression(argumentIndex);
20870                        if (FunctionUtility.isDirectRelation(option.getVendor(), functionCall.getFunctionName().toString(), functionCall.getArgs().size(), argumentIndex)) {
20871                                directExpressions.add(expr);
20872                        }
20873                        if (FunctionUtility.isIndirectRelation(option.getVendor(), functionCall.getFunctionName().toString(), functionCall.getArgs().size(), argumentIndex)) {
20874                                indirectExpressions.add(expr);
20875                        }
20876                }
20877        }
20878        
20879        private void getFunctionExpressions(List<TExpression> directExpressions, List<TExpression> indirectExpressions,
20880                        TCallStatement functionCall, int argumentIndex) {
20881                if (functionCall.getArgs() != null && argumentIndex < functionCall.getArgs().size()) {
20882                        TExpression expr = functionCall.getArgs().getExpression(argumentIndex);
20883                        if (FunctionUtility.isDirectRelation(option.getVendor(), functionCall.getRoutineName().toString(),
20884                                        functionCall.getArgs().size(), argumentIndex)) {
20885                                directExpressions.add(expr);
20886                        }
20887                        if (FunctionUtility.isIndirectRelation(option.getVendor(), functionCall.getRoutineName().toString(),
20888                                        functionCall.getArgs().size(), argumentIndex)) {
20889                                indirectExpressions.add(expr);
20890                        }
20891                }
20892        }
20893
20894        private void getFunctionExpressions(List<TExpression> directExpressions, List<TExpression> indirectExpressions,
20895                                                                                TMssqlExecute functionCall, String argumentName, int argumentIndex) {
20896                if (functionCall.getParameters() != null) {
20897                        for (int i = 0; i < functionCall.getParameters().size(); i++) {
20898                                TExecParameter param = functionCall.getParameters().getExecParameter(i);
20899                                if (param.getParameterName() != null) {
20900                                        if (DlineageUtil.compareColumnIdentifier(param.getParameterName().toString(), argumentName)) {
20901                                                TExpression expr = param.getParameterValue();
20902                                                directExpressions.add(expr);
20903                                        }
20904                                } else if (i == argumentIndex) {
20905                                        TExpression expr = param.getParameterValue();
20906                                        directExpressions.add(expr);
20907                                }
20908                        }
20909                }
20910        }
20911
20912        private void analyzeJoin(TJoin join, EffectType effectType) {
20913                if (join.getJoinItems() != null) {
20914                        for (int j = 0; j < join.getJoinItems().size(); j++) {
20915                                TJoinItem joinItem = join.getJoinItems().getJoinItem(j);
20916                                TExpression expr = joinItem.getOnCondition();
20917                                if (expr != null) {
20918                                        analyzeFilterCondition(null, expr, joinItem.getJoinType(), JoinClauseType.on, effectType);
20919                                }
20920                        }
20921                }
20922
20923                if (join.getJoin() != null) {
20924                        analyzeJoin(join.getJoin(), effectType);
20925                }
20926        }
20927
20928        private TSelectSqlStatement getParentSetSelectStmt(TSelectSqlStatement stmt) {
20929                TCustomSqlStatement parent = stmt.getParentStmt();
20930                if (parent == null)
20931                        return null;
20932                if (parent.getStatements() != null) {
20933                        for (int i = 0; i < parent.getStatements().size(); i++) {
20934                                TCustomSqlStatement temp = parent.getStatements().get(i);
20935                                if (temp instanceof TSelectSqlStatement) {
20936                                        TSelectSqlStatement select = (TSelectSqlStatement) temp;
20937                                        if (select.getLeftStmt() == stmt || select.getRightStmt() == stmt)
20938                                                return select;
20939                                }
20940                        }
20941                }
20942                if (parent instanceof TSelectSqlStatement) {
20943                        TSelectSqlStatement select = (TSelectSqlStatement) parent;
20944                        if (select.getLeftStmt() == stmt || select.getRightStmt() == stmt)
20945                                return select;
20946                }
20947                return null;
20948        }
20949
20950        private void createSelectSetResultColumns(SelectSetResultSet resultSet, TSelectSqlStatement stmt) {
20951                if (stmt.getSetOperatorType() != ESetOperatorType.none) {
20952                        createSelectSetResultColumns(resultSet, stmt.getLeftStmt());
20953                } else {
20954                        TResultColumnList columnList = stmt.getResultColumnList();
20955                        ResultSet subqueryResultSet = (ResultSet) modelManager.getModel(columnList);
20956                        if(subqueryResultSet!=null && subqueryResultSet.isDetermined()) {
20957                                for (int j = 0; j < subqueryResultSet.getColumns().size(); j++) {
20958                                        ResultColumn tableColumn = subqueryResultSet.getColumns().get(j);
20959                                        if (tableColumn.getRefColumnName() != null) {
20960                                                TObjectName columnName = new TObjectName();
20961                                                columnName.setString(tableColumn.getRefColumnName());
20962                                                modelFactory.createDeterminedResultColumn(
20963                                                                resultSet, columnName);
20964                                        } else {
20965                                                TObjectName columnName = new TObjectName();
20966                                                columnName.setString(tableColumn.getName());
20967                                                modelFactory.createDeterminedResultColumn(
20968                                                                resultSet, columnName);
20969                                        }
20970                                }
20971                                resultSet.setDetermined(true);
20972                                return;
20973                        }
20974                        
20975                        boolean isDetermined = true;
20976                        for (int i = 0; i < columnList.size(); i++) {
20977                                TResultColumn column = columnList.getResultColumn(i);
20978                                
20979                                if ("*".equals(column.getColumnNameOnly())) {
20980                                        TObjectName columnObject = column.getFieldAttr();
20981                                        TTable sourceTable = columnObject.getSourceTable();
20982                                        if (sourceTable != null) {
20983                                                Object tableModel = modelManager.getModel(sourceTable);
20984                                                if (tableModel instanceof Table && ((Table) tableModel).isCreateTable()) {
20985                                                        Table table = (Table) tableModel;
20986                                                        for (int j = 0; j < table.getColumns().size(); j++) {
20987                                                                TableColumn tableColumn = table.getColumns().get(j);
20988                                                                if (column.getExceptColumnList() != null) {
20989                                                                        boolean except = false;
20990                                                                        for (TObjectName objectName : column.getExceptColumnList()) {
20991                                                                                if (getColumnName(objectName.toString())
20992                                                                                                .equals(getColumnName(tableColumn.getName()))) {
20993                                                                                        except = true;
20994                                                                                        break;
20995                                                                                }
20996                                                                        }
20997                                                                        if (!except && tableColumn.isStruct()) {
20998                                                                                List<String> names = SQLUtil
20999                                                                                                .parseNames(tableColumn.getName());
21000                                                                                for (String name : names) {
21001                                                                                        for (TObjectName objectName : column
21002                                                                                                        .getExceptColumnList()) {
21003                                                                                                if (getColumnName(objectName.toString())
21004                                                                                                                .equals(getColumnName(name))) {
21005                                                                                                        except = true;
21006                                                                                                        break;
21007                                                                                                }
21008                                                                                        }
21009                                                                                        if (except) {
21010                                                                                                break;
21011                                                                                        }
21012                                                                                }
21013                                                                        }
21014                                                                        if (except) {
21015                                                                                continue;
21016                                                                        }
21017                                                                }
21018                                                                TObjectName columnName = new TObjectName();
21019                                                                columnName.setString(tableColumn.getName());
21020                                                                ResultColumn resultColumn = modelFactory.createResultColumn(
21021                                                                                resultSet, columnName);
21022                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
21023                                                                relation.setEffectType(EffectType.select);
21024                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
21025                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
21026                                                        }
21027                                                        continue;
21028                                                } else if (tableModel instanceof ResultSet
21029                                                                && ((ResultSet) tableModel).isDetermined()) {
21030                                                        ResultSet table = (ResultSet) tableModel;
21031                                                        for (int j = 0; j < table.getColumns().size(); j++) {
21032                                                                ResultColumn tableColumn = table.getColumns().get(j);
21033                                                                if (column.getExceptColumnList() != null) {
21034                                                                        boolean except = false;
21035                                                                        for (TObjectName objectName : column.getExceptColumnList()) {
21036                                                                                if (getColumnName(objectName.toString())
21037                                                                                                .equals(getColumnName(tableColumn.getName()))) {
21038                                                                                        except = true;
21039                                                                                        break;
21040                                                                                }
21041                                                                        }
21042                                                                        if (!except && tableColumn.isStruct()) {
21043                                                                                List<String> names = SQLUtil
21044                                                                                                .parseNames(tableColumn.getName());
21045                                                                                for (String name : names) {
21046                                                                                        for (TObjectName objectName : column
21047                                                                                                        .getExceptColumnList()) {
21048                                                                                                if (getColumnName(objectName.toString())
21049                                                                                                                .equals(getColumnName(name))) {
21050                                                                                                        except = true;
21051                                                                                                        break;
21052                                                                                                }
21053                                                                                        }
21054                                                                                        if (except) {
21055                                                                                                break;
21056                                                                                        }
21057                                                                                }
21058                                                                        }
21059                                                                        if (except) {
21060                                                                                continue;
21061                                                                        }
21062                                                                }
21063                                                                if (tableColumn.getRefColumnName() != null) {
21064                                                                        TObjectName columnName = new TObjectName();
21065                                                                        columnName.setString(tableColumn.getRefColumnName());
21066                                                                        ResultColumn resultColumn = modelFactory.createResultColumn(
21067                                                                                        resultSet, columnName);
21068                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
21069                                                                        relation.setEffectType(EffectType.select);
21070                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
21071                                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn));
21072                                                                } else {
21073                                                                        TObjectName columnName = new TObjectName();
21074                                                                        columnName.setString(tableColumn.getName());
21075                                                                        ResultColumn resultColumn = modelFactory.createResultColumn(
21076                                                                                        resultSet, columnName);
21077                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
21078                                                                        relation.setEffectType(EffectType.select);
21079                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
21080                                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn));
21081                                                                }
21082                                                        }
21083                                                        continue;
21084                                                }
21085                                                else {
21086                                                        isDetermined = false;
21087                                                }
21088                                        }
21089                                }
21090                                        
21091                                ResultColumn resultColumn = modelFactory.createSelectSetResultColumn(resultSet, column, i);
21092
21093                                if (resultColumn.getColumnObject() instanceof TResultColumn) {
21094                                        TResultColumn columnObject = (TResultColumn) resultColumn.getColumnObject();
21095                                        if (columnObject.getFieldAttr() != null) {
21096                                                if ("*".equals(getColumnName(columnObject.getFieldAttr()))) {
21097                                                        TObjectName fieldAttr = columnObject.getFieldAttr();
21098                                                        TTable sourceTable = fieldAttr.getSourceTable();
21099                                                        if (fieldAttr.getTableToken() != null && sourceTable != null) {
21100                                                                TObjectName[] columns = modelManager.getTableColumns(sourceTable);
21101                                                                for (int j = 0; j < columns.length; j++) {
21102                                                                        TObjectName columnName = columns[j];
21103                                                                        if (columnName == null) {
21104                                                                                continue;
21105                                                                        }
21106                                                                        if ("*".equals(getColumnName(columnName))) {
21107                                                                                continue;
21108                                                                        }
21109                                                                        resultColumn.bindStarLinkColumn(columnName);
21110                                                                }
21111
21112                                                                if (modelManager.getModel(sourceTable) instanceof Table) {
21113                                                                        Table tableModel = (Table) modelManager.getModel(sourceTable);
21114                                                                        if (tableModel != null && !tableModel.getColumns().isEmpty()) {
21115                                                                                for (int z = 0; z < tableModel.getColumns().size(); z++) {
21116                                                                                        if ("*".equals(
21117                                                                                                        getColumnName(tableModel.getColumns().get(z).getColumnObject()))) {
21118                                                                                                continue;
21119                                                                                        }
21120                                                                                        resultColumn.bindStarLinkColumn(
21121                                                                                                        tableModel.getColumns().get(z).getColumnObject());
21122                                                                                }
21123                                                                        }
21124                                                                } else if (modelManager.getModel(sourceTable) instanceof QueryTable) {
21125                                                                        QueryTable tableModel = (QueryTable) modelManager.getModel(sourceTable);
21126                                                                        if (tableModel != null && !tableModel.getColumns().isEmpty()) {
21127                                                                                for (ResultColumn item : tableModel.getColumns()) {
21128                                                                                        if (item.hasStarLinkColumn()) {
21129                                                                                                for (TObjectName starLinkColumn : item.getStarLinkColumnList()) {
21130                                                                                                        if ("*".equals(getColumnName(starLinkColumn))) {
21131                                                                                                                continue;
21132                                                                                                        }
21133                                                                                                        resultColumn.bindStarLinkColumn(starLinkColumn);
21134                                                                                                }
21135                                                                                        } else if (item.getColumnObject() instanceof TObjectName) {
21136                                                                                                TObjectName starLinkColumn = (TObjectName) item.getColumnObject();
21137                                                                                                if ("*".equals(getColumnName(starLinkColumn))) {
21138                                                                                                        continue;
21139                                                                                                }
21140                                                                                                resultColumn.bindStarLinkColumn(starLinkColumn);
21141                                                                                        }
21142                                                                                }
21143                                                                        }
21144                                                                }
21145
21146                                                        } else {
21147                                                                TTableList tables = stmt.getTables();
21148                                                                for (int k = 0; k < tables.size(); k++) {
21149                                                                        TTable tableElement = tables.getTable(k);
21150                                                                        TObjectName[] columns = modelManager.getTableColumns(tableElement);
21151                                                                        for (int j = 0; j < columns.length; j++) {
21152                                                                                TObjectName columnName = columns[j];
21153                                                                                if (columnName == null) {
21154                                                                                        continue;
21155                                                                                }
21156                                                                                if ("*".equals(getColumnName(columnName))) {
21157                                                                                        if (modelManager.getModel(tableElement) instanceof Table) {
21158                                                                                                Table tableModel = (Table) modelManager.getModel(tableElement);
21159                                                                                                if (tableModel != null && !tableModel.getColumns().isEmpty()) {
21160                                                                                                        for (int z = 0; z < tableModel.getColumns().size(); z++) {
21161                                                                                                                resultColumn.bindStarLinkColumn(
21162                                                                                                                                tableModel.getColumns().get(z).getColumnObject());
21163                                                                                                        }
21164                                                                                                }
21165                                                                                        } else if (modelManager.getModel(tableElement) instanceof QueryTable) {
21166                                                                                                QueryTable tableModel = (QueryTable) modelManager
21167                                                                                                                .getModel(tableElement);
21168                                                                                                if (tableModel != null && !tableModel.getColumns().isEmpty()) {
21169                                                                                                        for (ResultColumn item : tableModel.getColumns()) {
21170                                                                                                                if (item.hasStarLinkColumn()) {
21171                                                                                                                        for (TObjectName starLinkColumn : item
21172                                                                                                                                        .getStarLinkColumnList()) {
21173                                                                                                                                resultColumn.bindStarLinkColumn(starLinkColumn);
21174                                                                                                                        }
21175                                                                                                                } else if (item.getColumnObject() instanceof TObjectName) {
21176                                                                                                                        resultColumn.bindStarLinkColumn(
21177                                                                                                                                        (TObjectName) item.getColumnObject());
21178                                                                                                                }
21179                                                                                                        }
21180                                                                                                }
21181                                                                                        }
21182                                                                                        continue;
21183                                                                                }
21184                                                                                resultColumn.bindStarLinkColumn(columnName);
21185                                                                        }
21186                                                                }
21187                                                        }
21188                                                }
21189                                        }
21190                                }
21191                                
21192                                resultSet.setDetermined(isDetermined);
21193                        }
21194                }
21195        }
21196
21197        private void analyzeResultColumn(TResultColumn column, EffectType effectType) {
21198                // A SELECT * star column whose source resolves to a determined result set
21199                // is bound to a LinkedHashMap of per-column ResultColumns (see
21200                // ModelFactory.createStarResultColumn), and its data-flow relationships
21201                // are already created inline during star expansion in analyzeSelectStmt().
21202                // Re-analyzing the raw "*" here is redundant; skip it. (analyzeDataFlowRelation
21203                // keeps the same guard as a defensive backstop.)
21204                if (modelManager.getModel(column) instanceof LinkedHashMap) {
21205                        return;
21206                }
21207                TExpression expression = column.getExpr();
21208                if (expression.getExpressionType() == EExpressionType.sqlserver_proprietary_column_alias_t) {
21209                        expression = expression.getRightOperand();
21210                }
21211
21212                if (expression.getExpressionType() == EExpressionType.array_t) {
21213                        if (expression.getExprList() != null) {
21214                                for (TExpression expr : expression.getExprList()) {
21215                                        columnsInExpr visitor = new columnsInExpr();
21216                                        expr.inOrderTraverse(visitor);
21217                                        List<TObjectName> objectNames = visitor.getObjectNames();
21218
21219                                        List<TParseTreeNode> functions = visitor.getFunctions();
21220
21221                                        if (functions != null && !functions.isEmpty()) {
21222                                                analyzeFunctionDataFlowRelation(column, functions, effectType);
21223                                        }
21224
21225                                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
21226                                        if (subquerys != null && !subquerys.isEmpty()) {
21227                                                analyzeSubqueryDataFlowRelation(column, subquerys, effectType);
21228                                        }
21229
21230                                        analyzeDataFlowRelation(column, objectNames, column.getExceptColumnList(), effectType, functions);
21231
21232                                        List<TParseTreeNode> constants = visitor.getConstants();
21233                                        Object columnObject = modelManager.getModel(column);
21234                                        analyzeConstantDataFlowRelation(columnObject, constants, effectType, functions);
21235
21236                                        analyzeRecordSetRelation(functions, effectType);
21237                                        // analyzeResultColumnImpact( column, effectType, functions);
21238                                }
21239                        }
21240                        else {
21241                                List<TParseTreeNode> constants = new ArrayList<TParseTreeNode>();
21242                                TConstant constant = new TConstant();
21243                                constant.setString(expression.toString());
21244                                constants.add(constant);
21245                                Object columnObject = modelManager.getModel(column);
21246                                analyzeConstantDataFlowRelation(columnObject, constants, effectType, null);
21247                        }
21248                } else {
21249                        columnsInExpr visitor = new columnsInExpr();
21250                        expression.inOrderTraverse(visitor);
21251                        List<TObjectName> objectNames = visitor.getObjectNames();
21252
21253                        List<TParseTreeNode> functions = visitor.getFunctions();
21254
21255                        if (functions != null && !functions.isEmpty()) {
21256                                analyzeFunctionDataFlowRelation(column, functions, effectType);
21257                        }
21258
21259                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
21260                        if (subquerys != null && !subquerys.isEmpty()) {
21261                                analyzeSubqueryDataFlowRelation(column, subquerys, effectType);
21262                        }
21263
21264                        analyzeDataFlowRelation(column, objectNames, column.getExceptColumnList(), effectType, functions);
21265
21266                        List<TParseTreeNode> constants = visitor.getConstants();
21267                        Object columnObject = modelManager.getModel(column);
21268                        analyzeConstantDataFlowRelation(columnObject, constants, effectType, functions);
21269
21270                        analyzeRecordSetRelation(functions, effectType);
21271                        // analyzeResultColumnImpact( column, effectType, functions);
21272                }
21273        }
21274        
21275        
21276        private void analyzeValueColumn(Object object, TResultColumn column, EffectType effectType) {
21277                TExpression expression = column.getExpr();
21278                if (expression.getExpressionType() == EExpressionType.sqlserver_proprietary_column_alias_t) {
21279                        expression = expression.getRightOperand();
21280                }
21281
21282                if (expression.getExpressionType() == EExpressionType.array_t) {
21283                        if (expression.getExprList() != null) {
21284                                for (TExpression expr : expression.getExprList()) {
21285                                        columnsInExpr visitor = new columnsInExpr();
21286                                        expr.inOrderTraverse(visitor);
21287                                        List<TObjectName> objectNames = visitor.getObjectNames();
21288                                        analyzeDataFlowRelation(object, objectNames, column.getExceptColumnList(), effectType, null, null);
21289                                        List<TParseTreeNode> constants = visitor.getConstants();
21290                                        analyzeConstantDataFlowRelation(object, constants, effectType, null);
21291                                }
21292                        }
21293                        else {
21294                                List<TParseTreeNode> constants = new ArrayList<TParseTreeNode>();
21295                                TConstant constant = new TConstant();
21296                                constant.setString(expression.toString());
21297                                constants.add(constant);
21298                                Object columnObject = modelManager.getModel(column);
21299                                analyzeConstantDataFlowRelation(object, constants, effectType, null);
21300                        }
21301                } else {
21302                        columnsInExpr visitor = new columnsInExpr();
21303                        expression.inOrderTraverse(visitor);
21304                        List<TObjectName> objectNames = visitor.getObjectNames();
21305                        analyzeDataFlowRelation(object, objectNames, column.getExceptColumnList(), effectType, null, null);
21306                        List<TParseTreeNode> constants = visitor.getConstants();
21307                        analyzeConstantDataFlowRelation(object, constants, effectType, null);           }
21308        }
21309
21310        private void analyzeTableColumn(TableColumn tableColumn, TFunctionCall functionCall, EffectType effectType) {
21311                List<TParseTreeNode> functions = new ArrayList<TParseTreeNode>();
21312                functions.add(functionCall);
21313
21314                if (functions != null && !functions.isEmpty()) {
21315                        analyzeFunctionDataFlowRelation(tableColumn, functions, effectType);
21316                }
21317
21318                analyzeRecordSetRelation(functions, effectType);
21319        }
21320
21321        private void analyzeRecordSetRelation(List<TParseTreeNode> functions, EffectType effectType) {
21322                if (functions == null || functions.size() == 0)
21323                        return;
21324
21325                List<TFunctionCall> aggregateFunctions = new ArrayList<TFunctionCall>();
21326                for (TParseTreeNode function : functions) {
21327                        if (function instanceof TFunctionCall && isAggregateFunction((TFunctionCall) function)) {
21328                                aggregateFunctions.add((TFunctionCall) function);
21329                        }
21330                }
21331
21332                if (aggregateFunctions.size() == 0)
21333                        return;
21334
21335                for (int i = 0; i < aggregateFunctions.size(); i++) {
21336                        TFunctionCall function = aggregateFunctions.get(i);
21337
21338                        TCustomSqlStatement stmt = stmtStack.peek();
21339                        if (stmt instanceof TSelectSqlStatement) {
21340                                TSelectSqlStatement select = (TSelectSqlStatement) stmt;
21341                                if (select.getGroupByClause() != null) {
21342                                        if (select.getGroupByClause().isAllModifier()) {
21343                                                // GROUP BY ALL: implicit grouping columns are the
21344                                                // non-aggregate expressions in the SELECT list.
21345                                                TResultColumnList resultColumns = select.getResultColumnList();
21346                                                if (resultColumns != null) {
21347                                                        for (int j = 0; j < resultColumns.size(); j++) {
21348                                                                TResultColumn column = resultColumns.getResultColumn(j);
21349                                                                TExpression expr = column.getExpr();
21350                                                                if (expr == null)
21351                                                                        continue;
21352                                                                columnsInExpr aggVisitor = new columnsInExpr();
21353                                                                expr.inOrderTraverse(aggVisitor);
21354                                                                boolean containsAggregate = false;
21355                                                                for (TParseTreeNode funcNode : aggVisitor.getFunctions()) {
21356                                                                        if (funcNode instanceof TFunctionCall
21357                                                                                        && isAggregateFunction((TFunctionCall) funcNode)) {
21358                                                                                containsAggregate = true;
21359                                                                                break;
21360                                                                        }
21361                                                                }
21362                                                                if (!containsAggregate) {
21363                                                                        analyzeAggregate(function, expr);
21364                                                                }
21365                                                        }
21366                                                }
21367                                        } else {
21368                                                TGroupByItemList groupByList = select.getGroupByClause().getItems();
21369                                                for (int j = 0; j < groupByList.size(); j++) {
21370                                                        TGroupByItem groupBy = groupByList.getGroupByItem(j);
21371                                                        TExpression expr = groupBy.getExpr();
21372                                                        analyzeAggregate(function, expr);
21373                                                }
21374                                        }
21375
21376                                        if (select.getGroupByClause().getHavingClause() != null) {
21377                                                analyzeAggregate(function, select.getGroupByClause().getHavingClause());
21378                                        }
21379                                        // if ("COUNT".equalsIgnoreCase(function.getFunctionName().toString()))
21380                                        {
21381                                                analyzeAggregate(function, null);
21382                                        }
21383                                } else {
21384                                        analyzeAggregate(function, null);
21385                                }
21386                        }
21387                }
21388        }
21389
21390        private void analyzeDataFlowRelation(TParseTreeNode gspObject, List<TObjectName> objectNames,
21391                        TObjectNameList exceptColumnList, EffectType effectType, List<TParseTreeNode> functions) {
21392                Object columnObject = modelManager.getModel(gspObject);
21393                analyzeDataFlowRelation(columnObject, objectNames, exceptColumnList, effectType, functions, null);
21394        }
21395
21396        private DataFlowRelationship analyzeDataFlowRelation(Object modelObject, List<TObjectName> objectNames, EffectType effectType,
21397                        List<TParseTreeNode> functions) {
21398                return analyzeDataFlowRelation(modelObject, objectNames, null, effectType, functions, null);
21399        }
21400        
21401        private DataFlowRelationship analyzeDataFlowRelation(Object modelObject, List<TObjectName> objectNames, EffectType effectType,
21402                        List<TParseTreeNode> functions, Process process) {
21403                return analyzeDataFlowRelation(modelObject, objectNames, null, effectType, functions, process);
21404        }
21405
21406        private DataFlowRelationship analyzeDataFlowRelation(Object modelObject, List<TObjectName> objectNames,
21407                        TObjectNameList exceptColumnList, EffectType effectType, List<TParseTreeNode> functions, Process process) {
21408                return analyzeDataFlowRelation(modelObject, objectNames, exceptColumnList, effectType, functions, process, null);
21409        }
21410        
21411        private DataFlowRelationship analyzeDataFlowRelation(Object modelObject, List<TObjectName> objectNames,
21412                        TObjectNameList exceptColumnList, EffectType effectType, List<TParseTreeNode> functions, Process process, Integer valueIndex) {
21413                if (objectNames == null || objectNames.size() == 0)
21414                        return null;
21415
21416                // Reject model objects this method cannot turn into a relationship BEFORE
21417                // creating one, since createDataFlowRelation() registers the relation
21418                // globally and an early return afterwards would leak an empty relation.
21419                //
21420                // - LinkedHashMap: a SELECT * star column whose source resolves to a
21421                //   determined result set is expanded into per-column ResultColumns held
21422                //   in a LinkedHashMap by ModelFactory.createStarResultColumn(), with its
21423                //   data-flow relationships created inline during star expansion in
21424                //   analyzeSelectStmt(). Re-analyzing the raw "*" here is a no-op.
21425                // - null: modelManager.getModel() found no binding for the gsp object.
21426                if (modelObject == null || modelObject instanceof LinkedHashMap) {
21427                        return null;
21428                }
21429                // Lineage is best-effort: an unexpected model type should not abort the
21430                // whole statement's lineage (this used to throw UnsupportedOperationException).
21431                // Log it for diagnosis and skip just this column instead.
21432                if (!(modelObject instanceof ResultColumn) && !(modelObject instanceof TableColumn)) {
21433                        logger.warn("analyzeDataFlowRelation: unhandled model type "
21434                                        + modelObject.getClass().getName() + ", effectType=" + effectType);
21435                        return null;
21436                }
21437
21438                boolean isStar = false;
21439                boolean showStar = false;
21440
21441                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
21442                relation.setEffectType(effectType);
21443                relation.setProcess(process);
21444
21445                if (functions != null && !functions.isEmpty()) {
21446                        relation.setFunction(getFunctionName(functions.get(0)));
21447                }
21448
21449                int columnIndex = -1;
21450
21451                boolean isOut = false;
21452
21453                if (modelObject instanceof ResultColumn) {
21454                        relation.setTarget(new ResultColumnRelationshipElement((ResultColumn) modelObject));
21455
21456                        if ("*".equals(((ResultColumn) modelObject).getName())) {
21457                                isStar = true;
21458                                showStar = ((ResultColumn) modelObject).isShowStar();
21459                        }
21460
21461                        if (((ResultColumn) modelObject).getResultSet() != null) {
21462                                columnIndex = ((ResultColumn) modelObject).getResultSet().getColumns().indexOf(modelObject);
21463                        }
21464                } else if (modelObject instanceof TableColumn) {
21465                        Table table  = ((TableColumn) modelObject).getTable();
21466                        if(table.getSubType() == SubType.out && isNotInProcedure(table)){
21467                                isOut = true;
21468                                relation.addSource(new TableColumnRelationshipElement((TableColumn) modelObject));
21469                        }
21470                        else {
21471                                relation.setTarget(new TableColumnRelationshipElement((TableColumn) modelObject));
21472                        }
21473
21474                        if ("*".equals(((TableColumn) modelObject).getName())) {
21475                                isStar = true;
21476                        }
21477
21478                        if (((TableColumn) modelObject).getTable() != null) {
21479                                columnIndex = ((TableColumn) modelObject).getTable().getColumns().indexOf(modelObject);
21480                        }
21481                }
21482                // No trailing else: modelObject is guaranteed to be a ResultColumn or
21483                // TableColumn here (validated and logged above before relation creation).
21484
21485                for (int i = 0; i < objectNames.size(); i++) {
21486                        TObjectName columnName = objectNames.get(i);
21487                        if (columnName.toString().indexOf(".") == -1 && isConstant(columnName)) {
21488                                boolean isConstant = true;
21489                                if (columnName.getSourceTable() != null) {
21490                                        Table tableModel = modelManager.getTableByName(
21491                                                        DlineageUtil.getTableFullName(columnName.getSourceTable().getTableName().toString()));
21492                                        if (tableModel != null && tableModel.getColumns() != null) {
21493                                                for (int j = 0; j < tableModel.getColumns().size(); j++) {
21494                                                        if (DlineageUtil.compareColumnIdentifier(getColumnName(columnName),
21495                                                                        getColumnName(tableModel.getColumns().get(j).getName()))) {
21496                                                                isConstant = false;
21497                                                                break;
21498                                                        }
21499                                                }
21500                                        }
21501                                }
21502
21503                                if (isConstant) {
21504                                        if (option.isShowConstantTable()) {
21505                                                Table constantTable = modelFactory.createConstantsTable(stmtStack.peek());
21506                                                TableColumn tableColumn = modelFactory.createTableColumn(constantTable, columnName, false);
21507                                                if(tableColumn!=null) {
21508                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
21509                                                }
21510                                        }
21511                                        continue;
21512                                }
21513                        }
21514                        if (columnName.getDbObjectType() == EDbObjectType.variable) {
21515                                boolean find = false;
21516                                List<String> segments = SQLUtil.parseNames(columnName.toString());
21517                                if (segments.size() == 1) {
21518                                        Table variable = modelManager.getTableByName(DlineageUtil.getTableFullName(columnName.toString()));
21519                                        if (variable != null) {
21520                                                TableColumn columnModel = variable.getColumns().get(0);
21521                                                if(variable.getColumns().size()>0){
21522                                                        TableColumn matchColumn = matchColumn(variable.getColumns(), columnName);
21523                                                        if(matchColumn!=null){
21524                                                                columnModel = matchColumn;
21525                                                        }
21526                                                }
21527                                                if(isOut){
21528                                                        relation.setTarget(new TableColumnRelationshipElement(columnModel));
21529                                                }
21530                                                else {
21531                                                        relation.addSource(new TableColumnRelationshipElement(columnModel));
21532                                                }
21533                                                find = true;
21534                                        } else {
21535                                                if (columnName.toString().matches("\\$\\d+")) {
21536                                                        TCustomSqlStatement stmt = stmtStack.peek();
21537                                                        Procedure procedure = modelManager
21538                                                                        .getProcedureByName(DlineageUtil.getTableFullName(getProcedureParentName(stmt)));
21539                                                        if(procedure!=null) {
21540                                                                Variable cursorVariable = modelFactory.createVariable(procedure.getArguments().get(Integer.valueOf(columnName.toString().replace("$", "")) - 1).getName());
21541                                                                if (isOut) {
21542                                                                        relation.setTarget(new TableColumnRelationshipElement(cursorVariable.getColumns().get(0)));
21543                                                                } else {
21544                                                                        relation.addSource(new TableColumnRelationshipElement(cursorVariable.getColumns().get(0)));
21545                                                                }
21546                                                        }
21547                                                } else {
21548                                                        Variable cursorVariable = modelFactory.createVariable(columnName);
21549                                                        cursorVariable.setCreateTable(true);
21550                                                        cursorVariable.setSubType(SubType.record);
21551                                                        TableColumn variableProperty = null;
21552                                                        if(cursorVariable.getColumns() == null || cursorVariable.getColumns().isEmpty()) {
21553                                variableProperty = modelFactory.createTableColumn(cursorVariable, columnName,
21554                                        true);
21555                            }
21556                                                        else{
21557                                                                variableProperty = cursorVariable.getColumns().get(0);
21558                                                        }
21559                                                        if (isOut) {
21560                                                                relation.setTarget(new TableColumnRelationshipElement(variableProperty));
21561                                                        } else {
21562                                                                relation.addSource(new TableColumnRelationshipElement(variableProperty));
21563                                                        }
21564                                                }
21565                                                find = true;
21566                                        }
21567                                } else if (option.getVendor() == EDbVendor.dbvoracle && columnName.getTableToken()!=null) {
21568                                        Variable cursorVariable = modelFactory
21569                                                        .createVariable(columnName.getTableToken().toString());
21570
21571                                        TObjectName variableColumnName = new TObjectName();
21572                                        variableColumnName.setString(segments.get(segments.size() - 1));
21573
21574                                        if (cursorVariable.getColumns() != null) {
21575                                                for (int j = 0; j < cursorVariable.getColumns().size(); j++) {
21576                                                        if (getColumnName(variableColumnName)
21577                                                                        .equals(getColumnName(cursorVariable.getColumns().get(j).getColumnObject()))) {
21578                                                                TableColumn columnModel = cursorVariable.getColumns().get(j);
21579                                                                if (isOut) {
21580                                                                        relation.setTarget(new TableColumnRelationshipElement(columnModel));
21581                                                                } else {
21582                                                                        relation.addSource(new TableColumnRelationshipElement(columnModel));
21583                                                                }
21584                                                                find = true;
21585                                                        }
21586                                                }
21587                                        }
21588
21589                                        if (!find) {
21590                                                TableColumn variableColumn = new TableColumn(cursorVariable, variableColumnName);
21591                                                cursorVariable.addColumn(variableColumn);
21592                                                if (isOut) {
21593                                                        relation.setTarget(new TableColumnRelationshipElement(variableColumn));
21594                                                } else {
21595                                                        relation.addSource(new TableColumnRelationshipElement(variableColumn));
21596                                                }
21597                                        }
21598                                } else {
21599                                        Table variable = modelManager
21600                                                        .getTableByName(DlineageUtil.getTableFullName(segments.get(segments.size() - 2)));
21601                                        if (variable != null) {
21602                                                for (int j = 0; j < variable.getColumns().size(); j++) {
21603                                                        if (getColumnName(columnName)
21604                                                                        .equals(getColumnName(variable.getColumns().get(j).getColumnObject()))) {
21605                                                                TableColumn columnModel = variable.getColumns().get(j);
21606                                                                if (isOut) {
21607                                                                        relation.setTarget(new TableColumnRelationshipElement(columnModel));
21608                                                                } else {
21609                                                                        relation.addSource(new TableColumnRelationshipElement(columnModel));
21610                                                                }
21611                                                                find = true;
21612                                                        }
21613                                                }
21614                                        }
21615                                }
21616                                if (!find) {
21617                                        TCustomSqlStatement stmt = stmtStack.peek();
21618                                        if (getProcedureParentName(stmt) != null) {
21619                                                Procedure procedure = modelManager
21620                                                                .getProcedureByName(DlineageUtil.getTableFullName(getProcedureParentName(stmt)));
21621                                                if (procedure != null && procedure.getArguments() != null) {
21622                                                        for (Argument argument : procedure.getArguments()) {
21623                                                                if (DlineageUtil.getTableFullName(argument.getName())
21624                                                                                .equals(DlineageUtil.getTableFullName(columnName.toString()))) {
21625                                                                        relation.addSource(new ArgumentRelationshipElement(argument));
21626                                                                }
21627                                                        }
21628                                                }
21629                                        }
21630                                }
21631                                continue;
21632                        }
21633                        
21634                        // Handle sequence pseudocolumn syntax (sequence.NEXTVAL or sequence.CURRVAL)
21635                        // Used by Oracle, Snowflake, and accepted by other vendors for compatibility
21636                        if(("NEXTVAL".equalsIgnoreCase(columnName.getColumnNameOnly()) || "CURRVAL".equalsIgnoreCase(columnName.getColumnNameOnly()))){
21637                                List<String> segments = SQLUtil.parseNames(columnName.toString());
21638                                if (segments.size() > 1) {
21639                                        segments.remove(segments.size()-1);
21640                                        Table table = modelFactory.createTableByName(SQLUtil.mergeSegments(segments, 0), true);
21641                                        table.setSequence(true);
21642                                        TableColumn seqCursor = modelFactory.createTableColumn(table, columnName, true);
21643                                        relation.addSource(new TableColumnRelationshipElement(seqCursor));
21644                                        continue;
21645                                }
21646                        }
21647
21648                        {
21649                                if (columnName.getSourceTable() != null) {
21650
21651                                }
21652                                else {
21653                                        Table variable = modelManager.getTableByName(DlineageUtil.getTableFullName(columnName.toString()));
21654                                        if (variable == null) {
21655                                                variable = modelManager
21656                                                                .getTableByName(DlineageUtil.getTableFullName(columnName.getTableString()));
21657                                                if (variable != null && variable.isCursor()) {
21658                                                        TableColumn variableColumn = modelFactory.createInsertTableColumn(variable, columnName);
21659                                                        if (variableColumn != null) {
21660                                                                if(isOut){
21661                                                                        relation.setTarget(new TableColumnRelationshipElement(variableColumn));
21662                                                                }
21663                                                                else {
21664                                                                        relation.addSource(new TableColumnRelationshipElement(variableColumn));
21665                                                                }
21666                                                        } else {
21667                                                                if(isOut){
21668                                                                        relation.setTarget(new TableColumnRelationshipElement(variable.getColumns().get(0)));
21669                                                                }
21670                                                                else {
21671                                                                        relation.addSource(new TableColumnRelationshipElement(variable.getColumns().get(0)));
21672                                                                }
21673                                                        }
21674                                                        continue;
21675                                                }
21676                                        } else if (variable.isVariable() || variable.isCursor()) {
21677                                                TableColumn columnModel = variable.getColumns().get(0);
21678                                                if (valueIndex != null) {
21679                                                        if(isOut){
21680                                                                relation.setTarget(new TableColumnRelationshipElement(columnModel, valueIndex));
21681                                                        }
21682                                                        else {
21683                                                                relation.addSource(new TableColumnRelationshipElement(columnModel, valueIndex));
21684                                                        }
21685                                                } else {
21686                                                        if(isOut){
21687                                                                relation.setTarget(new TableColumnRelationshipElement(columnModel));
21688                                                        }
21689                                                        else {
21690                                                                relation.addSource(new TableColumnRelationshipElement(columnModel));
21691                                                        }
21692                                                }
21693                                                continue;
21694                                        }
21695                                }
21696                        }
21697                        
21698                        if (columnName.getColumnNameOnly().startsWith("@")
21699                                        && (option.getVendor() == EDbVendor.dbvmssql || option.getVendor() == EDbVendor.dbvazuresql)) {
21700                                continue;
21701                        }
21702
21703                        if (columnName.getColumnNameOnly().startsWith(":")
21704                                        && (option.getVendor() == EDbVendor.dbvhana || option.getVendor() == EDbVendor.dbvteradata)) {
21705                                Table variable = modelManager
21706                                                .getTableByName(DlineageUtil.getTableFullName(columnName.getColumnNameOnly().replace(":", "")));
21707                                if (variable != null) {
21708                                        for (int j = 0; j < variable.getColumns().size(); j++) {
21709                                                if (getColumnName(columnName).replace(":", "")
21710                                                                .equals(getColumnName(variable.getColumns().get(j).getColumnObject()))) {
21711                                                        TableColumn columnModel = variable.getColumns().get(j);
21712                                                        relation.addSource(new TableColumnRelationshipElement(columnModel));
21713                                                }
21714                                        }
21715                                }
21716                                continue;
21717                        }
21718
21719                        boolean linkedFirstTable = false;
21720                        
21721                        TCustomSqlStatement stmt = stmtStack.peek();
21722                        TTableList tableList = stmt.tables;
21723                        if ((tableList == null || tableList.size() == 0) && (hiveFromTables != null && hiveFromTables.size() > 0)) {
21724                                tableList = hiveFromTables;
21725                        }
21726
21727                        List<TTable> tables = new ArrayList<TTable>();
21728                        {
21729                                TTable table = columnName.getSourceTable();
21730
21731                                // 针对CursorVariable特殊处理
21732                                if (columnName.getTableToken() != null) {
21733
21734                                        Table tableModel = null;
21735                                        if (table != null && modelManager.getModel(table) instanceof Table) {
21736                                                tableModel = (Table) modelManager.getModel(table);
21737                                        }
21738
21739                                        if (tableModel == null) {
21740                                                tableModel = modelManager
21741                                                                .getTableByName(DlineageUtil.getTableFullName(columnName.getTableToken().toString()));
21742                                        }
21743
21744                                        if (tableModel == null) {
21745                                                TCustomSqlStatement currentStmt = ModelBindingManager.getGlobalStmtStack().peek();
21746                                                String procedureName = DlineageUtil.getProcedureParentName(currentStmt);
21747                                                String variableString = columnName.getTableToken().toString();
21748                                                if (variableString.startsWith(":")) {
21749                                                        variableString = variableString.substring(variableString.indexOf(":") + 1);
21750                                                }
21751                                                if (!SQLUtil.isEmpty(procedureName)) {
21752                                                        variableString = procedureName + "." + SQLUtil.getIdentifierNormalTableName(variableString);
21753                                                }
21754
21755                                                if (modelManager
21756                                                                .getTableByName(DlineageUtil.getTableFullName(variableString)) instanceof Variable) {
21757                                                        tableModel = modelManager.getTableByName(DlineageUtil.getTableFullName(variableString));
21758                                                }
21759                                        }
21760
21761                                        if (tableModel != null) {
21762
21763                                                if (table == null) {
21764                                                        table = tableModel.getTableObject();
21765                                                }
21766
21767                                                if (tableModel.isVariable()) {
21768                                                        if (!isStar && "*".equals(getColumnName(columnName))) {
21769                                                                TObjectName[] columns = modelManager.getTableColumns(table);
21770                                                                for (int j = 0; j < columns.length; j++) {
21771                                                                        TObjectName objectName = columns[j];
21772                                                                        if (objectName == null || "*".equals(getColumnName(objectName))) {
21773                                                                                continue;
21774                                                                        }
21775                                                                        TableColumn columnModel = modelFactory.createTableColumn(tableModel, objectName,
21776                                                                                        false);
21777                                                                        relation.addSource(new TableColumnRelationshipElement(columnModel));
21778                                                                }
21779                                                        } else {
21780                                                                if ("*".equals(getColumnName(columnName)) && !tableModel.getColumns().isEmpty()) {
21781
21782                                                                        for (int j = 0; j < tableModel.getColumns().size(); j++) {
21783                                                                                TableColumn columnModel = tableModel.getColumns().get(j);
21784                                                                                if (exceptColumnList != null) {
21785                                                                                        boolean flag = false;
21786                                                                                        for (TObjectName objectName : exceptColumnList) {
21787                                                                                                if (getColumnName(objectName)
21788                                                                                                                .equals(getColumnName(columnModel.getColumnObject()))) {
21789                                                                                                        flag = true;
21790                                                                                                        break;
21791                                                                                                }
21792                                                                                        }
21793                                                                                        if (flag) {
21794                                                                                                continue;
21795                                                                                        }
21796                                                                                }
21797                                                                                relation.addSource(new TableColumnRelationshipElement(columnModel));
21798                                                                        }
21799
21800                                                                        if (isStar && showStar) {
21801                                                                                TableColumn columnModel = modelFactory.createTableColumn(tableModel, columnName,
21802                                                                                                false);
21803                                                                                if (columnModel == null) {
21804                                                                                        if (tableModel.isCreateTable()) {
21805                                                                                                for (TableColumn tableColumn : tableModel.getColumns()) {
21806                                                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
21807                                                                                                        relation.setShowStarRelation(true);
21808                                                                                                }
21809                                                                                        }
21810                                                                                } else {
21811                                                                                        relation.addSource(new TableColumnRelationshipElement(columnModel));
21812                                                                                        relation.setShowStarRelation(true);
21813                                                                                }
21814                                                                        }
21815                                                                } else {
21816                                                                        TableColumn columnModel = modelFactory.createTableColumn(tableModel, columnName,
21817                                                                                        false);
21818                                                                        
21819                                                                        if(columnModel == null && containStarColumn(tableModel.getColumns())){
21820                                                                                columnModel = getStarColumn(tableModel.getColumns());
21821                                                                                if (columnModel != null && tableModel.getSubType() == SubType.record_type) {
21822                                                                                        if(!"*".equals(getColumnName(columnName))){
21823                                                                                                columnModel.bindStarLinkColumn(columnName);
21824                                                                                        }
21825                                                                                }
21826                                                                        }
21827                                                                        
21828                                                                        if (columnModel == null) {
21829                                                                                continue;
21830                                                                        }
21831                                                                        if (columnModel.hasStarLinkColumn()) {
21832                                                                                relation.addSource(new TableColumnRelationshipElement(columnModel,
21833                                                                                                columnModel.getStarLinkColumnNames()
21834                                                                                                                .indexOf(DlineageUtil.getColumnName(columnName))));
21835                                                                        } else {
21836                                                                                if (isOut) {
21837                                                                                        relation.setTarget(new TableColumnRelationshipElement(columnModel));
21838                                                                                } else {
21839                                                                                        relation.addSource(new TableColumnRelationshipElement(columnModel));
21840                                                                                }
21841                                                                        }
21842                                                                        if (columnName.getSourceTable() != null
21843                                                                                        && columnName.getSourceTable().getTableType() == ETableSource.function) {
21844                                                                                analyzeTableColumn(columnModel, columnName.getSourceTable().getFuncCall(),
21845                                                                                                effectType);
21846                                                                        }
21847                                                                }
21848                                                        }
21849                                                        continue;
21850                                                }
21851                                        }
21852                                }
21853
21854                                if (table == null) {
21855                                        table = modelManager.getTable(stmt, columnName);
21856                                }
21857
21858                                if (table == null) {
21859                                        if (columnName.getTableToken() != null || !"*".equals(getColumnName(columnName))) {
21860                                                table = columnName.getSourceTable();
21861                                        }
21862                                        
21863                                        if (table == null && !SQLUtil.isEmpty(columnName.getTableString())) {
21864                                                table = modelManager.getTableFromColumn(columnName);
21865                                        }
21866                                }
21867
21868                                if (table == null) {
21869                                        if (tableList != null) {
21870                                                for (int j = 0; j < tableList.size(); j++) {
21871                                                        if (table != null)
21872                                                                break;
21873
21874                                                        TTable tTable = tableList.getTable(j);
21875                                                        if (tTable.getTableType().name().startsWith("open")) {
21876                                                                continue;
21877                                                        }
21878
21879                                                        if (getTableLinkedColumns(tTable) != null && getTableLinkedColumns(tTable).size() > 0) {
21880                                                                for (int z = 0; z < getTableLinkedColumns(tTable).size(); z++) {
21881                                                                        TObjectName refer = getTableLinkedColumns(tTable).getObjectName(z);
21882                                                                        if ("*".equals(getColumnName(refer)))
21883                                                                                continue;
21884                                                                        // For BigQuery struct field access, match base column name from FieldPath
21885                                                                        String structFullName = getStructFieldFullName(columnName);
21886                                                                        if (structFullName != null) {
21887                                                                                String baseName = getStructFieldBaseName(columnName);
21888                                                                                if (baseName != null && DlineageUtil.getIdentifierNormalColumnName(getColumnName(refer))
21889                                                                                                .equals(DlineageUtil.getIdentifierNormalColumnName(baseName))) {
21890                                                                                        table = tTable;
21891                                                                                        break;
21892                                                                                }
21893                                                                        }
21894                                                                        if (getColumnName(refer).equals(getColumnName(columnName))) {
21895                                                                                table = tTable;
21896                                                                                break;
21897                                                                        }
21898                                                                }
21899                                                        }
21900
21901                                                        if (tTable.getLinkTable() != null) {
21902                                                                tTable = tTable.getLinkTable();
21903                                                                for (int z = 0; z < getTableLinkedColumns(tTable).size(); z++) {
21904                                                                        TObjectName refer = getTableLinkedColumns(tTable).getObjectName(z);
21905                                                                        if ("*".equals(getColumnName(refer)))
21906                                                                                continue;
21907                                                                        if (getColumnName(refer).equals(getColumnName(columnName))) {
21908                                                                                table = tTable;
21909                                                                                break;
21910                                                                        }
21911                                                                }
21912                                                        }
21913
21914                                                        if (table != null)
21915                                                                break;
21916
21917                                                        if (columnName.getTableToken() != null && (columnName.getTableToken().getAstext()
21918                                                                        .equalsIgnoreCase(tTable.getName())
21919                                                                        || columnName.getTableToken().getAstext().equalsIgnoreCase(tTable.getAliasName()))) {
21920                                                                table = tTable;
21921                                                                break;
21922                                                        }
21923                                                }
21924                                        }
21925
21926                                        if (table == null) {
21927                                                for (int j = 0; j < tableList.size(); j++) {
21928                                                        if (table != null)
21929                                                                break;
21930
21931                                                        TTable tTable = tableList.getTable(j);
21932                                                        Object model = ModelBindingManager.get().getModel(tTable);
21933                                                        if (model instanceof Table) {
21934                                                                Table tableModel = (Table) model;
21935                                                                for (int z = 0; tableModel.getColumns() != null
21936                                                                                && z < tableModel.getColumns().size(); z++) {
21937                                                                        TableColumn refer = tableModel.getColumns().get(z);
21938                                                                        if (getColumnName(refer.getName()).equals(getColumnName(columnName))) {
21939                                                                                table = tTable;
21940                                                                                break;
21941                                                                        }
21942                                                                        if (refer.hasStarLinkColumn()) {
21943                                                                                for (TObjectName linkColumn : refer.getStarLinkColumnList()) {
21944                                                                                        if (getColumnName(linkColumn).equals(getColumnName(columnName))) {
21945                                                                                                table = tTable;
21946                                                                                                break;
21947                                                                                        }
21948                                                                                }
21949                                                                        }
21950                                                                }
21951                                                        } else if (model instanceof QueryTable) {
21952                                                                QueryTable tableModel = (QueryTable) model;
21953                                                                for (int z = 0; tableModel.getColumns() != null
21954                                                                                && z < tableModel.getColumns().size(); z++) {
21955                                                                        ResultColumn refer = tableModel.getColumns().get(z);
21956                                                                        // Try FieldPath-based matching first
21957                                                                        String structFullName = getStructFieldFullName(columnName);
21958                                                                        if (structFullName != null) {
21959                                                                                String baseName = getStructFieldBaseName(columnName);
21960                                                                                if (baseName != null && DlineageUtil.getIdentifierNormalColumnName(refer.getName())
21961                                                                                                .equals(DlineageUtil.getIdentifierNormalColumnName(baseName))) {
21962                                                                                        table = tTable;
21963                                                                                        break;
21964                                                                                }
21965                                                                        }
21966                                                                        List<String> splits = SQLUtil.parseNames(columnName.toString());
21967                                                                        if (splits.size() > 1 && EDbVendor.dbvbigquery == getOption().getVendor()) {
21968                                                                                if (DlineageUtil.getIdentifierNormalColumnName(refer.getName())
21969                                                                                                .equals(DlineageUtil
21970                                                                                                                .getIdentifierNormalColumnName(getColumnName(splits.get(0))))) {
21971                                                                                        table = tTable;
21972                                                                                        break;
21973                                                                                }
21974                                                                        }
21975                                                                        else if (DlineageUtil.getIdentifierNormalColumnName(refer.getName()).equals(
21976                                                                                        DlineageUtil.getIdentifierNormalColumnName(getColumnName(columnName)))) {
21977                                                                                table = tTable;
21978                                                                                break;
21979                                                                        }
21980                                                                        if (refer.hasStarLinkColumn()) {
21981                                                                                for (TObjectName linkColumn : refer.getStarLinkColumnList()) {
21982                                                                                        if (getColumnName(linkColumn).equals(getColumnName(columnName))) {
21983                                                                                                table = tTable;
21984                                                                                                break;
21985                                                                                        }
21986                                                                                }
21987                                                                        }
21988                                                                }
21989                                                        }
21990                                                }
21991                                        }
21992                                }
21993
21994                                if (columnName.getTableToken() == null && "*".equals(getColumnName(columnName))) {
21995                                        if (!hasJoin(stmt)) {
21996                                                tables.add(table);
21997                                        } else {
21998                                                for (int j = 0; j < tableList.size(); j++) {
21999                                                        tables.add(tableList.getTable(j));
22000                                                }
22001                                        }
22002                                } else if (table != null) {
22003                                        tables.add(table);
22004                                }
22005
22006                                // 此处特殊处理，多表关联无法找到 column 所属的 Table, tTable.getLinkedColumns
22007                                // 也找不到，退而求其次采用第一个表
22008
22009                                if (stmt.getParentStmt() != null && isApplyJoin(stmt.getParentStmt())
22010                                                && (tableList != null && tableList.size() > 0)) {
22011                                        stmt = stmt.getParentStmt();
22012                                        TTable applyTable = tableList.getTable(0);
22013                                        if (modelManager.getModel(table) == null) {
22014                                                modelFactory.createTable(applyTable);
22015                                        }
22016                                }
22017
22018                                if (columnName.toString().indexOf(".")==-1 && isConstant(columnName)) {
22019                                        boolean isConstant = true;
22020                                        if (columnName.getSourceTable() != null) {
22021                                                Table tableModel = modelManager.getTableByName(
22022                                                                DlineageUtil.getTableFullName(columnName.getSourceTable().getTableName().toString()));
22023                                                if (tableModel != null && tableModel.getColumns() != null) {
22024                                                        for (int j = 0; j < tableModel.getColumns().size(); j++) {
22025                                                                if (DlineageUtil.compareColumnIdentifier(getColumnName(columnName),
22026                                                                                getColumnName(tableModel.getColumns().get(j).getName()))) {
22027                                                                        isConstant = false;
22028                                                                        break;
22029                                                                }
22030                                                        }
22031                                                }
22032                                        }
22033
22034                                        if (isConstant) {
22035                                                if (option.isShowConstantTable()) {
22036                                                        Table constantTable = modelFactory.createConstantsTable(stmtStack.peek());
22037                                                        TableColumn tableColumn = modelFactory.createTableColumn(constantTable, columnName, false);
22038                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
22039                                                }
22040                                                continue;
22041                                        }
22042                                }
22043
22044                                if (tableList != null && tableList.size() != 0 && tables.size() == 0
22045                                                && !(isBuiltInFunctionName(columnName) && isFromFunction(columnName))) {
22046                                        if (modelManager.getModel(stmt) instanceof ResultSet) {
22047                                                ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt);
22048                                                boolean find = false;
22049                                                for (ResultColumn resultColumn : resultSetModel.getColumns()) {
22050                                                        if(resultColumn.equals(modelObject)) {
22051                                                                continue;
22052                                                        }
22053                                                        if (!TSQLEnv.isAliasReferenceForbidden.get(option.getVendor())) {
22054                                                                if (getColumnName(columnName).equals(getColumnName(resultColumn.getName()))) {
22055                                                                        if (resultColumn.getColumnObject() != null) {
22056                                                                                int startToken = resultColumn.getColumnObject().getStartToken().posinlist;
22057                                                                                int endToken = resultColumn.getColumnObject().getEndToken().posinlist;
22058                                                                                if (columnName.getStartToken().posinlist >= startToken
22059                                                                                                && columnName.getEndToken().posinlist <= endToken) {
22060                                                                                        continue;
22061                                                                                }
22062                                                                        }
22063                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
22064                                                                        find = true;
22065                                                                        break;
22066                                                                }
22067                                                        }
22068                                                }
22069                                                if (find) {
22070                                                        continue;
22071                                                }
22072                                        }
22073
22074                                        // Structured-dataflow outer field resolution: an unresolved
22075                                        // reference like nodes1.key (or t.nodes1.key) may be a
22076                                        // struct-field projection produced by a structured generator
22077                                        // (e.g. Spark explode(from_json(...))) in a visible subquery.
22078                                        // Link directly to the matching FunctionResultColumn so the
22079                                        // exact source path (nodes[*].key) flows through.
22080                                        if (tryAppendStructuredFieldRelation(relation, columnName, tableList)) {
22081                                                continue;
22082                                        }
22083
22084                                        TObjectName pseudoTableName = new TObjectName();
22085                                        // Use qualified prefix from column name if available (e.g., sch.pk_constv2 from sch.pk_constv2.c_cdsl)
22086                                        // Otherwise fall back to default pseudo table name
22087                                        String qualifiedPrefix = getQualifiedPrefixFromColumn(columnName);
22088                                        pseudoTableName.setString(qualifiedPrefix != null ? qualifiedPrefix : "pseudo_table_include_orphan_column");
22089                                        Table pseudoTable = modelFactory.createTableByName(pseudoTableName);
22090                                        pseudoTable.setPseudo(true);
22091                                        TableColumn pseudoTableColumn = modelFactory.createTableColumn(pseudoTable, columnName, true);
22092
22093                                        // If not linking to first table and column has qualified prefix (3-part name like sch.pkg.col),
22094                                        // add the pseudo table column as source
22095                                        if (!isLinkOrphanColumnToFirstTable() && pseudoTableColumn != null && qualifiedPrefix != null) {
22096                                                if (isOut) {
22097                                                        relation.setTarget(new TableColumnRelationshipElement(pseudoTableColumn));
22098                                                } else {
22099                                                        relation.addSource(new TableColumnRelationshipElement(pseudoTableColumn));
22100                                                }
22101                                        }
22102
22103                                        if (isLinkOrphanColumnToFirstTable()) {
22104                                                TTable orphanTable = tableList.getTable(0);
22105                                                tables.add(orphanTable);
22106                                                Object tableModel = modelManager.getModel(orphanTable);
22107                                                if (tableModel == null) {
22108                                                        if(orphanTable.getSubquery()!=null) {
22109                                                                QueryTable queryTable = modelFactory.createQueryTable(orphanTable);
22110                                                                TSelectSqlStatement subquery = orphanTable.getSubquery();
22111                                                                analyzeSelectStmt(subquery);
22112                                                        }
22113                                                        else {
22114                                                                tableModel = modelFactory.createTable(orphanTable);
22115                                                        }
22116                                                }
22117                                                if (tableModel instanceof Table) {
22118                                                        TableColumn orphanColum = modelFactory.createTableColumn((Table) tableModel, columnName, false);
22119                                                        if(orphanColum!=null) {
22120                                                                ErrorInfo errorInfo = new ErrorInfo();
22121                                                                errorInfo.setErrorType(ErrorInfo.LINK_ORPHAN_COLUMN);
22122                                                                errorInfo.setErrorMessage("Link orphan column [" + columnName.toString()
22123                                                                                + "] to the first table [" + orphanTable.getFullNameWithAliasString() + "]");
22124                                                                errorInfo.setStartPosition(new Pair3<Long, Long, String>(columnName.getStartToken().lineNo,
22125                                                                                columnName.getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
22126                                                                errorInfo.setEndPosition(new Pair3<Long, Long, String>(columnName.getEndToken().lineNo,
22127                                                                                columnName.getEndToken().columnNo + columnName.getEndToken().getAstext().length(),
22128                                                                                ModelBindingManager.getGlobalHash()));
22129                                                                errorInfo.fillInfo(this);
22130                                                                errorInfos.add(errorInfo);
22131                                                        }
22132                                                        if (orphanTable.getSubquery() != null) {
22133                                                                TSelectSqlStatement subquery = orphanTable.getSubquery();
22134                                                                if (subquery.getResultColumnList().toString().endsWith("*") && subquery.getTables().size() == 1) {
22135                                                                        TTable subqueryTable = subquery.getTables().getTable(0);
22136                                                                        Object sourceTable = modelManager.getModel(subqueryTable);
22137                                                                        if(sourceTable instanceof Table) {
22138                                                                                modelFactory.createTableColumn((Table) sourceTable, columnName, false);
22139                                                                        }
22140                                                                        else if(sourceTable instanceof ResultSet) {
22141                                                                                modelFactory.createResultColumn((ResultSet) sourceTable, columnName, false);
22142                                                                        }
22143                                                                }
22144                                                        }
22145                                                        else if (orphanTable.getCTE()!=null && orphanTable.getCTE().getSubquery() != null) {
22146                                                                TSelectSqlStatement subquery = orphanTable.getCTE().getSubquery();
22147                                                                if (subquery.getResultColumnList().toString().endsWith("*") && subquery.getTables().size() == 1) {
22148                                                                        TTable subqueryTable = subquery.getTables().getTable(0);
22149                                                                        Object sourceTable = modelManager.getModel(subqueryTable);
22150                                                                        if(sourceTable instanceof Table) {
22151                                                                                modelFactory.createTableColumn((Table) sourceTable, columnName, false);
22152                                                                        }
22153                                                                        else if(sourceTable instanceof ResultSet) {
22154                                                                                modelFactory.createResultColumn((ResultSet) sourceTable, columnName, false);
22155                                                                        }
22156                                                                }
22157                                                        }
22158                                                }
22159                                                
22160                                                linkedFirstTable = true;
22161                                        }
22162                                }
22163                        }
22164
22165                        for (int k = 0; k < tables.size(); k++) {
22166                                TTable table = tables.get(k);
22167                                if (table != null) {
22168                                        Object object = modelManager.getModel(table);
22169                                        if(object instanceof PivotedTable) {
22170                                                TPivotClause clause = (TPivotClause)((PivotedTable)object).getGspObject();
22171                                                if(clause.getAliasClause()!=null) {
22172                                                        object = modelManager.getModel(clause.getAliasClause());
22173                                                }
22174                                        }
22175                                        if (object == null && table.getTableType() == ETableSource.objectname) {
22176                                                if (table.getCTE() != null) {
22177                                                        QueryTable queryTable = modelFactory.createQueryTable(table);
22178                                                        TSelectSqlStatement subquery = table.getCTE().getSubquery();
22179                                                        analyzeSelectStmt(subquery);
22180                                                        ResultSet resultSetModel = (ResultSet) modelManager.getModel(subquery);
22181
22182                                                        if (resultSetModel != null && resultSetModel != queryTable
22183                                                                        && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
22184                                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
22185                                                                impactRelation.setEffectType(EffectType.select);
22186                                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
22187                                                                                resultSetModel.getRelationRows()));
22188                                                                impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
22189                                                                                queryTable.getRelationRows()));
22190                                                        }
22191
22192                                                        if (resultSetModel != null && resultSetModel != queryTable) {
22193                                                                for (int j = 0; j < resultSetModel.getColumns().size(); j++) {
22194                                                                        ResultColumn sourceColumn = resultSetModel.getColumns().get(j);
22195                                                                        ResultColumn targetColumn = modelFactory.createSelectSetResultColumn(queryTable,
22196                                                                                        sourceColumn);
22197
22198                                                                        DataFlowRelationship queryRalation = modelFactory.createDataFlowRelation();
22199                                                                        queryRalation.setEffectType(EffectType.select);
22200                                                                        queryRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
22201                                                                        queryRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
22202                                                                }
22203                                                        }
22204                                                        
22205                                                        object = queryTable;
22206                                                        
22207                                                } else {
22208                                                        object = modelFactory.createTable(table);
22209                                                }
22210                                        }
22211                                        if (object instanceof Function) {
22212                                                relation.addSource(new ResultColumnRelationshipElement(((Function)object).getColumns().get(0)));
22213                                                continue;
22214                                        } else if (object instanceof ResultSet && !(object instanceof QueryTable)) {
22215                                                //Object tableModel = modelManager.getModel(columnName.getSourceTable());
22216                                                appendResultColumnRelationSource(modelObject, relation, columnIndex, columnName,
22217                                                                (ResultSet)object);
22218                                                if(table.getTableType() == ETableSource.function && !relation.getSources().isEmpty()) {
22219                                                        relation.getSources().stream().reduce((first, second) -> second).get().addTransform(Transform.FUNCTION, table);
22220                                                }
22221                                                continue;
22222                                        }
22223                                        else if (object instanceof Table) {
22224                                                Table tableModel = (Table) modelManager.getModel(table);
22225                                                if (tableModel != null) {
22226                                                        if (!isStar && "*".equals(getColumnName(columnName))) {
22227                                                                TObjectName[] columns = modelManager.getTableColumns(table);
22228                                                                for (int j = 0; j < columns.length; j++) {
22229                                                                        TObjectName objectName = columns[j];
22230                                                                        if (objectName == null || ("*".equals(getColumnName(objectName)) && tableModel.isDetermined())) {
22231                                                                                continue;
22232                                                                        }
22233                                                                        TableColumn columnModel = modelFactory.createTableColumn(tableModel, objectName,
22234                                                                                        false);
22235                                                                        if(columnModel == null) {
22236                                                                                continue;
22237                                                                        }
22238                                                                        relation.addSource(new TableColumnRelationshipElement(columnModel));
22239                                                                }
22240                                                        } else {
22241                                                                if ("*".equals(getColumnName(columnName)) && !tableModel.getColumns().isEmpty()) {
22242                                                                        Map<String, Pair<String, TExpression>> replaceAsIdentifierMap = new HashMap<String, Pair<String, TExpression>>();
22243                                                                        Map<String, TObjectName> replaceColumnMap = new HashMap<String, TObjectName>();
22244                                                                        if(modelObject instanceof ResultColumn && ((ResultColumn)modelObject).getColumnObject() instanceof TResultColumn) {
22245                                                                                TResultColumn resultColumn =  (TResultColumn )((ResultColumn)modelObject).getColumnObject();
22246                                                                                if(resultColumn.getReplaceExprAsIdentifiers()!=null && resultColumn.getReplaceExprAsIdentifiers().size()>0) {
22247                                                                                        for(TReplaceExprAsIdentifier replace: resultColumn.getReplaceExprAsIdentifiers()) {
22248                                                                                                replaceAsIdentifierMap.put(replace.getIdentifier().toString(), new Pair<String, TExpression>(resultColumn.getExpr().getExceptReplaceClause().toString(), replace.getExpr()));
22249                                                                                                replaceColumnMap.put(replace.getIdentifier().toString(), replace.getIdentifier());
22250                                                                                        }
22251                                                                                        ResultSet resultSet = ((ResultColumn)modelObject).getResultSet();
22252                                                                                        if(tableModel.isDetermined()) {
22253                                                                                                resultSet.getColumns().clear();
22254                                                                                        }
22255                                                                                }
22256                                                                        }
22257                                                                        
22258                                                                        for (int j = 0; j < tableModel.getColumns().size(); j++) {
22259                                                                                TableColumn columnModel = tableModel.getColumns().get(j);
22260                                                                                if (exceptColumnList != null) {
22261                                                                                        boolean flag = false;
22262                                                                                        for (TObjectName objectName : exceptColumnList) {
22263                                                                                                if (getColumnName(objectName)
22264                                                                                                                .equals(getColumnName(columnModel.getColumnObject()))) {
22265                                                                                                        flag = true;
22266                                                                                                        break;
22267                                                                                                }
22268                                                                                        }
22269                                                                                        if (flag) {
22270                                                                                                continue;
22271                                                                                        }
22272                                                                                }
22273                                                                                
22274                                                                                if (replaceAsIdentifierMap.containsKey(tableModel.getColumns().get(j).getName())) {
22275                                                                                        Pair<String, TExpression> expr = replaceAsIdentifierMap.get(tableModel.getColumns().get(j).getName());
22276                                                                                        ResultSet resultSet = ((ResultColumn)modelObject).getResultSet();
22277                                                                                        ResultColumn resultColumn = modelFactory.createResultColumn(resultSet, replaceColumnMap.get(tableModel.getColumns().get(j).getName()));
22278                                                                                        Transform transform = new Transform();
22279                                                                                        transform.setType(Transform.EXPRESSION);
22280                                                                                        TObjectName expression = new TObjectName();
22281                                                                                        expression.setString(expr.first);
22282                                                                                transform.setCode(expression);
22283                                                                                        resultColumn.setTransform(transform);
22284                                                                                        analyzeResultColumnExpressionRelation(resultColumn, expr.second);
22285                                                                                } else {
22286                                                                                        if(!replaceAsIdentifierMap.isEmpty()) {
22287                                                                                                ResultSet resultSet = ((ResultColumn) modelObject).getResultSet();
22288                                                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSet,
22289                                                                                                                columnModel.getColumnObject());
22290                                                                                                DataFlowRelationship relation1 = modelFactory.createDataFlowRelation();
22291                                                                                                relation1.setEffectType(effectType);
22292                                                                                                relation1.setProcess(process);
22293                                                                                                relation1.setTarget(new ResultColumnRelationshipElement(resultColumn));
22294                                                                                                relation1.addSource(new TableColumnRelationshipElement(columnModel));
22295                                                                                        }
22296                                                                                        else {
22297                                                                                                relation.addSource(
22298                                                                                                        new TableColumnRelationshipElement(columnModel));
22299                                                                                        }
22300                                                                                }
22301                                                                        }
22302
22303                                                                        if (isStar && showStar) {
22304                                                                                TableColumn columnModel = modelFactory.createTableColumn(tableModel, columnName,
22305                                                                                                false);
22306                                                                                if (columnModel == null) {
22307                                                                                        if(tableModel.isCreateTable()) {
22308                                                                                                for (TableColumn tableColumn : tableModel.getColumns()) {
22309                                                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
22310                                                                                                        relation.setShowStarRelation(true);
22311                                                                                                }
22312                                                                                        }
22313                                                                                } else {
22314                                                                                        relation.addSource(new TableColumnRelationshipElement(columnModel));
22315                                                                                        relation.setShowStarRelation(true);
22316                                                                                }
22317                                                                        }
22318                                                                } else {
22319                                                                        TableColumn columnModel = modelFactory.createTableColumn(tableModel, columnName,
22320                                                                                        false);
22321                                                                        if(columnModel == null) {
22322                                                                                if(tableModel.isCreateTable()) {
22323                                                                                        boolean flag = false;
22324                                                                                        // Try FieldPath-based matching first for BigQuery/Redshift struct fields
22325                                                                                        String structFullName = getStructFieldFullName(columnName);
22326                                                                                        if (structFullName != null && !flag) {
22327                                                                                                for (TableColumn tableColumn : tableModel.getColumns()) {
22328                                                                                                        if (tableColumn.isStruct()
22329                                                                                                                && DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName())
22330                                                                                                                        .equals(DlineageUtil.getIdentifierNormalColumnName(structFullName))) {
22331                                                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
22332                                                                                                                flag = true;
22333                                                                                                                if (modelObject instanceof ResultColumn) {
22334                                                                                                                        ((ResultColumn) modelObject).setStruct(true);
22335                                                                                                                } else if (modelObject instanceof TableColumn) {
22336                                                                                                                        ((TableColumn) modelObject).setStruct(true);
22337                                                                                                                }
22338                                                                                                                break;
22339                                                                                                        }
22340                                                                                                }
22341                                                                                        }
22342                                                                                        if (ModelBindingManager.getGlobalVendor() == EDbVendor.dbvbigquery || ModelBindingManager.getGlobalVendor() == EDbVendor.dbvredshift) {
22343                                                                                                for (TableColumn tableColumn : tableModel.getColumns()) {
22344                                                                                                        if(tableColumn.isStruct()) {
22345                                                                                                                List<String> names = SQLUtil.parseNames(tableColumn.getName());
22346                                                                                                                if (modelObject instanceof TableColumn) {
22347                                                                                                                        TableColumn targetColumn = (TableColumn) modelObject;
22348                                                                                                                        if (targetColumn.isStruct()) {
22349                                                                                                                                List<String> targetNames = SQLUtil
22350                                                                                                                                                .parseNames(targetColumn.getName());
22351                                                                                                                                if (!getColumnName(targetNames.get(0))
22352                                                                                                                                                .equals(getColumnName(names.get(0)))) {
22353                                                                                                                                        continue;
22354                                                                                                                                }
22355                                                                                                                        }
22356                                                                                                                }
22357                                                                                                                else if (modelObject instanceof ResultColumn) {
22358                                                                                                                        ResultColumn targetColumn = (ResultColumn) modelObject;
22359                                                                                                                        if (targetColumn.isStruct()) {
22360                                                                                                                                List<String> targetNames = SQLUtil
22361                                                                                                                                                .parseNames(targetColumn.getName());
22362                                                                                                                                if (!getColumnName(targetNames.get(0))
22363                                                                                                                                                .equals(getColumnName(names.get(0)))) {
22364                                                                                                                                        continue;
22365                                                                                                                                }
22366                                                                                                                        }
22367                                                                                                                }
22368                                                                                                                for(String name: names) {
22369                                                                                                                        if (getColumnName(name)
22370                                                                                                                                        .equals(getColumnName(modelObject.toString()))) {
22371                                                                                                                                relation.addSource(
22372                                                                                                                                                new TableColumnRelationshipElement(tableColumn));
22373                                                                                                                                flag = true;
22374                                                                                                                                if (modelObject instanceof ResultColumn) {
22375                                                                                                                                        ((ResultColumn)modelObject).setStruct(true);
22376                                                                                                                                }
22377                                                                                                                                else if (modelObject instanceof TableColumn) {
22378                                                                                                                                        ((TableColumn)modelObject).setStruct(true);
22379                                                                                                                                }
22380                                                                                                                                break;
22381                                                                                                                        }
22382                                                                                                                }
22383
22384                                                                                                                if (!flag && tableModel.getColumns().size() == 1 && tableModel
22385                                                                                                                                .getColumns().get(0).getSourceColumn() != null) {
22386                                                                                                                        TableColumn sourceColumn = tableModel
22387                                                                                                                                        .getColumns().get(0).getSourceColumn();
22388                                                                                                                        Table sourceTable = sourceColumn.getTable();
22389                                                                                                                        TObjectName sourceColumnName = new TObjectName();
22390                                                                                                                        sourceColumnName.setString(sourceColumn.getName() + "."
22391                                                                                                                                        + columnName.getColumnNameOnly());
22392                                                                                                                        TableColumn sourceTableColumn = modelFactory.createTableColumn(sourceTable, sourceColumnName, true);
22393                                                                                                                        relation.addSource(
22394                                                                                                                                        new TableColumnRelationshipElement(sourceTableColumn));
22395                                                                                                                        flag = true;
22396                                                                                                                        break;
22397                                                                                                                }
22398                                                                                                        }
22399                                                                                                        else if (getColumnName(tableColumn.getName())
22400                                                                                                                        .equals(getColumnName(modelObject.toString()))) {
22401                                                                                                                relation.addSource(
22402                                                                                                                                new TableColumnRelationshipElement(tableColumn));
22403                                                                                                                flag = true;
22404                                                                                                                break;
22405                                                                                                        }
22406                                                                                                }
22407                                                                                                if (modelObject instanceof TableColumn) {
22408                                                                                                        TableColumn column = (TableColumn) modelObject;
22409                                                                                                        for (TableColumn tableColumn : tableModel.getColumns()) {
22410                                                                                                                if (tableColumn.getColumnIndex() == null) {
22411                                                                                                                        continue;
22412                                                                                                                }
22413                                                                                                                if (tableColumn.getName().toLowerCase()
22414                                                                                                                                .indexOf(column.getName().toLowerCase()) == -1
22415                                                                                                                                && tableColumn.getName().toLowerCase()
22416                                                                                                                                                .indexOf(column.getColumnObject().toString()
22417                                                                                                                                                                .toLowerCase()) == -1) {
22418                                                                                                                        continue;
22419                                                                                                                }
22420                                                                                                                flag = true;
22421                                                                                                                relation.addSource(
22422                                                                                                                                new TableColumnRelationshipElement(tableColumn));
22423                                                                                                                relation.setShowStarRelation(true);
22424                                                                                                        }
22425                                                                                                        if (flag)
22426                                                                                                                break;
22427
22428                                                                                                } else if (modelObject instanceof ResultColumn) {
22429                                                                                                        ResultColumn column = (ResultColumn) modelObject;
22430                                                                                                        for (TableColumn tableColumn : tableModel.getColumns()) {
22431                                                                                                                if (tableColumn.getColumnIndex() == null) {
22432                                                                                                                        continue;
22433                                                                                                                }
22434                                                                                                                if (tableColumn.getName().toLowerCase()
22435                                                                                                                                .indexOf(column.getName().toLowerCase()) == -1
22436                                                                                                                                && tableColumn.getName().toLowerCase()
22437                                                                                                                                                .indexOf(column.getColumnObject().toString()
22438                                                                                                                                                                .toLowerCase()) == -1) {
22439                                                                                                                        continue;
22440                                                                                                                }
22441                                                                                                                flag = true;
22442                                                                                                                relation.addSource(
22443                                                                                                                                new TableColumnRelationshipElement(tableColumn));
22444                                                                                                                relation.setShowStarRelation(true);
22445                                                                                                        }
22446                                                                                                        if (flag)
22447                                                                                                                break;
22448
22449                                                                                                }
22450                                                                                        }
22451                                                                                        if (!flag 
22452                                                                                                        && (isStar 
22453                                                                                                                        || getColumnName(columnName).equals(getColumnName(tableModel.getName()))
22454                                                                                                                        || getColumnName(columnName).equals(getColumnName(tableModel.getAlias())))) {
22455                                                                                                for (TableColumn tableColumn : tableModel.getColumns()) {
22456                                                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
22457                                                                                                        relation.setShowStarRelation(true);
22458                                                                                                }
22459                                                                                        }
22460                                                                                }
22461                                                                        }
22462                                                                        else {
22463                                                                                if (linkedFirstTable || columnModel.getCandidateParents() != null) {
22464                                                                                        if (columnName.getCandidateTables() != null
22465                                                                                                        && columnName.getCandidateTables().size() > 1) {
22466                                                                                                List<Object> candidateParents = new ArrayList<Object>();
22467                                                                                                for(TTable tableItem: columnName.getCandidateTables()) {
22468                                                                                                        Object model = modelManager.getModel(tableItem);
22469                                                                                                        if(model!=null) {
22470                                                                                                                candidateParents.add(model);
22471                                                                                                        }
22472                                                                                                }
22473                                                                                                if (candidateParents.size() > 1) {
22474                                                                                                        columnModel.setCandidateParents(candidateParents);
22475                                                                                                }
22476                                                                                        }
22477                                                                                }
22478                                                                                relation.addSource(new TableColumnRelationshipElement(columnModel));
22479                                                                                relation.setShowStarRelation(true);
22480                                                                                if(modelObject instanceof TableColumn) {
22481                                                                                        TableColumn targetTableColumn = (TableColumn)modelObject;
22482                                                                                        if(targetTableColumn.getTable().getSubType() == SubType.unnest && targetTableColumn.getTable().getColumns().size() == 1) {
22483                                                                                                targetTableColumn.setSourceColumn(columnModel);
22484                                                                                                targetTableColumn.setStruct(true);
22485                                                                                        }
22486                                                                                }
22487                                                                                if (columnName.getSourceTable() != null
22488                                                                                                && columnName.getSourceTable().getTableType() == ETableSource.function) {
22489                                                                                        analyzeTableColumn(columnModel, columnName.getSourceTable().getFuncCall(),
22490                                                                                                        effectType);
22491                                                                                }
22492                                                                        }
22493                                                                }
22494                                                        }
22495                                                }
22496                                        } else if (modelManager.getModel(table) instanceof QueryTable) {
22497                                                QueryTable queryTable = (QueryTable) modelManager.getModel(table);
22498
22499                                                TObjectNameList cteColumns = null;
22500                                                TSelectSqlStatement subquery = null;
22501                                                if (queryTable.getTableObject().getCTE() != null) {
22502                                                        subquery = queryTable.getTableObject().getCTE().getSubquery();
22503                                                        cteColumns = queryTable.getTableObject().getCTE().getColumnList();
22504                                                } else if (queryTable.getTableObject().getAliasClause() != null
22505                                                                && queryTable.getTableObject().getAliasClause().getColumns() != null) {
22506
22507                                                } else if (queryTable.getTableObject().getTableExpr() != null
22508                                                                && queryTable.getTableObject().getTableExpr().getSubQuery() != null) {
22509                                                        subquery = queryTable.getTableObject().getTableExpr().getSubQuery();
22510                                                } else {
22511                                                        subquery = queryTable.getTableObject().getSubquery();
22512                                                }
22513
22514                                                if (cteColumns != null) {
22515                                                        for (int j = 0; j < cteColumns.size(); j++) {
22516                                                                modelFactory.createResultColumn(queryTable, cteColumns.getObjectName(j));
22517                                                        }
22518                                                }
22519
22520                                                if (subquery != null && subquery.isCombinedQuery()) {
22521                                                        SelectSetResultSet selectSetResultSetModel = (SelectSetResultSet) modelManager
22522                                                                        .getModel(subquery);
22523
22524                                                        if (selectSetResultSetModel != null
22525                                                                        && !selectSetResultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
22526                                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
22527                                                                impactRelation.setEffectType(EffectType.select);
22528                                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
22529                                                                                selectSetResultSetModel.getRelationRows()));
22530                                                                impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
22531                                                                                queryTable.getRelationRows()));
22532                                                        }
22533
22534                                                        if (selectSetResultSetModel != null) {
22535                                                                if (getColumnName(columnName).equals("*")) {
22536                                                                        Map<String, Pair<String, TExpression>> replaceAsIdentifierMap = new HashMap<String, Pair<String, TExpression>>();
22537                                                                        Map<String, TObjectName> replaceColumnMap = new HashMap<String, TObjectName>();
22538                                                                        if(modelObject instanceof ResultColumn && ((ResultColumn)modelObject).getColumnObject() instanceof TResultColumn) {
22539                                                                                TResultColumn resultColumn =  (TResultColumn )((ResultColumn)modelObject).getColumnObject();
22540                                                                                if(resultColumn.getReplaceExprAsIdentifiers()!=null && resultColumn.getReplaceExprAsIdentifiers().size()>0) {
22541                                                                                        for(TReplaceExprAsIdentifier replace: resultColumn.getReplaceExprAsIdentifiers()) {
22542                                                                                                replaceAsIdentifierMap.put(replace.getIdentifier().toString(), new Pair<String, TExpression>(resultColumn.getExpr().getExceptReplaceClause().toString(), replace.getExpr()));
22543                                                                                                replaceColumnMap.put(replace.getIdentifier().toString(), replace.getIdentifier());
22544                                                                                        }
22545                                                                                        ResultSet resultSet = ((ResultColumn)modelObject).getResultSet();
22546                                                                                        if(selectSetResultSetModel.isDetermined()) {
22547                                                                                                resultSet.getColumns().clear();
22548                                                                                        }
22549                                                                                }
22550                                                                        }
22551                                                                        
22552                                                                        
22553                                                                        for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) {
22554                                                                                ResultColumn sourceColumn = selectSetResultSetModel.getColumns().get(j);
22555                                                                                if (cteColumns != null) {
22556                                                                                        if (j < cteColumns.size()) {
22557                                                                                                ResultColumn targetColumn = queryTable.getColumns().get(j);
22558
22559                                                                                                if (exceptColumnList != null) {
22560                                                                                                        boolean flag = false;
22561                                                                                                        for (TObjectName objectName : exceptColumnList) {
22562                                                                                                                if (getColumnName(objectName)
22563                                                                                                                                .equals(getColumnName(targetColumn.getName()))) {
22564                                                                                                                        flag = true;
22565                                                                                                                        break;
22566                                                                                                                }
22567                                                                                                        }
22568                                                                                                        if (flag) {
22569                                                                                                                continue;
22570                                                                                                        }
22571                                                                                                }
22572
22573                                                                                                if (replaceAsIdentifierMap.containsKey(targetColumn.getName())) {
22574                                                                                                        Pair<String, TExpression> expr = replaceAsIdentifierMap.get(targetColumn.getName());
22575                                                                                                        ResultSet resultSet = ((ResultColumn)modelObject).getResultSet();
22576                                                                                                        ResultColumn resultColumn = modelFactory.createResultColumn(resultSet, replaceColumnMap.get(targetColumn.getName()));
22577                                                                                                        Transform transform = new Transform();
22578                                                                                                        transform.setType(Transform.EXPRESSION);
22579                                                                                                        TObjectName expression = new TObjectName();
22580                                                                                                        expression.setString(expr.first);
22581                                                                                                transform.setCode(expression);
22582                                                                                                        resultColumn.setTransform(transform);
22583                                                                                                        analyzeResultColumnExpressionRelation(resultColumn, expr.second);
22584                                                                                                } else {
22585                                                                                                        if(!replaceAsIdentifierMap.isEmpty()) {
22586                                                                                                                ResultSet resultSet = ((ResultColumn) modelObject).getResultSet();
22587                                                                                                                TObjectName resultColumnName = new TObjectName();
22588                                                                                                                resultColumnName.setString(targetColumn.getName());
22589                                                                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSet,
22590                                                                                                                                resultColumnName);
22591                                                                                                                DataFlowRelationship relation1 = modelFactory.createDataFlowRelation();
22592                                                                                                                relation1.setEffectType(effectType);
22593                                                                                                                relation1.setProcess(process);
22594                                                                                                                relation1.setTarget(new ResultColumnRelationshipElement(resultColumn));
22595                                                                                                                relation1.addSource(new ResultColumnRelationshipElement(targetColumn));
22596                                                                                                        }
22597                                                                                                        else {
22598                                                                                                                relation.addSource(
22599                                                                                                                        new ResultColumnRelationshipElement(targetColumn));
22600                                                                                                        }
22601                                                                                                }
22602                                                                                                
22603                                                                                        }
22604                                                                                } else {
22605                                                                                        ResultColumn targetColumn = modelFactory
22606                                                                                                        .createSelectSetResultColumn(queryTable, sourceColumn);
22607
22608                                                                                        DataFlowRelationship combinedQueryRelation = modelFactory
22609                                                                                                        .createDataFlowRelation();
22610                                                                                        combinedQueryRelation.setEffectType(effectType);
22611                                                                                        combinedQueryRelation
22612                                                                                                        .setTarget(new ResultColumnRelationshipElement(targetColumn));
22613                                                                                        combinedQueryRelation
22614                                                                                                        .addSource(new ResultColumnRelationshipElement(sourceColumn));
22615
22616                                                                                        relation.addSource(new ResultColumnRelationshipElement(targetColumn));
22617                                                                                }
22618                                                                        }
22619                                                                        break;
22620                                                                } else {
22621                                                                        boolean flag = false;
22622
22623                                                                        for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) {
22624                                                                                ResultColumn sourceColumn = selectSetResultSetModel
22625                                                                                                .getColumns().get(j);
22626                                                                                List<String> splits = SQLUtil.parseNames(columnName.toString());        
22627                                                                                if (splits.size() > 1 && EDbVendor.dbvbigquery == getOption().getVendor()) {
22628                                                                                        if (getColumnName(sourceColumn.getName())
22629                                                                                                        .equalsIgnoreCase(getColumnName(splits.get(0)))
22630                                                                                                        || getColumnName(sourceColumn.getName())
22631                                                                                                                        .equalsIgnoreCase(getColumnName(splits.get(1)))) {
22632                                                                                                ResultColumn targetColumn = modelFactory
22633                                                                                                                .createSelectSetResultColumn(queryTable, sourceColumn);
22634
22635                                                                                                DataFlowRelationship combinedQueryRelation = modelFactory
22636                                                                                                                .createDataFlowRelation();
22637                                                                                                combinedQueryRelation.setEffectType(effectType);
22638                                                                                                combinedQueryRelation.setTarget(
22639                                                                                                                new ResultColumnRelationshipElement(targetColumn));
22640                                                                                                combinedQueryRelation.addSource(
22641                                                                                                                new ResultColumnRelationshipElement(sourceColumn));
22642
22643                                                                                                relation.addSource(
22644                                                                                                                new ResultColumnRelationshipElement(targetColumn));
22645                                                                                                flag = true;
22646                                                                                                break;
22647                                                                                        }
22648                                                                                }
22649                                                                                else if (getColumnName(sourceColumn.getName())
22650                                                                                                .equalsIgnoreCase(getColumnName(columnName))) {
22651                                                                                        ResultColumn targetColumn = modelFactory
22652                                                                                                        .createSelectSetResultColumn(queryTable, sourceColumn);
22653
22654                                                                                        DataFlowRelationship combinedQueryRelation = modelFactory
22655                                                                                                        .createDataFlowRelation();
22656                                                                                        combinedQueryRelation.setEffectType(effectType);
22657                                                                                        combinedQueryRelation
22658                                                                                                        .setTarget(new ResultColumnRelationshipElement(targetColumn));
22659                                                                                        combinedQueryRelation
22660                                                                                                        .addSource(new ResultColumnRelationshipElement(sourceColumn));
22661
22662                                                                                        relation.addSource(new ResultColumnRelationshipElement(targetColumn));
22663                                                                                        flag = true;
22664                                                                                        break;
22665                                                                                }
22666                                                                                else if (sourceColumn instanceof SelectSetResultColumn && ((SelectSetResultColumn)sourceColumn).getAliasSet().size() > 1) {
22667                                                                                        for (String alias : ((SelectSetResultColumn)sourceColumn).getAliasSet()) {
22668                                                                                                if (getColumnName(alias).equalsIgnoreCase(getColumnName(columnName))) {
22669                                                                                                        ResultColumn targetColumn = modelFactory
22670                                                                                                                        .createSelectSetResultColumn(queryTable, sourceColumn);
22671
22672                                                                                                        DataFlowRelationship combinedQueryRelation = modelFactory
22673                                                                                                                        .createDataFlowRelation();
22674                                                                                                        combinedQueryRelation.setEffectType(effectType);
22675                                                                                                        combinedQueryRelation.setTarget(
22676                                                                                                                        new ResultColumnRelationshipElement(targetColumn));
22677                                                                                                        combinedQueryRelation.addSource(
22678                                                                                                                        new ResultColumnRelationshipElement(sourceColumn));
22679
22680                                                                                                        relation.addSource(
22681                                                                                                                        new ResultColumnRelationshipElement(targetColumn));
22682                                                                                                        flag = true;
22683                                                                                                        break;
22684                                                                                                }
22685                                                                                        }
22686                                                                                        if (flag) {
22687                                                                                                break;
22688                                                                                        }
22689                                                                                }
22690                                                                        }
22691
22692                                                                        if (flag) {
22693                                                                                break;
22694                                                                        } else if (columnIndex != -1) {
22695                                                                                for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) {
22696                                                                                        ResultColumn sourceColumn = selectSetResultSetModel.getColumns().get(j);
22697                                                                                        if (!sourceColumn.getStarLinkColumns().isEmpty()) {
22698                                                                                                if (cteColumns != null) {
22699                                                                                                        if (j < cteColumns.size()) {
22700                                                                                                                ResultColumn targetColumn = queryTable.getColumns().get(j);
22701                                                                                                                relation.addSource(
22702                                                                                                                                new ResultColumnRelationshipElement(targetColumn));
22703                                                                                                        }
22704                                                                                                } 
22705                                                                                                else {
22706                                                                                                        if(sourceColumn.hasStarLinkColumn()) {
22707                                                                                                                for (TObjectName linkColumn : sourceColumn.getStarLinkColumnList()) {
22708                                                                                                                        if (getColumnName(linkColumn).equals(getColumnName(columnName))) {
22709                                                                                                                                ResultColumn targetColumn = modelFactory
22710                                                                                                                                                .createSelectSetResultColumn(queryTable, sourceColumn);
22711
22712                                                                                                                                DataFlowRelationship combinedQueryRelation = modelFactory
22713                                                                                                                                                .createDataFlowRelation();
22714                                                                                                                                combinedQueryRelation.setEffectType(effectType);
22715                                                                                                                                combinedQueryRelation.setTarget(
22716                                                                                                                                                new ResultColumnRelationshipElement(targetColumn, linkColumn));
22717                                                                                                                                combinedQueryRelation.addSource(
22718                                                                                                                                                new ResultColumnRelationshipElement(sourceColumn));
22719
22720                                                                                                                                relation.addSource(
22721                                                                                                                                                new ResultColumnRelationshipElement(targetColumn, linkColumn));
22722                                                                                                                                flag = true;
22723                                                                                                                                break;
22724                                                                                                                        }
22725                                                                                                                }
22726                                                                                                        }
22727                                                                                                        
22728                                                                                                        if(!flag) {
22729                                                                                                                ResultColumn targetColumn = modelFactory
22730                                                                                                                                .createSelectSetResultColumn(queryTable, sourceColumn);
22731
22732                                                                                                                DataFlowRelationship combinedQueryRelation = modelFactory
22733                                                                                                                                .createDataFlowRelation();
22734                                                                                                                combinedQueryRelation.setEffectType(effectType);
22735                                                                                                                combinedQueryRelation.setTarget(
22736                                                                                                                                new ResultColumnRelationshipElement(targetColumn));
22737                                                                                                                combinedQueryRelation.addSource(
22738                                                                                                                                new ResultColumnRelationshipElement(sourceColumn));
22739
22740                                                                                                                relation.addSource(
22741                                                                                                                                new ResultColumnRelationshipElement(targetColumn));
22742                                                                                                        }
22743                                                                                                }
22744                                                                                                flag = true;
22745                                                                                                break;
22746                                                                                        }
22747                                                                                }
22748                                                                        }
22749
22750                                                                        if (flag) {
22751                                                                                break;
22752                                                                        } else if (columnIndex < selectSetResultSetModel.getColumns().size()
22753                                                                                        && columnIndex != -1) {
22754                                                                                ResultColumn sourceColumn = selectSetResultSetModel.getColumns()
22755                                                                                                .get(columnIndex);
22756                                                                                if (cteColumns != null) {
22757                                                                                        boolean flag1 = false;
22758                                                                                        for (ResultColumn targetColumn : queryTable.getColumns()) {
22759                                                                                                if (getColumnName(targetColumn.getName())
22760                                                                                                                .equalsIgnoreCase(getColumnName(columnName))) {
22761                                                                                                        relation.addSource(
22762                                                                                                                        new ResultColumnRelationshipElement(targetColumn));
22763                                                                                                        flag1 = true;
22764                                                                                                        break;
22765                                                                                                }
22766                                                                                        }
22767                                                                                        if (!flag1 && columnIndex < cteColumns.size()){
22768                                                                                                ResultColumn targetColumn = queryTable.getColumns()
22769                                                                                                                .get(columnIndex);
22770                                                                                                relation.addSource(
22771                                                                                                                new ResultColumnRelationshipElement(targetColumn));
22772                                                                                        }
22773                                                                                } else {
22774                                                                                        ResultColumn targetColumn = modelFactory
22775                                                                                                        .createSelectSetResultColumn(queryTable, sourceColumn);
22776
22777                                                                                        DataFlowRelationship combinedQueryRelation = modelFactory
22778                                                                                                        .createDataFlowRelation();
22779                                                                                        combinedQueryRelation.setEffectType(effectType);
22780                                                                                        combinedQueryRelation
22781                                                                                                        .setTarget(new ResultColumnRelationshipElement(targetColumn));
22782                                                                                        combinedQueryRelation
22783                                                                                                        .addSource(new ResultColumnRelationshipElement(sourceColumn));
22784
22785                                                                                        relation.addSource(new ResultColumnRelationshipElement(targetColumn));
22786                                                                                }
22787                                                                                flag = true;
22788                                                                                break;
22789                                                                        }
22790
22791                                                                        if (flag) {
22792                                                                                break;
22793                                                                        }
22794                                                                }
22795                                                        } else if (cteColumns != null) {
22796                                                                if (getColumnName(columnName).equals("*")) {
22797                                                                        Map<String, Pair<String, TExpression>> replaceAsIdentifierMap = new HashMap<String, Pair<String, TExpression>>();
22798                                                                        Map<String, TObjectName> replaceColumnMap = new HashMap<String, TObjectName>();
22799                                                                        if(modelObject instanceof ResultColumn && ((ResultColumn)modelObject).getColumnObject() instanceof TResultColumn) {
22800                                                                                TResultColumn resultColumn =  (TResultColumn )((ResultColumn)modelObject).getColumnObject();
22801                                                                                if(resultColumn.getReplaceExprAsIdentifiers()!=null && resultColumn.getReplaceExprAsIdentifiers().size()>0) {
22802                                                                                        for(TReplaceExprAsIdentifier replace: resultColumn.getReplaceExprAsIdentifiers()) {
22803                                                                                                replaceAsIdentifierMap.put(replace.getIdentifier().toString(), new Pair<String, TExpression>(resultColumn.getExpr().getExceptReplaceClause().toString(), replace.getExpr()));
22804                                                                                                replaceColumnMap.put(replace.getIdentifier().toString(), replace.getIdentifier());
22805                                                                                        }
22806                                                                                        ResultSet resultSet = ((ResultColumn)modelObject).getResultSet();
22807                                                                                        
22808                                                                                        if (columnName.getSourceColumn() != null) {
22809                                                                                                Object model = modelManager.getModel(columnName.getSourceColumn());
22810                                                                                                if (model instanceof ResultColumn && ((ResultColumn)model).getResultSet().isDetermined()) {
22811                                                                                                        resultSet.getColumns().clear();
22812                                                                                                }
22813                                                                                        } else if (columnName.getSourceTable() != null) {
22814                                                                                                Object tableModel = modelManager.getModel(columnName.getSourceTable());
22815                                                                                                if (tableModel instanceof Table && ((Table)tableModel).isDetermined()) {
22816                                                                                                        resultSet.getColumns().clear();
22817                                                                                                }
22818                                                                                        }
22819                                                                                }
22820                                                                        }
22821                                                                        
22822                                                                        for (int j = 0; j < cteColumns.size(); j++) {
22823                                                                                ResultColumn targetColumn = queryTable.getColumns().get(j);
22824
22825                                                                                if (exceptColumnList != null) {
22826                                                                                        boolean flag = false;
22827                                                                                        for (TObjectName objectName : exceptColumnList) {
22828                                                                                                if (getColumnName(objectName)
22829                                                                                                                .equals(getColumnName(targetColumn.getName()))) {
22830                                                                                                        flag = true;
22831                                                                                                        break;
22832                                                                                                }
22833                                                                                        }
22834                                                                                        if (flag) {
22835                                                                                                continue;
22836                                                                                        }
22837                                                                                }
22838
22839                                                                                if (replaceAsIdentifierMap.containsKey(targetColumn.getName())) {
22840                                                                                        Pair<String, TExpression> expr = replaceAsIdentifierMap.get(targetColumn.getName());
22841                                                                                        ResultSet resultSet = ((ResultColumn)modelObject).getResultSet();
22842                                                                                        ResultColumn resultColumn = modelFactory.createResultColumn(resultSet, replaceColumnMap.get(targetColumn.getName()));
22843                                                                                        Transform transform = new Transform();
22844                                                                                        transform.setType(Transform.EXPRESSION);
22845                                                                                        TObjectName expression = new TObjectName();
22846                                                                                        expression.setString(expr.first);
22847                                                                                transform.setCode(expression);
22848                                                                                        resultColumn.setTransform(transform);
22849                                                                                        analyzeResultColumnExpressionRelation(resultColumn, expr.second);
22850                                                                                } else {
22851                                                                                        if(!replaceAsIdentifierMap.isEmpty()) {
22852                                                                                                ResultSet resultSet = ((ResultColumn) modelObject).getResultSet();
22853                                                                                                TObjectName resultColumnName = new TObjectName();
22854                                                                                                resultColumnName.setString(targetColumn.getName());
22855                                                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSet,
22856                                                                                                                resultColumnName);
22857                                                                                                DataFlowRelationship relation1 = modelFactory.createDataFlowRelation();
22858                                                                                                relation1.setEffectType(effectType);
22859                                                                                                relation1.setProcess(process);
22860                                                                                                relation1.setTarget(new ResultColumnRelationshipElement(resultColumn));
22861                                                                                                relation1.addSource(new ResultColumnRelationshipElement(targetColumn));
22862                                                                                        }
22863                                                                                        else {
22864                                                                                                relation.addSource(
22865                                                                                                        new ResultColumnRelationshipElement(targetColumn));
22866                                                                                        }
22867                                                                                }
22868                                                                        }
22869                                                                        break;
22870                                                                } else {
22871                                                                        boolean flag = false;
22872
22873                                                                        for (int j = 0; j < cteColumns.size(); j++) {
22874                                                                                TObjectName sourceColumn = cteColumns.getObjectName(j);
22875
22876                                                                                if (getColumnName(sourceColumn).equalsIgnoreCase(getColumnName(columnName))) {
22877                                                                                        ResultColumn targetColumn = queryTable.getColumns().get(j);
22878
22879                                                                                        relation.addSource(new ResultColumnRelationshipElement(targetColumn));
22880                                                                                        flag = true;
22881                                                                                        break;
22882                                                                                }
22883                                                                        }
22884
22885                                                                        if (flag) {
22886                                                                                break;
22887                                                                        }
22888                                                                }
22889                                                        }
22890
22891                                                        if (columnName.getSourceColumn() != null) {
22892                                                                Object model = modelManager.getModel(columnName.getSourceColumn());
22893                                                                if (model instanceof ResultColumn) {
22894                                                                        ResultColumn resultColumn = (ResultColumn) model;
22895                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
22896                                                                }
22897                                                        } else if (columnName.getSourceTable() != null) {
22898                                                                Object tableModel = modelManager.getModel(columnName.getSourceTable());
22899                                                                if (tableModel instanceof Table) {
22900                                                                        Object model = modelManager
22901                                                                                        .getModel(new Pair<Table, TObjectName>((Table) tableModel, columnName));
22902                                                                        if (model instanceof TableColumn) {
22903                                                                                relation.addSource(new TableColumnRelationshipElement((TableColumn) model));
22904                                                                        }
22905                                                                }
22906                                                        }
22907                                                } else {
22908                                                        List<ResultColumn> columns = queryTable.getColumns();
22909                                                        if (getColumnName(columnName).equals("*")) {
22910                                                                Map<String, Pair<String, TExpression>> replaceAsIdentifierMap = new HashMap<String, Pair<String, TExpression>>();
22911                                                                Map<String, TObjectName> replaceColumnMap = new HashMap<String, TObjectName>();
22912                                                                if(modelObject instanceof ResultColumn && ((ResultColumn)modelObject).getColumnObject() instanceof TResultColumn) {
22913                                                                        TResultColumn resultColumn =  (TResultColumn )((ResultColumn)modelObject).getColumnObject();
22914                                                                        if(resultColumn.getReplaceExprAsIdentifiers()!=null && resultColumn.getReplaceExprAsIdentifiers().size()>0) {
22915                                                                                for(TReplaceExprAsIdentifier replace: resultColumn.getReplaceExprAsIdentifiers()) {
22916                                                                                        replaceAsIdentifierMap.put(replace.getIdentifier().toString(), new Pair<String, TExpression>(resultColumn.getExpr().getExceptReplaceClause().toString(), replace.getExpr()));
22917                                                                                        replaceColumnMap.put(replace.getIdentifier().toString(), replace.getIdentifier());
22918                                                                                }
22919                                                                                ResultSet resultSet = ((ResultColumn)modelObject).getResultSet();
22920                                                                                if(queryTable.isDetermined()) {
22921                                                                                        resultSet.getColumns().clear();
22922                                                                                }
22923                                                                        }
22924                                                                }
22925                                                                
22926                                                                int index = 0;
22927                                                                for (int j = 0; j < queryTable.getColumns().size(); j++) {
22928                                                                        ResultColumn targetColumn = queryTable.getColumns().get(j);
22929                                                                        if (exceptColumnList != null) {
22930                                                                                boolean flag = false;
22931                                                                                for (TObjectName objectName : exceptColumnList) {
22932                                                                                        if (getColumnName(objectName)
22933                                                                                                        .equals(getColumnName(targetColumn.getName()))) {
22934                                                                                                flag = true;
22935                                                                                                break;
22936                                                                                        }
22937                                                                                }
22938                                                                                if (flag) {
22939                                                                                        continue;
22940                                                                                }
22941                                                                        }
22942                                                                        
22943                                                                        if (replaceAsIdentifierMap.containsKey(targetColumn.getName())) {
22944                                                                                Pair<String, TExpression> expr = replaceAsIdentifierMap.get(targetColumn.getName());
22945                                                                                ResultSet resultSet = ((ResultColumn)modelObject).getResultSet();
22946                                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSet, replaceColumnMap.get(targetColumn.getName()));
22947                                                                                Transform transform = new Transform();
22948                                                                                transform.setType(Transform.EXPRESSION);
22949                                                                                TObjectName expression = new TObjectName();
22950                                                                                expression.setString(expr.first);
22951                                                                        transform.setCode(expression);
22952                                                                                resultColumn.setTransform(transform);
22953                                                                                analyzeResultColumnExpressionRelation(resultColumn, expr.second);
22954                                                                        } else {
22955                                                                                if(!replaceAsIdentifierMap.isEmpty()) {
22956                                                                                        ResultSet resultSet = ((ResultColumn) modelObject).getResultSet();
22957                                                                                        TObjectName resultColumnName = new TObjectName();
22958                                                                                        resultColumnName.setString(targetColumn.getName());
22959                                                                                        ResultColumn resultColumn = modelFactory.createResultColumn(resultSet,
22960                                                                                                        resultColumnName);
22961                                                                                        DataFlowRelationship relation1 = modelFactory.createDataFlowRelation();
22962                                                                                        relation1.setEffectType(effectType);
22963                                                                                        relation1.setProcess(process);
22964                                                                                        relation1.setTarget(new ResultColumnRelationshipElement(resultColumn));
22965                                                                                        relation1.addSource(new ResultColumnRelationshipElement(targetColumn));
22966                                                                                }
22967                                                                                else {
22968                                                                                        relation.addSource(
22969                                                                                                new ResultColumnRelationshipElement(targetColumn));
22970                                                                                }
22971                                                                        }
22972                                                                        index++;
22973                                                                }
22974                                                        } else {
22975                                                                if (table.getCTE() != null) {
22976                                                                        
22977                                                                        if (modelObject instanceof TableColumn) {
22978                                                                                Table modelTable = ((TableColumn) modelObject).getTable();
22979                                                                                if (modelTable.getSubType() == SubType.unnest) {
22980                                                                                        boolean find = false;
22981                                                                                        for (k = 0; k < columns.size(); k++) {
22982                                                                                                ResultColumn column = columns.get(k);
22983                                                                                                if (column.isStruct()) {
22984                                                                                                        List<String> names = SQLUtil.parseNames(column.getName());
22985                                                                                                        for (String name : names) {
22986                                                                                                                if (getColumnName(name).equals(getColumnName(columnName))) {
22987                                                                                                                        DataFlowRelationship unnestRelation = modelFactory.createDataFlowRelation();
22988                                                                                                                        unnestRelation.setEffectType(effectType);
22989                                                                                                                        unnestRelation.setProcess(process);
22990                                                                                                                        unnestRelation.addSource(new ResultColumnRelationshipElement(
22991                                                                                                                                        column, columnName));
22992                                                                                                                        TObjectName unnestTableColumnName = new TObjectName();
22993                                                                                                                        unnestTableColumnName.setString(names.get(names.size()-1));
22994                                                                                                                        TableColumn unnestTableColumn = modelFactory.createTableColumn(modelTable, unnestTableColumnName, true);
22995                                                                                                                        unnestRelation.setTarget(new TableColumnRelationshipElement(unnestTableColumn));
22996                                                                                                                        find = true;
22997                                                                                                                }
22998                                                                                                        }
22999                                                                                                        List<String> names1 = SQLUtil.parseNames(column.getName());
23000                                                                                                        if (names.size() == 1 && names1.size() >= 1) {
23001                                                                                                                for (String name : names1) {
23002                                                                                                                        if (getColumnName(name)
23003                                                                                                                                        .equals(getColumnName(column.getName()))) {
23004                                                                                                                                DataFlowRelationship unnestRelation = modelFactory.createDataFlowRelation();
23005                                                                                                                                unnestRelation.setEffectType(effectType);
23006                                                                                                                                unnestRelation.setProcess(process);
23007                                                                                                                                unnestRelation.addSource(new ResultColumnRelationshipElement(
23008                                                                                                                                                column, columnName));
23009                                                                                                                                TObjectName unnestTableColumnName = new TObjectName();
23010                                                                                                                                unnestTableColumnName.setString(names1.get(names.size()-1));
23011                                                                                                                                TableColumn unnestTableColumn = modelFactory.createTableColumn(modelTable, unnestTableColumnName, true);
23012                                                                                                                                unnestRelation.setTarget(new TableColumnRelationshipElement(unnestTableColumn));
23013                                                                                                                                find = true;
23014                                                                                                                        }
23015                                                                                                                }
23016                                                                                                        }
23017                                                                                                }
23018                                                                                        }
23019                                                                                        if (find) {
23020                                                                                                modelTable.getColumns().remove(modelObject);
23021                                                                                                break;
23022                                                                                        }
23023                                                                                }
23024                                                                        }
23025                                                                        
23026                                                                        for (k = 0; k < columns.size(); k++) {
23027                                                                                ResultColumn column = columns.get(k);
23028                                                                                if ("*".equals(column.getName())) {
23029                                                                                        if (!containsStarColumn(column, columnName)) {
23030                                                                                                column.bindStarLinkColumn(columnName);
23031                                                                                        }
23032                                                                                        relation.addSource(new ResultColumnRelationshipElement(column, columnName));
23033                                                                                } else if (DlineageUtil.compareColumnIdentifier(getColumnName(columnName),
23034                                                                                                DlineageUtil.getIdentifierNormalColumnName(column.getName()))) {
23035                                                                                        if (!column.equals(modelObject)) {
23036                                                                                                relation.addSource(
23037                                                                                                                new ResultColumnRelationshipElement(column, columnName));
23038                                                                                        }
23039                                                                                        break;
23040                                                                                } else if(column.isStruct()) {
23041                                                                                        List<String> names = SQLUtil.parseNames(column.getName());
23042                                                                                        for(String name: names) {
23043                                                                                                if (getColumnName(name)
23044                                                                                                                .equals(getColumnName(columnName))) {
23045                                                                                                        relation.addSource(
23046                                                                                                                        new ResultColumnRelationshipElement(column, columnName));
23047                                                                                                }
23048                                                                                        }
23049                                                                                        List<String> names1 = SQLUtil.parseNames(column.getName());
23050                                                                                        if (names.size() == 1 && names1.size() >= 1) {
23051                                                                                                for(String name: names1) {
23052                                                                                                        if (getColumnName(name)
23053                                                                                                                        .equals(getColumnName(column.getName()))) {
23054                                                                                                                relation.addSource(
23055                                                                                                                                new ResultColumnRelationshipElement(column, columnName));
23056                                                                                                        }
23057                                                                                                }
23058                                                                                        }
23059                                                                                }
23060                                                                        }
23061                                                                } else if (table.getAliasClause() != null
23062                                                                                && table.getAliasClause().getColumns() != null) {
23063                                                                        for (k = 0; k < columns.size(); k++) {
23064                                                                                ResultColumn column = columns.get(k);
23065                                                                                List<String> splits = SQLUtil.parseNames(columnName.toString());        
23066                                                                                if ("*".equals(column.getName())) {
23067                                                                                        if (!containsStarColumn(column, columnName)) {
23068                                                                                                column.bindStarLinkColumn(columnName);
23069                                                                                        }
23070                                                                                        relation.addSource(new ResultColumnRelationshipElement(column, columnName));
23071                                                                                } else if (splits.size() > 1 && EDbVendor.dbvbigquery == getOption().getVendor()) {
23072                                                                                        if (DlineageUtil.compareColumnIdentifier(getColumnName(splits.get(0)),
23073                                                                                                        DlineageUtil.getIdentifierNormalColumnName(column.getName()))) {
23074                                                                                                if (!column.equals(modelObject)) {
23075                                                                                                        relation.addSource(new ResultColumnRelationshipElement(column,
23076                                                                                                                        columnName));
23077                                                                                                }
23078                                                                                                break;
23079                                                                                        }
23080                                                                                } else if (DlineageUtil.compareColumnIdentifier(getColumnName(columnName),
23081                                                                                                DlineageUtil.getIdentifierNormalColumnName(column.getName()))) {
23082                                                                                        if (!column.equals(modelObject)) {
23083                                                                                                relation.addSource(
23084                                                                                                                new ResultColumnRelationshipElement(column, columnName));
23085                                                                                        }
23086                                                                                        break;
23087                                                                                }
23088                                                                        }
23089                                                                } else if (table.getSubquery() != null || (table.getTableExpr() != null
23090                                                                                && table.getTableExpr().getSubQuery() != null)) {
23091                                                                        TSelectSqlStatement select = table.getSubquery();
23092                                                                        if (select == null) {
23093                                                                                select = table.getTableExpr().getSubQuery();
23094                                                                        }
23095                                                                        if (columnName.getSourceTable() != null) {
23096                                                                                Object tableModel = modelManager.getModel(columnName.getSourceTable());
23097                                                                                appendResultColumnRelationSource(modelObject, relation, columnIndex, columnName,
23098                                                                                                tableModel);
23099                                                                        } else if (columnName.getObjectToken() != null
23100                                                                                        && !SQLUtil.isEmpty(table.getAliasName())) {
23101                                                                                if (DlineageUtil.compareTableIdentifier(columnName.getObjectToken().toString(),
23102                                                                                                table.getAliasName())) {
23103                                                                                        Object tableModel = modelManager.getModel(table);
23104                                                                                        appendResultColumnRelationSource(modelObject, relation, columnIndex,
23105                                                                                                        columnName, tableModel);
23106                                                                                }
23107                                                                        } else if(columns!=null) {
23108                                                                                for (k = 0; k < columns.size(); k++) {
23109                                                                                        ResultColumn column = columns.get(k);
23110                                                                                        List<String> splits = SQLUtil.parseNames(columnName.toString());        
23111                                                                                        if ("*".equals(column.getName())) {
23112                                                                                                if (!containsStarColumn(column, columnName)) {
23113                                                                                                        column.bindStarLinkColumn(columnName);
23114                                                                                                }
23115                                                                                                relation.addSource(new ResultColumnRelationshipElement(column, columnName));
23116                                                                                        } else if (splits.size() > 1 && EDbVendor.dbvbigquery == getOption().getVendor()) {
23117                                                                                                if (DlineageUtil.compareColumnIdentifier(getColumnName(splits.get(0)),
23118                                                                                                                DlineageUtil.getIdentifierNormalColumnName(column.getName()))) {
23119                                                                                                        if (!column.equals(modelObject)) {
23120                                                                                                                relation.addSource(new ResultColumnRelationshipElement(column,
23121                                                                                                                                columnName));
23122                                                                                                        }
23123                                                                                                        break;
23124                                                                                                }
23125                                                                                        } else if (DlineageUtil.compareColumnIdentifier(getColumnName(columnName),
23126                                                                                                        DlineageUtil.getIdentifierNormalColumnName(column.getName()))) {
23127                                                                                                if (!column.equals(modelObject)) {
23128                                                                                                        relation.addSource(
23129                                                                                                                        new ResultColumnRelationshipElement(column, columnName));
23130                                                                                                }
23131                                                                                                break;
23132                                                                                        }
23133                                                                                }
23134                                                                        }
23135                                                                } else if (table.getOutputMerge() != null) {
23136                                                                        if (columnName.getSourceColumn() != null) {
23137                                                                                Object model = modelManager.getModel(columnName.getSourceColumn());
23138                                                                                if (model instanceof ResultColumn) {
23139                                                                                        ResultColumn resultColumn = (ResultColumn) model;
23140                                                                                        if ("*".equals(resultColumn.getName())
23141                                                                                                        && !containsStarColumn(resultColumn, columnName)) {
23142                                                                                                resultColumn.bindStarLinkColumn(columnName);
23143                                                                                        }
23144                                                                                        relation.addSource(
23145                                                                                                        new ResultColumnRelationshipElement(resultColumn, columnName));
23146                                                                                }
23147                                                                        } else if (columnName.getSourceTable() != null) {
23148                                                                                Object tableModel = modelManager.getModel(columnName.getSourceTable());
23149                                                                                appendResultColumnRelationSource(modelObject, relation, columnIndex, columnName,
23150                                                                                                tableModel);
23151                                                                        } else if (columnName.getObjectToken() != null
23152                                                                                        && !SQLUtil.isEmpty(table.getAliasName())) {
23153                                                                                if (DlineageUtil.compareTableIdentifier(columnName.getObjectToken().toString(),
23154                                                                                                table.getAliasName())) {
23155                                                                                        Object tableModel = modelManager.getModel(table);
23156                                                                                        appendResultColumnRelationSource(modelObject, relation, columnIndex,
23157                                                                                                        columnName, tableModel);
23158                                                                                }
23159                                                                        }
23160                                                                }
23161                                                        }
23162                                                }
23163                                        }
23164                                }
23165                        }
23166                        if (relation.getSources().size() == 0 && isKeyword(columnName)) {
23167                                Table constantTable = modelFactory.createConstantsTable(stmtStack.peek());
23168                                TableColumn constantColumn = modelFactory.createTableColumn(constantTable, columnName, true);
23169                                relation.addSource(new ConstantRelationshipElement(constantColumn));
23170                        }
23171
23172                        if (relation.getSources().size() > 0) {
23173                                for (RelationshipElement<?> sourceItem: relation.getSources()) {
23174                                        Object source = sourceItem.getElement();
23175                                        ImpactRelationship impactRelation = null;
23176                                        if (source instanceof ResultColumn
23177                                                        && !((ResultColumn) source).getResultSet().getRelationRows().getHoldRelations().isEmpty()) {
23178                                                impactRelation = modelFactory.createImpactRelation();
23179                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
23180                                                                ((ResultColumn) source).getResultSet().getRelationRows()));
23181                                                impactRelation.setEffectType(effectType);
23182                                        } else if (source instanceof TableColumn
23183                                                        && !((TableColumn) source).getTable().getRelationRows().getHoldRelations().isEmpty()) {
23184                                                impactRelation = modelFactory.createImpactRelation();
23185                                                impactRelation.addSource(new RelationRowsRelationshipElement<TableRelationRows>(
23186                                                                ((TableColumn) source).getTable().getRelationRows()));
23187                                                impactRelation.setEffectType(effectType);;
23188                                        }
23189
23190                                        if (impactRelation == null) {
23191                                                continue;
23192                                        }
23193
23194                                        Object target = relation.getTarget().getElement();
23195                                        if (target instanceof ResultColumn) {
23196                                                impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
23197                                                                ((ResultColumn) target).getResultSet().getRelationRows()));
23198                                        } else if (target instanceof TableColumn) {
23199                                                impactRelation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(
23200                                                                ((TableColumn) target).getTable().getRelationRows()));
23201                                        }
23202
23203                                        if (impactRelation.getSources().iterator().next().getElement() == impactRelation.getTarget().getElement()) {
23204                                                modelManager.removeRelation(impactRelation);
23205                                        }
23206                                }
23207                        }
23208                }
23209                return relation;
23210        }
23211
23212        private boolean isNotInProcedure(Table table) {
23213        TStoredProcedureSqlStatement stmt = getProcedureParent(stmtStack.peek());
23214                Procedure procedure = this.modelFactory.createProcedure(stmt);
23215                if(procedure!=null && procedure.getName().equals(table.getParent())){
23216                        return false;
23217                }
23218                return true;
23219        }
23220
23221        private boolean hasJoin(TCustomSqlStatement stmt) {
23222                if (stmt.getJoins() == null || stmt.getJoins().size() == 0)
23223                        return false;
23224                if (stmt.getJoins().size() > 1) {
23225                        return true;
23226                }
23227                TJoinItemList joinItems = stmt.getJoins().getJoin(0).getJoinItems();
23228                if (joinItems == null || joinItems.size() == 0) {
23229                        return false;
23230                }
23231                return true;
23232        }
23233
23234        private boolean isInQuery(TSelectSqlStatement query, TResultColumn column) {
23235                if(query == null)
23236                        return false;
23237                TResultColumnList columns = query.getResultColumnList();
23238                if (columns != null) {
23239                        for (int i = 0; i < columns.size(); i++) {
23240                                if (columns.getResultColumn(i).equals(column)) {
23241                                        return true;
23242                                }
23243                        }
23244                }
23245                return false;
23246        }
23247
23248        private void appendResultColumnRelationSource(Object modelObject, DataFlowRelationship relation, int columnIndex,
23249                        TObjectName columnName, Object tableModel) {
23250                if (tableModel instanceof Table) {
23251                        Object model = modelManager.getModel(new Pair<Table, TObjectName>((Table) tableModel, columnName));
23252                        if (model instanceof TableColumn) {
23253                                relation.addSource(new TableColumnRelationshipElement((TableColumn) model));
23254                        }
23255                } else if (tableModel instanceof ResultSet) {
23256                        List<ResultColumn> queryColumns = ((ResultSet) tableModel).getColumns();
23257                        boolean flag = false;
23258                        for (int l = 0; l < queryColumns.size(); l++) {
23259                                ResultColumn column = queryColumns.get(l);
23260                                if (DlineageUtil.compareColumnIdentifier(getColumnName(columnName),
23261                                                DlineageUtil.getIdentifierNormalColumnName(column.getName()))) {
23262                                        if (!column.equals(modelObject)) {
23263                                                relation.addSource(new ResultColumnRelationshipElement(column, columnName));
23264                                                flag = true;
23265                                        }
23266                                        break;
23267                                }
23268                        }
23269                        if (!flag) {
23270                                for (int l = 0; l < queryColumns.size(); l++) {
23271                                        ResultColumn column = queryColumns.get(l);
23272                                        if ("*".equals(column.getName())) {
23273                                                if (!containsStarColumn(column, columnName)) {
23274                                                        column.bindStarLinkColumn(columnName);
23275                                                }
23276                                                relation.addSource(new ResultColumnRelationshipElement(column, columnName));
23277                                                flag = true;
23278                                                break;
23279                                        }
23280                                }
23281                        }
23282                        if (!flag && columnIndex < queryColumns.size() && columnIndex != -1) {
23283                                relation.addSource(new ResultColumnRelationshipElement(queryColumns.get(columnIndex), columnName));
23284                        }
23285                }
23286        }
23287
23288        private boolean isApplyJoin(TCustomSqlStatement stmt) {
23289                if (stmt.getJoins() == null || stmt.getJoins().size() == 0)
23290                        return false;
23291                TJoinItemList joinItems = stmt.getJoins().getJoin(0).getJoinItems();
23292                if (joinItems == null || joinItems.size() == 0) {
23293                        return false;
23294                }
23295                if (joinItems.getJoinItem(0).getJoinType() == EJoinType.crossapply
23296                                || joinItems.getJoinItem(0).getJoinType() == EJoinType.outerapply)
23297                        return true;
23298                return false;
23299        }
23300
23301        private boolean containsStarColumn(ResultColumn resultColumn, TObjectName columnName) {
23302                String targetColumnName = getColumnName(columnName);
23303                if (resultColumn.hasStarLinkColumn()) {
23304                        return resultColumn.getStarLinkColumns().containsKey(targetColumnName);
23305                }
23306                return false;
23307        }
23308
23309        private void analyzeAggregate(TFunctionCall function, TExpression expr) {
23310                TCustomSqlStatement stmt = stmtStack.peek();
23311                ResultSet resultSet = (ResultSet) modelManager.getModel(stmt.getResultColumnList());
23312                if (resultSet == null) {
23313                        return;
23314                }
23315
23316                if (expr != null) {
23317                        columnsInExpr visitor = new columnsInExpr();
23318                        expr.inOrderTraverse(visitor);
23319                        List<TObjectName> objectNames = visitor.getObjectNames();
23320                        for (int j = 0; j < objectNames.size(); j++) {
23321                                TObjectName columnName = objectNames.get(j);
23322
23323                                if (columnName.getDbObjectType() == EDbObjectType.variable) {
23324                                        continue;
23325                                }
23326
23327                                if (columnName.getColumnNameOnly().startsWith("@")
23328                                                && (option.getVendor() == EDbVendor.dbvmssql || option.getVendor() == EDbVendor.dbvazuresql)) {
23329                                        continue;
23330                                }
23331
23332                                if (columnName.getColumnNameOnly().startsWith(":")
23333                                                && (option.getVendor() == EDbVendor.dbvhana || option.getVendor() == EDbVendor.dbvteradata)) {
23334                                        continue;
23335                                }
23336
23337                                Object targetModel0 = modelManager.getModel(function.getFunctionName());
23338                                if (targetModel0 == null && function.getFunctionType() == EFunctionType.array_agg_t && modelManager.getModel(function) instanceof Function) {
23339                                        Function functionModel = (Function) modelManager.getModel(function);
23340                                        if (functionModel.getColumns().size() == 1) {
23341                                                targetModel0 = ((Function) functionModel).getColumns().get(0);
23342                                        }
23343                                }
23344                                if (!(targetModel0 instanceof ResultColumn)) {
23345                                        continue;
23346                                }
23347                                ResultColumn targetResultColumn0 = (ResultColumn) targetModel0;
23348                                AbstractRelationship relation = modelFactory.createRecordSetRelation();
23349                                relation.setEffectType(EffectType.function);
23350                                relation.setFunction(function.getFunctionName().toString());
23351                                relation.setTarget(new ResultColumnRelationshipElement(targetResultColumn0));
23352                                TTable table = modelManager.getTable(stmt, columnName);
23353                                if (table != null) {
23354                                        if (modelManager.getModel(table) instanceof Table) {
23355                                                Table tableModel = (Table) modelManager.getModel(table);
23356                                                if (tableModel != null) {
23357                                                        TableColumn columnModel = modelFactory.createTableColumn(tableModel, columnName, false);
23358                                                        if (columnModel != null) {
23359                                                                relation.addSource(
23360                                                                                new TableColumnRelationshipElement(columnModel, columnName.getLocation()));
23361                                                        }
23362                                                }
23363                                        } else if (modelManager.getModel(table) instanceof QueryTable) {
23364                                                Object model = modelManager.getModel(columnName.getSourceColumn());
23365                                                if (model instanceof ResultColumn) {
23366                                                        ResultColumn resultColumn = (ResultColumn) model;
23367                                                        if (resultColumn != null) {
23368                                                                relation.addSource(
23369                                                                                new ResultColumnRelationshipElement(resultColumn, columnName.getLocation()));
23370                                                        }
23371                                                }
23372                                        }
23373                                }
23374                        }
23375
23376                        List<TParseTreeNode> functions = visitor.getFunctions();
23377                        for (int j = 0; j < functions.size(); j++) {
23378                                TParseTreeNode functionObj = functions.get(j);
23379                                Object functionModel = modelManager.getModel(functionObj);
23380                                if (functionModel == null) {
23381                                        functionModel = createFunction(functionObj);
23382                                }
23383                                if (functionModel instanceof Function) {
23384                                        Object targetModel1 = modelManager.getModel(function.getFunctionName());
23385                                        if (targetModel1 == null && function.getFunctionType() == EFunctionType.array_agg_t && modelManager.getModel(function) instanceof Function) {
23386                                                Function functionModel1 = (Function) modelManager.getModel(function);
23387                                                if (functionModel1.getColumns().size() == 1) {
23388                                                        targetModel1 = ((Function) functionModel1).getColumns().get(0);
23389                                                }
23390                                        }
23391                                        if (!(targetModel1 instanceof ResultColumn)) {
23392                                                continue;
23393                                        }
23394                                        ResultColumn targetRC_func = (ResultColumn) targetModel1;
23395                                        AbstractRelationship relation;
23396                                        if ("COUNT".equalsIgnoreCase(function.getFunctionName().toString())) {
23397                                                // relation = modelFactory.createDataFlowRelation();
23398                                                relation = modelFactory.createRecordSetRelation();
23399                                        } else {
23400                                                relation = modelFactory.createRecordSetRelation();
23401                                        }
23402                                        relation.setEffectType(EffectType.function);
23403                                        relation.setFunction(function.getFunctionName().toString());
23404                                        relation.setTarget(new ResultColumnRelationshipElement(targetRC_func));
23405
23406                                        if (functionObj instanceof TFunctionCall) {
23407                                                ResultColumn resultColumn = (ResultColumn) modelManager
23408                                                                .getModel(((TFunctionCall) functionObj).getFunctionName());
23409                                                if (resultColumn != null) {
23410                                                        ResultColumnRelationshipElement element = new ResultColumnRelationshipElement(resultColumn,
23411                                                                        ((TFunctionCall) functionObj).getFunctionName().getLocation());
23412                                                        relation.addSource(element);
23413                                                }
23414                                        }
23415                                        if (functionObj instanceof TCaseExpression) {
23416                                                ResultColumn resultColumn = (ResultColumn) modelManager
23417                                                                .getModel(((TCaseExpression) functionObj).getWhenClauseItemList());
23418                                                if (resultColumn != null) {
23419                                                        ResultColumnRelationshipElement element = new ResultColumnRelationshipElement(resultColumn);
23420                                                        relation.addSource(element);
23421                                                }
23422                                        }
23423                                } else if (functionModel instanceof Table) {
23424                                        Object targetModel2 = modelManager.getModel(function.getFunctionName());
23425                                        if (targetModel2 == null && function.getFunctionType() == EFunctionType.array_agg_t && modelManager.getModel(function) instanceof Function) {
23426                                                Function functionModel1 = (Function) modelManager.getModel(function);
23427                                                if (functionModel1.getColumns().size() == 1) {
23428                                                        targetModel2 = ((Function) functionModel1).getColumns().get(0);
23429                                                }
23430                                        }
23431                                        if (!(targetModel2 instanceof ResultColumn)) {
23432                                                continue;
23433                                        }
23434                                        ResultColumn targetRC_table = (ResultColumn) targetModel2;
23435                                        TableColumn tableColumn = modelFactory.createTableColumn((Table) functionModel,
23436                                                        function.getFunctionName(), false);
23437                                        AbstractRelationship relation;
23438                                        if ("COUNT".equalsIgnoreCase(function.getFunctionName().toString())) {
23439                                                // relation = modelFactory.createDataFlowRelation();
23440                                                relation = modelFactory.createRecordSetRelation();
23441                                        } else {
23442                                                relation = modelFactory.createRecordSetRelation();
23443                                        }
23444                                        relation.setEffectType(EffectType.function);
23445                                        relation.setFunction(function.getFunctionName().toString());
23446                                        relation.setTarget(new ResultColumnRelationshipElement(targetRC_table));
23447                                        TableColumnRelationshipElement element = new TableColumnRelationshipElement(tableColumn);
23448                                        relation.addSource(element);
23449                                }
23450                        }
23451                }
23452
23453                if (expr == null || "COUNT".equalsIgnoreCase(function.getFunctionName().toString())) {
23454                        if ("COUNT".equalsIgnoreCase(function.getFunctionName().toString())
23455                                        // https://e.gitee.com/gudusoft/issues/list?issue=I4L5EO
23456                                        // 对于非 count() 函数，当有group by clause时， RelationRows 不参与indirect dataflow,
23457                                        // 让位与group by clause中的column.
23458                                        || ((TSelectSqlStatement) stmt).getGroupByClause() == null) {
23459                                TTableList tables = stmt.getTables();
23460                                if (tables != null) {
23461                                        for (int i = 0; i < tables.size(); i++) {
23462                                                TTable table = tables.getTable(i);
23463                                                if (modelManager.getModel(table) == null && table.getSubquery() == null) {
23464                                                        modelFactory.createTable(table);
23465                                                }
23466                                                if (modelManager.getModel(table) instanceof Table) {
23467                                                        Object targetModel3 = modelManager.getModel(function.getFunctionName());
23468                                                        if (targetModel3 == null && function.getFunctionType() == EFunctionType.array_agg_t && modelManager.getModel(function) instanceof Function) {
23469                                                                Function functionModel = (Function) modelManager.getModel(function);
23470                                                                if (functionModel.getColumns().size() == 1) {
23471                                                                        targetModel3 = functionModel.getColumns().get(0);
23472                                                                }
23473                                                        }
23474                                                        if (!(targetModel3 instanceof ResultColumn)) {
23475                                                                continue;
23476                                                        }
23477                                                        ResultColumn targetRC_tbl = (ResultColumn) targetModel3;
23478                                                        Table tableModel = (Table) modelManager.getModel(table);
23479                                                        AbstractRelationship relation;
23480                                                        if ("COUNT".equalsIgnoreCase(function.getFunctionName().toString())) {
23481                                                                // relation = modelFactory.createDataFlowRelation();
23482                                                                relation = modelFactory.createRecordSetRelation();
23483                                                        } else {
23484                                                                relation = modelFactory.createRecordSetRelation();
23485                                                        }
23486                                                        relation.setEffectType(EffectType.function);
23487                                                        relation.setFunction(function.getFunctionName().toString());
23488                                                        relation.setTarget(new ResultColumnRelationshipElement(targetRC_tbl));
23489
23490                                                        if (relation instanceof DataFlowRelationship && option.isShowCountTableColumn()) {
23491                                                                List<TExpression> expressions = new ArrayList<TExpression>();
23492                                                                getFunctionExpressions(expressions, new ArrayList<TExpression>(), function);
23493                                                                for (int j = 0; j < expressions.size(); j++) {
23494                                                                        columnsInExpr visitor = new columnsInExpr();
23495                                                                        expressions.get(j).inOrderTraverse(visitor);
23496                                                                        List<TObjectName> objectNames = visitor.getObjectNames();
23497                                                                        if (objectNames != null) {
23498                                                                                for (TObjectName columnName : objectNames) {
23499                                                                                        TTable tempTable = modelManager.getTable(stmt, columnName);
23500                                                                                        if (table.equals(tempTable)) {
23501                                                                                                TableColumn tableColumn = modelFactory.createTableColumn(tableModel,
23502                                                                                                                columnName, false);
23503                                                                                                TableColumnRelationshipElement element = new TableColumnRelationshipElement(
23504                                                                                                                tableColumn);
23505                                                                                                relation.addSource(element);
23506                                                                                        }
23507                                                                                }
23508                                                                        }
23509                                                                }
23510                                                        }
23511                                                        if (relation.getSources().size() == 0) {
23512                                                                RelationRowsRelationshipElement element = new RelationRowsRelationshipElement<TableRelationRows>(
23513                                                                                tableModel.getRelationRows());
23514                                                                relation.addSource(element);
23515                                                        }
23516                                                } else if (modelManager.getModel(table) instanceof QueryTable) {
23517                                                        Object targetModel4 = modelManager.getModel(function.getFunctionName());
23518                                                        if (targetModel4 == null && function.getFunctionType() == EFunctionType.array_agg_t && modelManager.getModel(function) instanceof Function) {
23519                                                                Function functionModel = (Function) modelManager.getModel(function);
23520                                                                targetModel4 = functionModel.getColumns().get(0);
23521                                                        }
23522                                                        if (!(targetModel4 instanceof ResultColumn)) {
23523                                                                continue;
23524                                                        }
23525                                                        ResultColumn targetRC_qt = (ResultColumn) targetModel4;
23526                                                        QueryTable tableModel = (QueryTable) modelManager.getModel(table);
23527                                                        AbstractRelationship relation;
23528                                                        if ("COUNT".equalsIgnoreCase(function.getFunctionName().toString())) {
23529                                                                // relation = modelFactory.createDataFlowRelation();
23530                                                                relation = modelFactory.createRecordSetRelation();
23531                                                        } else {
23532                                                                relation = modelFactory.createRecordSetRelation();
23533                                                        }
23534                                                        relation.setEffectType(EffectType.function);
23535                                                        relation.setFunction(function.getFunctionName().toString());
23536                                                        relation.setTarget(new ResultColumnRelationshipElement(targetRC_qt));
23537                                                        RelationRowsRelationshipElement element = new RelationRowsRelationshipElement<ResultSetRelationRows>(
23538                                                                        tableModel.getRelationRows());
23539                                                        relation.addSource(element);
23540                                                }
23541                                        }
23542                                }
23543                        }
23544
23545                        if (stmt.getWhereClause() == null || stmt.getWhereClause().getCondition() == null) {
23546                                return;
23547                        }
23548
23549                        columnsInExpr visitor = new columnsInExpr();
23550                        stmt.getWhereClause().getCondition().inOrderTraverse(visitor);
23551                        List<TObjectName> objectNames = visitor.getObjectNames();
23552                        for (int j = 0; j < objectNames.size(); j++) {
23553                                TObjectName columnName = objectNames.get(j);
23554                                if (columnName.getDbObjectType() == EDbObjectType.variable) {
23555                                        Variable tableModel;
23556                                        if (columnName.toString().indexOf(".") != -1) {
23557                                                List<String> splits = SQLUtil.parseNames(columnName.toString());
23558                                                tableModel = modelFactory.createVariable(splits.get(splits.size() - 2));
23559                                        } else {
23560                                                tableModel = modelFactory.createVariable(columnName);
23561                                        }
23562                                        tableModel.setCreateTable(true);
23563                                        tableModel.setSubType(SubType.record);
23564                                        TObjectName variableProperties = new TObjectName();
23565                                        variableProperties.setString("*");
23566                                        modelFactory.createTableColumn(tableModel, variableProperties, true);
23567                                }
23568
23569//                              if (columnName.getColumnNameOnly().startsWith("@")
23570//                                              && (option.getVendor() == EDbVendor.dbvmssql || option.getVendor() == EDbVendor.dbvazuresql)) {
23571//                                      continue;
23572//                              }
23573//
23574//                              if (columnName.getColumnNameOnly().startsWith(":") && (option.getVendor() == EDbVendor.dbvhana || option.getVendor() == EDbVendor.dbvteradata)) {
23575//                                      continue;
23576//                              }
23577
23578                                Object targetModel5 = modelManager.getModel(function.getFunctionName());
23579                                if (targetModel5 == null && function.getFunctionType() == EFunctionType.array_agg_t && modelManager.getModel(function) instanceof Function) {
23580                                        Function functionModel = (Function) modelManager.getModel(function);
23581                                        if (functionModel.getColumns().size() == 1) {
23582                                                targetModel5 = functionModel.getColumns().get(0);
23583                                        }
23584                                }
23585                                if (!(targetModel5 instanceof ResultColumn)) {
23586                                        continue;
23587                                }
23588                                AbstractRelationship relation = modelFactory.createRecordSetRelation();
23589                                relation.setEffectType(EffectType.function);
23590                                relation.setFunction(function.getFunctionName().toString());
23591                                relation.setTarget(new ResultColumnRelationshipElement((ResultColumn) targetModel5));
23592
23593                                TTable table = modelManager.getTable(stmt, columnName);
23594                                if (table != null) {
23595                                        if (modelManager.getModel(table) instanceof Table) {
23596                                                Table tableModel = (Table) modelManager.getModel(table);
23597                                                if (tableModel != null) {
23598                                                        TableColumn columnModel = modelFactory.createTableColumn(tableModel, columnName, false);
23599                                                        if(columnModel == null) {
23600                                                                continue;
23601                                                        }
23602                                                        relation.addSource(
23603                                                                        new TableColumnRelationshipElement(columnModel, columnName.getLocation()));
23604                                                }
23605                                        } else if (modelManager.getModel(table) instanceof QueryTable) {
23606                                                Object model = modelManager.getModel(columnName.getSourceColumn());
23607                                                if (model instanceof ResultColumn) {
23608                                                        ResultColumn resultColumn = (ResultColumn) model;
23609                                                        if (resultColumn != null) {
23610                                                                relation.addSource(
23611                                                                                new ResultColumnRelationshipElement(resultColumn, columnName.getLocation()));
23612                                                        }
23613                                                }
23614                                        }
23615                                }
23616                        }
23617                }
23618        }
23619
23620        private void analyzeFilterCondition(Object modelObject, TExpression expr, EJoinType joinType,
23621                        JoinClauseType joinClauseType, EffectType effectType) {
23622                if (expr == null) {
23623                        return;
23624                }
23625
23626                TCustomSqlStatement stmt = stmtStack.peek();
23627
23628                columnsInExpr visitor = new columnsInExpr();
23629                expr.inOrderTraverse(visitor);
23630
23631                List<TObjectName> objectNames = visitor.getObjectNames();
23632                List<TParseTreeNode> functions = visitor.getFunctions();
23633                List<TResultColumn> resultColumns = visitor.getResultColumns();
23634                List<TParseTreeNode> constants = visitor.getConstants();
23635
23636                ImpactRelationship relation = modelFactory.createImpactRelation();
23637                relation.setEffectType(effectType);
23638                relation.setJoinClauseType(joinClauseType);
23639                if (modelObject instanceof ResultColumn) {
23640                        relation.setTarget(new ResultColumnRelationshipElement((ResultColumn) modelObject));
23641                } else {
23642                        ResultSet resultSet = (ResultSet) modelManager.getModel(stmt.getResultColumnList());
23643                        if (resultSet == null && stmt instanceof TUpdateSqlStatement) {
23644                                resultSet = (ResultSet) modelManager.getModel(stmt);
23645                        }
23646                        if (resultSet == null && stmt instanceof TMergeSqlStatement) {
23647                                TSelectSqlStatement subquery = ((TMergeSqlStatement) stmt).getUsingTable().getSubquery();
23648                                if (subquery != null) {
23649                                        resultSet = (ResultSet) modelManager.getModel(((TMergeSqlStatement) stmt).getUsingTable());
23650                                }
23651                                else {
23652                                        resultSet = modelFactory.createQueryTable(((TMergeSqlStatement) stmt).getUsingTable());
23653                                }
23654                        }
23655                        if (resultSet != null) {
23656                                relation.setTarget(
23657                                                new RelationRowsRelationshipElement<ResultSetRelationRows>(resultSet.getRelationRows()));
23658                        }
23659                        if (stmt instanceof TDeleteSqlStatement) {
23660                                Table table = (Table) modelManager.getModel(((TDeleteSqlStatement) stmt).getTargetTable());
23661                                if (table != null) {
23662                                        relation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(table.getRelationRows()));
23663                                }
23664                        }
23665                }
23666                if (relation.getTarget() != null) {
23667
23668                        if (constants != null && constants.size() > 0) {
23669                                if (option.isShowConstantTable()) {
23670                                        Table constantTable = modelFactory.createConstantsTable(stmtStack.peek());
23671                                        for (int i = 0; i < constants.size(); i++) {
23672                                                TParseTreeNode constant = constants.get(i);
23673                                                if (constant instanceof TConstant) {
23674                                                        TableColumn constantColumn = modelFactory.createTableColumn(constantTable,
23675                                                                        (TConstant) constant);
23676                                                        relation.addSource(new ConstantRelationshipElement(constantColumn));
23677                                                } else if (constant instanceof TObjectName) {
23678                                                        TableColumn constantColumn = modelFactory.createTableColumn(constantTable,
23679                                                                        (TObjectName) constant, false);
23680                                                        if(constantColumn == null) {
23681                                                                continue;
23682                                                        }
23683                                                        relation.addSource(new ConstantRelationshipElement(constantColumn));
23684                                                }
23685                                        }
23686                                }
23687                        }
23688
23689                        for (int j = 0; j < objectNames.size(); j++) {
23690                                TObjectName columnName = objectNames.get(j);
23691                                if (columnName.getDbObjectType() == EDbObjectType.variable) {
23692                                        Variable variable = modelFactory.createVariable(columnName);
23693                                        variable.setSubType(SubType.record);
23694                                        if (variable.getColumns().isEmpty()) {
23695                                                TObjectName variableProperties = new TObjectName();
23696                                                variableProperties.setString("*");
23697                                                modelFactory.createTableColumn(variable, variableProperties, true);
23698                                        }
23699                                        relation.addSource(new TableColumnRelationshipElement(variable.getColumns().get(0), columnName.getLocation()));
23700                                        continue;
23701                                }
23702
23703                                if (columnName.getColumnNameOnly().startsWith("@")
23704                                                && (option.getVendor() == EDbVendor.dbvmssql || option.getVendor() == EDbVendor.dbvazuresql)) {
23705                                        continue;
23706                                }
23707
23708                                if (columnName.getColumnNameOnly().startsWith(":")
23709                                                && (option.getVendor() == EDbVendor.dbvhana || option.getVendor() == EDbVendor.dbvteradata)) {
23710                                        continue;
23711                                }
23712
23713                                TTable table = modelManager.getTable(stmt, columnName);
23714
23715                                if (table == null) {
23716                                        table = columnName.getSourceTable();
23717                                }
23718
23719                                if (table == null && stmt.tables != null) {
23720                                        for (int k = 0; k < stmt.tables.size(); k++) {
23721                                                if (table != null)
23722                                                        break;
23723
23724                                                TTable tTable = stmt.tables.getTable(k);
23725                                                if (tTable.getTableType().name().startsWith("open")) {
23726                                                        continue;
23727                                                } else if (getTableLinkedColumns(tTable) != null && getTableLinkedColumns(tTable).size() > 0) {
23728                                                        for (int z = 0; z < getTableLinkedColumns(tTable).size(); z++) {
23729                                                                TObjectName refer = getTableLinkedColumns(tTable).getObjectName(z);
23730                                                                if ("*".equals(getColumnName(refer)))
23731                                                                        continue;
23732                                                                if (getColumnName(refer).equals(getColumnName(columnName))) {
23733                                                                        table = tTable;
23734                                                                        break;
23735                                                                }
23736                                                        }
23737                                                } else if (columnName.getTableToken() != null
23738                                                                && (columnName.getTableToken().getAstext().equalsIgnoreCase(tTable.getName())
23739                                                                                || columnName.getTableToken().getAstext().equalsIgnoreCase(tTable.getAliasName()))) {
23740                                                        table = tTable;
23741                                                        break;
23742                                                }
23743                                        }
23744
23745                                        if (table == null) {
23746                                                for (int k = 0; k < stmt.tables.size(); k++) {
23747                                                        if (table != null)
23748                                                                break;
23749
23750                                                        TTable tTable = stmt.tables.getTable(k);
23751                                                        Object model = ModelBindingManager.get().getModel(tTable);
23752                                                        if (model instanceof Table) {
23753                                                                Table tableModel = (Table) model;
23754                                                                for (int z = 0; tableModel.getColumns() != null
23755                                                                                && z < tableModel.getColumns().size(); z++) {
23756                                                                        TableColumn refer = tableModel.getColumns().get(z);
23757                                                                        if (getColumnName(refer.getName()).equals(getColumnName(columnName))) {
23758                                                                                table = tTable;
23759                                                                                break;
23760                                                                        }
23761                                                                        if (refer.hasStarLinkColumn()) {
23762                                                                                for (TObjectName linkColumn : refer.getStarLinkColumnList()) {
23763                                                                                        if (getColumnName(linkColumn).equals(getColumnName(columnName))) {
23764                                                                                                table = tTable;
23765                                                                                                break;
23766                                                                                        }
23767                                                                                }
23768                                                                        }
23769                                                                }
23770                                                        } else if (model instanceof QueryTable) {
23771                                                                QueryTable tableModel = (QueryTable) model;
23772                                                                for (int z = 0; tableModel.getColumns() != null
23773                                                                                && z < tableModel.getColumns().size(); z++) {
23774                                                                        ResultColumn refer = tableModel.getColumns().get(z);
23775                                                                        if (DlineageUtil.getIdentifierNormalColumnName(refer.getName()).equals(
23776                                                                                        DlineageUtil.getIdentifierNormalColumnName(getColumnName(columnName)))) {
23777                                                                                table = tTable;
23778                                                                                break;
23779                                                                        }
23780                                                                        if (refer.hasStarLinkColumn()) {
23781                                                                                for (TObjectName linkColumn : refer.getStarLinkColumnList()) {
23782                                                                                        if (getColumnName(linkColumn).equals(getColumnName(columnName))) {
23783                                                                                                table = tTable;
23784                                                                                                break;
23785                                                                                        }
23786                                                                                }
23787                                                                        }
23788                                                                }
23789                                                        }
23790                                                }
23791                                        }
23792                                }
23793
23794                                if (table == null && stmt.tables != null && stmt.tables.size() != 0
23795                                                && !(isBuiltInFunctionName(columnName) && isFromFunction(columnName))) {
23796                                        
23797                                        if (modelManager.getModel(stmt) instanceof ResultSet) {
23798                                                ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt);
23799                                                boolean find = false;
23800                                                for (ResultColumn resultColumn : resultSetModel.getColumns()) {
23801                                                        if(resultColumn.equals(modelObject)) {
23802                                                                continue;
23803                                                        }
23804                                                        if (!TSQLEnv.isAliasReferenceForbidden.get(option.getVendor())) {
23805                                                                if (getColumnName(columnName).equals(getColumnName(resultColumn.getName()))) {
23806                                                                        if (resultColumn.getColumnObject() != null) {
23807                                                                                int startToken = resultColumn.getColumnObject().getStartToken().posinlist;
23808                                                                                int endToken = resultColumn.getColumnObject().getEndToken().posinlist;
23809                                                                                if (columnName.getStartToken().posinlist >= startToken
23810                                                                                                && columnName.getEndToken().posinlist <= endToken) {
23811                                                                                        continue;
23812                                                                                }
23813                                                                        }
23814                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
23815                                                                        find = true;
23816                                                                        break;
23817                                                                }
23818                                                        }
23819                                                }
23820                                                if (find) {
23821                                                        continue;
23822                                                }
23823                                        }
23824
23825                                        TObjectName pseudoTableName = new TObjectName();
23826                                        // Use qualified prefix from column name if available (e.g., sch.pk_constv2 from sch.pk_constv2.c_cdsl)
23827                                        // Otherwise fall back to default pseudo table name
23828                                        String qualifiedPrefix = getQualifiedPrefixFromColumn(columnName);
23829                                        pseudoTableName.setString(qualifiedPrefix != null ? qualifiedPrefix : "pseudo_table_include_orphan_column");
23830                                        Table pseudoTable = modelFactory.createTableByName(pseudoTableName);
23831                                        pseudoTable.setPseudo(true);
23832                                        TableColumn pseudoTableColumn = modelFactory.createTableColumn(pseudoTable, columnName, true);
23833
23834                                        // If not linking to first table and column has qualified prefix (3-part name like sch.pkg.col),
23835                                        // add the pseudo table column as source
23836                                        if (!isLinkOrphanColumnToFirstTable() && pseudoTableColumn != null && qualifiedPrefix != null) {
23837                                                relation.addSource(new TableColumnRelationshipElement(pseudoTableColumn));
23838                                        }
23839
23840                                        if (isLinkOrphanColumnToFirstTable()) {
23841                                                TTable orphanTable = stmt.tables.getTable(0);
23842                                                table = stmt.tables.getTable(0);
23843                                                Object tableModel = modelManager.getModel(table);
23844                                                if (tableModel == null) {
23845                                                        tableModel = modelFactory.createTable(orphanTable);
23846                                                }
23847                                                if (tableModel instanceof Table) {
23848                                                        modelFactory.createTableColumn((Table) tableModel, columnName, false);
23849                                                        ErrorInfo errorInfo = new ErrorInfo();
23850                                                        errorInfo.setErrorType(ErrorInfo.LINK_ORPHAN_COLUMN);
23851                                                        errorInfo.setErrorMessage("Link orphan column [" + columnName.toString()
23852                                                                        + "] to the first table [" + orphanTable.getFullNameWithAliasString() + "]");
23853                                                        errorInfo.setStartPosition(new Pair3<Long, Long, String>(columnName.getStartToken().lineNo,
23854                                                                        columnName.getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
23855                                                        errorInfo.setEndPosition(new Pair3<Long, Long, String>(columnName.getEndToken().lineNo,
23856                                                                        columnName.getEndToken().columnNo + columnName.getEndToken().getAstext().length(),
23857                                                                        ModelBindingManager.getGlobalHash()));
23858                                                        errorInfo.fillInfo(this);
23859                                                        errorInfos.add(errorInfo);
23860                                                }
23861                                        }
23862                                }
23863
23864                                if (table != null) {
23865                                        if (modelManager.getModel(table) instanceof Table) {
23866                                                Table tableModel = (Table) modelManager.getModel(table);
23867                                                if (tableModel != null) {
23868                                                        TableColumn columnModel = modelFactory.createTableColumn(tableModel, columnName, false);
23869                                                        if(columnModel!=null) {
23870                                                                TableColumnRelationshipElement element = new TableColumnRelationshipElement(columnModel,
23871                                                                                columnName.getLocation());
23872                                                                relation.addSource(element);
23873                                                        }
23874                                                }
23875                                        } else if (modelManager.getModel(table) instanceof QueryTable) {
23876                                                QueryTable tableModel = (QueryTable)modelManager.getModel(table);
23877                                                if (table.getSubquery() != null && table.getSubquery().isCombinedQuery()) {
23878                                                        TSelectSqlStatement subquery = table.getSubquery();
23879                                                        List<ResultSet> resultSets = new ArrayList<ResultSet>();
23880                                                        if (!subquery.getLeftStmt().isCombinedQuery()) {
23881                                                                ResultSet sourceResultSet = (ResultSet) modelManager
23882                                                                                .getModel(subquery.getLeftStmt().getResultColumnList());
23883                                                                resultSets.add(sourceResultSet);
23884                                                        } else {
23885                                                                ResultSet sourceResultSet = (ResultSet) modelManager.getModel(subquery.getLeftStmt());
23886                                                                resultSets.add(sourceResultSet);
23887                                                        }
23888
23889                                                        if (!subquery.getRightStmt().isCombinedQuery()) {
23890                                                                ResultSet sourceResultSet = (ResultSet) modelManager
23891                                                                                .getModel(subquery.getRightStmt().getResultColumnList());
23892                                                                resultSets.add(sourceResultSet);
23893                                                        } else {
23894                                                                ResultSet sourceResultSet = (ResultSet) modelManager.getModel(subquery.getRightStmt());
23895                                                                resultSets.add(sourceResultSet);
23896                                                        }
23897
23898                                                        for (ResultSet sourceResultSet : resultSets) {
23899                                                                if (sourceResultSet != null && columnName.getSourceColumn() != null) {
23900                                                                        for (int k = 0; k < sourceResultSet.getColumns().size(); k++) {
23901                                                                                if (getColumnName(sourceResultSet.getColumns().get(k).getName()).equals(
23902                                                                                                getColumnName(columnName.getSourceColumn().getColumnNameOnly()))) {
23903                                                                                        Set<TObjectName> starLinkColumnSet = sourceResultSet.getColumns().get(k)
23904                                                                                                        .getStarLinkColumns().get(getColumnName(columnName));
23905                                                                                        if (starLinkColumnSet != null && !starLinkColumnSet.isEmpty()) {
23906                                                                                                ResultColumn column = modelFactory.createResultColumn(sourceResultSet,
23907                                                                                                                starLinkColumnSet.iterator().next(), true);
23908                                                                                                relation.addSource(new ResultColumnRelationshipElement(column));
23909                                                                                        } else {
23910                                                                                                relation.addSource(new ResultColumnRelationshipElement(
23911                                                                                                                sourceResultSet.getColumns().get(k)));
23912                                                                                        }
23913                                                                                }
23914                                                                        }
23915                                                                }
23916                                                        }
23917                                                } else {
23918                                                        Object model = modelManager.getModel(columnName.getSourceColumn());
23919                                                        if (model instanceof ResultColumn) {
23920                                                                ResultColumn resultColumn = (ResultColumn) model;
23921                                                                if (resultColumn != null) {
23922                                                                        if (resultColumn.hasStarLinkColumn()) {
23923                                                                                Set<TObjectName> starLinkColumnSet = resultColumn.getStarLinkColumns()
23924                                                                                                .get(getColumnName(columnName));
23925                                                                                if (starLinkColumnSet != null && !starLinkColumnSet.isEmpty()) {
23926                                                                                        ResultColumn column = modelFactory.createResultColumn(
23927                                                                                                        resultColumn.getResultSet(), starLinkColumnSet.iterator().next(), true);
23928                                                                                        relation.addSource(new ResultColumnRelationshipElement(column));
23929                                                                                } else {
23930                                                                                        resultColumn.bindStarLinkColumn(columnName);
23931                                                                                        ResultColumn column = modelFactory
23932                                                                                                        .createResultColumn(resultColumn.getResultSet(), columnName, true);
23933                                                                                        relation.addSource(new ResultColumnRelationshipElement(column));
23934                                                                                }
23935                                                                        } else {
23936                                                                                ResultColumnRelationshipElement element = new ResultColumnRelationshipElement(
23937                                                                                                resultColumn, columnName.getLocation());
23938                                                                                relation.addSource(element);
23939                                                                        }
23940                                                                }
23941                                                        }
23942                                                        else{
23943                                                                boolean find = false;
23944                                                                for (int i = 0; i < tableModel.getColumns().size(); i++) {
23945                                                                        ResultColumn resultColumn = tableModel.getColumns().get(i);
23946                                                                        if (DlineageUtil.getIdentifierNormalColumnName(resultColumn.getName()).equals(
23947                                                                                        DlineageUtil.getIdentifierNormalColumnName(getColumnName(columnName)))) {
23948                                                                                ResultColumnRelationshipElement element = new ResultColumnRelationshipElement(
23949                                                                                                resultColumn, columnName.getLocation());
23950                                                                                relation.addSource(element);
23951                                                                                find = true;
23952                                                                                break;
23953                                                                        }
23954                                                                        else if (resultColumn.getName().endsWith("*")) {
23955                                                                                resultColumn.bindStarLinkColumn(columnName);
23956                                                                        }
23957                                                                }
23958                                                                if(!find){
23959                                                                        ResultColumn resultColumn = new ResultColumn(tableModel, columnName);
23960                                                                        ResultColumnRelationshipElement element = new ResultColumnRelationshipElement(
23961                                                                                                resultColumn, columnName.getLocation());
23962                                                                                relation.addSource(element);
23963                                                                }
23964                                                        }
23965                                                }
23966                                        }
23967                                }
23968                        }
23969
23970                        for (int j = 0; j < functions.size(); j++) {
23971                                TParseTreeNode functionObj = functions.get(j);
23972                                Object functionModel = modelManager.getModel(functionObj);
23973                                if (functionModel == null) {
23974                                        functionModel = createFunction(functionObj);
23975                                }
23976                                if (functionModel instanceof Function) {
23977                                        if (functionObj instanceof TFunctionCall) {
23978                                                ResultColumn resultColumn = (ResultColumn) modelManager
23979                                                                .getModel(((TFunctionCall) functionObj).getFunctionName());
23980                                                if (resultColumn != null) {
23981                                                        ResultColumnRelationshipElement element = new ResultColumnRelationshipElement(resultColumn,
23982                                                                        ((TFunctionCall) functionObj).getFunctionName().getLocation());
23983                                                        relation.addSource(element);
23984                                                }
23985                                        }
23986                                        if (functionObj instanceof TCaseExpression) {
23987                                                ResultColumn resultColumn = (ResultColumn) modelManager
23988                                                                .getModel(((TCaseExpression) functionObj).getWhenClauseItemList());
23989                                                if (resultColumn != null) {
23990                                                        ResultColumnRelationshipElement element = new ResultColumnRelationshipElement(resultColumn);
23991                                                        relation.addSource(element);
23992                                                }
23993                                        }
23994                                } else if (functionModel instanceof Table) {
23995                                        TableColumn tableColumn = modelFactory.createTableColumn((Table) functionModel,
23996                                                        ((TFunctionCall) functionObj).getFunctionName(), false);
23997                                        TableColumnRelationshipElement element = new TableColumnRelationshipElement(tableColumn);
23998                                        relation.addSource(element);
23999                                }
24000                        }
24001
24002                        for (int j = 0; j < resultColumns.size(); j++) {
24003                                TResultColumn resultColumn = resultColumns.get(j);
24004                                if (modelManager.getModel(resultColumn) instanceof ResultColumn) {
24005                                        ResultColumn resultColumnModel = (ResultColumn) modelManager.getModel(resultColumn);
24006                                        relation.addSource(new ResultColumnRelationshipElement(resultColumnModel, ESqlClause.selectList));
24007                                }
24008                        }
24009                }
24010
24011                if (isShowJoin() && joinClauseType != null) {
24012                        joinInExpr joinVisitor = new joinInExpr(joinType, joinClauseType, effectType);
24013                        expr.inOrderTraverse(joinVisitor);
24014                }
24015        }
24016
24017        public void dispose() {
24018                accessedSubqueries.clear();
24019                accessedStatements.clear();
24020                stmtStack.clear();
24021                viewDDLMap.clear();
24022                procedureDDLMap.clear();
24023                structObjectMap.clear();
24024                appendResultSets.clear();
24025                appendStarColumns.clear();
24026                appendTableStarColumns.clear();
24027                modelManager.DISPLAY_ID.clear();
24028                modelManager.DISPLAY_NAME.clear();
24029                tableIds.clear();
24030                ModelBindingManager.remove();
24031        }
24032
24033        class joinTreatColumnsInExpr implements IExpressionVisitor {
24034
24035                private List<TObjectName> objectNames = new ArrayList<TObjectName>();
24036                
24037                private TTable table;
24038
24039                public joinTreatColumnsInExpr(TTable table) {
24040                        this.table = table;
24041                }
24042
24043                public List<TObjectName> getObjectNames() {
24044                        return objectNames;
24045                }
24046
24047                boolean is_compare_condition(EExpressionType t) {
24048                        return t == EExpressionType.simple_comparison_t;
24049                }
24050
24051                @Override
24052                public boolean exprVisit(TParseTreeNode pNode, boolean isLeafNode) {
24053                        TExpression expr = (TExpression) pNode;
24054                        if (is_compare_condition(expr.getExpressionType())) {
24055                                TExpression leftExpr = expr.getLeftOperand();
24056                                columnsInExpr leftVisitor = new columnsInExpr();
24057                                leftExpr.inOrderTraverse(leftVisitor);
24058                                List<TObjectName> leftObjectNames = leftVisitor.getObjectNames();
24059
24060                                TExpression rightExpr = expr.getRightOperand();
24061                                columnsInExpr rightVisitor = new columnsInExpr();
24062                                rightExpr.inOrderTraverse(rightVisitor);
24063                                List<TObjectName> rightObjectNames = rightVisitor.getObjectNames();
24064
24065                                if (!leftObjectNames.isEmpty() && !rightObjectNames.isEmpty()) {
24066                                        for (TObjectName column : leftObjectNames) {
24067                                                if (column.getSourceTable() != null && column.getSourceTable().equals(table)) {
24068                                                        objectNames.add(column);
24069                                                        return false;
24070                                                }
24071                                        }
24072                                        for (TObjectName column : rightObjectNames) {
24073                                                if (column.getSourceTable() != null && column.getSourceTable().equals(table)) {
24074                                                        objectNames.add(column);
24075                                                        return false;
24076                                                }
24077                                        }
24078                                }
24079                                return false;
24080                        }
24081                        return true;
24082                }
24083        }
24084        
24085        class columnsInExpr implements IExpressionVisitor {
24086
24087                private List<TParseTreeNode> constants = new ArrayList<TParseTreeNode>();
24088                private List<TObjectName> objectNames = new ArrayList<TObjectName>();
24089                private List<TParseTreeNode> functions = new ArrayList<TParseTreeNode>();
24090                private List<TResultColumn> resultColumns = new ArrayList<TResultColumn>();
24091                private List<TSelectSqlStatement> subquerys = new ArrayList<TSelectSqlStatement>();
24092                private boolean skipFunction = false;
24093
24094                public void setSkipFunction(boolean skipFunction) {
24095                        this.skipFunction = skipFunction;
24096                }
24097
24098                public List<TParseTreeNode> getFunctions() {
24099                        return functions;
24100                }
24101
24102                public List<TSelectSqlStatement> getSubquerys() {
24103                        return subquerys;
24104                }
24105
24106                public List<TParseTreeNode> getConstants() {
24107                        return constants;
24108                }
24109
24110                public List<TObjectName> getObjectNames() {
24111                        return objectNames;
24112                }
24113
24114                public List<TResultColumn> getResultColumns() {
24115                        return resultColumns;
24116                }
24117
24118                @Override
24119                public boolean exprVisit(TParseTreeNode pNode, boolean isLeafNode) {
24120                        TExpression lcexpr = (TExpression) pNode;
24121                        // Handle named argument expressions (e.g., "INPUT => value" in Snowflake FLATTEN)
24122                        // The left operand is the parameter name, NOT a column reference.
24123                        // Only traverse the right operand (the value).
24124                        if (lcexpr.getExpressionType() == EExpressionType.assignment_t) {
24125                                // Skip left operand (parameter name) - only traverse right operand (value)
24126                                if (lcexpr.getRightOperand() != null) {
24127                                        lcexpr.getRightOperand().inOrderTraverse(this);
24128                                }
24129                                return false; // Don't continue default traversal
24130                        }
24131                        if (lcexpr.getExpressionType() == EExpressionType.simple_constant_t) {
24132                                if (lcexpr.getConstantOperand() != null) {
24133                                        if(lcexpr.getConstantOperand().getInt64_expression()!=null 
24134                                                        && lcexpr.getConstantOperand().getInt64_expression().getExpressionType() == EExpressionType.function_t) {
24135                                                lcexpr.getConstantOperand().getInt64_expression().inOrderTraverse(this);
24136                                        }
24137                                        else {
24138                                                constants.add(lcexpr.getConstantOperand());
24139                                        }
24140                                }
24141                        } else if (lcexpr.getExpressionType() == EExpressionType.array_t) {
24142                                if(lcexpr.getObjectOperand()!=null) {
24143                                        TObjectName object = lcexpr.getObjectOperand();
24144                                        objectNames.add(object);
24145                                } else if (lcexpr.getExprList() != null) {
24146                                        for (int j = 0; j < lcexpr.getExprList().size(); j++) {
24147                                                TExpression expr = lcexpr.getExprList().getExpression(j);
24148                                                if (expr != null)
24149                                                        expr.inOrderTraverse(this);
24150                                        }
24151                                }
24152                        } else if (lcexpr.getExpressionType() == EExpressionType.simple_object_name_t) {
24153                                if (lcexpr.getObjectOperand() != null && !(isBuiltInFunctionName(lcexpr.getObjectOperand())
24154                                                && isFromFunction(lcexpr.getObjectOperand()))) {
24155                                        TObjectName object = lcexpr.getObjectOperand();
24156                                        // Skip named argument parameter names (e.g., INPUT in "INPUT => value")
24157                                        // These are function parameter names, NOT column references
24158                                        if (object.getObjectType() == TObjectName.ttobjNamedArgParameter) {
24159                                                // Skip - this is a named argument parameter name
24160                                        } else if (object.getDbObjectType() == EDbObjectType.column
24161                                                        || object.getDbObjectType() == EDbObjectType.column_alias
24162                                                        || object.getDbObjectType() == EDbObjectType.alias
24163                                                        || object.getDbObjectType() == EDbObjectType.unknown
24164                                                        || object.getDbObjectType() == EDbObjectType.variable) {
24165                                                objectNames.add(object);
24166                                        } else if (object.getDbObjectType() == EDbObjectType.notAColumn
24167                                                        || object.getDbObjectType() == EDbObjectType.date_time_part ) {
24168                                                constants.add(object);
24169                                        }
24170                                }
24171                        } else if (lcexpr.getExpressionType() == EExpressionType.between_t) {
24172                                if (lcexpr.getBetweenOperand() != null && lcexpr.getBetweenOperand().getObjectOperand() != null) {
24173                                        TObjectName object = lcexpr.getBetweenOperand().getObjectOperand();
24174                                        if (object.getDbObjectType() == EDbObjectType.column
24175                                                        || object.getDbObjectType() == EDbObjectType.column_alias
24176                                                        || object.getDbObjectType() == EDbObjectType.alias
24177                                                        || object.getDbObjectType() == EDbObjectType.unknown
24178                                                        || object.getDbObjectType() == EDbObjectType.variable) {
24179                                                objectNames.add(object);
24180                                        }
24181                                }
24182                        } else if (lcexpr.getExpressionType() == EExpressionType.object_access_t) {
24183                                if (lcexpr.getObjectAccess() != null) {
24184                                        TObjectNameList objects = lcexpr.getObjectAccess().getAttributes();
24185                                        TFunctionCall function = lcexpr.getObjectAccess().getObjectExpr().getFunctionCall();
24186                                        if (objects != null && function != null) {
24187                                                for (TObjectName object : objects) {
24188                                                        TGSqlParser sqlparser = new TGSqlParser(option.getVendor());
24189                                                        sqlparser.sqltext = "select " + function.getFunctionName().toString() + "."
24190                                                                        + object.getColumnNameOnly() + " from " + function.getFunctionName().toString();
24191                                                        if (sqlparser.parse() == 0) {
24192                                                                TObjectName objectName = sqlparser.sqlstatements.get(0).getResultColumnList()
24193                                                                                .getResultColumn(0).getFieldAttr();
24194                                                                objectNames.add(objectName);
24195                                                        }
24196                                                }
24197                                        }
24198                                }
24199                        } else if (lcexpr.getExpressionType() == EExpressionType.function_t || lcexpr.getExpressionType() == EExpressionType.fieldselection_t) {
24200                                TFunctionCall func = lcexpr.getFunctionCall();
24201                                if (func == null) {
24202                                        return true;
24203                                }
24204                                if (skipFunction) {
24205                                        if (func.getArgs() != null) {
24206                                                for (int k = 0; k < func.getArgs().size(); k++) {
24207                                                        TExpression expr = func.getArgs().getExpression(k);
24208                                                        if (expr != null)
24209                                                                expr.inOrderTraverse(this);
24210                                                }
24211                                        }
24212
24213                                        if (func.getTrimArgument() != null) {
24214                                                TTrimArgument args = func.getTrimArgument();
24215                                                TExpression expr = args.getStringExpression();
24216                                                if (expr != null) {
24217                                                        expr.inOrderTraverse(this);
24218                                                }
24219                                                expr = args.getTrimCharacter();
24220                                                if (expr != null) {
24221                                                        expr.inOrderTraverse(this);
24222                                                }
24223                                        }
24224
24225                                        if (func.getAgainstExpr() != null) {
24226                                                func.getAgainstExpr().inOrderTraverse(this);
24227                                        }
24228//                                      if (func.getBetweenExpr() != null) {
24229//                                              func.getBetweenExpr().inOrderTraverse(this);
24230//                                      }
24231                                        if (func.getExpr1() != null) {
24232                                                func.getExpr1().inOrderTraverse(this);
24233                                        }
24234                                        if (func.getExpr2() != null) {
24235                                                func.getExpr2().inOrderTraverse(this);
24236                                        }
24237                                        if (func.getExpr3() != null) {
24238                                                func.getExpr3().inOrderTraverse(this);
24239                                        }
24240                                        if (func.getParameter() != null) {
24241                                                func.getParameter().inOrderTraverse(this);
24242                                        }
24243                                } else {
24244                                        functions.add(func);
24245                                }
24246
24247                        } else if (lcexpr.getExpressionType() == EExpressionType.case_t) {
24248                                TCaseExpression expr = lcexpr.getCaseExpression();
24249                                if (skipFunction) {
24250                                        TExpression defaultExpr = expr.getElse_expr();
24251                                        if (defaultExpr != null) {
24252                                                defaultExpr.inOrderTraverse(this);
24253                                        }
24254                                        TWhenClauseItemList list = expr.getWhenClauseItemList();
24255                                        for (int i = 0; i < list.size(); i++) {
24256                                                TWhenClauseItem element = (TWhenClauseItem) list.getElement(i);
24257                                                (((TWhenClauseItem) element).getReturn_expr()).inOrderTraverse(this);
24258
24259                                        }
24260                                } else {
24261                                        functions.add(expr);
24262                                }
24263                        } else if (lcexpr.getSubQuery() != null) {
24264                                TSelectSqlStatement select = lcexpr.getSubQuery();
24265                                analyzeSelectStmt(select);
24266                                subquerys.add(select);
24267                                if (select.getResultColumnList() != null && select.getResultColumnList().size() > 0) {
24268                                        for (TResultColumn column : select.getResultColumnList()) {
24269                                                resultColumns.add(column);
24270                                        }
24271                                }
24272                        }
24273                        return true;
24274                }
24275        }
24276
24277        class joinInExpr implements IExpressionVisitor {
24278
24279                private EJoinType joinType;
24280                private JoinClauseType joinClauseType;
24281                private EffectType effectType;
24282
24283                public joinInExpr(EJoinType joinType, JoinClauseType joinClauseType, EffectType effectType) {
24284                        this.joinType = joinType;
24285                        this.joinClauseType = joinClauseType;
24286                        this.effectType = effectType;
24287                }
24288
24289                boolean is_compare_condition(EExpressionType t) {
24290                        return ((t == EExpressionType.simple_comparison_t) || (t == EExpressionType.group_comparison_t)
24291                                        || (t == EExpressionType.in_t) || (t == EExpressionType.pattern_matching_t)
24292                                        || (t == EExpressionType.left_join_t) || (t == EExpressionType.right_join_t));
24293                }
24294
24295                @Override
24296                public boolean exprVisit(TParseTreeNode pNode, boolean isLeafNode) {
24297                        TExpression expr = (TExpression) pNode;
24298                        if (is_compare_condition(expr.getExpressionType())) {
24299                                TExpression leftExpr = expr.getLeftOperand();
24300                                columnsInExpr leftVisitor = new columnsInExpr();
24301                                leftExpr.inOrderTraverse(leftVisitor);
24302                                List<TObjectName> leftObjectNames = leftVisitor.getObjectNames();
24303                                List<TParseTreeNode> leftObjects = leftVisitor.getFunctions();
24304                                leftObjects.addAll(leftObjectNames);
24305                                
24306                                TExpression rightExpr = expr.getRightOperand();
24307                                columnsInExpr rightVisitor = new columnsInExpr();
24308                                rightExpr.inOrderTraverse(rightVisitor);
24309                                List<TObjectName> rightObjectNames = rightVisitor.getObjectNames();
24310                                List<TParseTreeNode> rightObjects = rightVisitor.getFunctions();
24311                                rightObjects.addAll(rightObjectNames);
24312
24313                                if (!leftObjects.isEmpty() && !rightObjects.isEmpty()) {
24314                                        TCustomSqlStatement stmt = stmtStack.peek();
24315
24316                                        for (int i = 0; i < leftObjects.size(); i++) {
24317                                                TParseTreeNode leftObject = leftObjects.get(i);
24318                                                TTable leftTable = null;
24319                                                TFunctionCall leftFunction = null;
24320                                                TObjectName leftObjectName = null;
24321                                                if (leftObject instanceof TObjectName) {
24322                                                        leftObjectName = (TObjectName)leftObject;
24323
24324                                                        if (leftObjectName.getDbObjectType() == EDbObjectType.variable) {
24325                                                                continue;
24326                                                        }
24327
24328                                                        if (leftObjectName.getColumnNameOnly().startsWith("@")
24329                                                                        && (option.getVendor() == EDbVendor.dbvmssql
24330                                                                                        || option.getVendor() == EDbVendor.dbvazuresql)) {
24331                                                                continue;
24332                                                        }
24333
24334                                                        if (leftObjectName.getColumnNameOnly().startsWith(":")
24335                                                                        && (option.getVendor() == EDbVendor.dbvhana
24336                                                                                        || option.getVendor() == EDbVendor.dbvteradata)) {
24337                                                                continue;
24338                                                        }
24339
24340                                                        leftTable = modelManager.getTable(stmt, leftObjectName);
24341
24342                                                        if (leftTable == null) {
24343                                                                leftTable = leftObjectName.getSourceTable();
24344                                                        }
24345
24346                                                        if (leftTable == null) {
24347                                                                leftTable = modelManager.guessTable(stmt, leftObjectName);
24348                                                        }
24349                                                }
24350                                                else if(leftObject instanceof TFunctionCall){
24351                                                        leftFunction = (TFunctionCall)leftObject;
24352                                                }
24353
24354                                                if (leftTable != null || leftFunction != null) {
24355                                                        for (int j = 0; j < rightObjects.size(); j++) {
24356                                                                JoinRelationship joinRelation = modelFactory.createJoinRelation();
24357                                                                joinRelation.setEffectType(effectType);
24358                                                                if (joinType != null) {
24359                                                                        joinRelation.setJoinType(joinType);
24360                                                                } else {
24361                                                                        if (expr.getLeftOperand().isOracleOuterJoin()) {
24362                                                                                joinRelation.setJoinType(right);
24363                                                                        } else if (expr.getRightOperand().isOracleOuterJoin()) {
24364                                                                                joinRelation.setJoinType(EJoinType.left);
24365                                                                        } else if (expr.getExpressionType() == EExpressionType.left_join_t) {
24366                                                                                joinRelation.setJoinType(EJoinType.left);
24367                                                                        } else if (expr.getExpressionType() == EExpressionType.right_join_t) {
24368                                                                                joinRelation.setJoinType(right);
24369                                                                        } else {
24370                                                                                joinRelation.setJoinType(EJoinType.inner);
24371                                                                        }
24372                                                                }
24373
24374                                                                joinRelation.setJoinClauseType(joinClauseType);
24375                                                                joinRelation.setJoinCondition(expr.toString());
24376
24377
24378                                                                if (leftTable != null) {
24379                                                                        if (modelManager.getModel(leftTable) instanceof Table) {
24380                                                                                Table tableModel = (Table) modelManager.getModel(leftTable);
24381                                                                                if (tableModel != null) {
24382                                                                                        TableColumn columnModel = modelFactory.createTableColumn(tableModel,
24383                                                                                                        leftObjectName, false);
24384                                                                                        if (columnModel != null) {
24385                                                                                                joinRelation.addSource(new TableColumnRelationshipElement(columnModel));
24386                                                                                        }
24387                                                                                }
24388                                                                        } else if (modelManager.getModel(leftTable) instanceof QueryTable) {
24389                                                                                QueryTable table = (QueryTable) modelManager.getModel(leftTable);
24390                                                                                TSelectSqlStatement subquery = table.getTableObject().getSubquery();
24391                                                                                if (subquery != null && subquery.isCombinedQuery()) {
24392                                                                                        ResultColumn resultColumn = matchResultColumn(table.getColumns(),
24393                                                                                                        leftObjectName);
24394                                                                                        if (resultColumn != null) {
24395                                                                                                joinRelation
24396                                                                                                                .addSource(new ResultColumnRelationshipElement(resultColumn));
24397                                                                                        }
24398                                                                                } else if (leftObjectName.getSourceColumn() != null) {
24399                                                                                        Object model = modelManager.getModel(leftObjectName);
24400                                                                                        if (model == null) {
24401                                                                                                model = modelFactory.createResultColumn(table, leftObjectName);
24402                                                                                        }
24403                                                                                        if (model instanceof ResultColumn) {
24404                                                                                                ResultColumn resultColumn = (ResultColumn) model;
24405                                                                                                if (resultColumn != null) {
24406                                                                                                        joinRelation.addSource(
24407                                                                                                                        new ResultColumnRelationshipElement(resultColumn));
24408                                                                                                }
24409                                                                                        } else if (model instanceof LinkedHashMap) {
24410                                                                                                String columnName = getColumnNameOnly(leftObjectName.toString());
24411                                                                                                LinkedHashMap<String, ResultColumn> resultColumns = (LinkedHashMap<String, ResultColumn>) model;
24412                                                                                                if (resultColumns.containsKey(columnName)) {
24413                                                                                                        ResultColumn resultColumn = resultColumns.get(columnName);
24414                                                                                                        joinRelation.addSource(
24415                                                                                                                        new ResultColumnRelationshipElement(resultColumn));
24416                                                                                                }
24417                                                                                        }
24418                                                                                } else {
24419                                                                                        ResultColumn resultColumn = matchResultColumn(table.getColumns(),
24420                                                                                                        leftObjectName);
24421                                                                                        if (resultColumn != null) {
24422                                                                                                joinRelation
24423                                                                                                                .addSource(new ResultColumnRelationshipElement(resultColumn));
24424                                                                                        }
24425                                                                                }
24426                                                                        }
24427                                                                }
24428                                                                else if(leftFunction!=null) {
24429                                                                        Object functionObj = createFunction(leftFunction);
24430                                                                        if(functionObj instanceof Function) {
24431                                                                                Function function = (Function)functionObj;
24432                                                                                joinRelation.addSource(new ResultColumnRelationshipElement(function.getColumns().get(0)));
24433                                                                        }
24434                                                                }
24435
24436                                                                TParseTreeNode rightObject = rightObjects.get(j);
24437                                                                if(rightObject instanceof TObjectName) {
24438                                                                        TObjectName rightObjectName = (TObjectName)rightObject;
24439
24440                                                                        if (rightObjectName.getDbObjectType() == EDbObjectType.variable) {
24441                                                                                continue;
24442                                                                        }
24443
24444                                                                        if (rightObjectName.getColumnNameOnly().startsWith("@")
24445                                                                                        && (option.getVendor() == EDbVendor.dbvmssql
24446                                                                                                        || option.getVendor() == EDbVendor.dbvazuresql)) {
24447                                                                                continue;
24448                                                                        }
24449
24450                                                                        if (rightObjectName.getColumnNameOnly().startsWith(":")
24451                                                                                        && (option.getVendor() == EDbVendor.dbvhana
24452                                                                                                        || option.getVendor() == EDbVendor.dbvteradata)) {
24453                                                                                continue;
24454                                                                        }
24455
24456                                                                        TTable rightTable = modelManager.getTable(stmt, rightObjectName);
24457                                                                        if (rightTable == null) {
24458                                                                                rightTable = rightObjectName.getSourceTable();
24459                                                                        }
24460
24461                                                                        if (rightTable == null) {
24462                                                                                rightTable = modelManager.guessTable(stmt, rightObjectName);
24463                                                                        }
24464
24465                                                                        if (modelManager.getModel(rightTable) instanceof Table) {
24466                                                                                Table tableModel = (Table) modelManager.getModel(rightTable);
24467                                                                                if (tableModel != null) {
24468                                                                                        TableColumn columnModel = modelFactory.createTableColumn(tableModel,
24469                                                                                                        rightObjectName, false);
24470                                                                                        if(columnModel != null) {
24471                                                                                                joinRelation.setTarget(new TableColumnRelationshipElement(columnModel));
24472                                                                                        }
24473                                                                                }
24474                                                                        } else if (modelManager.getModel(rightTable) instanceof QueryTable) {
24475                                                                                QueryTable table = (QueryTable) modelManager.getModel(rightTable);
24476                                                                                TSelectSqlStatement subquery = table.getTableObject().getSubquery();
24477                                                                                if (subquery != null && subquery.isCombinedQuery()) {
24478                                                                                        ResultColumn resultColumn = matchResultColumn(table.getColumns(),
24479                                                                                                        rightObjectName);
24480                                                                                        if (resultColumn != null) {
24481                                                                                                joinRelation.setTarget(new ResultColumnRelationshipElement(resultColumn));
24482                                                                                        }
24483                                                                                } else if (rightObjectName.getSourceColumn() != null) {
24484                                                                                        Object model = modelManager.getModel(rightObjectName);
24485                                                                                        if (model == null) {
24486                                                                                                model = modelManager
24487                                                                                                                .getModel(rightObjectName.getSourceColumn());
24488                                                                                        }
24489                                                                                        if (model instanceof ResultColumn) {
24490                                                                                                joinRelation.setTarget(new ResultColumnRelationshipElement((ResultColumn)model));
24491                                                                                        }
24492                                                                                        else if (model instanceof LinkedHashMap) {
24493                                                                                                String columnName = getColumnNameOnly(rightObjectName.toString());
24494                                                                                                LinkedHashMap<String, ResultColumn> resultColumns = (LinkedHashMap<String, ResultColumn>)model;
24495                                                                                                if (resultColumns.containsKey(columnName)) {
24496                                                                                                        ResultColumn resultColumn = resultColumns.get(columnName);
24497                                                                                                        joinRelation.setTarget(new ResultColumnRelationshipElement(resultColumn));
24498                                                                                                }
24499                                                                                        }
24500                                                                                } else {
24501                                                                                        ResultColumn resultColumn = matchResultColumn(table.getColumns(),
24502                                                                                                        rightObjectName);
24503                                                                                        if (resultColumn != null) {
24504                                                                                                joinRelation.setTarget(new ResultColumnRelationshipElement(resultColumn));
24505                                                                                        }
24506                                                                                }
24507                                                                        }
24508                                                                }
24509                                                                else if(rightObject instanceof TFunctionCall) {
24510                                                                        Object functionObj = createFunction(rightObject);
24511                                                                        if(functionObj instanceof Function) {
24512                                                                                Function function = (Function)functionObj;
24513                                                                                joinRelation.setTarget(new ResultColumnRelationshipElement(function.getColumns().get(0)));
24514                                                                        }
24515                                                                }
24516                                                        }
24517                                                }
24518                                        }
24519                                }
24520                        }
24521                        return true;
24522                }
24523        }
24524
24525        @Deprecated
24526        public static Dataflow getSqlflowJSONModel(dataflow dataflow) {
24527                EDbVendor vendor = ModelBindingManager.getGlobalVendor();
24528                if(vendor == null) {
24529                        throw new IllegalArgumentException("getSqlflowJSONModel(dataflow dataflow) is deprecated, please call method getSqlflowJSONModel(dataflow dataflow, EDbVendor vendor).");
24530                }
24531                return getSqlflowJSONModel(vendor, dataflow, false);
24532        }
24533
24534        public static Dataflow getSqlflowJSONModel(dataflow dataflow, EDbVendor vendor) {
24535                return getSqlflowJSONModel(vendor, dataflow, false);
24536        }
24537
24538        public static Dataflow getSqlflowJSONModel(EDbVendor vendor, dataflow dataflow, boolean normalizeIdentifier) {
24539                Dataflow model = new Dataflow();
24540                
24541                if (dataflow.getErrors() != null && !dataflow.getErrors().isEmpty()) {
24542                        List<Error> errorList = new ArrayList<Error>();
24543                        for (error error : dataflow.getErrors()) {
24544                                Error err = new Error();
24545                                err.setErrorMessage(error.getErrorMessage());
24546                                err.setErrorType(error.getErrorType());
24547                                err.setCoordinates(Coordinate.parse(error.getCoordinate()));
24548                                err.setFile(err.getFile());
24549                                err.setOriginCoordinates(Coordinate.parse(error.getOriginCoordinate()));
24550                                errorList.add(err);
24551                        }
24552                        model.setErrors(errorList.toArray(new Error[0]));
24553                }
24554
24555                Sqlflow sqlflow = MetadataUtil.convertDataflowToMetadata(vendor, dataflow);
24556                sqlflow.setErrorMessages(null);
24557                model.setDbobjs(sqlflow);
24558                model.setOrientation(dataflow.getOrientation());
24559
24560
24561                List<gudusoft.gsqlparser.dlineage.dataflow.model.json.Process> processes = new ArrayList<gudusoft.gsqlparser.dlineage.dataflow.model.json.Process>();
24562                if(dataflow.getProcesses()!=null){
24563                        for(process process: dataflow.getProcesses()){
24564                                gudusoft.gsqlparser.dlineage.dataflow.model.json.Process processModel = new gudusoft.gsqlparser.dlineage.dataflow.model.json.Process();
24565                                processModel.setId(process.getId());
24566                                processModel.setName(process.getName());
24567                                processModel.setProcedureId(process.getProcedureId());
24568                                processModel.setProcedureName(process.getProcedureName());
24569                                processModel.setType(process.getType());
24570                                processModel.setCoordinate(process.getCoordinate());
24571                                processModel.setDatabase(process.getDatabase());
24572                                processModel.setSchema(process.getSchema());
24573                                processModel.setServer(process.getServer());
24574                                processModel.setQueryHashId(process.getQueryHashId());
24575                                if (process.getTransforms() != null && !process.getTransforms().isEmpty()) {
24576                                        List<gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform> transforms = new ArrayList<gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform>();
24577                                        for (transform transform : process.getTransforms()) {
24578                                                gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform item = new gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform();
24579                                                item.setCode(transform.getCode());
24580                                                item.setType(transform.getType());
24581                                                item.setCoordinate(transform.getCoordinate(true));
24582                                                transforms.add(item);
24583                                        }
24584                                        processModel.setTransforms(transforms
24585                                                        .toArray(new gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform[0]));
24586                                }
24587                                processes.add(processModel);
24588                        }
24589                }
24590                model.setProcesses(processes.toArray(new gudusoft.gsqlparser.dlineage.dataflow.model.json.Process[0]));
24591
24592                List<gudusoft.gsqlparser.dlineage.dataflow.model.json.Relationship> relations = new ArrayList<gudusoft.gsqlparser.dlineage.dataflow.model.json.Relationship>();
24593                if (dataflow.getRelationships() != null) {
24594                        for (relationship relation : dataflow.getRelationships()) {
24595                                gudusoft.gsqlparser.dlineage.dataflow.model.json.Relationship relationModel;
24596                                if (relation.getType().equals("join")) {
24597                                        gudusoft.gsqlparser.dlineage.dataflow.model.json.JoinRelationship joinRelationModel = new gudusoft.gsqlparser.dlineage.dataflow.model.json.JoinRelationship();
24598                                        joinRelationModel.setCondition(relation.getCondition());
24599                                        joinRelationModel.setJoinType(relation.getJoinType());
24600                                        joinRelationModel.setClause(relation.getClause());
24601                                        relationModel = joinRelationModel;
24602                                } else {
24603                                        relationModel = new gudusoft.gsqlparser.dlineage.dataflow.model.json.Relationship();
24604                                }
24605
24606                                relationModel.setId(relation.getId());
24607                                relationModel.setProcessId(relation.getProcessId());
24608                                relationModel.setProcessType(relation.getProcessType());
24609                                relationModel.setType(relation.getType());
24610                                relationModel.setEffectType(relation.getEffectType());
24611                                relationModel.setPartition(relation.getPartition());
24612                                relationModel.setFunction(relation.getFunction());
24613                                relationModel.setProcedureId(relation.getProcedureId());
24614                                relationModel.setSqlHash(relation.getSqlHash());
24615                                relationModel.setCondition(relation.getCondition());
24616                                relationModel.setSqlComment(relation.getSqlComment());
24617                                relationModel.setTimestampMax(relation.getTimestampMax());
24618                                relationModel.setTimestampMin(relation.getTimestampMin());
24619                                if (Boolean.TRUE.equals(relation.getBuiltIn())) {
24620                                        relationModel.setBuiltIn(relation.getBuiltIn());
24621                                }
24622                                relationModel.setCallStmt(relation.getCallStmt());
24623                                relationModel.setCallCoordinate(relation.getCallCoordinate());
24624                                
24625                                if (relation.getTarget() != null && relation.getSources() != null && !relation.getSources().isEmpty()) {
24626                                        {
24627                                                gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement targetModel = new gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement();
24628                                                targetColumn target = relation.getTarget();
24629                                                if (normalizeIdentifier) {
24630                                                        targetModel.setColumn(SQLUtil.getIdentifierNormalColumnName(vendor, target.getColumn()));
24631                                                        targetModel.setParentName(
24632                                                                        SQLUtil.getIdentifierNormalTableName(vendor, target.getParent_name()));
24633                                                        targetModel.setTargetName(
24634                                                                        SQLUtil.getIdentifierNormalColumnName(vendor, target.getTarget_name()));
24635                                                } else {
24636                                                        targetModel.setColumn(target.getColumn());
24637                                                        targetModel.setParentName(target.getParent_name());
24638                                                        targetModel.setTargetName(target.getTarget_name());
24639                                                }
24640                                                targetModel.setId(target.getId());
24641                                                targetModel.setTargetId(target.getTarget_id());
24642                                                targetModel.setParentId(target.getParent_id());
24643                                                targetModel.setCoordinates(Coordinate.parse(target.getCoordinate()));
24644                                                targetModel.setFunction(target.getFunction());
24645                                                targetModel.setType(target.getType());
24646                                                relationModel.setTarget(targetModel);
24647                                        }
24648
24649                                        List<gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement> sourceModels = new ArrayList<gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement>();
24650                                        for (sourceColumn source : relation.getSources()) {
24651                                                gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement sourceModel = new gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement();
24652                                                if (normalizeIdentifier) {
24653                                                        sourceModel.setColumn(SQLUtil.getIdentifierNormalColumnName(vendor, source.getColumn()));
24654                                                        sourceModel.setParentName(
24655                                                                        SQLUtil.getIdentifierNormalTableName(vendor, source.getParent_name()));
24656                                                        sourceModel.setSourceName(
24657                                                                        SQLUtil.getIdentifierNormalColumnName(vendor, source.getSource_name()));
24658                                                } else {
24659                                                        sourceModel.setColumn(source.getColumn());
24660                                                        sourceModel.setParentName(source.getParent_name());
24661                                                        sourceModel.setSourceName(source.getSource_name());
24662                                                }
24663                                                sourceModel.setColumnType(source.getColumn_type());
24664                                                sourceModel.setId(source.getId());
24665                                                sourceModel.setParentId(source.getParent_id());
24666                                                sourceModel.setSourceId(source.getSource_id());
24667                                                sourceModel.setCoordinates(Coordinate.parse(source.getCoordinate()));
24668                                                sourceModel.setClauseType(source.getClauseType());
24669                                                sourceModel.setType(source.getType());
24670                                                sourceModels.add(sourceModel);
24671                                                if (source.getTransforms() != null && !source.getTransforms().isEmpty()) {
24672                                                        List<gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform> transforms = new ArrayList<gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform>();
24673                                                        for (transform transform : source.getTransforms()) {
24674                                                                gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform item = new gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform();
24675                                                                item.setCode(transform.getCode());
24676                                                                item.setType(transform.getType());
24677                                                                item.setCoordinate(transform.getCoordinate(true));
24678                                                                transforms.add(item);
24679                                                        }
24680                                                        sourceModel.setTransforms(transforms
24681                                                                        .toArray(new gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform[0]));
24682                                                }
24683                                                
24684                                                if (source.getCandidateParents() != null && !source.getCandidateParents().isEmpty()) {
24685                                                        List<gudusoft.gsqlparser.dlineage.dataflow.model.json.CandidateTable> candidateParents = new ArrayList<gudusoft.gsqlparser.dlineage.dataflow.model.json.CandidateTable>();
24686                                                        for (candidateTable candidateTable : source.getCandidateParents()) {
24687                                                                gudusoft.gsqlparser.dlineage.dataflow.model.json.CandidateTable item = new gudusoft.gsqlparser.dlineage.dataflow.model.json.CandidateTable();
24688                                                                item.setId(candidateTable.getId());
24689                                                                if (normalizeIdentifier) {
24690                                                                        item.setName(
24691                                                                                        SQLUtil.getIdentifierNormalTableName(vendor, candidateTable.getName()));
24692                                                                } else {
24693                                                                        item.setName(candidateTable.getName());
24694                                                                }
24695                                                                candidateParents.add(item);
24696                                                        }
24697                                                        sourceModel.setCandidateParents(candidateParents
24698                                                                        .toArray(new gudusoft.gsqlparser.dlineage.dataflow.model.json.CandidateTable[0]));
24699                                                }
24700                                        }
24701                                        relationModel.setSources(sourceModels
24702                                                        .toArray(new gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement[0]));
24703                                        relations.add(relationModel);
24704                                } else if (relation.getCaller() != null && relation.getCallees() != null
24705                                                && !relation.getCallees().isEmpty()) {
24706                                        {
24707                                                gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement targetModel = new gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement();
24708                                                targetColumn target = relation.getCaller();
24709                                                if (normalizeIdentifier) {
24710                                                        targetModel.setName(SQLUtil.getIdentifierNormalColumnName(vendor, target.getName()));
24711                                                } else {
24712                                                        targetModel.setName(target.getName());
24713                                                }
24714                                                targetModel.setId(target.getId());
24715                                                targetModel.setCoordinates(Coordinate.parse(target.getCoordinate()));
24716                                                targetModel.setType(target.getType());
24717                                                relationModel.setCaller(targetModel);
24718                                        }
24719
24720                                        List<gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement> sourceModels = new ArrayList<gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement>();
24721                                        for (sourceColumn source : relation.getCallees()) {
24722                                                gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement sourceModel = new gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement();
24723                                                if (normalizeIdentifier) {
24724                                                        sourceModel.setName(SQLUtil.getIdentifierNormalColumnName(vendor, source.getName()));
24725                                                } else {
24726                                                        sourceModel.setName(source.getName());
24727                                                }
24728                                                sourceModel.setId(source.getId());
24729                                                sourceModel.setCoordinates(Coordinate.parse(source.getCoordinate()));
24730                                                sourceModel.setType(source.getType());
24731                                                sourceModels.add(sourceModel);
24732                                        }
24733                                        relationModel.setCallees(sourceModels
24734                                                        .toArray(new gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement[0]));
24735                                        relations.add(relationModel);
24736                                }
24737                        }
24738                }
24739                model.setRelationships(relations.toArray(new gudusoft.gsqlparser.dlineage.dataflow.model.json.Relationship[0]));
24740                return model;
24741        }
24742
24743        public static String getVersion() {
24744                return "3.1.4";
24745        }
24746
24747        public static String getReleaseDate() {
24748                return "2023-03-04";
24749        }
24750
24751        public static void main(String[] args) {
24752                if (args.length < 1) {
24753                        System.out.println(
24754                                        "Usage: java DataFlowAnalyzer [/f <path_to_sql_file>] [/d <path_to_directory_includes_sql_files>] [/s [/text]] [/json] [/traceView] [/t <database type>] [/o <output file path>][/version]");
24755                        System.out.println("/f: Option, specify the sql file path to analyze fdd relation.");
24756                        System.out.println("/d: Option, specify the sql directory path to analyze fdd relation.");
24757                        System.out.println("/j: Option, analyze the join relation.");
24758                        System.out.println("/s: Option, simple output, ignore the intermediate results.");
24759                        System.out.println("/i: Option, ignore all result sets.");
24760                        System.out.println("/traceView: Option, analyze the source tables of views.");
24761                        System.out.println("/text: Option, print the plain text format output.");
24762                        System.out.println("/json: Option, print the json format output.");
24763                        System.out.println(
24764                                        "/t: Option, set the database type. Support oracle, mysql, mssql, db2, netezza, teradata, informix, sybase, postgresql, hive, greenplum and redshift, the default type is oracle");
24765                        System.out.println("/o: Option, write the output stream to the specified file.");
24766                        System.out.println("/log: Option, generate a dataflow.log file to log information.");
24767                        return;
24768                }
24769
24770                File sqlFiles = null;
24771
24772                List<String> argList = Arrays.asList(args);
24773
24774                if (argList.indexOf("/version") != -1) {
24775                        System.out.println("Version: " + DataFlowAnalyzer.getVersion());
24776                        System.out.println("Release Date: " + DataFlowAnalyzer.getReleaseDate());
24777                        return;
24778                }
24779
24780                if (argList.indexOf("/f") != -1 && argList.size() > argList.indexOf("/f") + 1) {
24781                        sqlFiles = new File(args[argList.indexOf("/f") + 1]);
24782                        if (!sqlFiles.exists() || !sqlFiles.isFile()) {
24783                                System.out.println(sqlFiles + " is not a valid file.");
24784                                return;
24785                        }
24786                } else if (argList.indexOf("/d") != -1 && argList.size() > argList.indexOf("/d") + 1) {
24787                        sqlFiles = new File(args[argList.indexOf("/d") + 1]);
24788                        if (!sqlFiles.exists() || !sqlFiles.isDirectory()) {
24789                                System.out.println(sqlFiles + " is not a valid directory.");
24790                                return;
24791                        }
24792                } else {
24793                        System.out.println("Please specify a sql file path or directory path to analyze dlineage.");
24794                        return;
24795                }
24796
24797                EDbVendor vendor = EDbVendor.dbvoracle;
24798
24799                int index = argList.indexOf("/t");
24800
24801                if (index != -1 && args.length > index + 1) {
24802                        vendor = TGSqlParser.getDBVendorByName(args[index + 1]);
24803                }
24804
24805                String outputFile = null;
24806
24807                index = argList.indexOf("/o");
24808
24809                if (index != -1 && args.length > index + 1) {
24810                        outputFile = args[index + 1];
24811                }
24812
24813                FileOutputStream writer = null;
24814                if (outputFile != null) {
24815                        try {
24816                                writer = new FileOutputStream(outputFile);
24817                                System.setOut(new PrintStream(writer));
24818                        } catch (FileNotFoundException e) {
24819                                logger.error("output file is not found.", e);
24820                        }
24821                }
24822
24823                boolean simple = argList.indexOf("/s") != -1;
24824                boolean ignoreResultSets = argList.indexOf("/i") != -1;
24825                boolean showJoin = argList.indexOf("/j") != -1;
24826                boolean textFormat = false;
24827                boolean jsonFormat = false;
24828                if (simple) {
24829                        textFormat = argList.indexOf("/text") != -1;
24830                }
24831
24832                boolean traceView = argList.indexOf("/traceView") != -1;
24833                if (traceView) {
24834                        simple = true;
24835                }
24836
24837                jsonFormat = argList.indexOf("/json") != -1;
24838
24839                DataFlowAnalyzer dlineage = new DataFlowAnalyzer(sqlFiles, vendor, simple);
24840
24841                dlineage.setShowJoin(showJoin);
24842                dlineage.setIgnoreRecordSet(ignoreResultSets);
24843                // dlineage.setShowImplicitSchema(true);
24844
24845                if (simple && !jsonFormat) {
24846                        dlineage.setTextFormat(textFormat);
24847                }
24848
24849                String result = dlineage.generateDataFlow();
24850
24851//              dataflow dataflow = ProcessUtility.generateTableLevelLineage(dlineage, dlineage.getDataFlow());
24852//              System.out.println(result);
24853
24854                if (jsonFormat) {
24855                        // Map jsonResult = new LinkedHashMap();
24856                        Dataflow model = getSqlflowJSONModel(vendor, dlineage.getDataFlow(), true);
24857                        // jsonResult.put("data", BeanUtils.bean2Map(model));
24858                        result = JSON.toJSONString(model);
24859                } else if (traceView) {
24860                        result = dlineage.traceView();
24861                }
24862
24863                if (result != null) {
24864                        System.out.println(result);
24865
24866                        if (writer != null && result.length() < 1024 * 1024) {
24867                                System.err.println(result);
24868                        }
24869                }
24870
24871                try {
24872                        if (writer != null) {
24873                                writer.close();
24874                        }
24875                } catch (IOException e) {
24876                        logger.error("close writer failed.", e);
24877                }
24878
24879                boolean log = argList.indexOf("/log") != -1;
24880
24881                PrintStream systemSteam = System.err;
24882                ByteArrayOutputStream sw = new ByteArrayOutputStream();
24883                PrintStream pw = new PrintStream(sw);
24884                System.setErr(pw);
24885
24886
24887                List<ErrorInfo> errors = dlineage.getErrorMessages();
24888                if (!errors.isEmpty()) {
24889                        System.err.println("Error log:\n");
24890                        for (int i = 0; i < errors.size(); i++) {
24891                                System.err.println(errors.get(i).getErrorMessage());
24892                        }
24893                }
24894
24895                if (sw != null) {
24896                        String errorMessage = sw.toString().trim();
24897                        if (errorMessage.length() > 0) {
24898                                if (log) {
24899                                        try {
24900                                                pw = new PrintStream(new File(".", "dataflow.log"));
24901                                                pw.print(errorMessage);
24902                                        } catch (FileNotFoundException e) {
24903                                                logger.error("error log file is not found.", e);
24904                                        }
24905                                }
24906
24907                                System.setErr(systemSteam);
24908                                System.err.println(errorMessage);
24909                        }
24910                }
24911        }
24912
24913        public List<ErrorInfo> getErrorMessages() {
24914                return errorInfos;
24915        }
24916
24917        public String traceView() {
24918                StringBuilder buffer = new StringBuilder();
24919                dataflow dataflow = this.getDataFlow();
24920                Map<table, Set<table>> traceViewMap = new LinkedHashMap<table, Set<table>>();
24921                if (dataflow != null && dataflow.getViews() != null) {
24922                        List<relationship> relations = dataflow.getRelationships();
24923                        Map<String, table> viewMap = new HashMap<String, table>();
24924                        Map<String, table> tableMap = new HashMap<String, table>();
24925                        for (table view : dataflow.getViews()) {
24926                                viewMap.put(view.getId(), view);
24927                                tableMap.put(view.getId(), view);
24928                        }
24929                        for (table table : dataflow.getTables()) {
24930                                tableMap.put(table.getId(), table);
24931                        }
24932                        for (relationship relation : relations) {
24933                                if (!RelationshipType.fdd.name().equals(relation.getType())) {
24934                                        continue;
24935                                }
24936                                String parentId = relation.getTarget().getParent_id();
24937                                if (viewMap.containsKey(parentId)) {
24938                                        if (!traceViewMap.containsKey(viewMap.get(parentId))) {
24939                                                traceViewMap.put(viewMap.get(parentId), new LinkedHashSet<table>());
24940                                        }
24941
24942                                        for (sourceColumn sourceColumn : relation.getSources()) {
24943                                                traceViewMap.get(viewMap.get(parentId)).add(tableMap.get(sourceColumn.getParent_id()));
24944                                        }
24945                                }
24946                        }
24947
24948                        Map<table, Set<table>> viewTableMap = new LinkedHashMap<table, Set<table>>();
24949                        for (table view : traceViewMap.keySet()) {
24950                                Set<table> tables = new LinkedHashSet<table>();
24951                                traverseViewSourceTables(tables, view, traceViewMap);
24952                                viewTableMap.put(view, tables);
24953                        }
24954
24955                        for (table view : viewTableMap.keySet()) {
24956                                buffer.append(view.getFullName());
24957                                for (table table : viewTableMap.get(view)) {
24958                                        buffer.append(",").append(table.getFullName());
24959                                }
24960                                buffer.append(System.getProperty("line.separator"));
24961                        }
24962                }
24963                return buffer.toString().trim();
24964        }
24965
24966        private void traverseViewSourceTables(Set<table> tables, table view, Map<table, Set<table>> traceViewMap) {
24967                Set<table> sourceTables = traceViewMap.get(view);
24968                for (table sourceTable : sourceTables) {
24969                        if (sourceTable.isTable()) {
24970                                tables.add(sourceTable);
24971                        } else if (sourceTable.isView()) {
24972                                traverseViewSourceTables(tables, sourceTable, traceViewMap);
24973                        }
24974                }
24975        }
24976
24977        protected List<SqlInfo> convertSQL(EDbVendor vendor, String json) {
24978                List<SqlInfo> sqlInfos = new ArrayList<SqlInfo>();
24979                List sqlContents = (List) JSON.parseObject(json);
24980                for (int j = 0; j < sqlContents.size(); j++) {
24981                        Map sqlContent = (Map) sqlContents.get(j);
24982                        String sql = (String) sqlContent.get("sql");
24983                        String fileName = (String) sqlContent.get("fileName");
24984                        String filePath = (String) sqlContent.get("filePath");
24985                        if (sql != null && sql.trim().startsWith("{")) {
24986                                if (sql.indexOf("createdBy") != -1) {
24987                                        if (this.sqlenv == null) {
24988                                                TSQLEnv[] sqlenvs = new TJSONSQLEnvParser(option.getDefaultServer(),
24989                                                                option.getDefaultDatabase(), option.getDefaultSchema()).parseSQLEnv(vendor, sql);
24990                                                if (sqlenvs != null && sqlenvs.length > 0) {
24991                                                        this.sqlenv = sqlenvs[0];
24992                                                }
24993                                        }
24994                                        if (sql.toLowerCase().indexOf("sqldep") != -1 || sql.toLowerCase().indexOf("grabit") != -1) {
24995                                                Map queryObject = (Map) JSON.parseObject(sql);
24996                                                List querys = (List) queryObject.get("queries");
24997                                                if (querys != null) {
24998                                                        for (int i = 0; i < querys.size(); i++) {
24999                                                                Map object = (Map) querys.get(i);
25000                                                                SqlInfo info = new SqlInfo();
25001                                                                info.setSql(JSON.toJSONString(object));
25002                                                                info.setFileName(fileName);
25003                                                                info.setFilePath(filePath);
25004                                                                info.setOriginIndex(i);
25005                                                                sqlInfos.add(info);
25006                                                        }
25007                                                        queryObject.remove("queries");
25008                                                        SqlInfo info = new SqlInfo();
25009                                                        info.setSql(JSON.toJSONString(queryObject));
25010                                                        info.setFileName(fileName);
25011                                                        info.setFilePath(filePath);
25012                                                        info.setOriginIndex(querys.size());
25013                                                        sqlInfos.add(info);
25014                                                } else {
25015                                                        SqlInfo info = new SqlInfo();
25016                                                        info.setSql(JSON.toJSONString(queryObject));
25017                                                        info.setFileName(fileName);
25018                                                        info.setFilePath(filePath);
25019                                                        info.setOriginIndex(0);
25020                                                        sqlInfos.add(info);
25021                                                }
25022                                        } else if (sql.toLowerCase().indexOf("sqlflow") != -1) {
25023                                                Map sqlflow = (Map) JSON.parseObject(sql);
25024                                                List<Map> servers = (List<Map>) sqlflow.get("servers");
25025                                                if (servers != null) {
25026                                                        for (Map queryObject : servers) {
25027                                                                String name = (String) queryObject.get("name");
25028                                                                String dbVendor = (String) queryObject.get("dbVendor");
25029                                                                List querys = (List) queryObject.get("queries");
25030                                                                if (querys != null) {
25031                                                                        for (int i = 0; i < querys.size(); i++) {
25032                                                                                Map object = (Map) querys.get(i);
25033                                                                                SqlInfo info = new SqlInfo();
25034                                                                                info.setSql(JSON.toJSONString(object));
25035                                                                                info.setFileName(fileName);
25036                                                                                info.setFilePath(filePath);
25037                                                                                info.setOriginIndex(i);
25038                                                                                info.setDbVendor(dbVendor);
25039                                                                                info.setServer(name);
25040                                                                                sqlInfos.add(info);
25041                                                                        }
25042                                                                        queryObject.remove("queries");
25043                                                                        Map serverObject = new IndexedLinkedHashMap();
25044                                                                        serverObject.put("createdBy", sqlflow.get("createdBy"));
25045                                                                        serverObject.put("servers", Arrays.asList(queryObject));
25046                                                                        SqlInfo info = new SqlInfo();
25047                                                                        info.setSql(JSON.toJSONString(serverObject));
25048                                                                        info.setFileName(fileName);
25049                                                                        info.setFilePath(filePath);
25050                                                                        info.setOriginIndex(querys.size());
25051                                                                        info.setDbVendor(dbVendor);
25052                                                                        info.setServer(filePath);
25053                                                                        sqlInfos.add(info);
25054                                                                } else {
25055                                                                        SqlInfo info = new SqlInfo();
25056                                                                        info.setSql(JSON.toJSONString(queryObject));
25057                                                                        info.setFileName(fileName);
25058                                                                        info.setFilePath(filePath);
25059                                                                        info.setOriginIndex(0);
25060                                                                        sqlInfos.add(info);
25061                                                                }
25062                                                        }
25063                                                }
25064                                                
25065                                                List<Map> errorMessages =  (List<Map>) sqlflow.get("errorMessages");
25066                                                if(errorMessages!=null && !errorMessages.isEmpty()) {
25067                                                        for(Map error: errorMessages){
25068                                                                ErrorInfo errorInfo = new ErrorInfo();
25069                                                                errorInfo.setErrorType(ErrorInfo.METADATA_ERROR);
25070                                                                errorInfo.setErrorMessage((String)error.get("errorMessage"));
25071                                                                errorInfo.setFileName(fileName);
25072                                                                errorInfo.setFilePath(filePath);
25073                                                                errorInfo.setStartPosition(new Pair3<Long, Long, String>(-1L, -1L,
25074                                                                                ModelBindingManager.getGlobalHash()));
25075                                                                errorInfo.setEndPosition(new Pair3<Long, Long, String>(-1L, -1L,
25076                                                                                ModelBindingManager.getGlobalHash()));
25077                                                                errorInfo.setOriginStartPosition(new Pair<Long, Long>(-1L, -1L));
25078                                                                errorInfo.setOriginEndPosition(new Pair<Long, Long>(-1L, -1L));
25079                                                                metadataErrors.add(errorInfo);
25080                                                        }
25081                                                }
25082                                        }
25083                                }
25084                        } else if (sql != null) {
25085                                SqlInfo info = new SqlInfo();
25086                                info.setSql(sql);
25087                                info.setFileName(fileName);
25088                                info.setFilePath(filePath);
25089                                info.setOriginIndex(0);
25090                                sqlInfos.add(info);
25091                        }
25092                }
25093                return sqlInfos;
25094        }
25095
25096        public void setTextFormat(boolean textFormat) {
25097                option.setTextFormat(textFormat);
25098        }
25099
25100        public boolean isBuiltInFunctionName(TObjectName object) {
25101                if (object == null || object.getGsqlparser() == null)
25102                        return false;
25103                try {
25104                        EDbVendor vendor = object.getGsqlparser().getDbVendor();
25105                        if (vendor == EDbVendor.dbvteradata) {
25106                                boolean result = TERADATA_BUILTIN_FUNCTIONS.contains(object.toString().toUpperCase());
25107                                if (result) {
25108                                        return true;
25109                                }
25110                        }
25111
25112                        List<String> versions = functionChecker.getAvailableDbVersions(vendor);
25113                        if (versions != null && versions.size() > 0) {
25114                                for (int i = 0; i < versions.size(); i++) {
25115                                        boolean result = functionChecker.isBuiltInFunction(object.toString(),
25116                                                        object.getGsqlparser().getDbVendor(), versions.get(i));
25117                                        if (result) {
25118                                                return result;
25119                                        }
25120                                }
25121
25122                                // boolean result =
25123                                // TERADATA_BUILTIN_FUNCTIONS.contains(object.toString());
25124                                // if (result) {
25125                                // return true;
25126                                // }
25127                        }
25128                } catch (Exception e) {
25129                }
25130
25131                return false;
25132        }
25133        
25134        public boolean isBuiltInFunctionName(String functionName) {
25135                if (functionName == null)
25136                        return false;
25137                try {
25138                        EDbVendor vendor = getOption().getVendor();
25139                        if (vendor == EDbVendor.dbvteradata) {
25140                                boolean result = TERADATA_BUILTIN_FUNCTIONS.contains(functionName.toUpperCase());
25141                                if (result) {
25142                                        return true;
25143                                }
25144                        }
25145
25146                        List<String> versions = functionChecker.getAvailableDbVersions(vendor);
25147                        if (versions != null && versions.size() > 0) {
25148                                for (int i = 0; i < versions.size(); i++) {
25149                                        boolean result = functionChecker.isBuiltInFunction(functionName.toUpperCase(),
25150                                                        vendor, versions.get(i));
25151                                        if (result) {
25152                                                return result;
25153                                        }
25154                                }
25155
25156                                // boolean result =
25157                                // TERADATA_BUILTIN_FUNCTIONS.contains(object.toString());
25158                                // if (result) {
25159                                // return true;
25160                                // }
25161                        }
25162                } catch (Exception e) {
25163                }
25164
25165                return false;
25166        }
25167
25168        public boolean isKeyword(TObjectName object) {
25169                if (object == null || object.getGsqlparser() == null)
25170                        return false;
25171                try {
25172                        EDbVendor vendor = object.getGsqlparser().getDbVendor();
25173
25174                        List<String> versions = keywordChecker.getAvailableDbVersions(vendor);
25175                        if (versions != null && versions.size() > 0) {
25176                                for (int i = 0; i < versions.size(); i++) {
25177                                        List<String> segments = SQLUtil.parseNames(object.toString());
25178                                        boolean result = keywordChecker.isKeyword(segments.get(segments.size() - 1),
25179                                                        object.getGsqlparser().getDbVendor(), versions.get(i), true);
25180                                        if (result) {
25181                                                return result;
25182                                        }
25183                                }
25184                        }
25185                } catch (Exception e) {
25186                }
25187
25188                return false;
25189        }
25190        
25191        public boolean isKeyword(String objectName) {
25192                if (objectName == null)
25193                        return false;
25194                try {
25195                        EDbVendor vendor = getOption().getVendor();
25196
25197                        List<String> versions = keywordChecker.getAvailableDbVersions(vendor);
25198                        if (versions != null && versions.size() > 0) {
25199                                for (int i = 0; i < versions.size(); i++) {
25200                                        List<String> segments = SQLUtil.parseNames(objectName);
25201                                        boolean result = keywordChecker.isKeyword(segments.get(segments.size() - 1),
25202                                                        vendor, versions.get(i), false);
25203                                        if (result) {
25204                                                return result;
25205                                        }
25206                                }
25207                        }
25208                } catch (Exception e) {
25209                }
25210
25211                return false;
25212        }
25213
25214        public boolean isAggregateFunction(TFunctionCall func) {
25215                if (func == null)
25216                        return false;
25217                return Arrays
25218                                .asList(new String[] { "AVG", "COUNT", "MAX", "MIN", "SUM", "COLLECT", "CORR", "COVAR_POP",
25219                                                "COVAR_SAMP", "CUME_DIST", "DENSE_RANK", "FIRST", "GROUP_ID", "GROUPING", "GROUPING_ID", "LAST",
25220                                                "LISTAGG", "MEDIAN", "PERCENT_RANK", "PERCENTILE_CONT", "PERCENTILE_DISC", "RANK",
25221                                                "STATS_BINOMIAL_TEST", "STATS_CROSSTAB", "STATS_F_TEST", "STATS_KS_TEST", "STATS_MODE",
25222                                                "STATS_MW_TEST", "STATS_ONE_WAY_ANOVA", "STATS_WSR_TEST", "STDDEV", "STDDEV_POP", "STDDEV_SAMP",
25223                                                "SYS_XMLAGG", "VAR_ POP", "VAR_ SAMP", "VARI ANCE", "XMLAGG", "ARRAY_AGG" })
25224                                .contains(func.getFunctionName().toString().toUpperCase());
25225        }
25226
25227        public boolean isConstant(TObjectName object) {
25228                if (object == null || object.getGsqlparser() == null)
25229                        return false;
25230                List<String> constants = Arrays.asList(new String[] { "NEXTVAL", "CURRVAL", "SYSDATE", "CENTURY", "YEAR",
25231                                "MONTH", "DAY", "HOUR", "MINUTE", "SECOND" });
25232                List<String> segments = SQLUtil.parseNames(object.toString());
25233                // sequence.NEXTVAL or sequence.CURRVAL is a sequence reference, not a constant
25234                // This syntax is used by Oracle, Snowflake, and accepted by other vendors for compatibility
25235                String columnNameOnly = object.getColumnNameOnly();
25236                if (segments.size() > 1 && ("NEXTVAL".equalsIgnoreCase(columnNameOnly) || "CURRVAL".equalsIgnoreCase(columnNameOnly))) {
25237                        return false;
25238                }
25239                boolean result = constants.indexOf(segments.get(segments.size() - 1).toUpperCase()) != -1;
25240                if (result) {
25241                        return result;
25242                }
25243                if (isKeyword(object)) {
25244                        return true;
25245                }
25246                return false;
25247        }
25248
25249        private Pair3<Long, Long, Integer> convertCoordinate(Pair3<Long, Long, String> position) {
25250//              if (ModelBindingManager.getGlobalOption()!=null && ModelBindingManager.getGlobalOption().isIgnoreCoordinate()) {
25251//                      return new Pair3<>(-1L, -1L, -1);
25252//              }
25253                return new Pair3<Long, Long, Integer>(position.first, position.second,
25254                                ModelBindingManager.getGlobalSqlInfo().getIndexOf(position.third));
25255        }
25256
25257        /**
25258         * Analyze MDX SELECT statement to generate data lineage.
25259         * Maps MDX cube as source table, measures/dimensions as columns,
25260         * and creates dataflow relationships to the result set.
25261         */
25262        private void analyzeMdxSelectStmt(gudusoft.gsqlparser.stmt.mdx.TMdxSelect stmt) {
25263                // Get cube name from FROM clause
25264                gudusoft.gsqlparser.nodes.mdx.TMdxIdentifierNode cube = stmt.getCube();
25265                if (cube == null) {
25266                        return;
25267                }
25268
25269                String cubeName = getMdxIdentifierName(cube);
25270                Table cubeTable = modelFactory.createTableByName(cubeName, false);
25271
25272                // Collect all MDX identifier references from axes and WHERE clause
25273                List<String> measureNames = new ArrayList<String>();
25274                List<String> dimensionNames = new ArrayList<String>();
25275
25276                // Process axes (COLUMNS, ROWS, etc.)
25277                if (stmt.getAxes() != null) {
25278                        for (int i = 0; i < stmt.getAxes().size(); i++) {
25279                                gudusoft.gsqlparser.nodes.mdx.TMdxAxisNode axis = stmt.getAxes().getElement(i);
25280                                if (axis.getExpNode() != null) {
25281                                        collectMdxReferences(axis.getExpNode(), measureNames, dimensionNames);
25282                                }
25283                        }
25284                }
25285
25286                // Process WHERE clause (slicer dimension)
25287                if (stmt.getWhere() != null && stmt.getWhere().getFilter() != null) {
25288                        collectMdxReferences(stmt.getWhere().getFilter(), measureNames, dimensionNames);
25289                }
25290
25291                // Process WITH MEMBER definitions
25292                if (stmt.getWiths() != null) {
25293                        for (int i = 0; i < stmt.getWiths().size(); i++) {
25294                                gudusoft.gsqlparser.nodes.mdx.TMdxWithNode withNode = stmt.getWiths().getElement(i);
25295                                if (withNode.getNameNode() != null) {
25296                                        String withName = getMdxIdentifierName(withNode.getNameNode());
25297                                        // Calculated members are treated as derived measures
25298                                        if (withName.toLowerCase().startsWith("[measures].")
25299                                                        || withName.toLowerCase().startsWith("measures.")) {
25300                                                measureNames.add(withName);
25301                                        }
25302                                }
25303                                // Also collect references used in the WITH expression
25304                                if (withNode.getExprNode() != null) {
25305                                        collectMdxReferences(withNode.getExprNode(), measureNames, dimensionNames);
25306                                }
25307                        }
25308                }
25309
25310                // Create columns on the cube table for all referenced measures and dimensions
25311                Set<String> addedColumns = new LinkedHashSet<String>();
25312                for (String measure : measureNames) {
25313                        if (addedColumns.add(measure)) {
25314                                modelFactory.createTableColumn(cubeTable, measure);
25315                        }
25316                }
25317                for (String dimension : dimensionNames) {
25318                        if (addedColumns.add(dimension)) {
25319                                modelFactory.createTableColumn(cubeTable, dimension);
25320                        }
25321                }
25322
25323                // Create result set with all referenced columns
25324                ResultSet resultSet = modelFactory.createResultSet(stmt, false);
25325                if (resultSet != null) {
25326                        for (String colName : addedColumns) {
25327                                TableColumn sourceCol = findTableColumnByName(cubeTable, colName);
25328                                if (sourceCol != null) {
25329                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
25330                                        relation.setEffectType(EffectType.select);
25331                                        relation.addSource(new TableColumnRelationshipElement(sourceCol));
25332                                        relation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
25333                                                        resultSet.getRelationRows()));
25334                                }
25335                        }
25336                }
25337        }
25338
25339        private TableColumn findTableColumnByName(Table table, String name) {
25340                for (TableColumn col : table.getColumns()) {
25341                        if (col.getName().equals(name)) {
25342                                return col;
25343                        }
25344                }
25345                return null;
25346        }
25347
25348        /**
25349         * Extract the display name from an MDX identifier node.
25350         * E.g., [Measures].[Departures NEAT] -> [Measures].[Departures NEAT]
25351         */
25352        private String getMdxIdentifierName(gudusoft.gsqlparser.nodes.mdx.TMdxIdentifierNode idNode) {
25353                StringBuilder sb = new StringBuilder();
25354                for (int i = 0; i < idNode.getSegmentList().size(); i++) {
25355                        if (i > 0) sb.append(".");
25356                        gudusoft.gsqlparser.nodes.mdx.IMdxIdentifierSegment seg = idNode.getSegmentList().getElement(i);
25357                        if (seg.getQuoting() == gudusoft.gsqlparser.nodes.mdx.EMdxQuoting.QUOTED) {
25358                                sb.append("[").append(seg.getName()).append("]");
25359                        } else {
25360                                sb.append(seg.getName());
25361                        }
25362                }
25363                return sb.toString();
25364        }
25365
25366        /**
25367         * Recursively collect measure and dimension references from MDX expression tree.
25368         * Uses iterative DFS to avoid StackOverflow on deeply nested expressions.
25369         */
25370        private void collectMdxReferences(gudusoft.gsqlparser.nodes.mdx.TMdxExpNode expr,
25371                        List<String> measures, List<String> dimensions) {
25372                Deque<gudusoft.gsqlparser.nodes.mdx.TMdxExpNode> stack = new ArrayDeque<gudusoft.gsqlparser.nodes.mdx.TMdxExpNode>();
25373                stack.push(expr);
25374
25375                while (!stack.isEmpty()) {
25376                        gudusoft.gsqlparser.nodes.mdx.TMdxExpNode current = stack.pop();
25377                        if (current == null) continue;
25378
25379                        if (current instanceof gudusoft.gsqlparser.nodes.mdx.TMdxIdentifierNode) {
25380                                gudusoft.gsqlparser.nodes.mdx.TMdxIdentifierNode idNode =
25381                                                (gudusoft.gsqlparser.nodes.mdx.TMdxIdentifierNode) current;
25382                                String name = getMdxIdentifierName(idNode);
25383                                if (name.toLowerCase().startsWith("[measures].")
25384                                                || name.toLowerCase().startsWith("measures.")) {
25385                                        measures.add(name);
25386                                } else if (idNode.getSegmentList().size() > 1) {
25387                                        // Multi-segment identifiers that aren't measures are dimensions
25388                                        dimensions.add(name);
25389                                }
25390                        } else if (current instanceof gudusoft.gsqlparser.nodes.mdx.TMdxSetNode) {
25391                                gudusoft.gsqlparser.nodes.mdx.TMdxSetNode setNode =
25392                                                (gudusoft.gsqlparser.nodes.mdx.TMdxSetNode) current;
25393                                if (setNode.getTupleList() != null) {
25394                                        for (int i = 0; i < setNode.getTupleList().size(); i++) {
25395                                                stack.push(setNode.getTupleList().getElement(i));
25396                                        }
25397                                }
25398                        } else if (current instanceof gudusoft.gsqlparser.nodes.mdx.TMdxTupleNode) {
25399                                gudusoft.gsqlparser.nodes.mdx.TMdxTupleNode tupleNode =
25400                                                (gudusoft.gsqlparser.nodes.mdx.TMdxTupleNode) current;
25401                                if (tupleNode.getExprList() != null) {
25402                                        for (int i = 0; i < tupleNode.getExprList().size(); i++) {
25403                                                stack.push(tupleNode.getExprList().getElement(i));
25404                                        }
25405                                }
25406                        } else if (current instanceof gudusoft.gsqlparser.nodes.mdx.TMdxBinOpNode) {
25407                                gudusoft.gsqlparser.nodes.mdx.TMdxBinOpNode binOp =
25408                                                (gudusoft.gsqlparser.nodes.mdx.TMdxBinOpNode) current;
25409                                if (binOp.getRightExprNode() != null) stack.push(binOp.getRightExprNode());
25410                                if (binOp.getLeftExprNode() != null) stack.push(binOp.getLeftExprNode());
25411                        } else if (current instanceof gudusoft.gsqlparser.nodes.mdx.TMdxFunctionNode) {
25412                                gudusoft.gsqlparser.nodes.mdx.TMdxFunctionNode funcNode =
25413                                                (gudusoft.gsqlparser.nodes.mdx.TMdxFunctionNode) current;
25414                                if (funcNode.getArguments() != null) {
25415                                        for (int i = 0; i < funcNode.getArguments().size(); i++) {
25416                                                stack.push(funcNode.getArguments().getElement(i));
25417                                        }
25418                                }
25419                        }
25420                }
25421        }
25422
25423        /**
25424         * Analyze a Power Query M-language document for data lineage.
25425         *
25426         * Delegates to TPowerQueryAnalyzer to extract navigation chains and
25427         * NativeQuery embedded SQL, then feeds the results back through the
25428         * standard SQL analysis pipeline so the dataflow model contains
25429         * regular table/column lineage.
25430         */
25431        private void analyzePowerQueryDocumentStmt(TPowerQueryDocumentStmt pqStmt) {
25432                TPowerQueryAnalyzer pqAnalyzer = new TPowerQueryAnalyzer(pqStmt);
25433
25434                if (option.getPowerQueryInnerVendor() != null) {
25435                        pqAnalyzer.withExplicitInnerVendor(option.getPowerQueryInnerVendor());
25436                }
25437
25438                PowerQueryLineageResult result = pqAnalyzer.analyze();
25439
25440                if (result.isEmpty()) {
25441                        for (String w : result.getWarnings()) {
25442                                logger.warn("Power Query: " + w);
25443                        }
25444                        return;
25445                }
25446
25447                for (PowerQueryLineageResult.NativeQueryRef nq : result.getNativeQueryReferences()) {
25448                        if (nq.innerParser != null && nq.innerParseReturnCode == 0) {
25449                                for (int i = 0; i < nq.innerParser.sqlstatements.size(); i++) {
25450                                        analyzeCustomSqlStmt(nq.innerParser.sqlstatements.get(i));
25451                                }
25452                        }
25453                }
25454
25455                for (PowerQueryLineageResult.NavigationRef nav : result.getNavigationReferences()) {
25456                        if (nav.syntheticSelect != null && nav.resolvedVendor != null) {
25457                                TGSqlParser synParser = new TGSqlParser(nav.resolvedVendor);
25458                                synParser.sqltext = nav.syntheticSelect;
25459                                int rc = synParser.parse();
25460                                if (rc == 0) {
25461                                        for (int i = 0; i < synParser.sqlstatements.size(); i++) {
25462                                                analyzeCustomSqlStmt(synParser.sqlstatements.get(i));
25463                                        }
25464                                }
25465                        }
25466                }
25467
25468                for (String w : result.getWarnings()) {
25469                        logger.warn("Power Query: " + w);
25470                }
25471        }
25472
25473}