Source code

001
002package gudusoft.gsqlparser.dlineage;
003
004import gudusoft.gsqlparser.*;
005import gudusoft.gsqlparser.common.structured.StructuredAdapterContext;
006import gudusoft.gsqlparser.common.structured.StructuredDataflowDescriptor;
007import gudusoft.gsqlparser.common.structured.StructuredDataflowRegistry;
008import gudusoft.gsqlparser.common.structured.StructuredFieldBinding;
009import gudusoft.gsqlparser.common.structured.StructuredValueSource;
010import gudusoft.gsqlparser.dlineage.dataflow.listener.DataFlowHandleListener;
011import gudusoft.gsqlparser.dlineage.dataflow.metadata.MetadataReader;
012import gudusoft.gsqlparser.dlineage.dataflow.metadata.grabit.GrabitMetadataAnalyzer;
013import gudusoft.gsqlparser.dlineage.dataflow.metadata.sqldep.SQLDepMetadataAnalyzer;
014import gudusoft.gsqlparser.dlineage.dataflow.metadata.sqlflow.SqlflowMetadataAnalyzer;
015import gudusoft.gsqlparser.dlineage.dataflow.metadata.sqlflow.sharded.SqlflowShardedMetadataAnalyzer;
016import gudusoft.gsqlparser.dlineage.impl.powerquery.PowerQueryLineageResult;
017import gudusoft.gsqlparser.dlineage.impl.powerquery.TPowerQueryAnalyzer;
018import gudusoft.gsqlparser.dlineage.dataflow.model.*;
019import gudusoft.gsqlparser.dlineage.dataflow.model.Process;
020import gudusoft.gsqlparser.dlineage.dataflow.model.JoinRelationship.JoinClauseType;
021import gudusoft.gsqlparser.dlineage.dataflow.model.json.Coordinate;
022import gudusoft.gsqlparser.dlineage.dataflow.model.json.Dataflow;
023import gudusoft.gsqlparser.dlineage.dataflow.model.json.Error;
024import gudusoft.gsqlparser.dlineage.dataflow.model.xml.*;
025import gudusoft.gsqlparser.dlineage.dataflow.sqlenv.SQLEnvParser;
026import gudusoft.gsqlparser.dlineage.metadata.MetadataUtil;
027import gudusoft.gsqlparser.dlineage.metadata.Sqlflow;
028import gudusoft.gsqlparser.dlineage.util.*;
029import gudusoft.gsqlparser.nodes.TTable;
030import gudusoft.gsqlparser.nodes.*;
031import gudusoft.gsqlparser.nodes.couchbase.TObjectConstruct;
032import gudusoft.gsqlparser.nodes.couchbase.TPair;
033import gudusoft.gsqlparser.nodes.functions.TJsonObjectFunction;
034import gudusoft.gsqlparser.nodes.hive.THiveTransformClause;
035import gudusoft.gsqlparser.nodes.hive.THiveTransformClause.ETransformType;
036import gudusoft.gsqlparser.resolver2.ScopeBuildResult;
037import gudusoft.gsqlparser.resolver2.TSQLResolver2;
038import gudusoft.gsqlparser.sqlenv.*;
039import gudusoft.gsqlparser.sqlenv.parser.TJSONSQLEnvParser;
040import gudusoft.gsqlparser.stmt.*;
041import gudusoft.gsqlparser.stmt.db2.TDb2CallStmt;
042import gudusoft.gsqlparser.stmt.db2.TDb2ReturnStmt;
043import gudusoft.gsqlparser.stmt.db2.TDb2SqlVariableDeclaration;
044import gudusoft.gsqlparser.stmt.hive.THiveLoad;
045import gudusoft.gsqlparser.stmt.mssql.*;
046import gudusoft.gsqlparser.stmt.mysql.TLoadDataStmt;
047import gudusoft.gsqlparser.stmt.oracle.*;
048import gudusoft.gsqlparser.stmt.powerquery.TPowerQueryDocumentStmt;
049import gudusoft.gsqlparser.stmt.redshift.TRedshiftCopy;
050import gudusoft.gsqlparser.stmt.redshift.TRedshiftDeclare;
051import gudusoft.gsqlparser.stmt.snowflake.*;
052import gudusoft.gsqlparser.stmt.teradata.TTeradataCreateProcedure;
053import gudusoft.gsqlparser.util.*;
054import gudusoft.gsqlparser.util.json.JSON;
055
056import javax.script.ScriptEngine;
057import javax.script.ScriptEngineManager;
058import javax.script.ScriptException;
059import java.io.*;
060import java.util.zip.GZIPInputStream;
061import java.util.*;
062import java.util.concurrent.atomic.AtomicInteger;
063import java.util.regex.Matcher;
064import java.util.regex.Pattern;
065import java.util.stream.Collectors;
066
067import static gudusoft.gsqlparser.EJoinType.right;
068
069@SuppressWarnings("rawtypes")
070public class DataFlowAnalyzer implements IDataFlowAnalyzer {
071
072        private static final Logger logger = LoggerFactory.getLogger(DataFlowAnalyzer.class);
073
074        private static final List<String> TERADATA_BUILTIN_FUNCTIONS = Arrays
075                        .asList(new String[] { "ACCOUNT", "CURRENT_DATE", "CURRENT_ROLE", "CURRENT_TIME", "CURRENT_TIMESTAMP",
076                                        "CURRENT_USER", "DATABASE", "DATE", "PROFILE", "ROLE", "SESSION", "TIME", "USER", "SYSDATE", });
077
078        private static final List<String> CONSTANT_BUILTIN_FUNCTIONS = Arrays.asList(new String[] { "ACCOUNT",
079                        "CURRENT_DATE", "CURRENT_ROLE", "CURRENT_TIME", "CURRENT_TIMESTAMP", "CURRENT_USER", "DATABASE", "DATE",
080                        "PROFILE", "ROLE", "SESSION", "TIME", "USER", "SYSDATE", "GETDATE" });
081
082        private Stack<TCustomSqlStatement> stmtStack = new Stack<TCustomSqlStatement>();
083        private List<ResultSet> appendResultSets = new ArrayList<ResultSet>();
084        private Set<TCustomSqlStatement> accessedStatements = new HashSet<TCustomSqlStatement>();
085        private Set<TSelectSqlStatement> accessedSubqueries = new HashSet<TSelectSqlStatement>();
086        private Map<String, TCustomSqlStatement> viewDDLMap = new HashMap<String, TCustomSqlStatement>();
087        private Map<String, TCustomSqlStatement> procedureDDLMap = new HashMap<String, TCustomSqlStatement>();
088        private Map<TTable, TObjectNameList> structObjectMap = new HashMap<TTable, TObjectNameList>();
089
090        private SqlInfo[] sqlInfos;
091        private List<ErrorInfo> errorInfos = new ArrayList<ErrorInfo>();
092        private final List<DynamicSqlSite> dynamicSqlSites = new ArrayList<DynamicSqlSite>();
093        private IndexedLinkedHashMap<String, List<SqlInfo>> sqlInfoMap = new IndexedLinkedHashMap<String, List<SqlInfo>>();
094        private TSQLEnv sqlenv = null;
095        private ModelBindingManager modelManager = new ModelBindingManager();
096        private ModelFactory modelFactory = new ModelFactory(modelManager);
097        private PipelinedFunctionAnalyzer pipelinedAnalyzer;
098        private List<Long> tableIds = new ArrayList<Long>();
099        private TTableList hiveFromTables;
100        private dataflow dataflow;
101        private String dataflowString;
102        private Option option = new Option();
103
104        {
105                modelManager.TABLE_COLUMN_ID = option.getStartId();
106                modelManager.RELATION_ID = option.getStartId();
107                ModelBindingManager.set(modelManager);
108                ModelBindingManager.setGlobalStmtStack(stmtStack);
109                ModelBindingManager.setGlobalOption(option);
110                ModelBindingManager.setGlobalSqlInfo(sqlInfoMap);
111        }
112
113    private void initPipelinedFunctionAnalyzer() {
114        // Only initialize pipelinedAnalyzer for Oracle
115        if (option.getVendor() == EDbVendor.dbvoracle)
116        {
117            pipelinedAnalyzer = new PipelinedFunctionAnalyzer(modelManager, modelFactory, option);
118        }
119        else{
120            pipelinedAnalyzer = null;
121        }
122    }
123
124        public DataFlowAnalyzer(String sqlContent, Option option) {
125                SqlInfo[] sqlInfos = new SqlInfo[1];
126                SqlInfo info = new SqlInfo();
127                info.setSql(sqlContent);
128                info.setOriginIndex(0);
129                sqlInfos[0] = info;
130                this.option = option;
131                ModelBindingManager.setGlobalOption(this.option);
132                this.sqlInfos = convertSQL(option.getVendor(), JSON.toJSONString(sqlInfos)).toArray(new SqlInfo[0]);
133    }
134
135    public DataFlowAnalyzer(String sqlContent, EDbVendor dbVendor, boolean simpleOutput, String defaultServer,
136                        String defaultDatabase, String defaltSchema) {
137                SqlInfo[] sqlInfos = new SqlInfo[1];
138                SqlInfo info = new SqlInfo();
139                info.setSql(sqlContent);
140                info.setOriginIndex(0);
141                sqlInfos[0] = info;
142                option.setVendor(dbVendor);
143                option.setSimpleOutput(simpleOutput);
144                option.setDefaultServer(defaultServer);
145                option.setDefaultDatabase(defaultDatabase);
146                option.setDefaultSchema(defaltSchema);
147                this.sqlInfos = convertSQL(dbVendor, JSON.toJSONString(sqlInfos)).toArray(new SqlInfo[0]);
148        }
149
150        public DataFlowAnalyzer(String sqlContent, EDbVendor dbVendor, boolean simpleOutput) {
151                SqlInfo[] sqlInfos = new SqlInfo[1];
152                SqlInfo info = new SqlInfo();
153                info.setSql(sqlContent);
154                info.setOriginIndex(0);
155                sqlInfos[0] = info;
156                option.setVendor(dbVendor);
157                option.setSimpleOutput(simpleOutput);
158                this.sqlInfos = convertSQL(dbVendor, JSON.toJSONString(sqlInfos)).toArray(new SqlInfo[0]);
159    }
160
161        public DataFlowAnalyzer(String[] sqlContents, Option option) {
162                SqlInfo[] sqlInfos = new SqlInfo[sqlContents.length];
163                for (int i = 0; i < sqlContents.length; i++) {
164                        SqlInfo info = new SqlInfo();
165                        info.setSql(sqlContents[i]);
166                        info.setOriginIndex(0);
167                        sqlInfos[i] = info;
168                }
169                this.option = option;
170                ModelBindingManager.setGlobalOption(this.option);
171                this.sqlInfos = convertSQL(option.getVendor(), JSON.toJSONString(sqlInfos)).toArray(new SqlInfo[0]);
172        }
173
174        public DataFlowAnalyzer(String[] sqlContents, EDbVendor dbVendor, boolean simpleOutput, String defaultServer,
175                        String defaultDatabase, String defaltSchema) {
176                SqlInfo[] sqlInfos = new SqlInfo[sqlContents.length];
177                for (int i = 0; i < sqlContents.length; i++) {
178                        SqlInfo info = new SqlInfo();
179                        info.setSql(sqlContents[i]);
180                        info.setOriginIndex(0);
181                        sqlInfos[i] = info;
182                }
183                option.setVendor(dbVendor);
184                option.setSimpleOutput(simpleOutput);
185                option.setDefaultServer(defaultServer);
186                option.setDefaultDatabase(defaultDatabase);
187                option.setDefaultSchema(defaltSchema);
188                this.sqlInfos = convertSQL(dbVendor, JSON.toJSONString(sqlInfos)).toArray(new SqlInfo[0]);
189        }
190
191        public DataFlowAnalyzer(String[] sqlContents, EDbVendor dbVendor, boolean simpleOutput) {
192                SqlInfo[] sqlInfos = new SqlInfo[sqlContents.length];
193                for (int i = 0; i < sqlContents.length; i++) {
194                        SqlInfo info = new SqlInfo();
195                        info.setSql(sqlContents[i]);
196                        info.setOriginIndex(0);
197                        sqlInfos[i] = info;
198                }
199                option.setVendor(dbVendor);
200                option.setSimpleOutput(simpleOutput);
201                this.sqlInfos = convertSQL(dbVendor, JSON.toJSONString(sqlInfos)).toArray(new SqlInfo[0]);
202        }
203
204        public DataFlowAnalyzer(SqlInfo[] sqlInfos, Option option) {
205                this.sqlInfos = sqlInfos;
206                this.option = option;
207                ModelBindingManager.setGlobalOption(this.option);
208        }
209
210        public DataFlowAnalyzer(SqlInfo[] sqlInfos, EDbVendor dbVendor, boolean simpleOutput) {
211                this.sqlInfos = sqlInfos;
212                option.setVendor(dbVendor);
213                option.setSimpleOutput(simpleOutput);
214        }
215
216        public DataFlowAnalyzer(File[] sqlFiles, Option option) {
217                SqlInfo[] sqlInfos = new SqlInfo[sqlFiles.length];
218                for (int i = 0; i < sqlFiles.length; i++) {
219                        SqlInfo info = new SqlInfo();
220                        info.setSql(SQLUtil.getFileContent(sqlFiles[i]));
221                        info.setFileName(sqlFiles[i].getName());
222                        info.setFilePath(sqlFiles[i].getAbsolutePath());
223                        info.setOriginIndex(0);
224                        sqlInfos[i] = info;
225                }
226                this.sqlInfos = sqlInfos;
227                this.option = option;
228                ModelBindingManager.setGlobalOption(this.option);
229        }
230
231        public DataFlowAnalyzer(File[] sqlFiles, EDbVendor dbVendor, boolean simpleOutput) {
232                SqlInfo[] sqlInfos = new SqlInfo[sqlFiles.length];
233                for (int i = 0; i < sqlFiles.length; i++) {
234                        SqlInfo info = new SqlInfo();
235                        info.setSql(SQLUtil.getFileContent(sqlFiles[i]));
236                        info.setFileName(sqlFiles[i].getName());
237                        info.setFilePath(sqlFiles[i].getAbsolutePath());
238                        info.setOriginIndex(0);
239                        sqlInfos[i] = info;
240                }
241                this.sqlInfos = sqlInfos;
242                option.setVendor(dbVendor);
243                option.setSimpleOutput(simpleOutput);
244        }
245
246        public DataFlowAnalyzer(File sqlFile, Option option) {
247                File[] children = SQLUtil.listFiles(sqlFile);
248                SqlInfo[] sqlInfos = new SqlInfo[children.length];
249                for (int i = 0; i < children.length; i++) {
250                        SqlInfo info = new SqlInfo();
251                        info.setSql(SQLUtil.getFileContent(children[i]));
252                        info.setFileName(children[i].getName());
253                        info.setFilePath(children[i].getAbsolutePath());
254                        info.setOriginIndex(0);
255                        sqlInfos[i] = info;
256                }
257                this.sqlInfos = sqlInfos;
258                this.option = option;
259                ModelBindingManager.setGlobalOption(this.option);
260        }
261
262        public DataFlowAnalyzer(File sqlFile, EDbVendor dbVendor, boolean simpleOutput) {
263                File[] children = SQLUtil.listFiles(sqlFile);
264                List<SqlInfo> sqlInfos = new ArrayList<>();
265                for (int i = 0; i < children.length; i++) {
266                        SqlInfo info = new SqlInfo();
267                        info.setSql(SQLUtil.getFileContent(children[i]));
268            if(children[i].getName().toLowerCase().endsWith(".csv")){
269                if(!MetadataReader.isMetadata(info.getSql())){
270                    continue;
271                }
272            }
273            else if(children[i].getName().toLowerCase().endsWith(".json")){
274                if (!(MetadataReader.isGrabit(info.getSql()) || MetadataReader.isSqlflow(info.getSql()) || MetadataReader.isSqlflowSharded(info.getSql()))){
275                    continue;
276                }
277            }
278                        info.setFileName(children[i].getName());
279                        info.setFilePath(children[i].getAbsolutePath());
280                        info.setOriginIndex(0);
281                        sqlInfos.add(info);
282                }
283                this.sqlInfos = sqlInfos.toArray(new SqlInfo[0]);
284                option.setVendor(dbVendor);
285                option.setSimpleOutput(simpleOutput);
286        }
287
288        public boolean isIgnoreRecordSet() {
289                return option.isIgnoreRecordSet();
290        }
291
292        public void setIgnoreRecordSet(boolean ignoreRecordSet) {
293                option.setIgnoreRecordSet(ignoreRecordSet);
294        }
295
296        public boolean isSimpleShowTopSelectResultSet() {
297                return option.isSimpleShowTopSelectResultSet();
298        }
299
300        public void setSimpleShowTopSelectResultSet(boolean simpleShowTopSelectResultSet) {
301                option.setSimpleShowTopSelectResultSet(simpleShowTopSelectResultSet);
302        }
303
304        public boolean isSimpleShowFunction() {
305                return option.isSimpleShowFunction();
306        }
307
308        public void setSimpleShowFunction(boolean simpleShowFunction) {
309                option.setSimpleShowFunction(simpleShowFunction);
310        }
311
312        public boolean isShowJoin() {
313                return option.isShowJoin();
314        }
315
316        public void setShowJoin(boolean showJoin) {
317                option.setShowJoin(showJoin);
318        }
319
320        public void setShowCallRelation(boolean showCallRelation) {
321                option.setShowCallRelation(showCallRelation);
322        }
323
324        public boolean isShowCallRelation() {
325                return option.isShowCallRelation();
326        }
327
328        public boolean isShowImplicitSchema() {
329                return option.isShowImplicitSchema();
330        }
331
332        public void setShowImplicitSchema(boolean showImplicitSchema) {
333                option.setShowImplicitSchema(showImplicitSchema);
334        }
335
336        public boolean isShowConstantTable() {
337                return option.isShowConstantTable();
338        }
339
340        public void setShowConstantTable(boolean showConstantTable) {
341                option.setShowConstantTable(showConstantTable);
342        }
343
344        public boolean isShowCountTableColumn() {
345                return option.isShowCountTableColumn();
346        }
347
348        public void setShowCountTableColumn(boolean showCountTableColumn) {
349                option.setShowCountTableColumn(showCountTableColumn);
350        }
351
352        public boolean isTransform() {
353                return option.isTransform();
354        }
355
356        public void setTransform(boolean transform) {
357                option.setTransform(transform);
358                if (option.isTransformCoordinate()) {
359                        option.setTransform(true);
360                }
361        }
362
363        public boolean isTransformCoordinate() {
364                return option.isTransformCoordinate();
365        }
366
367        public void setTransformCoordinate(boolean transformCoordinate) {
368                option.setTransformCoordinate(transformCoordinate);
369                if (transformCoordinate) {
370                        option.setTransform(true);
371                }
372        }
373
374        public boolean isLinkOrphanColumnToFirstTable() {
375                return option.isLinkOrphanColumnToFirstTable();
376        }
377
378        public void setLinkOrphanColumnToFirstTable(boolean linkOrphanColumnToFirstTable) {
379                option.setLinkOrphanColumnToFirstTable(linkOrphanColumnToFirstTable);
380        }
381
382        public boolean isIgnoreTemporaryTable() {
383                return option.isIgnoreTemporaryTable();
384        }
385
386        public void setIgnoreTemporaryTable(boolean ignoreTemporaryTable) {
387                option.setIgnoreTemporaryTable(ignoreTemporaryTable);
388        }
389        
390        public boolean isIgnoreCoordinate() {
391                return option.isIgnoreCoordinate();
392        }
393
394        public void setIgnoreCoordinate(boolean ignoreCoordinate) {
395                option.setIgnoreCoordinate(ignoreCoordinate);
396        }
397
398        public void setHandleListener(DataFlowHandleListener listener) {
399                option.setHandleListener(listener);
400        }
401
402        public void setSqlEnv(TSQLEnv sqlenv) {
403                this.sqlenv = sqlenv;
404        }
405
406        public void setOption(Option option) {
407                this.option = option;
408                ModelBindingManager.setGlobalOption(this.option);
409        }
410
411        public Option getOption() {
412                return option;
413        }
414
415        public synchronized String chechSyntax() {
416                StringBuilder builder = new StringBuilder();
417                if (sqlInfos != null) {
418                        for (SqlInfo sqlInfo : sqlInfos) {
419                                String content = sqlInfo.getSql();
420                                if (content != null && content.indexOf("<dlineage") != -1) {
421                                        try {
422                                                XML2Model.loadXML(dataflow.class, content);
423                                                continue;
424                                        } catch (Exception e) {
425                                                builder.append("Parsing dataflow ").append("occurs errors.\n").append(e.getMessage())
426                                                                .append("\n");
427                                        }
428                                }
429                                if (content != null && content.trim().startsWith("{")) {
430                                        Map queryObject = (Map) JSON.parseObject(content);
431                                        content = (String) queryObject.get("sourceCode");
432                                }
433                                if (MetadataReader.isMetadata(content)) {
434                                        continue;
435                                }
436                                TGSqlParser sqlparser = new TGSqlParser(option.getVendor());
437                                sqlparser.sqltext = content;
438                                int result = sqlparser.parse();
439                                if (result != 0) {
440                                        builder.append("Parsing sql ").append("occurs errors.\n").append(sqlparser.getErrormessage())
441                                                        .append("\n");
442                                }
443                        }
444                }
445                return builder.toString();
446        }
447
448        public synchronized String generateDataFlow(boolean withExtraInfo) {
449                if (ModelBindingManager.get() == null) {
450                        ModelBindingManager.set(modelManager);
451                }
452
453        initPipelinedFunctionAnalyzer();
454
455                dataflow = analyzeSqlScript();
456
457                if (dataflow != null && !withExtraInfo && dataflow.getResultsets() != null) {
458                        for (table t : dataflow.getResultsets()) {
459                                t.setIsTarget(null);
460                                if (t.getColumns() != null) {
461                                        for (column t1 : t.getColumns()) {
462                                                t1.setIsFunction(null);
463                                        }
464                                }
465                        }
466                }
467                
468                if(dataflow!=null && dataflow.getRelationships()!=null && option.getFilterRelationTypes()!=null && !option.getFilterRelationTypes().isEmpty()) {
469                        List<relationship> relationships = new ArrayList<relationship>();
470                        for(relationship relationship: dataflow.getRelationships()) {
471                                if(option.getFilterRelationTypes().contains(relationship.getType())) {
472                                        relationships.add(relationship);
473                                }
474                        }
475                        dataflow.setRelationships(relationships);
476                }
477
478                if (option.getHandleListener() != null) {
479                        option.getHandleListener().endAnalyze(dataflow);
480                }
481
482                if (option.isOutput()) {
483                        if (option.getHandleListener() != null) {
484                                option.getHandleListener().startOutputDataFlowXML();
485                        }
486                        if (dataflow != null) {
487                                if (option.isTextFormat()) {
488                                        dataflowString = getTextOutput(dataflow);
489                                } else {
490                                        try {
491                                                dataflowString = XML2Model.saveXML(dataflow);
492                                        }catch (Exception e){
493                                                logger.error("Output dataflow to xml failed.", e);
494                                                dataflowString = null;
495                                        }
496                                }
497                        }
498                        if (option.getHandleListener() != null) {
499                                option.getHandleListener().endOutputDataFlowXML(dataflowString == null ? 0 : dataflowString.length());
500                        }
501                }
502
503                return dataflowString;
504        }
505
506        private dataflow removeDuplicateColumns(dataflow dataflow) {
507                List<table> tables = new ArrayList<table>();
508                if (dataflow.getTables() != null) {
509                        tables.addAll(dataflow.getTables());
510                }
511                if (dataflow.getViews() != null) {
512                        tables.addAll(dataflow.getViews());
513                }
514                if (dataflow.getStages() != null) {
515                        tables.addAll(dataflow.getStages());
516                }
517                if (dataflow.getDatasources() != null) {
518                        tables.addAll(dataflow.getDatasources());
519                }
520                if (dataflow.getStreams() != null) {
521                        tables.addAll(dataflow.getStreams());
522                }
523                if (dataflow.getPaths() != null) {
524                        tables.addAll(dataflow.getPaths());
525                }
526                if (dataflow.getVariables() != null) {
527                        tables.addAll(dataflow.getVariables());
528                }
529                if (dataflow.getResultsets() != null) {
530                        tables.addAll(dataflow.getResultsets());
531                }
532                for (table table : tables) {
533                        if (table.getColumns() == null) {
534                                continue;
535                        }
536                        Set<String> columnIds = new HashSet<String>();
537                        Iterator<column> iter = table.getColumns().iterator();
538                        while(iter.hasNext()) {
539                                column column = iter.next();
540                                String id = column.getId();
541                                if (columnIds.contains(id)) {
542                                        iter.remove();
543                                } else {
544                                        columnIds.add(id);
545                                }
546                        }
547                }
548                return dataflow;
549        }
550
551        public synchronized String generateDataFlow() {
552                return generateDataFlow(false);
553        }
554
555        public synchronized String generateSqlInfos() {
556                return JSON.toJSONString(sqlInfoMap);
557        }
558
559        public Map<String, List<SqlInfo>> getSqlInfos() {
560                return sqlInfoMap;
561        }
562
563        public Map getHashSQLMap() {
564                return modelManager.getHashSQLMap();
565        }
566        
567        public Map getDynamicSQLMap() {
568                return modelManager.getDynamicSQLMap();
569        }
570
571        /**
572         * @deprecated please use SqlInfoHelper.getSelectedDbObjectInfo
573         */
574        public DbObjectPosition getSelectedDbObjectInfo(Coordinate start, Coordinate end) {
575                if (start == null || end == null) {
576                        throw new IllegalArgumentException("Coordinate can't be null.");
577                }
578
579                String hashCode = start.getHashCode();
580
581                if (hashCode == null) {
582                        throw new IllegalArgumentException("Coordinate hashcode can't be null.");
583                }
584
585                int dbObjectStartLine = (int) start.getX() - 1;
586                int dbObjectStarColumn = (int) start.getY() - 1;
587                int dbObjectEndLine = (int) end.getX() - 1;
588                int dbObjectEndColumn = (int) end.getY() - 1;
589                List<SqlInfo> sqlInfoList;
590                if (hashCode.matches("\\d+")) {
591                        sqlInfoList = sqlInfoMap.getValueAtIndex(Integer.valueOf(hashCode));
592                } else {
593                        sqlInfoList = sqlInfoMap.get(hashCode);
594                }
595                for (int j = 0; j < sqlInfoList.size(); j++) {
596                        SqlInfo sqlInfo = sqlInfoList.get(j);
597                        int startLine = sqlInfo.getLineStart();
598                        int endLine = sqlInfo.getLineEnd();
599                        if (dbObjectStartLine >= startLine && dbObjectStartLine <= endLine) {
600                                DbObjectPosition position = new DbObjectPosition();
601                                position.setFile(sqlInfo.getFileName());
602                                position.setFilePath(sqlInfo.getFilePath());
603                                position.setSql(sqlInfo.getSql());
604                                position.setIndex(sqlInfo.getOriginIndex());
605                                List<Pair<Integer, Integer>> positions = position.getPositions();
606                                positions.add(new Pair<Integer, Integer>(
607                                                dbObjectStartLine - startLine + sqlInfo.getOriginLineStart() + 1, dbObjectStarColumn + 1));
608                                positions.add(new Pair<Integer, Integer>(dbObjectEndLine - startLine + sqlInfo.getOriginLineStart() + 1,
609                                                dbObjectEndColumn + 1));
610                                return position;
611                        }
612                }
613                return null;
614        }
615
616        public static dataflow mergeTables(dataflow dataflow, Long startId) {
617                return mergeTables(dataflow, startId, new Option());
618        }
619
620        public static dataflow mergeTables(dataflow dataflow, Long startId, Option option) {
621                List<table> tableCopy = new ArrayList<table>();
622                List<table> viewCopy = new ArrayList<table>();
623                List<table> databaseCopy = new ArrayList<table>();
624                List<table> schemaCopy = new ArrayList<table>();
625                List<table> stageCopy = new ArrayList<table>();
626                List<table> dataSourceCopy = new ArrayList<table>();
627                List<table> streamCopy = new ArrayList<table>();
628                List<table> fileCopy = new ArrayList<table>();
629                List<table> variableCopy = new ArrayList<table>();
630                List<table> cursorCopy = new ArrayList<table>();
631                List<table> resultSetCopy = new ArrayList<table>();
632                if (dataflow.getTables() != null) {
633                        tableCopy.addAll(dataflow.getTables());
634                }
635                dataflow.setTables(tableCopy);
636                if (dataflow.getViews() != null) {
637                        viewCopy.addAll(dataflow.getViews());
638                }
639                dataflow.setViews(viewCopy);
640                if (dataflow.getDatabases() != null) {
641                        databaseCopy.addAll(dataflow.getDatabases());
642                }
643                dataflow.setDatabases(databaseCopy);
644                if (dataflow.getSchemas() != null) {
645                        schemaCopy.addAll(dataflow.getSchemas());
646                }
647                dataflow.setSchemas(schemaCopy);
648                if (dataflow.getStages() != null) {
649                        stageCopy.addAll(dataflow.getStages());
650                }
651                dataflow.setStages(stageCopy);
652                if (dataflow.getDatasources() != null) {
653                        dataSourceCopy.addAll(dataflow.getDatasources());
654                }
655                dataflow.setDatasources(dataSourceCopy);
656                if (dataflow.getStreams() != null) {
657                        streamCopy.addAll(dataflow.getStreams());
658                }
659                dataflow.setStreams(streamCopy);
660                if (dataflow.getPaths() != null) {
661                        fileCopy.addAll(dataflow.getPaths());
662                }
663                dataflow.setPaths(fileCopy);
664                if (dataflow.getVariables() != null) {
665                        variableCopy.addAll(dataflow.getVariables());
666                }
667                dataflow.setVariables(variableCopy);
668                if (dataflow.getResultsets() != null) {
669                        resultSetCopy.addAll(dataflow.getResultsets());
670                }
671                dataflow.setResultsets(resultSetCopy);
672
673                Map<String, List<table>> tableMap = new HashMap<String, List<table>>();
674                Map<String, String> tableTypeMap = new HashMap<String, String>();
675                Map<String, TMssqlCreateType> mssqlTypeMap = new HashMap<String, TMssqlCreateType>();
676                Map<String, String> tableIdMap = new HashMap<String, String>();
677
678                Map<String, List<column>> columnMap = new HashMap<String, List<column>>();
679                Map<String, Set<String>> tableColumnMap = new HashMap<String, Set<String>>();
680                Map<String, String> columnIdMap = new HashMap<String, String>();
681                Map<String, column> columnMergeIdMap = new HashMap<String, column>();
682
683                List<table> tables = new ArrayList<table>();
684                tables.addAll(dataflow.getTables());
685                tables.addAll(dataflow.getViews());
686                tables.addAll(dataflow.getDatabases());
687                tables.addAll(dataflow.getSchemas());
688                tables.addAll(dataflow.getStages());
689                tables.addAll(dataflow.getDatasources());
690                tables.addAll(dataflow.getStreams());
691                tables.addAll(dataflow.getPaths());
692                tables.addAll(dataflow.getResultsets());
693                tables.addAll(dataflow.getVariables());
694                
695                Set<String> columnIds = new HashSet<String>();
696
697                for (table table : tables) {
698                        String qualifiedTableName = DlineageUtil.getQualifiedTableName(table);
699                        String tableFullName = DlineageUtil.getIdentifierNormalTableName(qualifiedTableName);
700                        if ("variable".endsWith(table.getType()) && !SQLUtil.isEmpty(table.getParent())) {
701                                tableFullName = table.getParent() + "." + tableFullName;
702                        }
703                        
704                        if (!tableMap.containsKey(tableFullName)) {
705                                tableMap.put(tableFullName, new ArrayList<table>());
706                        }
707
708                        tableMap.get(tableFullName).add(table);
709
710                        if (!tableTypeMap.containsKey(tableFullName)) {
711                                tableTypeMap.put(tableFullName, table.getType());
712                        } else if ("view".equals(table.getSubType())) {
713                                tableTypeMap.put(tableFullName, table.getType());
714                        } else if ("database".equals(table.getSubType())) {
715                                tableTypeMap.put(tableFullName, table.getType());
716                        } else if ("schema".equals(table.getSubType())) {
717                                tableTypeMap.put(tableFullName, table.getType());
718                        } else if ("stage".equals(table.getSubType())) {
719                                tableTypeMap.put(tableFullName, table.getType());
720                        } else if ("sequence".equals(table.getSubType())) {
721                                tableTypeMap.put(tableFullName, table.getType());
722                        } else if ("datasource".equals(table.getSubType())) {
723                                tableTypeMap.put(tableFullName, table.getType());
724                        } else if ("stream".equals(table.getSubType())) {
725                                tableTypeMap.put(tableFullName, table.getType());
726                        } else if ("file".equals(table.getSubType())) {
727                                tableTypeMap.put(tableFullName, table.getType());
728                        } else if ("table".equals(tableTypeMap.get(tableFullName))) {
729                                tableTypeMap.put(tableFullName, table.getType());
730                        } else if ("variable".equals(tableTypeMap.get(tableFullName))) {
731                                tableTypeMap.put(tableFullName, table.getType());
732                        }
733
734                        if (table.getColumns() != null) {
735                                if (!tableColumnMap.containsKey(tableFullName)) {
736                                        tableColumnMap.put(tableFullName, new LinkedHashSet<String>());
737                                }
738                                for (column column : table.getColumns()) {
739                                        String columnFullName = tableFullName + "."
740                                                        + (column.getQualifiedTable() != null
741                                                                        ? (DlineageUtil.getIdentifierNormalTableName(column.getQualifiedTable()) + ".")
742                                                                        : "")
743                                                        + ("false".equals(table.getIsTarget()) ? DlineageUtil.normalizeColumnName(column.getName())
744                                                                        : DlineageUtil.getIdentifierNormalColumnName(column.getName()));
745
746                                        if (!columnMap.containsKey(columnFullName)) {
747                                                columnMap.put(columnFullName, new ArrayList<column>());
748                                                tableColumnMap.get(tableFullName).add(columnFullName);
749                                        }
750
751                                        columnMap.get(columnFullName).add(column);
752                                        columnIds.add(column.getId());
753                                }
754                        }
755                }
756
757                Set<String> relationParentIds = new HashSet<String>();
758                if (dataflow.getRelationships() != null) {
759                        for (relationship rel : dataflow.getRelationships()) {
760                                if (rel.getSources() != null) {
761                                        for (sourceColumn src : rel.getSources()) {
762                                                if (src.getParent_id() != null) {
763                                                        relationParentIds.add(src.getParent_id());
764                                                }
765                                        }
766                                }
767                                if (rel.getTarget() != null) {
768                                        if (rel.getTarget().getParent_id() != null) {
769                                                relationParentIds.add(rel.getTarget().getParent_id());
770                                        }
771                                }
772                        }
773                }
774
775                Iterator<String> tableNameIter = tableMap.keySet().iterator();
776                while (tableNameIter.hasNext()) {
777                        String tableName = tableNameIter.next();
778                        List<table> tableList = tableMap.get(tableName);
779                        table table;
780                        if (tableList.size() > 1) {
781                                table standardTable = tableList.get(0);
782                                // Function允许重名，不做合并处理
783                                if (standardTable.isFunction()) {
784                                        continue;
785                                }
786                                
787                                // Variable允许重名，不做合并处理
788                                if (standardTable.isVariable()) {
789                                        continue;
790                                }
791                                
792                                // 临时表不做合并处理
793                                if(SQLUtil.isTempTable(standardTable)) {
794                                        continue;
795                                }
796
797                                String type = tableTypeMap.get(tableName);
798                                table = new table();
799                                table.setId(String.valueOf(++startId));
800                                table.setServer(standardTable.getServer());
801                                table.setDatabase(standardTable.getDatabase());
802                                table.setSchema(standardTable.getSchema());
803                                table.setName(standardTable.getName());
804                                table.setDisplayName(standardTable.getDisplayName());
805                                table.setParent(standardTable.getParent());
806                                table.setMore(standardTable.getMore());
807                                if (standardTable.getCandidateTables() != null && !standardTable.getCandidateTables().isEmpty()) {
808                                        table.setCandidateTables(new ArrayList<String>(standardTable.getCandidateTables()));
809                                }
810                                table.setColumns(new ArrayList<column>());
811                                String subType = null;
812                                for(table item: tableList){
813                                        if (item.getSubType() != null) {
814                                                subType = item.getSubType();
815                                                break;
816                                        }
817                                }
818                                if (subType != null) {
819                                        table.setSubType(subType);
820                                } else {
821                                        table.setSubType(standardTable.getSubType());
822                                }
823                                Set<String> processIds = new LinkedHashSet<String>();
824                                for (int k = 0; k < tableList.size(); k++) {
825                                        if (tableList.get(k).getProcessIds() != null) {
826                                                processIds.addAll(tableList.get(k).getProcessIds());
827                                        }
828                                }
829                                if (!processIds.isEmpty()) {
830                                        table.setProcessIds(new ArrayList<String>(processIds));
831                                }
832                                Set<String> alias = new LinkedHashSet<String>();
833                                for (int k = 0; k < tableList.size(); k++) {
834                                        if (tableList.get(k).getAlias() != null) {
835                                                alias.addAll(Arrays.asList(tableList.get(k).getAlias().split("\\s*,\\s*")));
836                                        }
837                                }
838                                if (!alias.isEmpty()) {
839                                        String aliasString = Arrays.toString(alias.toArray(new String[0]));
840                                        table.setAlias(aliasString.substring(1, aliasString.length() - 1));
841                                }
842                                table.setType(type);
843                                for (table item : tableList) {
844                                        if (!SQLUtil.isEmpty(table.getCoordinate()) && !SQLUtil.isEmpty(item.getCoordinate())) {
845                                                if (table.getCoordinate().indexOf(item.getCoordinate()) == -1) {
846                                                        table.appendCoordinate(item.getCoordinate());
847                                                }
848                                        } else if (!SQLUtil.isEmpty(item.getCoordinate())) {
849                                                table.setCoordinate(item.getCoordinate());
850                                        }
851
852                                        if (item.getStarStmt() != null) {
853                                                table.setStarStmt(item.getStarStmt());
854                                        }
855
856                                        tableIdMap.put(item.getId(), table.getId());
857
858                                        if (item.isView()) {
859                                                dataflow.getViews().remove(item);
860                                        } else if (item.isDatabaseType()) {
861                                                dataflow.getDatabases().remove(item);
862                                        } else if (item.isSchemaType()) {
863                                                dataflow.getSchemas().remove(item);
864                                        } else if (item.isStage()) {
865                                                dataflow.getStages().remove(item);
866                                        } else if (item.isDataSource()) {
867                                                dataflow.getDatasources().remove(item);
868                                        } else if (item.isStream()) {
869                                                dataflow.getStreams().remove(item);
870                                        } else if (item.isFile()) {
871                                                dataflow.getPaths().remove(item);
872                                        } else if (item.isVariable()) {
873                                                dataflow.getVariables().remove(item);
874                                        } else if (item.isTable()) {
875                                                dataflow.getTables().remove(item);
876                                        } else if (item.isResultSet()) {
877                                                dataflow.getResultsets().remove(item);
878                                        }
879                                }
880
881                                if (table.isView()) {
882                                        dataflow.getViews().add(table);
883                                } else if (table.isDatabaseType()) {
884                                        dataflow.getDatabases().add(table);
885                                } else if (table.isSchemaType()) {
886                                        dataflow.getSchemas().add(table);
887                                } else if (table.isStage()) {
888                                        dataflow.getStages().add(table);
889                                } else if (table.isDataSource()) {
890                                        dataflow.getDatasources().add(table);
891                                } else if (table.isStream()) {
892                                        dataflow.getStreams().add(table);
893                                } else if (table.isFile()) {
894                                        dataflow.getPaths().add(table);
895                                } else if (table.isVariable()) {
896                                        dataflow.getVariables().add(table);
897                                } else if (table.isResultSet()) {
898                                        dataflow.getResultsets().add(table);
899                                } else {
900                                        dataflow.getTables().add(table);
901                                }
902                        } else {
903                                table = tableList.get(0);
904                                if(Boolean.TRUE.toString().equals(table.getIsDetermined())){
905                                        continue;
906                                }
907                                
908                                if (option.isIgnoreUnusedSynonym() && SubType.synonym.name().equals(table.getSubType())) {
909                                        boolean hasSourceRelation = relationParentIds.contains(table.getId());
910                                        if (!hasSourceRelation) {
911                                                dataflow.getTables().remove(table);
912                                                tableColumnMap.get(tableName).clear();
913                                                continue;
914                                        }
915                                }
916                        }
917
918                        Set<String> columns = tableColumnMap.get(tableName);
919                        Iterator<String> columnIter = columns.iterator();
920                        List<column> mergeColumns = new ArrayList<column>();
921                        while (columnIter.hasNext()) {
922                                String columnName = columnIter.next();
923                                List<column> columnList = columnMap.get(columnName);
924                                List<column> functions = new ArrayList<column>();
925                                for (column t : columnList) {
926                                        if (Boolean.TRUE.toString().equals(t.getIsFunction())) {
927                                                functions.add(t);
928                                        }
929                                }
930                                if (functions != null && !functions.isEmpty()) {
931                                        for (column function : functions) {
932                                                mergeColumns.add(function);
933                                                columnIdMap.put(function.getId(), function.getId());
934                                                columnMergeIdMap.put(function.getId(), function);
935                                        }
936
937                                        columnList.removeAll(functions);
938                                }
939                                if (!columnList.isEmpty()) {
940                                        column firstColumn = columnList.iterator().next();
941                                        if (columnList.size() > 1) {
942                                                column mergeColumn = new column();
943                                                mergeColumn.setId(String.valueOf(++startId));
944                                                mergeColumn.setName(firstColumn.getName());
945                                                mergeColumn.setDisplayName(firstColumn.getDisplayName());
946                                                mergeColumn.setSource(firstColumn.getSource());
947                                                mergeColumn.setQualifiedTable(firstColumn.getQualifiedTable());
948                                                mergeColumn.setDataType(firstColumn.getDataType());
949                                                mergeColumn.setForeignKey(firstColumn.isForeignKey());
950                                                mergeColumn.setPrimaryKey(firstColumn.isPrimaryKey());
951                                                mergeColumn.setUnqiueKey(firstColumn.isUnqiueKey());
952                                                mergeColumn.setIndexKey(firstColumn.isIndexKey());
953                                                mergeColumns.add(mergeColumn);
954                                                for (column item : columnList) {
955                                                        mergeColumn.appendCoordinate(item.getCoordinate());
956                                                        columnIdMap.put(item.getId(), mergeColumn.getId());
957                                                }
958                                                columnMergeIdMap.put(mergeColumn.getId(), mergeColumn);
959                                                columnIds.add(mergeColumn.getId());
960                                        } else {
961                                                mergeColumns.add(firstColumn);
962                                                columnIdMap.put(firstColumn.getId(), firstColumn.getId());
963                                                columnMergeIdMap.put(firstColumn.getId(), firstColumn);
964                                        }
965                                }
966                        }
967                        table.setColumns(mergeColumns);
968                }
969
970                if (dataflow.getRelationships() != null) {
971                        Map<String, relationship> mergeRelations = new LinkedHashMap<String, relationship>();
972                        for (int i = 0; i < dataflow.getRelationships().size(); i++) {
973                                relationship relation = dataflow.getRelationships().get(i);
974                                
975                                if("crud".equals(relation.getType()) && option.getAnalyzeMode() == AnalyzeMode.crud) {
976                                        String jsonString = JSON.toJSONString(relation, true);
977                                        String key = SHA256.getMd5(jsonString);
978                                        if (!mergeRelations.containsKey(key)) {
979                                                mergeRelations.put(key, relation);
980                                        }
981                                        continue;
982                                }
983                                
984                                targetColumn target = relation.getTarget();
985                                if ("call".equals(relation.getType())) {
986                                        target = relation.getCaller();
987                                }
988                                if (target != null && tableIdMap.containsKey(target.getParent_id())) {
989                                        target.setParent_id(tableIdMap.get(target.getParent_id()));
990                                }
991
992                                if (columnIdMap.containsKey(target.getId())) {
993                                        target.setId(columnIdMap.get(target.getId()));
994                                        target.setCoordinate(columnMergeIdMap.get(target.getId()).getCoordinate());
995                                }
996                                else if(option.isIgnoreUnusedSynonym() && EffectType.synonym.name().equals(relation.getEffectType())){
997                                        continue;
998                                }
999                                
1000                                if (!"call".equals(relation.getType()) && !columnIds.contains(target.getId())) {
1001                                        continue;
1002                                }
1003
1004                                List<sourceColumn> sources = relation.getSources();
1005                                if ("call".equals(relation.getType())) {
1006                                        sources = relation.getCallees();
1007                                }
1008                                Set<sourceColumn> sourceSet = new LinkedHashSet<sourceColumn>();
1009                                if (sources != null) {
1010                                        for (sourceColumn source : sources) {
1011                                                if (!"call".equals(relation.getType()) && !columnIds.contains(source.getId())) {
1012                                                        continue;
1013                                                }
1014                                                if (tableIdMap.containsKey(source.getParent_id())) {
1015                                                        source.setParent_id(tableIdMap.get(source.getParent_id()));
1016                                                }
1017                                                if (tableIdMap.containsKey(source.getSource_id())) {
1018                                                        source.setSource_id(tableIdMap.get(source.getSource_id()));
1019                                                }
1020                                                if (columnIdMap.containsKey(source.getId())) {
1021                                                        source.setId(columnIdMap.get(source.getId()));
1022                                                        source.setCoordinate(columnMergeIdMap.get(source.getId()).getCoordinate());
1023                                                }
1024                                        }
1025
1026                                        sourceSet.addAll(sources);
1027                                        if ("call".equals(relation.getType())) {
1028                                                relation.setCallees(new ArrayList<sourceColumn>(sourceSet));
1029                                        } else {
1030                                                relation.setSources(new ArrayList<sourceColumn>(sourceSet));
1031                                        }
1032                                }
1033
1034                                String jsonString = JSON.toJSONString(relation, true);
1035                String key = SHA256.getMd5(jsonString);
1036                                if (!mergeRelations.containsKey(key)) {
1037                                        mergeRelations.put(key, relation);
1038                                }
1039                        }
1040
1041                        dataflow.setRelationships(new ArrayList<relationship>(mergeRelations.values()));
1042                }
1043
1044                tableMap.clear();
1045                tableTypeMap.clear();
1046                tableIdMap.clear();
1047                columnMap.clear();
1048                tableColumnMap.clear();
1049                columnIdMap.clear();
1050                columnMergeIdMap.clear();
1051                tables.clear();
1052                return dataflow;
1053        }
1054
1055        public synchronized dataflow getDataFlow() {
1056                if (dataflow != null) {
1057                        return dataflow;
1058                } else if (dataflowString != null) {
1059                        return XML2Model.loadXML(dataflow.class, dataflowString);
1060                }
1061                return null;
1062        }
1063
1064        List<ErrorInfo> metadataErrors = new ArrayList<>();
1065        
1066        private synchronized dataflow analyzeSqlScript() {
1067                init();
1068
1069                try {
1070                        dataflow dataflow = new dataflow();
1071                        
1072                        if (sqlInfos != null) {
1073                                if (option.getHandleListener() != null) {
1074                                        if (sqlInfos.length == 1) {
1075                                                option.getHandleListener().startAnalyze(null, sqlInfos[0].getSql().length(), false);
1076                                        } else {
1077                                                option.getHandleListener().startAnalyze(null, sqlInfos.length, true);
1078                                        }
1079                                }
1080
1081                                if (sqlenv == null) {
1082                                        if (option.getHandleListener() != null) {
1083                                                option.getHandleListener().startParseSQLEnv();
1084                                        }
1085                                        TSQLEnv[] sqlenvs = new SQLEnvParser(option.getDefaultServer(), option.getDefaultDatabase(),
1086                                                        option.getDefaultSchema()).parseSQLEnv(option.getVendor(), sqlInfos);
1087                                        if (sqlenvs != null && sqlenvs.length > 0) {
1088                                                sqlenv = sqlenvs[0];
1089                                        }
1090                                        if (option.getHandleListener() != null) {
1091                                                option.getHandleListener().endParseSQLEnv();
1092                                        }
1093                                }
1094                                TGSqlParser sqlparser = new TGSqlParser(option.getVendor());
1095                                Map<String, Pair3<StringBuilder, AtomicInteger, String>> databaseMap = new LinkedHashMap<String, Pair3<StringBuilder, AtomicInteger, String>>();
1096                                for (int i = 0; i < sqlInfos.length; i++) {
1097                                        SqlInfo sqlInfo = sqlInfos[i];
1098                                        if (sqlInfo == null) {
1099                                                sqlInfoMap.put(String.valueOf(i), new ArrayList<SqlInfo>());
1100                                                continue;
1101                                        }
1102                                        String sql = sqlInfo.getSql();
1103                                        if (SQLUtil.isEmpty(sql) && sqlInfo.getFileName() != null
1104                                                        && new File(sqlInfo.getFileName()).exists()) {
1105                                                sql = SQLUtil.getFileContent(sqlInfo.getFileName());
1106                                        }
1107                                        if (SQLUtil.isEmpty(sql) && sqlInfo.getFilePath() != null
1108                                                        && new File(sqlInfo.getFilePath()).exists()) {
1109                                                sql = SQLUtil.getFileContent(sqlInfo.getFilePath());
1110                                        }
1111                                        String sqlTrim = null;
1112                                        if (sql != null) {
1113                                                sqlTrim = sql.substring(0, Math.min(sql.length(), 512)).trim();
1114                                        }
1115                                        if(sql!=null && sqlTrim.startsWith("<") && sqlTrim.indexOf("<dlineage")!=-1){
1116                                                dataflow temp = XML2Model.loadXML(dataflow.class, sql);
1117                                                if(sqlInfos.length == 1){
1118                                                        dataflow = temp;
1119                                                }
1120                                                else {
1121                                                        if (temp.getTables() != null) {
1122                                                                dataflow.getTables().addAll(temp.getTables());
1123                                                        }
1124                                                        if (temp.getViews() != null) {
1125                                                                dataflow.getViews().addAll(temp.getViews());
1126                                                        }
1127                                                        if (temp.getResultsets() != null) {
1128                                                                dataflow.getResultsets().addAll(temp.getResultsets());
1129                                                        }
1130                                                        if (temp.getRelationships() != null) {
1131                                                                dataflow.getRelationships().addAll(temp.getRelationships());
1132                                                        }
1133                                                        if (temp.getErrors() != null) {
1134                                                                dataflow.getErrors().addAll(temp.getErrors());
1135                                                        }
1136                                                }
1137                                        }
1138                                        else if (sql != null && sqlTrim.startsWith("{")) {
1139                                                EDbVendor vendor = SQLUtil.isEmpty(sqlInfo.getDbVendor()) ? option.getVendor()
1140                                                                : EDbVendor.valueOf(sqlInfo.getDbVendor());
1141                                                TSQLEnv[] sqlenvs = new TJSONSQLEnvParser(option.getDefaultServer(),
1142                                                                option.getDefaultDatabase(), option.getDefaultSchema()).parseSQLEnv(vendor, sql);
1143                                                if (sqlenvs != null && sqlenvs.length > 0) {
1144                                                        if (sqlenv == null) {
1145                                                                sqlenv = sqlenvs[0];
1146                                                        } else {
1147                                                                sqlenv = SQLEnvParser.mergeSQLEnv(Arrays.asList(sqlenv, sqlenvs[0]));
1148                                                        }
1149                                                }
1150                                                if (sqlenv != null) {
1151                                                        if (MetadataReader.isGrabit(sql) || MetadataReader.isSqlflow(sql) || MetadataReader.isSqlflowSharded(sql)) {
1152                                                                String hash = SHA256.getMd5(sql);
1153                                                                String fileHash = SHA256.getMd5(hash);
1154                                                                if (!sqlInfoMap.containsKey(fileHash)) {
1155                                                                        sqlInfoMap.put(fileHash, new ArrayList<SqlInfo>());
1156                                                                        sqlInfoMap.get(fileHash).add(sqlInfo);
1157                                                                }
1158                                                                ModelBindingManager.setGlobalHash(fileHash);
1159                                                                dataflow temp = null;
1160
1161                                                                if (MetadataReader.isGrabit(sql)) {
1162                                                                        temp = new GrabitMetadataAnalyzer().analyzeMetadata(option.getVendor(), sql);
1163                                                                } else if (MetadataReader.isSqlflowSharded(sql)) {
1164                                                                        String baseDir = null;
1165                                                                        if (sqlInfo.getFilePath() != null) {
1166                                                                                baseDir = new File(sqlInfo.getFilePath()).getParent();
1167                                                                        }
1168                                                                        temp = new SqlflowShardedMetadataAnalyzer(baseDir).analyzeMetadata(option.getVendor(), sql);
1169                                                                } else {
1170                                                                        temp = new SqlflowMetadataAnalyzer(sqlenv).analyzeMetadata(option.getVendor(), sql);
1171                                                                }
1172//                                                              if (temp.getPackages() != null) {
1173//                                                                      dataflow.getPackages().addAll(temp.getPackages());
1174//                                                              }
1175//                                                              if (temp.getProcedures() != null) {
1176//                                                                      dataflow.getProcedures().addAll(temp.getProcedures());
1177//                                                              }
1178                                                                if (temp.getTables() != null) {
1179                                                                        dataflow.getTables().addAll(temp.getTables());
1180                                                                }
1181                                                                if (temp.getViews() != null) {
1182                                                                        dataflow.getViews().addAll(temp.getViews());
1183                                                                }
1184                                                                if (temp.getResultsets() != null) {
1185                                                                        dataflow.getResultsets().addAll(temp.getResultsets());
1186                                                                }
1187                                                                if (temp.getRelationships() != null) {
1188                                                                        dataflow.getRelationships().addAll(temp.getRelationships());
1189                                                                }
1190                                                                if (temp.getErrors() != null) {
1191                                                                        dataflow.getErrors().addAll(temp.getErrors());
1192                                                                }
1193                                                                if (sql.indexOf("createdBy") != -1) {
1194                                                                        if (sql.toLowerCase().indexOf("sqldep") != -1
1195                                                                                        || sql.toLowerCase().indexOf("grabit") != -1) {
1196                                                                                Map jsonObject = (Map) JSON.parseObject(sql);
1197                                                                                List<Map> queries = (List<Map>) jsonObject.get("queries");
1198                                                                                if (queries != null) {
1199                                                                                        for (int j = 0; j < queries.size(); j++) {
1200                                                                                                Map queryObject = queries.get(j);
1201                                                                                                appendSqlInfo(databaseMap, j, sqlInfo, queryObject);
1202                                                                                        }
1203                                                                                }
1204                                                                        } else if (sql.toLowerCase().indexOf("sqlflow") != -1) {
1205                                                                                Map sqlflow = (Map) JSON.parseObject(sql);
1206                                                                                if ("sqlflow-sharded".equals(sqlflow.get("format"))) {
1207                                                                                        String baseDir = null;
1208                                                                                        if (sqlInfo.getFilePath() != null) {
1209                                                                                                baseDir = new File(sqlInfo.getFilePath()).getParent();
1210                                                                                        }
1211                                                                                        String sourceCompression = (String) sqlflow.get("sourceCompression");
1212                                                                                        List<Map> servers = (List<Map>) sqlflow.get("servers");
1213                                                                                        if (servers != null) {
1214                                                                                                for (Map serverObject : servers) {
1215                                                                                                        List<Map> databases = (List<Map>) serverObject.get("databases");
1216                                                                                                        if (databases != null) {
1217                                                                                                                for (Map database : databases) {
1218                                                                                                                        Map source = (Map) database.get("source");
1219                                                                                                                        if (source != null) {
1220                                                                                                                                String sourcePath = (String) source.get("path");
1221                                                                                                                                if (sourcePath != null && baseDir != null) {
1222                                                                    File sourceFile = new File(baseDir, sourcePath);
1223                                                                                                                                        String fullPath = sourceFile.getAbsolutePath();
1224                                                                                                                                        if ("block".equals(sourceCompression)) {
1225                                                                                                                                                readGzipBlockSource(fullPath, databaseMap, sqlInfo);
1226                                                                                                                                        } else {
1227                                                                                                                                                String sourceContent = SQLUtil.getFileContent(fullPath);
1228                                                                                                                                                if (sourceContent != null) {
1229                                                                                                                                                        String[] lines = sourceContent.split("\\r?\\n");
1230                                                                                                                                                        for (int j = 0; j < lines.length; j++) {
1231                                                                                                                                                                String line = lines[j].trim();
1232                                                                                                                                                                if (line.isEmpty()) {
1233                                                                                                                                                                        continue;
1234                                                                                                                                                                }
1235                                                                                                                                                                try {
1236                                                                                                                                                                        Map sourceObject = (Map) JSON.parseObject(line);
1237                                                                                                                                                                        String sourceCode = (String) sourceObject.get("sourceCode");
1238                                                                                                                                                                        if (sourceCode != null && !sourceCode.isEmpty()) {
1239                                                                                                                                                                                SqlInfo sourceSqlInfo = new SqlInfo();
1240                                                                                                                                                                                sourceSqlInfo.setFileName(sourceFile.getName());
1241                                                                                                                                                                                sourceSqlInfo.setFilePath(sourceFile.getAbsolutePath());
1242                                                                                                                                                                                sourceSqlInfo.setSql(sourceCode);
1243                                                                                                                                                                                sourceSqlInfo.setOriginIndex(j);
1244                                                                                                                                                                                appendSqlInfo(databaseMap, j, sourceSqlInfo, sourceObject);
1245                                                                                                                                                                        }
1246                                                                                                                                                                } catch (Exception e) {
1247                                                                                                                                                                        logger.warn("Parse source jsonl line failed.", e);
1248                                                                                                                                                                }
1249                                                                                                                                                        }
1250                                                                                                                                                }
1251                                                                                                                                        }
1252                                                                                                                                }
1253                                                                                                                        }
1254                                                                                                                }
1255                                                                                                        }
1256                                                                                                }
1257                                                                                        }
1258                                                                                } else {
1259                                                                                        List<Map> servers = (List<Map>) sqlflow.get("servers");
1260                                                                                        if (servers != null) {
1261                                                                                                for (Map serverObject : servers) {
1262                                                                                                        List<Map> queries = (List<Map>) serverObject.get("queries");
1263                                                                                                        if (queries != null) {
1264                                                                                                                for (int j = 0; j < queries.size(); j++) {
1265                                                                                                                        Map queryObject = queries.get(j);
1266                                                                                                                        appendSqlInfo(databaseMap, j, sqlInfo, queryObject);
1267                                                                                                                }
1268                                                                                                        }
1269                                                                                                }
1270                                                                                        }
1271                                                                                }
1272                                                                                List<Map> errorMessages =  (List<Map>) sqlflow.get("errorMessages");
1273                                                                                if(errorMessages!=null && !errorMessages.isEmpty()) {
1274                                                                                        for(Map error: errorMessages){
1275                                                                                                ErrorInfo errorInfo = new ErrorInfo();
1276                                                                                                errorInfo.setErrorType(ErrorInfo.METADATA_ERROR);
1277                                                                                                errorInfo.setErrorMessage((String)error.get("errorMessage"));
1278                                                                                                errorInfo.setFileName(sqlInfo.getFileName());
1279                                                                                                errorInfo.setFilePath(sqlInfo.getFilePath());
1280                                                                                                errorInfo.setStartPosition(new Pair3<Long, Long, String>(-1L, -1L,
1281                                                                                                                ModelBindingManager.getGlobalHash()));
1282                                                                                                errorInfo.setEndPosition(new Pair3<Long, Long, String>(-1L, -1L,
1283                                                                                                                ModelBindingManager.getGlobalHash()));
1284                                                                                                errorInfo.setOriginStartPosition(new Pair<Long, Long>(-1L, -1L));
1285                                                                                                errorInfo.setOriginEndPosition(new Pair<Long, Long>(-1L, -1L));
1286                                                                                                metadataErrors.add(errorInfo);
1287                                                                                        }
1288                                                                                }
1289                                                                        }
1290                                                                }
1291                                                        } else {
1292                                                                Map queryObject = (Map) JSON.parseObject(sql);
1293                                                                appendSqlInfo(databaseMap, i, sqlInfo, queryObject);
1294                                                        }
1295                                                } else {
1296                                                        Map queryObject = (Map) JSON.parseObject(sql);
1297                                                        appendSqlInfo(databaseMap, i, sqlInfo, queryObject);
1298                                                }
1299                                        } else {
1300                                                ModelBindingManager.removeGlobalDatabase();
1301                                                ModelBindingManager.removeGlobalSchema();
1302                                                ModelBindingManager.removeGlobalHash();
1303
1304                                                String content = sql;
1305
1306                                                if (content == null) {
1307                                                        continue;
1308                                                }
1309
1310                                                String delimiterChar = String.valueOf(sqlparser.getDelimiterChar());
1311
1312                                                if (sqlInfos.length > 1) {
1313                                                        String endTrim = SQLUtil.endTrim(content);
1314                                                        if (endTrim.endsWith(delimiterChar) || endTrim.endsWith(";")) {
1315                                                                content += "\n";
1316                                                        } else if (option.getVendor() == EDbVendor.dbvredshift
1317                                                                        || option.getVendor() == EDbVendor.dbvgaussdb
1318                                                                        || option.getVendor() == EDbVendor.dbvedb
1319                                                                        || option.getVendor() == EDbVendor.dbvpostgresql
1320                                                                        || option.getVendor() == EDbVendor.dbvmysql
1321                                                                        || option.getVendor() == EDbVendor.dbvteradata) {
1322                                                                content += ("\n\n-- " + TBaseType.sqlflow_stmt_delimiter_str + "\n\n");
1323                                                        } else {
1324                                                                content = endTrim + ";" + "\n";
1325                                                        }
1326                                                }
1327
1328                                                sqlInfo.setSql(content);
1329
1330                                                if (MetadataReader.isMetadata(content)) {
1331                                                        String hash = SHA256.getMd5(content);
1332                                                        ModelBindingManager.setGlobalHash(hash);
1333                                                        dataflow temp = new SQLDepMetadataAnalyzer().analyzeMetadata(option.getVendor(), content);
1334                                                        if (temp.getProcedures() != null) {
1335                                                                dataflow.getProcedures().addAll(temp.getProcedures());
1336                                                        }
1337                                                        if (temp.getTables() != null) {
1338                                                                dataflow.getTables().addAll(temp.getTables());
1339                                                        }
1340                                                        if (temp.getViews() != null) {
1341                                                                dataflow.getViews().addAll(temp.getViews());
1342                                                        }
1343                                                        if (temp.getResultsets() != null) {
1344                                                                dataflow.getResultsets().addAll(temp.getResultsets());
1345                                                        }
1346                                                        if (temp.getRelationships() != null) {
1347                                                                dataflow.getRelationships().addAll(temp.getRelationships());
1348                                                        }
1349                                                        if (temp.getErrors() != null) {
1350                                                                dataflow.getErrors().addAll(temp.getErrors());
1351                                                        }
1352                                                        String fileHash = SHA256.getMd5(hash);
1353                                                        if (!sqlInfoMap.containsKey(fileHash)) {
1354                                                                sqlInfoMap.put(fileHash, new ArrayList<SqlInfo>());
1355                                                                sqlInfoMap.get(fileHash).add(sqlInfo);
1356                                                        }
1357                                                } else {
1358                                                        String sqlHash = SHA256.getMd5(content);
1359                                                        String fileHash = SHA256.getMd5(sqlHash);
1360                                                        if (!sqlInfoMap.containsKey(fileHash)) {
1361                                                                sqlInfoMap.put(fileHash, new ArrayList<SqlInfo>());
1362                                                        }
1363
1364                                                        String database = TSQLEnv.DEFAULT_DB_NAME;
1365                                                        String schema = TSQLEnv.DEFAULT_SCHEMA_NAME;
1366                                                        if (sqlenv != null) {
1367                                                                // Prefer the per-analysis Option default database over the attached
1368                                                                // env's default catalog, so a shared multi-catalog TSQLEnv can be
1369                                                                // reused across analyses with different default databases without
1370                                                                // the env's default catalog deciding name resolution.
1371                                                                if (!SQLUtil.isEmpty(option.getDefaultDatabase())
1372                                                                                && !TSQLEnv.DEFAULT_DB_NAME.equals(option.getDefaultDatabase())) {
1373                                                                        database = option.getDefaultDatabase();
1374                                                                } else {
1375                                                                        database = sqlenv.getDefaultCatalogName();
1376                                                                }
1377                                                                if (database == null) {
1378                                                                        database = TSQLEnv.DEFAULT_DB_NAME;
1379                                                                }
1380                                                                schema = sqlenv.getDefaultSchemaName();
1381                                                                if (schema == null) {
1382                                                                        schema = TSQLEnv.DEFAULT_SCHEMA_NAME;
1383                                                                }
1384                                                        }
1385
1386                                                        boolean supportCatalog = TSQLEnv.supportCatalog(option.getVendor());
1387                                                        boolean supportSchema = TSQLEnv.supportSchema(option.getVendor());
1388                                                        StringBuilder builder = new StringBuilder();
1389                                                        if (supportCatalog) {
1390                                                                builder.append(database);
1391                                                        }
1392                                                        if (supportSchema) {
1393                                                                if (builder.length() > 0) {
1394                                                                        builder.append(".");
1395                                                                }
1396                                                                builder.append(schema);
1397                                                        }
1398                                                        String group = builder.toString();
1399                                                        SqlInfo sqlInfoItem = new SqlInfo();
1400                                                        sqlInfoItem.setFileName(sqlInfo.getFileName());
1401                                                        sqlInfoItem.setFilePath(sqlInfo.getFilePath());
1402                                                        sqlInfoItem.setSql(sqlInfo.getSql());
1403                                                        sqlInfoItem.setOriginIndex(0);
1404                                                        sqlInfoItem.setOriginLineStart(0);
1405                                                        int lineEnd = sqlInfo.getSql().split("\n").length - 1;
1406                                                        sqlInfoItem.setOriginLineEnd(lineEnd);
1407                                                        sqlInfoItem.setIndex(0);
1408                                                        sqlInfoItem.setLineStart(0);
1409                                                        sqlInfoItem.setLineEnd(lineEnd);
1410                                                        sqlInfoItem.setHash(SHA256.getMd5(sqlHash));
1411                                                        sqlInfoItem.setGroup(group);
1412                                                        sqlInfoMap.get(fileHash).add(sqlInfoItem);
1413                                                        if (!databaseMap.containsKey(sqlHash)) {
1414                                                                databaseMap.put(sqlHash, new Pair3<StringBuilder, AtomicInteger, String>(
1415                                                                                new StringBuilder(), new AtomicInteger(), group));
1416                                                        }
1417                                                        databaseMap.get(sqlHash).first.append(sqlInfoItem.getSql());
1418                                                        databaseMap.get(sqlHash).second.incrementAndGet();
1419                                                }
1420                                        }
1421                                }
1422
1423                                boolean supportCatalog = TSQLEnv.supportCatalog(option.getVendor());
1424                                boolean supportSchema = TSQLEnv.supportSchema(option.getVendor());
1425
1426                                Iterator<String> schemaIter = databaseMap.keySet().iterator();
1427                                while (schemaIter.hasNext()) {
1428                                        if (option.getHandleListener() != null && option.getHandleListener().isCanceled()) {
1429                                                break;
1430                                        }
1431                                        String key = schemaIter.next();
1432                                        String group = databaseMap.get(key).third;
1433                                        String[] split = SQLUtil.parseNames(group).toArray(new String[0]);
1434
1435                                        ModelBindingManager.removeGlobalDatabase();
1436                                        ModelBindingManager.removeGlobalSchema();
1437                                        ModelBindingManager.removeGlobalSQLEnv();
1438                                        ModelBindingManager.removeGlobalHash();
1439
1440                                        String defaultDatabase = null;
1441                                        String defaultSchema = null;
1442                                        if(sqlenv!=null){
1443                                                defaultDatabase = sqlenv.getDefaultCatalogName();
1444                                                defaultSchema = sqlenv.getDefaultSchemaName();
1445                                        }
1446                                        if (supportCatalog && supportSchema) {
1447                                                if (split.length >= 2) {
1448                                                        if (!TSQLEnv.DEFAULT_DB_NAME.equals(split[split.length - 2])) {
1449                                                                ModelBindingManager.setGlobalDatabase(split[split.length - 2]);
1450                                                                sqlenv.setDefaultCatalogName(ModelBindingManager.getGlobalDatabase());
1451                                                        }
1452                                                }
1453                                                if (split.length >= 1) {
1454                                                        if (!TSQLEnv.DEFAULT_SCHEMA_NAME.equals(split[split.length - 1])) {
1455                                                                ModelBindingManager.setGlobalSchema(split[split.length - 1]);
1456                                                                sqlenv.setDefaultSchemaName(ModelBindingManager.getGlobalSchema());
1457                                                        }
1458                                                }
1459                                        } else if (supportCatalog) {
1460                                                if (!TSQLEnv.DEFAULT_DB_NAME.equals(split[split.length - 1])) {
1461                                                        ModelBindingManager.setGlobalDatabase(split[split.length - 1]);
1462                                                        sqlenv.setDefaultCatalogName(ModelBindingManager.getGlobalDatabase());
1463                                                }
1464                                        } else if (supportSchema) {
1465                                                if (!TSQLEnv.DEFAULT_SCHEMA_NAME.equals(split[split.length - 1])) {
1466                                                        ModelBindingManager.setGlobalSchema(split[split.length - 1]);
1467                                                        sqlenv.setDefaultSchemaName(ModelBindingManager.getGlobalSchema());
1468                                                }
1469                                        }
1470                                        if (option.getHandleListener() != null) {
1471                                                option.getHandleListener().startParse(null, databaseMap.get(key).first.toString());
1472                                        }
1473
1474                                        if (sqlenv == null) {
1475                                                sqlenv = new TSQLEnv(option.getVendor()) {
1476
1477                                                        @Override
1478                                                        public void initSQLEnv() {
1479                                                                // TODO Auto-generated method stub
1480
1481                                                        }
1482                                                };
1483                                        }
1484                                        ModelBindingManager.setGlobalSQLEnv(sqlenv);
1485                                        sqlparser.sqltext = databaseMap.get(key).first.toString();
1486                                        ModelBindingManager.setGlobalHash(SHA256.getMd5(key));
1487                                        analyzeAndOutputResult(sqlparser);
1488                                        if(sqlenv!=null){
1489                                                sqlenv.setDefaultCatalogName(defaultDatabase);
1490                                                sqlenv.setDefaultSchemaName(defaultSchema);
1491                                        }
1492                                }
1493
1494                                materializeSqlEnvSynonyms();
1495
1496                                appendProcesses(dataflow);
1497                                appendOraclePackages(dataflow);
1498                                appendProcedures(dataflow);
1499                                appendTables(dataflow);
1500                                appendViews(dataflow);
1501                                appendResultSets(dataflow);
1502                                appendRelations(dataflow);
1503                                appendErrors(dataflow);
1504                        }
1505
1506                        dataflow = handleDataflowExecProcedure(dataflow);
1507
1508                        if (dataflow != null && option.getAnalyzeMode() != AnalyzeMode.crud) {
1509                                if (!isShowJoin()) {
1510                                        dataflow = mergeTables(dataflow, modelManager.TABLE_COLUMN_ID, option);
1511                                } else {
1512                                        dataflow = removeDuplicateColumns(dataflow);
1513                                }
1514                        }
1515
1516                        if (dataflow != null && option.isNormalizeOutput()) {
1517                                dataflow = getNormalizeDataflow(dataflow);
1518                        }
1519
1520                        if (dataflow != null && option.isIgnoreCoordinate()) {
1521                                dataflow = filterDataflowCoordinate(dataflow);
1522                        }
1523                        
1524                        if (option.isSimpleOutput() || option.isIgnoreRecordSet()) {
1525                                List<String> showTypes = new ArrayList<>();
1526                                if(option.getSimpleShowRelationTypes()!=null) {
1527                                        showTypes.addAll(option.getSimpleShowRelationTypes());
1528                                }
1529                                if(showTypes.isEmpty()) {
1530                                        showTypes.add("fdd");
1531                                }
1532                                if(option.isShowCallRelation()) {
1533                                        showTypes.add("call");
1534                                }
1535                                if(option.isShowERDiagram()) {
1536                                        showTypes.add("er");
1537                                }
1538                                dataflow simpleDataflow = getSimpleDataflow(dataflow, option.isSimpleOutput(), showTypes);
1539                                if (simpleDataflow.getResultsets() != null) {
1540                                        for (table t : simpleDataflow.getResultsets()) {
1541                                                t.setIsTarget(null);
1542                                        }
1543                                }
1544                                return simpleDataflow;
1545                        } else {
1546                                return dataflow;
1547                        }
1548                        
1549                } catch (Exception e) {
1550                        logger.error("analyze sql failed.", e);
1551                        ErrorInfo errorInfo = new ErrorInfo();
1552                        errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
1553                        if (e.getMessage() == null) {
1554                                if (e.getStackTrace() != null && e.getStackTrace().length > 0) {
1555                                        errorInfo.setErrorMessage(e.getClass().getSimpleName() + ": " + e.getStackTrace()[0].toString());
1556                                } else {
1557                                        errorInfo.setErrorMessage(e.getClass().getSimpleName());
1558                                }
1559                        } else {
1560                                errorInfo.setErrorMessage(e.getClass().getSimpleName() + ": " + e.getMessage());
1561                        }
1562                        errorInfo.fillInfo(this);
1563                        errorInfos.add(errorInfo);
1564                }
1565
1566                modelManager.reset();
1567                return null;
1568        }
1569
1570        private dataflow handleDataflowExecProcedure(dataflow dataflow) {
1571                Set<String> procedures = new HashSet<String>();
1572                for (procedure procedure : dataflow.getProcedures()) {
1573                        procedures.add(DlineageUtil.getIdentifierNormalTableName(procedure.getName()));
1574                }
1575                List<table> removeResultSets = new ArrayList<table>();
1576                
1577                Map<String, table> allMap = new HashMap<String, table>();
1578                
1579                if (dataflow.getResultsets() != null) {
1580                        for (table t : dataflow.getResultsets()) {
1581                                allMap.put(t.getId().toLowerCase(), t);
1582                        }
1583                }
1584
1585                if (dataflow.getTables() != null) {
1586                        for (table t : dataflow.getTables()) {
1587                                allMap.put(t.getId().toLowerCase(), t);
1588                        }
1589                }
1590
1591                if (dataflow.getViews() != null) {
1592                        for (table t : dataflow.getViews()) {
1593                                allMap.put(t.getId().toLowerCase(), t);
1594                        }
1595                }
1596
1597                if (dataflow.getVariables() != null) {
1598                        for (table t : dataflow.getVariables()) {
1599                                allMap.put(t.getId().toLowerCase(), t);
1600                        }
1601                }
1602
1603                
1604                for (table resultSet : dataflow.getResultsets()) {
1605                        String resultSetName = DlineageUtil.getIdentifierNormalTableName(resultSet.getName());
1606                        if (!(ResultSetType.function.name().equals(resultSet.getType())
1607                                        && modelManager.getFunctionTable(resultSetName) != null && procedures.contains(resultSetName))) {
1608                                continue;
1609                        }
1610
1611                        Set<Object> functionTables = modelManager
1612                                        .getFunctionTable(DlineageUtil.getIdentifierNormalTableName(resultSet.getName()));
1613                        List<ResultSet> functionResults = functionTables.stream().filter(t -> t instanceof ResultSet)
1614                                        .map(t -> (ResultSet) t).collect(Collectors.toList());
1615                        if (!(resultSet.getColumns().size() == 1
1616                                        && DlineageUtil.getIdentifierNormalTableName(resultSet.getColumns().get(0).getName()).equals(resultSetName))
1617                                        || functionResults.isEmpty()) {
1618                                continue;
1619                        }
1620
1621                        column column = resultSet.getColumns().get(0);
1622                        String sourceColumnId = column.getId();
1623                        boolean reserveResultSet = false;
1624                        List<relationship> appendRelations = new ArrayList<relationship>();
1625                        Set<relationship> removeRelations = new HashSet<relationship>();
1626                        
1627                        for (relationship relationship : dataflow.getRelationships()) {
1628                                targetColumn targetColumn = relationship.getTarget();
1629                                List<sourceColumn> sourceColumns = relationship.getSources();
1630                                List<sourceColumn> newSourceColumns = new ArrayList<sourceColumn>();
1631                                for (sourceColumn sourceColumn : sourceColumns) {
1632                                        if (!sourceColumn.getId().equals(sourceColumnId)) {
1633                                                continue;
1634                                        }
1635                                        for (ResultSet sourceResultSet : functionResults) {
1636                                                for (ResultColumn resultColumn : sourceResultSet.getColumns()) {
1637                                                        if (!DlineageUtil.compareColumnIdentifier(resultColumn.getName(),
1638                                                                        targetColumn.getColumn())) {
1639                                                                if (targetColumn.getColumn().endsWith("*")) {
1640                                                                        table parent = allMap.get(targetColumn.getParent_id());
1641                                                                        if (parent != null && parent.getColumns().size() > 1) {
1642                                                                                for (column item : parent.getColumns()) {
1643                                                                                        if (DlineageUtil.compareColumnIdentifier(resultColumn.getName(),
1644                                                                                                        item.getName())) {
1645                                                                                                relationship newRelation = appendStarRelation(dataflow, relationship, item, resultColumn);
1646                                                                                                appendRelations.add(newRelation);
1647                                                                                                removeRelations.add(relationship);
1648                                                                                                newSourceColumns.add(newRelation.getSources().get(0));
1649                                                                                                if (resultColumn.getResultSet() != null && resultColumn.getResultSet().isTarget()) {
1650                                                                                                        dataflow.getResultsets().stream().filter(
1651                                                                                                                        t -> t.getId().equals(String.valueOf(resultColumn.getResultSet().getId())))
1652                                                                                                                        .forEach(t -> t.setIsTarget(Boolean.FALSE.toString()));
1653                                                                                                }
1654                                                                                        }
1655                                                                                }
1656                                                                        }
1657                                                                }
1658                                                                continue;
1659                                                        }
1660                                                        
1661                                                        sourceColumn sourceColumn1 = new sourceColumn();
1662                                                        sourceColumn1.setId(String.valueOf(resultColumn.getId()));
1663                                                        sourceColumn1.setColumn(resultColumn.getName());
1664                                                        sourceColumn1.setParent_id(String.valueOf(resultColumn.getResultSet().getId()));
1665                                                        sourceColumn1.setParent_name(getResultSetName(resultColumn.getResultSet()));
1666                                                        if (resultColumn.getStartPosition() != null
1667                                                                        && resultColumn.getEndPosition() != null) {
1668                                                                sourceColumn1.setCoordinate(convertCoordinate(resultColumn.getStartPosition())
1669                                                                                + "," + convertCoordinate(resultColumn.getEndPosition()));
1670                                                        }
1671                                                        newSourceColumns.add(sourceColumn1);
1672                                                        
1673                                                        if (resultColumn.getResultSet() != null && resultColumn.getResultSet().isTarget()) {
1674                                                                dataflow.getResultsets().stream().filter(
1675                                                                                t -> t.getId().equals(String.valueOf(resultColumn.getResultSet().getId())))
1676                                                                                .forEach(t -> t.setIsTarget(Boolean.FALSE.toString()));
1677                                                        }
1678                                                }
1679                                        }
1680
1681                                        if (!newSourceColumns.isEmpty()) {
1682                                                if (removeRelations.isEmpty()) {
1683                                                        relationship.setSources(newSourceColumns);
1684                                                }
1685                                        }
1686                                        else {
1687                                                reserveResultSet = true;
1688                                        }
1689                                }
1690                        }
1691                        
1692                        dataflow.getRelationships().addAll(appendRelations);
1693                        dataflow.getRelationships().removeAll(removeRelations);
1694                        
1695                        if(!reserveResultSet) {
1696                                removeResultSets.add(resultSet);
1697                        }
1698                }
1699                dataflow.getResultsets().removeAll(removeResultSets);
1700                return dataflow;
1701        }
1702
1703        private relationship appendStarRelation(dataflow dataflow,
1704                        relationship relationship, column item, ResultColumn resultColumn) {
1705                relationship relationElement = new relationship();
1706                relationElement.setType(relationship.getType());
1707                relationElement.setEffectType(relationship.getEffectType());
1708                relationElement.setSqlHash(relationship.getSqlHash());
1709                relationElement.setSqlComment(relationship.getSqlComment());
1710                relationElement.setProcedureId(relationship.getProcedureId());
1711                relationElement.setId(String.valueOf(++ModelBindingManager.get().RELATION_ID));
1712                relationElement.setProcessId(relationship.getProcessId());
1713                relationElement.setProcessType(relationship.getProcessType());
1714                
1715                targetColumn targetColumn1 = new targetColumn();
1716                targetColumn1.setId(String.valueOf(item.getId()));
1717                targetColumn1.setColumn(item.getName());
1718                targetColumn1.setParent_id(relationship.getTarget().getParent_id());
1719                targetColumn1.setParent_name(relationship.getTarget().getParent_name());
1720                targetColumn1.setParent_alias(relationship.getTarget().getParent_alias());
1721                if (relationship.getTarget().getCoordinate() != null) {
1722                        targetColumn1.setCoordinate(item.getCoordinate());
1723                }
1724                relationElement.setTarget(targetColumn1);       
1725                
1726                sourceColumn sourceColumn1 = new sourceColumn();
1727                sourceColumn1.setId(String.valueOf(resultColumn.getId()));
1728                sourceColumn1.setColumn(resultColumn.getName());
1729                sourceColumn1.setParent_id(String.valueOf(resultColumn.getResultSet().getId()));
1730                sourceColumn1.setParent_name(getResultSetName(resultColumn.getResultSet()));
1731                if (resultColumn.getStartPosition() != null
1732                                && resultColumn.getEndPosition() != null) {
1733                        sourceColumn1.setCoordinate(convertCoordinate(resultColumn.getStartPosition())
1734                                        + "," + convertCoordinate(resultColumn.getEndPosition()));
1735                }
1736                relationElement.addSource(sourceColumn1);
1737                return relationElement;
1738        }
1739
1740        private dataflow getNormalizeDataflow(dataflow instance) {
1741                List<table> tables = new ArrayList<>();
1742                if (instance.getResultsets() != null) {
1743                        for (table t : instance.getResultsets()) {
1744                                tables.add(t);
1745                        }
1746                }
1747
1748                if (instance.getTables() != null) {
1749                        for (table t : instance.getTables()) {
1750                                tables.add(t);
1751                        }
1752                }
1753
1754                if (instance.getViews() != null) {
1755                        for (table t : instance.getViews()) {
1756                                tables.add(t);
1757                        }
1758                }
1759
1760                if (instance.getPaths() != null) {
1761                        for (table t : instance.getPaths()) {
1762                                tables.add(t);
1763                        }
1764                }
1765
1766                if (instance.getStages() != null) {
1767                        for (table t : instance.getStages()) {
1768                                tables.add(t);
1769                        }
1770                }
1771
1772                if (instance.getSequences() != null) {
1773                        for (table t : instance.getSequences()) {
1774                                tables.add(t);
1775                        }
1776                }
1777
1778                if (instance.getDatasources() != null) {
1779                        for (table t : instance.getDatasources()) {
1780                                tables.add(t);
1781                        }
1782                }
1783
1784                if (instance.getDatabases() != null) {
1785                        for (table t : instance.getDatabases()) {
1786                                tables.add(t);
1787                        }
1788                }
1789
1790                if (instance.getSchemas() != null) {
1791                        for (table t : instance.getSchemas()) {
1792                                tables.add(t);
1793                        }
1794                }
1795
1796                if (instance.getStreams() != null) {
1797                        for (table t : instance.getStreams()) {
1798                                tables.add(t);
1799                        }
1800                }
1801
1802                if (instance.getVariables() != null) {
1803                        for (table t : instance.getVariables()) {
1804                                tables.add(t);
1805                        }
1806                }
1807
1808                Map<String, String> idNameMap = new HashMap();
1809
1810                for(table table: tables){
1811                        if(table.getDatabase()!=null) {
1812                                table.setDatabase(DlineageUtil.getIdentifierNormalName(table.getDatabase(), ESQLDataObjectType.dotCatalog));
1813                        }
1814                        if(table.getSchema()!=null) {
1815                                table.setSchema(DlineageUtil.getIdentifierNormalName(table.getSchema(), ESQLDataObjectType.dotSchema));
1816                        }
1817                        if(table.getName()!=null) {
1818                                table.setName(DlineageUtil.getIdentifierNormalName(table.getName(), ESQLDataObjectType.dotTable));
1819                                idNameMap.put(table.getId(), table.getName());
1820                        }
1821                        if(table.getColumns()!=null){
1822                                for(column column: table.getColumns()){
1823                                        column.setName(DlineageUtil.getIdentifierNormalName(column.getName(), ESQLDataObjectType.dotColumn));
1824                                        idNameMap.put(column.getId(), column.getName());
1825                                }
1826                        }
1827                }
1828                if(instance.getPackages()!=null){
1829                        for(oraclePackage oraclePackage: instance.getPackages()){
1830                                if(oraclePackage.getDatabase()!=null) {
1831                                        oraclePackage.setDatabase(DlineageUtil.getIdentifierNormalName(oraclePackage.getDatabase(), ESQLDataObjectType.dotCatalog));
1832                                }
1833                                if(oraclePackage.getSchema()!=null) {
1834                                        oraclePackage.setSchema(DlineageUtil.getIdentifierNormalName(oraclePackage.getSchema(), ESQLDataObjectType.dotSchema));
1835                                }
1836                                if(oraclePackage.getName()!=null) {
1837                                        oraclePackage.setName(DlineageUtil.getIdentifierNormalName(oraclePackage.getName(), ESQLDataObjectType.dotTable));
1838                                        idNameMap.put(oraclePackage.getId(), oraclePackage.getName());
1839                                }
1840                                if(oraclePackage.getArguments()!=null){
1841                                        for(argument argument: oraclePackage.getArguments()){
1842                                                argument.setName(DlineageUtil.getIdentifierNormalName(argument.getName(), ESQLDataObjectType.dotColumn));
1843                                                idNameMap.put(argument.getId(), argument.getName());
1844                                        }
1845                                }
1846                                if(oraclePackage.getProcedures()!=null){
1847                                        for(procedure procedure: oraclePackage.getProcedures()){
1848                                                if(procedure.getDatabase()!=null) {
1849                                                        procedure.setDatabase(DlineageUtil.getIdentifierNormalName(procedure.getDatabase(), ESQLDataObjectType.dotCatalog));
1850                                                }
1851                                                if(procedure.getSchema()!=null) {
1852                                                        procedure.setSchema(DlineageUtil.getIdentifierNormalName(procedure.getSchema(), ESQLDataObjectType.dotSchema));
1853                                                }
1854                                                if(procedure.getName()!=null) {
1855                                                        procedure.setName(DlineageUtil.getIdentifierNormalName(procedure.getName(), ESQLDataObjectType.dotTable));
1856                                                        idNameMap.put(procedure.getId(), procedure.getName());
1857                                                }
1858                                                for(argument argument: procedure.getArguments()){
1859                                                        argument.setName(DlineageUtil.getIdentifierNormalName(argument.getName(), ESQLDataObjectType.dotColumn));
1860                                                        idNameMap.put(argument.getId(), argument.getName());
1861                                                }
1862                                        }
1863                                }
1864                        }
1865                }
1866                if(instance.getProcedures()!=null){
1867                        for(procedure procedure: instance.getProcedures()){
1868                                if(procedure.getDatabase()!=null) {
1869                                        procedure.setDatabase(DlineageUtil.getIdentifierNormalName(procedure.getDatabase(), ESQLDataObjectType.dotCatalog));
1870                                }
1871                                if(procedure.getSchema()!=null) {
1872                                        procedure.setSchema(DlineageUtil.getIdentifierNormalName(procedure.getSchema(), ESQLDataObjectType.dotSchema));
1873                                }
1874                                if(procedure.getName()!=null) {
1875                                        procedure.setName(DlineageUtil.getIdentifierNormalName(procedure.getName(), ESQLDataObjectType.dotTable));
1876                                        idNameMap.put(procedure.getId(), procedure.getName());
1877                                }
1878                                for(argument argument: procedure.getArguments()){
1879                                        argument.setName(DlineageUtil.getIdentifierNormalName(argument.getName(), ESQLDataObjectType.dotColumn));
1880                                        idNameMap.put(argument.getId(), argument.getName());
1881                                }
1882                        }
1883                }
1884                if (instance.getProcesses() != null) {
1885                        for (process process : instance.getProcesses()) {
1886                                if(process.getDatabase()!=null) {
1887                                        process.setDatabase(DlineageUtil.getIdentifierNormalName(process.getDatabase(), ESQLDataObjectType.dotCatalog));
1888                                }
1889                                if(process.getSchema()!=null) {
1890                                        process.setSchema(DlineageUtil.getIdentifierNormalName(process.getSchema(), ESQLDataObjectType.dotSchema));
1891                                }
1892                                if (process.getProcedureId() != null) {
1893                                        process.setProcedureName(DlineageUtil.getIdentifierNormalName(process.getProcedureName(), ESQLDataObjectType.dotTable));
1894                                        idNameMap.put(process.getId(), process.getName());
1895                                }
1896                        }
1897                }
1898
1899                if(instance.getRelationships()!=null){
1900                        for(relationship relation: instance.getRelationships()){
1901                                targetColumn targetColumn = relation.getTarget();
1902                                if(targetColumn != null) {
1903                                        targetColumn.setColumn(idNameMap.get(targetColumn.getId()));
1904                                        targetColumn.setTarget_name(idNameMap.get(targetColumn.getTarget_id()));
1905                                        targetColumn.setParent_name(idNameMap.get(targetColumn.getParent_id()));
1906                                }
1907                                List<sourceColumn> sourceColumns = relation.getSources();
1908                                if(sourceColumns!=null){
1909                                        for(sourceColumn sourceColumn: sourceColumns){
1910                                                sourceColumn.setColumn(idNameMap.get(sourceColumn.getId()));
1911                                                sourceColumn.setSource_name(idNameMap.get(sourceColumn.getSource_id()));
1912                                                sourceColumn.setParent_name(idNameMap.get(sourceColumn.getParent_id()));
1913                                        }
1914                                }
1915                                targetColumn = relation.getCaller();
1916                                if(targetColumn != null) {
1917                                        targetColumn.setName(idNameMap.get(targetColumn.getId()));
1918                                }
1919                                sourceColumns = relation.getCallees();
1920                                if(sourceColumns!=null){
1921                                        for(sourceColumn sourceColumn: sourceColumns){
1922                                                sourceColumn.setName(idNameMap.get(sourceColumn.getId()));
1923                                        }
1924                                }
1925                        }
1926                }
1927
1928
1929                return instance;
1930        }
1931
1932        private dataflow filterDataflowCoordinate(dataflow instance) {
1933                List<table> tables = new ArrayList<>();
1934                if (instance.getResultsets() != null) {
1935                        for (table t : instance.getResultsets()) {
1936                                tables.add(t);
1937                        }
1938                }
1939
1940                if (instance.getTables() != null) {
1941                        for (table t : instance.getTables()) {
1942                                tables.add(t);
1943                        }
1944                }
1945
1946                if (instance.getViews() != null) {
1947                        for (table t : instance.getViews()) {
1948                                tables.add(t);
1949                        }
1950                }
1951
1952                if (instance.getPaths() != null) {
1953                        for (table t : instance.getPaths()) {
1954                                tables.add(t);
1955                        }
1956                }
1957
1958                if (instance.getStages() != null) {
1959                        for (table t : instance.getStages()) {
1960                                tables.add(t);
1961                        }
1962                }
1963
1964                if (instance.getSequences() != null) {
1965                        for (table t : instance.getSequences()) {
1966                                tables.add(t);
1967                        }
1968                }
1969
1970                if (instance.getDatasources() != null) {
1971                        for (table t : instance.getDatasources()) {
1972                                tables.add(t);
1973                        }
1974                }
1975
1976                if (instance.getDatabases() != null) {
1977                        for (table t : instance.getDatabases()) {
1978                                tables.add(t);
1979                        }
1980                }
1981
1982                if (instance.getSchemas() != null) {
1983                        for (table t : instance.getSchemas()) {
1984                                tables.add(t);
1985                        }
1986                }
1987
1988                if (instance.getStreams() != null) {
1989                        for (table t : instance.getStreams()) {
1990                                tables.add(t);
1991                        }
1992                }
1993
1994                if (instance.getVariables() != null) {
1995                        for (table t : instance.getVariables()) {
1996                                tables.add(t);
1997                        }
1998                }
1999
2000                for(table table: tables){
2001                        table.clearCoordinate();
2002                        if(table.getColumns()!=null){
2003                                for(column column: table.getColumns()){
2004                                        column.clearCoordinate();
2005                                }
2006                        }
2007                }
2008                if(instance.getPackages()!=null){
2009                        for(oraclePackage oraclePackage: instance.getPackages()){
2010                                oraclePackage.setCoordinate(null);
2011                                if(oraclePackage.getProcedures()!=null){
2012                                        for(procedure procedure: oraclePackage.getProcedures()){
2013                                                procedure.setCoordinate(null);
2014                                                for(argument argument: procedure.getArguments()){
2015                                                        argument.setCoordinate(null);
2016                                                }
2017                                        }
2018                                }
2019                        }
2020                }
2021                if(instance.getProcedures()!=null){
2022                        for(procedure procedure: instance.getProcedures()){
2023                                procedure.setCoordinate(null);
2024                                for(argument argument: procedure.getArguments()){
2025                                        argument.setCoordinate(null);
2026                                }
2027                        }
2028                }
2029                if (instance.getProcesses() != null) {
2030                        for (process process : instance.getProcesses()) {
2031                                process.setCoordinate(null);
2032                        }
2033                }
2034
2035                if(instance.getRelationships()!=null){
2036                        for(relationship relation: instance.getRelationships()){
2037                                targetColumn targetColumn = relation.getTarget();
2038                                if(targetColumn != null) {
2039                                        targetColumn.setCoordinate(null);
2040                                }
2041                                List<sourceColumn> sourceColumns = relation.getSources();
2042                                if(sourceColumns!=null){
2043                                        for(sourceColumn sourceColumn: sourceColumns){
2044                                                sourceColumn.setCoordinate(null);
2045                                        }
2046                                }
2047                                targetColumn = relation.getCaller();
2048                                if(targetColumn != null) {
2049                                        targetColumn.setCoordinate(null);
2050                                }
2051                                sourceColumns = relation.getCallees();
2052                                if(sourceColumns!=null){
2053                                        for(sourceColumn sourceColumn: sourceColumns){
2054                                                sourceColumn.setCoordinate(null);
2055                                        }
2056                                }
2057                        }
2058                }
2059
2060                return instance;
2061        }
2062
2063        private void appendSqlInfo(Map<String, Pair3<StringBuilder, AtomicInteger, String>> databaseMap, int index,
2064                        SqlInfo sqlInfo, Map queryObject) {
2065                EDbVendor vendor = option.getVendor();
2066                if (!SQLUtil.isEmpty(sqlInfo.getDbVendor())) {
2067                        vendor = EDbVendor.valueOf(sqlInfo.getDbVendor());
2068                }
2069
2070                boolean supportCatalog = TSQLEnv.supportCatalog(vendor);
2071                boolean supportSchema = TSQLEnv.supportSchema(vendor);
2072
2073                String groupName = (String) queryObject.get("groupName");
2074                if (DlineageUtil.isProcedureExcluded(groupName)) {
2075                        return;
2076                }
2077
2078                String content = (String) queryObject.get("sourceCode");
2079                if (SQLUtil.isEmpty(content)) {
2080                        return;
2081                }
2082                StringBuilder builder = new StringBuilder();
2083                if (supportCatalog) {
2084                        String database = (String) queryObject.get("database");
2085                        if (database.indexOf(".") != -1) {
2086                                String delimitedChar = TSQLEnv.delimitedChar(vendor);
2087                                database = delimitedChar + SQLUtil.trimColumnStringQuote(database) + delimitedChar;
2088                        }
2089                        builder.append(database);
2090                }
2091                if (supportSchema) {
2092                        String schema = (String) queryObject.get("schema");
2093                        if (schema.indexOf(".") != -1) {
2094                                String delimitedChar = TSQLEnv.delimitedChar(vendor);
2095                                schema = delimitedChar + SQLUtil.trimColumnStringQuote(schema) + delimitedChar;
2096                        }
2097                        if (builder.length() > 0) {
2098                                builder.append(".");
2099                        }
2100                        builder.append(schema);
2101                }
2102                String group = builder.toString();
2103                String sqlHash = SHA256.getMd5(content);
2104                String hash = SHA256.getMd5(sqlHash);
2105                if (!databaseMap.containsKey(sqlHash)) {
2106                        databaseMap.put(sqlHash,
2107                                        new Pair3<StringBuilder, AtomicInteger, String>(new StringBuilder(), new AtomicInteger(), group));
2108                }
2109                String delimiterChar = String.valueOf(TGSqlParser.getDelimiterChar(option.getVendor()));
2110                StringBuilder buffer = new StringBuilder(content);
2111                if (content.trim().endsWith(delimiterChar) || content.trim().endsWith(";")) {
2112                        buffer.append("\n");
2113                } else if(vendor == EDbVendor.dbvredshift
2114                                || vendor == EDbVendor.dbvgaussdb
2115                                || vendor == EDbVendor.dbvedb
2116                                || vendor == EDbVendor.dbvpostgresql
2117                                || vendor == EDbVendor.dbvmysql
2118                                || vendor == EDbVendor.dbvteradata){
2119                        buffer.append("\n\n-- " + TBaseType.sqlflow_stmt_delimiter_str + "\n\n");
2120                } else{
2121                        SQLUtil.endTrim(buffer);
2122                        buffer.append(";").append("\n");
2123                }
2124
2125                int lineStart = databaseMap.get(sqlHash).first.toString().split("\n", -1).length - 1;
2126                if (databaseMap.get(sqlHash).first.toString().length() == 0) {
2127                        lineStart = 0;
2128                }
2129                databaseMap.get(sqlHash).first.append(buffer.toString());
2130                SqlInfo sqlInfoItem = new SqlInfo();
2131                sqlInfoItem.setServer(sqlInfo.getServer());
2132                sqlInfoItem.setDbVendor(sqlInfo.getDbVendor());
2133                sqlInfoItem.setFileName(sqlInfo.getFileName());
2134                sqlInfoItem.setFilePath(sqlInfo.getFilePath());
2135                sqlInfoItem.setSql(buffer.toString());
2136                sqlInfoItem.setOriginIndex(index);
2137                sqlInfoItem.setOriginLineStart(0);
2138                sqlInfoItem.setOriginLineEnd(buffer.toString().split("\n", -1).length - 1);
2139                sqlInfoItem.setIndex(databaseMap.get(sqlHash).second.getAndIncrement());
2140                sqlInfoItem.setLineStart(lineStart);
2141                sqlInfoItem.setLineEnd(databaseMap.get(sqlHash).first.toString().split("\n", -1).length - 1);
2142                sqlInfoItem.setGroup(group);
2143                sqlInfoItem.setHash(hash);
2144
2145                if (!sqlInfoMap.containsKey(hash)) {
2146                        sqlInfoMap.put(hash, new ArrayList<SqlInfo>());
2147                }
2148                sqlInfoMap.get(hash).add(sqlInfoItem);
2149        }
2150
2151        private void readGzipBlockSource(String fullPath, Map<String, Pair3<StringBuilder, AtomicInteger, String>> databaseMap, SqlInfo sqlInfo) {
2152                try (FileInputStream fis = new FileInputStream(fullPath);
2153                         GZIPInputStream gzis = new GZIPInputStream(fis);
2154                         BufferedReader reader = new BufferedReader(new InputStreamReader(gzis, "UTF-8"))) {
2155                        String line;
2156                        int j = 0;
2157                        while ((line = reader.readLine()) != null) {
2158                                line = line.trim();
2159                                if (line.isEmpty()) {
2160                                        continue;
2161                                }
2162                                try {
2163                                        Map sourceObject = (Map) JSON.parseObject(line);
2164                                        String sourceCode = (String) sourceObject.get("sourceCode");
2165                                        if (sourceCode != null && !sourceCode.isEmpty()) {
2166                                                SqlInfo sourceSqlInfo = new SqlInfo();
2167                                                sourceSqlInfo.setFileName(sqlInfo.getFileName());
2168                                                sourceSqlInfo.setFilePath(sqlInfo.getFilePath());
2169                                                sourceSqlInfo.setSql(sourceCode);
2170                                                sourceSqlInfo.setOriginIndex(j);
2171                                                appendSqlInfo(databaseMap, j, sourceSqlInfo, null);
2172                                        }
2173                                } catch (Exception e) {
2174                                        logger.warn("Parse gzip source jsonl line failed.", e);
2175                                }
2176                                j++;
2177                        }
2178                } catch (Exception e) {
2179                        logger.warn("Read gzip source file failed: " + fullPath, e);
2180                }
2181        }
2182
2183        static String getTextOutput(dataflow dataflow) {
2184                StringBuffer buffer = new StringBuffer();
2185                List<relationship> relations = dataflow.getRelationships();
2186                if (relations != null) {
2187                        for (int i = 0; i < relations.size(); i++) {
2188                                relationship relation = relations.get(i);
2189                                targetColumn target = relation.getTarget();
2190                                List<sourceColumn> sources = relation.getSources();
2191                                if (target != null && sources != null && sources.size() > 0) {
2192                                        buffer.append(target.getColumn()).append(" depends on: ");
2193                                        Set<String> columnSet = new LinkedHashSet<String>();
2194                                        for (int j = 0; j < sources.size(); j++) {
2195                                                sourceColumn sourceColumn = sources.get(j);
2196                                                String columnName = sourceColumn.getColumn();
2197                                                if (sourceColumn.getParent_name() != null && sourceColumn.getParent_name().length() > 0) {
2198                                                        columnName = sourceColumn.getParent_name() + "." + columnName;
2199                                                }
2200                                                columnSet.add(columnName);
2201                                        }
2202                                        String[] columns = columnSet.toArray(new String[0]);
2203                                        for (int j = 0; j < columns.length; j++) {
2204                                                buffer.append(columns[j]);
2205                                                if (j == columns.length - 1) {
2206                                                        buffer.append("\n");
2207                                                } else
2208                                                        buffer.append(", ");
2209                                        }
2210                                }
2211                        }
2212                }
2213                return buffer.toString();
2214        }
2215
2216        private String mergeRelationType(List<Pair<sourceColumn, List<String>>> typePaths) {
2217                RelationshipType relationType = RelationshipType.join;
2218                for (int i = 0; i < typePaths.size(); i++) {
2219                        List<String> path = typePaths.get(i).second;
2220                        RelationshipType type = RelationshipType.valueOf(getRelationType(path));
2221                        if (type.ordinal() < relationType.ordinal()) {
2222                                relationType = type;
2223                        }
2224                }
2225                return relationType.name();
2226        }
2227
2228        private String getRelationType(List<String> typePaths) {
2229                if (typePaths.contains("join"))
2230                        return "join";
2231                if (typePaths.contains("fdr"))
2232                        return "fdr";
2233                if (typePaths.contains("frd"))
2234                        return "frd";
2235                if (typePaths.contains("fddi"))
2236                        return "fddi";
2237                return "fdd";
2238        }
2239
2240        public dataflow getSimpleDataflow(dataflow instance, boolean simpleOutput) throws Exception {
2241                return getSimpleDataflow(instance, simpleOutput, Arrays.asList("fdd"));
2242        }
2243        
2244        public dataflow getSimpleDataflow(dataflow instance, boolean simpleOutput, List<String> types) throws Exception {
2245                ModelBindingManager.setGlobalVendor(option.getVendor());
2246                allMap.clear();
2247                targetTables.clear();
2248                resultSetMap.clear();
2249                tableMap.clear();
2250                viewMap.clear();
2251                cursorMap.clear();
2252                variableMap.clear();
2253                fileMap.clear();
2254                stageMap.clear();
2255                sequenceMap.clear();
2256                dataSourceMap.clear();
2257                databaseMap.clear();
2258                schemaMap.clear();
2259                streamMap.clear();
2260                dataflow simple = new dataflow();
2261                List<relationship> simpleRelations = new ArrayList<relationship>();
2262                List<relationship> relations = instance.getRelationships();
2263                if (instance.getResultsets() != null) {
2264                        for (table t : instance.getResultsets()) {
2265                                resultSetMap.put(t.getId().toLowerCase(), t);
2266                                allMap.put(t.getId().toLowerCase(), t);
2267                        }
2268                }
2269
2270                if (instance.getTables() != null) {
2271                        for (table t : instance.getTables()) {
2272                                tableMap.put(t.getId().toLowerCase(), t);
2273                                allMap.put(t.getId().toLowerCase(), t);
2274                        }
2275                }
2276
2277                if (instance.getViews() != null) {
2278                        for (table t : instance.getViews()) {
2279                                viewMap.put(t.getId().toLowerCase(), t);
2280                                allMap.put(t.getId().toLowerCase(), t);
2281                        }
2282                }
2283
2284                if (instance.getPaths() != null) {
2285                        for (table t : instance.getPaths()) {
2286                                fileMap.put(t.getId().toLowerCase(), t);
2287                                allMap.put(t.getId().toLowerCase(), t);
2288                        }
2289                }
2290
2291                if (instance.getStages() != null) {
2292                        for (table t : instance.getStages()) {
2293                                stageMap.put(t.getId().toLowerCase(), t);
2294                                allMap.put(t.getId().toLowerCase(), t);
2295                        }
2296                }
2297                
2298                if (instance.getSequences() != null) {
2299                        for (table t : instance.getSequences()) {
2300                                sequenceMap.put(t.getId().toLowerCase(), t);
2301                                allMap.put(t.getId().toLowerCase(), t);
2302                        }
2303                }
2304
2305                if (instance.getDatasources() != null) {
2306                        for (table t : instance.getDatasources()) {
2307                                dataSourceMap.put(t.getId().toLowerCase(), t);
2308                                allMap.put(t.getId().toLowerCase(), t);
2309                        }
2310                }
2311
2312                if (instance.getDatabases() != null) {
2313                        for (table t : instance.getDatabases()) {
2314                                databaseMap.put(t.getId().toLowerCase(), t);
2315                                allMap.put(t.getId().toLowerCase(), t);
2316                        }
2317                }
2318
2319                if (instance.getSchemas() != null) {
2320                        for (table t : instance.getSchemas()) {
2321                                schemaMap.put(t.getId().toLowerCase(), t);
2322                                allMap.put(t.getId().toLowerCase(), t);
2323                        }
2324                }
2325
2326                if (instance.getStreams() != null) {
2327                        for (table t : instance.getStreams()) {
2328                                streamMap.put(t.getId().toLowerCase(), t);
2329                                allMap.put(t.getId().toLowerCase(), t);
2330                        }
2331                }
2332
2333                if (instance.getVariables() != null) {
2334                        for (table t : instance.getVariables()) {
2335                                if(SubType.cursor.name().equals(t.getSubType())){
2336                                        cursorMap.put(t.getId().toLowerCase(), t);
2337                                }
2338                                else {
2339                                        variableMap.put(t.getId().toLowerCase(), t);
2340                                }
2341                                allMap.put(t.getId().toLowerCase(), t);
2342                        }
2343                }
2344
2345                if (relations != null) {
2346                        
2347                        List<relationship> filterRelations = new ArrayList<>();
2348                        for (relationship relationElem : relations) {
2349                                if (!types.contains(relationElem.getType()))
2350                                        continue;
2351                                else {
2352                                        filterRelations.add(relationElem);
2353                                }
2354                        }
2355                        
2356                        relations = filterRelations;
2357                        
2358                        Map<String, Set<relationship>> targetIdRelationMap = new HashMap<String, Set<relationship>>();
2359                        for (relationship relation : relations) {
2360                                if (relation.getTarget() != null) {
2361                                        String key = relation.getTarget().getParent_id() + "." + relation.getTarget().getId();
2362                                        if (!targetIdRelationMap.containsKey(key)) {
2363                                                targetIdRelationMap.put(key, new TreeSet<relationship>(new Comparator<relationship>() {
2364                                                        @Override
2365                                                        public int compare(relationship o1, relationship o2) {
2366                                                                return o1.getId().compareTo(o2.getId());
2367                                                        }
2368                                                }));
2369                                        }
2370                                        targetIdRelationMap.get(key).add(relation);
2371                                }
2372                        }
2373
2374                        Iterator<String> keys = targetIdRelationMap.keySet().iterator();
2375                        while (keys.hasNext()) {
2376                                String key = keys.next();
2377                                if (targetIdRelationMap.get(key).size() > 500) {
2378                                        keys.remove();
2379                                }
2380                        }
2381
2382                        for (relationship relationElem : relations) {
2383                                if (RelationshipType.call.name().equals(relationElem.getType())) {
2384                                        continue;
2385                                }
2386                                if (RelationshipType.er.name().equals(relationElem.getType())) {
2387                                        continue;
2388                                }
2389                                targetColumn target = relationElem.getTarget();
2390                                String targetParent = target.getParent_id();
2391                                if (isTarget(instance, targetParent, simpleOutput)) {
2392                                        List<Pair<sourceColumn, List<String>>> relationSources = new ArrayList<Pair<sourceColumn, List<String>>>();
2393                                        findSourceRelations(target, instance, targetIdRelationMap, relationElem, relationSources,
2394                                                        new String[] { relationElem.getType() }, simpleOutput);
2395                                        if (relationSources.size() > 0) {
2396                                                Map<sourceColumn, List<Pair<sourceColumn, List<String>>>> columnMap = new LinkedHashMap<sourceColumn, List<Pair<sourceColumn, List<String>>>>();
2397                                                for (Pair<sourceColumn, List<String>> t : relationSources) {
2398                                                        sourceColumn key = ((Pair<sourceColumn, List<String>>) t).first;
2399                                                        if (!columnMap.containsKey(key)) {
2400                                                                columnMap.put(key, new ArrayList<Pair<sourceColumn, List<String>>>());
2401                                                        }
2402                                                        columnMap.get(key).add(t);
2403                                                }
2404                                                Iterator<sourceColumn> iter = columnMap.keySet().iterator();
2405                                                Map<String, List<sourceColumn>> relationSourceMap = new HashMap<String, List<sourceColumn>>();
2406                                                while (iter.hasNext()) {
2407                                                        sourceColumn column = iter.next();
2408                                                        String relationType = mergeRelationType(columnMap.get(column));
2409                                                        if (!relationSourceMap.containsKey(relationType)) {
2410                                                                relationSourceMap.put(relationType, new ArrayList<sourceColumn>());
2411                                                        }
2412                                                        relationSourceMap.get(relationType).add(column);
2413                                                }
2414
2415                                                Iterator<String> sourceIter = relationSourceMap.keySet().iterator();
2416                                                while (sourceIter.hasNext()) {
2417                                                        String relationType = sourceIter.next();
2418                                                        relationship simpleRelation = (relationship) relationElem.clone();
2419                                                        simpleRelation.setSources(relationSourceMap.get(relationType));
2420                                                        simpleRelation.setType(relationType);
2421                                                        simpleRelation.setId(String.valueOf(++ModelBindingManager.get().RELATION_ID));
2422                                                        simpleRelations.add(simpleRelation);
2423                                                }
2424                                        }
2425                                }
2426                        }
2427                }
2428
2429                simple.setProcedures(instance.getProcedures());
2430                simple.setPackages(instance.getPackages());
2431                simple.setProcesses(instance.getProcesses());
2432                simple.setErrors(instance.getErrors());
2433                List<table> tables = new ArrayList<table>();
2434                for (table t : instance.getTables()) {
2435                        if (!SQLUtil.isTempTable(t)) {
2436                                tables.add(t);
2437                        }
2438                        else {
2439                                if (option.isIgnoreTemporaryTable()) {
2440                                        continue;
2441                                }
2442                                else {
2443                                        tables.add(t);
2444                                }
2445                        }
2446                }
2447                simple.setStages(instance.getStages());
2448                simple.setSequences(instance.getSequences());
2449                simple.setDatasources(instance.getDatasources());
2450                simple.setStreams(instance.getStreams());
2451                simple.setPaths(instance.getPaths());
2452                simple.setTables(tables);
2453                simple.setViews(instance.getViews());
2454                if(option.isSimpleShowVariable()) {
2455                        simple.setVariables(instance.getVariables());
2456                }
2457                else if(option.isSimpleShowCursor()) {
2458                        simple.setVariables(instance.getVariables().stream().filter(t->SubType.cursor.name().equals(t.getSubType())).collect(Collectors.toList()));
2459                }
2460                if (instance.getResultsets() != null) {
2461                        List<table> resultSets = new ArrayList<table>();
2462                        for (int i = 0; i < instance.getResultsets().size(); i++) {
2463                                table resultSet = instance.getResultsets().get(i);
2464                                if (isTargetResultSet(instance, resultSet.getId(), simpleOutput)) {
2465                                        // special handle function #524 #296
2466                                        resultSets.add(resultSet);
2467                                }
2468                        }
2469                        simple.setResultsets(resultSets);
2470                }
2471
2472                List<table> functions = new ArrayList<table>();
2473                if (option.isShowCallRelation()) {
2474                        for (int i = 0; i < relations.size(); i++) {
2475                                relationship relationElem = relations.get(i);
2476                                if (!RelationshipType.call.name().equals(relationElem.getType())) {
2477                                        continue;
2478                                }
2479                                simpleRelations.add(relationElem);
2480                                for (sourceColumn callee : relationElem.getCallees()) {
2481                                        String calleeId = callee.getId();
2482                                        if (resultSetMap.containsKey(calleeId)) {
2483                                                table function = resultSetMap.get(calleeId);
2484                                                function.setIsTarget("true");
2485                                                functions.add(function);
2486                                        }
2487                                }
2488                        }
2489                }
2490
2491                if (option.isShowERDiagram()) {
2492                        for (int i = 0; i < relations.size(); i++) {
2493                                relationship relationElem = relations.get(i);
2494                                if (!RelationshipType.er.name().equals(relationElem.getType())) {
2495                                        continue;
2496                                }
2497                                simpleRelations.add(relationElem);
2498                                for (sourceColumn callee : relationElem.getCallees()) {
2499                                        String calleeId = callee.getId();
2500                                        if (resultSetMap.containsKey(calleeId)) {
2501                                                table function = resultSetMap.get(calleeId);
2502                                                function.setIsTarget("true");
2503                                                functions.add(function);
2504                                        }
2505                                }
2506                        }
2507                }
2508                
2509                if (!functions.isEmpty()) {
2510                        if (simple.getResultsets() == null) {
2511                                simple.setResultsets(functions);
2512                        } else {
2513                                simple.getResultsets().addAll(functions);
2514                        }
2515                }
2516
2517                simple.setRelationships(simpleRelations);
2518                simple.setOrientation(instance.getOrientation());
2519                targetTables.clear();
2520                resultSetMap.clear();
2521                tableMap.clear();
2522                viewMap.clear();
2523                cursorMap.clear();
2524                variableMap.clear();
2525                fileMap.clear();
2526                stageMap.clear();
2527                dataSourceMap.clear();
2528                databaseMap.clear();
2529                schemaMap.clear();
2530                streamMap.clear();
2531                return simple;
2532        }
2533
2534        private void findSourceRelations(targetColumn target, dataflow instance, Map<String, Set<relationship>> sourceIdRelationMap,
2535                        relationship targetRelation, List<Pair<sourceColumn, List<String>>> relationSources, String[] pathTypes, boolean simpleOutput) {
2536                findStarSourceRelations(target, instance, null, sourceIdRelationMap, targetRelation, relationSources, pathTypes,
2537                                new HashSet<String>(), new LinkedHashSet<transform>(), new LinkedHashSet<candidateTable>(), 0, simpleOutput);
2538        }
2539
2540        private void findStarSourceRelations(targetColumn target, dataflow instance, targetColumn starRelationTarget,
2541                        Map<String, Set<relationship>> sourceIdRelationMap, relationship targetRelation,
2542                        List<Pair<sourceColumn, List<String>>> relationSources, String[] pathTypes, Set<String> paths,
2543                        Set<transform> transforms, Set<candidateTable> candidateTables, int level, boolean simpleOutput) {
2544                if (targetRelation != null && targetRelation.getSources() != null) {
2545                        
2546                        //获取source为*的Column Parent
2547                        String starParentId = null;
2548                        for (int i = 0; i < targetRelation.getSources().size(); i++) {
2549                                sourceColumn source = targetRelation.getSources().get(i);
2550                                if (starRelationTarget != null && "*".equals(source.getColumn())) {
2551                                        starParentId = source.getParent_id();
2552                                }
2553                        }
2554                        
2555                        for (int i = 0; i < targetRelation.getSources().size(); i++) {
2556                                sourceColumn source = targetRelation.getSources().get(i);
2557                                if (starRelationTarget != null && !"*".equals(source.getColumn())
2558                                                && !DlineageUtil.getIdentifierNormalColumnName(starRelationTarget.getColumn())
2559                                                                .equals(DlineageUtil.getIdentifierNormalColumnName(source.getColumn()))) {
2560                                        table parent = allMap.get(source.getParent_id());
2561                                        if (parent != null && isFunction(parent)) {
2562                                                // function返回值未知，不对星号做处理
2563                                        } 
2564                                        else if (parent == null) {
2565                                                continue;
2566                                        } else if(starParentId!=null && starParentId.equals(parent.getId())){
2567                                                //如果source和 * column的parent相同，则跳过
2568                                                continue;
2569                                        }
2570                                }
2571
2572                                String sourceColumnId = source.getId();
2573                                String sourceParentId = source.getParent_id();
2574                                if (sourceParentId == null || sourceColumnId == null) {
2575                                        continue;
2576                                }
2577                                if (isTarget(instance, sourceParentId, simpleOutput)) {
2578                                        List<transform> transforms2 = new ArrayList<transform>(transforms.size());
2579                                        transforms2.addAll(transforms);
2580                                        Collections.reverse(transforms2);
2581                                        
2582                                        List<candidateTable> candidateTables2 = new ArrayList<candidateTable>(candidateTables.size());
2583                                        candidateTables2.addAll(candidateTables);
2584                                        
2585                                        sourceColumn sourceColumnCopy = DlineageUtil.copySourceColumn(source);
2586                                        for (transform t : transforms2) {
2587                                                sourceColumnCopy.addTransform(t);
2588                                        }
2589                                        
2590                                        for (candidateTable t : candidateTables2) {
2591                                                sourceColumnCopy.addCandidateParent(t);
2592                                        }
2593                                        
2594                                        if(Boolean.TRUE.equals(target.isStruct()) && Boolean.TRUE.equals(source.isStruct())) {
2595                                                List<String> targetColumns = SQLUtil.parseNames(target.getColumn());
2596                                                List<String> sourceColumns = SQLUtil.parseNames(source.getColumn());
2597                                                if(!DlineageUtil.getIdentifierNormalColumnName(targetColumns.get(targetColumns.size()-1))
2598                                                                .equals(DlineageUtil.getIdentifierNormalColumnName(sourceColumns.get(sourceColumns.size()-1)))) {
2599                                                        continue;
2600                                                }
2601                                        }
2602                                        relationSources.add(new Pair<sourceColumn, List<String>>(sourceColumnCopy, Arrays.asList(pathTypes)));
2603                                } else {
2604                                        Set<relationship> sourceRelations = sourceIdRelationMap
2605                                                        .get(source.getParent_id() + "." + source.getId());
2606                                        if (sourceRelations != null) {
2607                                                if (paths.contains(source.getParent_id() + "." + source.getId())) {
2608                                                        continue;
2609                                                } else {
2610                                                        paths.add(source.getParent_id() + "." + source.getId());
2611                                                        if (source.getTransforms() != null) {
2612                                                                transforms.addAll(source.getTransforms());
2613                                                        }
2614                                                        if (source.getCandidateParents() != null) {
2615                                                                candidateTables.addAll(source.getCandidateParents());
2616                                                        }
2617                                                }
2618                                                for (relationship relation : sourceRelations) {
2619                                                        LinkedHashSet<transform> transforms2 = new LinkedHashSet<transform>(transforms.size());
2620                                                        transforms2.addAll(transforms);
2621                                                        LinkedHashSet<candidateTable> candidateTables2 = new LinkedHashSet<candidateTable>(candidateTables.size());
2622                                                        candidateTables2.addAll(candidateTables);
2623                                                        String[] types = new String[pathTypes.length + 1];
2624                                                        types[0] = relation.getType();
2625                                                        System.arraycopy(pathTypes, 0, types, 1, pathTypes.length);
2626                                                        if (!"*".equals(source.getColumn())) {
2627                                                                findStarSourceRelations(target, instance, null, sourceIdRelationMap, relation, relationSources,
2628                                                                                types, paths, transforms2, candidateTables2, level + 1, simpleOutput);
2629                                                        } else {
2630                                                                findStarSourceRelations(target, instance,
2631                                                                                starRelationTarget == null ? targetRelation.getTarget() : starRelationTarget,
2632                                                                                sourceIdRelationMap, relation, relationSources, types, paths, transforms, candidateTables2,
2633                                                                                level + 1, simpleOutput);
2634                                                        }
2635                                                }
2636                                        }
2637                                }
2638                        }
2639                }
2640        }
2641
2642        private Map<String, Boolean> targetTables = new HashMap<String, Boolean>();
2643        private Map<String, table> resultSetMap = new HashMap<String, table>();
2644        private Map<String, table> tableMap = new HashMap<String, table>();
2645        private Map<String, table> viewMap = new HashMap<String, table>();
2646        private Map<String, table> cursorMap = new HashMap<String, table>();
2647        private Map<String, table> variableMap = new HashMap<String, table>();
2648        private Map<String, table> fileMap = new HashMap<String, table>();
2649        private Map<String, table> stageMap = new HashMap<String, table>();
2650        private Map<String, table> sequenceMap = new HashMap<String, table>();
2651        private Map<String, table> dataSourceMap = new HashMap<String, table>();
2652        private Map<String, table> databaseMap = new HashMap<String, table>();
2653        private Map<String, table> schemaMap = new HashMap<String, table>();
2654        private Map<String, table> streamMap = new HashMap<String, table>();
2655        private Map<String, table> allMap = new HashMap<String, table>();
2656
2657        private boolean isTarget(dataflow instance, String targetParentId, boolean simpleOutput) {
2658                if (targetTables.containsKey(targetParentId))
2659                        return targetTables.get(targetParentId);
2660                if (isTable(instance, targetParentId)) {
2661                        targetTables.put(targetParentId, true);
2662                        return true;
2663                } else if (isView(instance, targetParentId)) {
2664                        targetTables.put(targetParentId, true);
2665                        return true;
2666                } else if (isFile(instance, targetParentId)) {
2667                        targetTables.put(targetParentId, true);
2668                        return true;
2669                } else if (isDatabase(instance, targetParentId)) {
2670                        targetTables.put(targetParentId, true);
2671                        return true;
2672                } else if (isSchema(instance, targetParentId)) {
2673                        targetTables.put(targetParentId, true);
2674                        return true;
2675                } else if (isStage(instance, targetParentId)) {
2676                        targetTables.put(targetParentId, true);
2677                        return true;
2678                } else if (isSequence(instance, targetParentId)) {
2679                        targetTables.put(targetParentId, true);
2680                        return true;
2681                } else if (isDataSource(instance, targetParentId)) {
2682                        targetTables.put(targetParentId, true);
2683                        return true;
2684                } else if (isStream(instance, targetParentId)) {
2685                        targetTables.put(targetParentId, true);
2686                        return true;
2687                } else if (isCursor(instance, targetParentId) && option.isSimpleShowCursor()) {
2688                        targetTables.put(targetParentId, true);
2689                        return true;
2690                } else if ((isVariable(instance, targetParentId) || isCursor(instance, targetParentId)) && option.isSimpleShowVariable()) {
2691                        targetTables.put(targetParentId, true);
2692                        return true;
2693                } else if (isTargetResultSet(instance, targetParentId, simpleOutput)) {
2694                        targetTables.put(targetParentId, true);
2695                        return true;
2696                }
2697                targetTables.put(targetParentId, false);
2698                return false;
2699        }
2700
2701        private boolean isTargetResultSet(dataflow instance, String targetParent, boolean simpleOutput) {
2702                if (resultSetMap.containsKey(targetParent.toLowerCase())) {
2703                        table result = resultSetMap.get(targetParent.toLowerCase());
2704                        boolean isTarget = result.isTarget();
2705                        Option option = ModelBindingManager.getGlobalOption();
2706                        if (option != null && option.isSqlflowIgnoreFunction() && isFunction(result)) {
2707                                return false;
2708                        }
2709                        if (isTarget && simpleOutput) {
2710                                if (option != null && option.isSimpleShowFunction() && isFunction(result)) {
2711                                        return true;
2712
2713                                } else if (option != null && option.isSimpleShowTopSelectResultSet()) {
2714                                        return true;
2715                                }
2716                                if (ResultSetType.of(result.getType()) != null && option.containsResultSetType(ResultSetType.of(result.getType()))) {
2717                                        return true;
2718                                }
2719                        } else
2720                                return isTarget;
2721                }
2722                return false;
2723        }
2724
2725        private boolean isFunction(table resultSet) {
2726                if("function".equals(resultSet.getType())){
2727                        return true;
2728                }
2729                else if("resultset".equals(resultSet.getType()) && "function".equals(resultSet.getSubType())){
2730                        return true;
2731                }
2732                return false;
2733        }
2734
2735        private boolean isView(dataflow instance, String targetParent) {
2736                if (viewMap.containsKey(targetParent.toLowerCase())) {
2737                        return true;
2738                }
2739                return false;
2740        }
2741
2742        private boolean isCursor(dataflow instance, String targetParent) {
2743                if (cursorMap.containsKey(targetParent.toLowerCase())) {
2744                        return true;
2745                }
2746                return false;
2747        }
2748
2749        private boolean isVariable(dataflow instance, String targetParent) {
2750                if (variableMap.containsKey(targetParent.toLowerCase())) {
2751                        return true;
2752                }
2753                return false;
2754        }
2755
2756        private boolean isFile(dataflow instance, String targetParent) {
2757                if (fileMap.containsKey(targetParent.toLowerCase())) {
2758                        return true;
2759                }
2760                return false;
2761        }
2762
2763        private boolean isStage(dataflow instance, String targetParent) {
2764                if (stageMap.containsKey(targetParent.toLowerCase())) {
2765                        return true;
2766                }
2767                return false;
2768        }
2769        
2770        private boolean isSequence(dataflow instance, String targetParent) {
2771                if (sequenceMap.containsKey(targetParent.toLowerCase())) {
2772                        return true;
2773                }
2774                return false;
2775        }
2776
2777        private boolean isDataSource(dataflow instance, String targetParent) {
2778                if (dataSourceMap.containsKey(targetParent.toLowerCase())) {
2779                        return true;
2780                }
2781                return false;
2782        }
2783
2784        private boolean isDatabase(dataflow instance, String targetParent) {
2785                if (databaseMap.containsKey(targetParent.toLowerCase())) {
2786                        return true;
2787                }
2788                return false;
2789        }
2790
2791        private boolean isSchema(dataflow instance, String targetParent) {
2792                if (schemaMap.containsKey(targetParent.toLowerCase())) {
2793                        return true;
2794                }
2795                return false;
2796        }
2797
2798        private boolean isStream(dataflow instance, String targetParent) {
2799                if (streamMap.containsKey(targetParent.toLowerCase())) {
2800                        return true;
2801                }
2802                return false;
2803        }
2804
2805        private boolean isTable(dataflow instance, String targetParent) {
2806                if (tableMap.containsKey(targetParent.toLowerCase())) {
2807                        if (SQLUtil.isTempTable(tableMap.get(targetParent))) {
2808                                if (option.isIgnoreTemporaryTable()) {
2809                                        return false;
2810                                }
2811                        }
2812                        if (tableMap.get(targetParent).isFunction()) {
2813                                if (option != null && option.isSimpleShowFunction()) {
2814                                        return true;
2815                                } else {
2816                                        return false;
2817                                }
2818                        }
2819                        if (SubType.synonym.name().equals(tableMap.get(targetParent).getSubType())) {
2820                                if (option != null && option.isSimpleShowSynonym()) {
2821                                        return true;
2822                                } else {
2823                                        return false;
2824                                }
2825                        }
2826                        return true;
2827                } else {
2828                        return false;
2829                }
2830        }
2831
2832        private void init() {
2833                metadataErrors.clear();
2834                sqlInfoMap.clear();
2835                errorInfos.clear();
2836                dynamicSqlSites.clear();
2837                dataflow = null;
2838                dataflowString = null;
2839                ModelBindingManager.removeGlobalDatabase();
2840                ModelBindingManager.removeGlobalSchema();
2841                ModelBindingManager.removeGlobalVendor();
2842                ModelBindingManager.removeGlobalSQLEnv();
2843                ModelBindingManager.removeGlobalHash();
2844                appendResultSets.clear();
2845                appendStarColumns.clear();
2846                appendTableStarColumns.clear();
2847                modelManager.TABLE_COLUMN_ID = option.getStartId();
2848                modelManager.RELATION_ID = option.getStartId();
2849                modelManager.DISPLAY_ID.clear();
2850                modelManager.DISPLAY_NAME.clear();
2851                tableIds.clear();
2852                ModelBindingManager.setGlobalVendor(option.getVendor());
2853                modelManager.reset();
2854        }
2855
2856        private String getErrorMessage(TSyntaxError error, String errorType) {
2857                String s = "", hint = "Syntax error";
2858                if (ErrorInfo.SYNTAX_HINT.equals(errorType)) {
2859                        hint = "Syntax hint";
2860                }
2861                if (error.hint.length() > 0)
2862                        hint = error.hint;
2863                s = s + hint + "(" + error.errorno + ") near: " + error.tokentext;
2864                s = s + "(" + error.lineNo;
2865                s = s + "," + error.columnNo + ")";
2866                return s;
2867        }
2868
2869//      boolean OLD_ENABLE_RESOLVER = TBaseType.isEnableResolver();
2870        private void analyzeAndOutputResult(TGSqlParser sqlparser) {
2871                try {
2872                        accessedSubqueries.clear();
2873                        accessedStatements.clear();
2874                        stmtStack.clear();
2875                        viewDDLMap.clear();
2876                        procedureDDLMap.clear();
2877                        structObjectMap.clear();
2878                        try {
2879                                if(sqlenv!=null) {
2880                                        sqlparser.setSqlEnv(sqlenv);
2881                                }
2882                                int result = sqlparser.parse();
2883                                if (result != 0) {
2884                                        ArrayList<TSyntaxError> errors = sqlparser.getSyntaxErrors();
2885                                        if (errors != null && !errors.isEmpty()) {
2886                                                for (int i = 0; i < errors.size(); i++) {
2887                                                        TSyntaxError error = errors.get(i);
2888                                                        ErrorInfo errorInfo = new ErrorInfo();
2889                                                        errorInfo.setErrorType(ErrorInfo.SYNTAX_ERROR);
2890                                                        errorInfo.setErrorMessage(getErrorMessage(error, ErrorInfo.SYNTAX_ERROR));
2891                                                        errorInfo.setStartPosition(new Pair3<Long, Long, String>(error.lineNo, error.columnNo,
2892                                                                        ModelBindingManager.getGlobalHash()));
2893                                                        String[] segments = error.tokentext.split("\n");
2894                                                        if (segments.length <= 1) {
2895                                                                errorInfo.setEndPosition(new Pair3<Long, Long, String>(error.lineNo,
2896                                                                                error.columnNo + error.tokentext.length(),
2897                                                                                ModelBindingManager.getGlobalHash()));
2898                                                        } else {
2899                                                                errorInfo.setEndPosition(
2900                                                                                new Pair3<Long, Long, String>(error.lineNo + segments.length - 1,
2901                                                                                                (long) segments[segments.length - 1].length() + 1,
2902                                                                                                ModelBindingManager.getGlobalHash()));
2903                                                        }
2904                                                        ;
2905                                                        errorInfo.fillInfo(this);
2906                                                        errorInfos.add(errorInfo);
2907                                                }
2908                                        }
2909                                }
2910
2911                                if (option.getHandleListener() != null) {
2912                                        option.getHandleListener().endParse(result == 0);
2913                                }
2914                        } catch (Exception e) {
2915                                logger.error("analyze sql failed.", e);
2916                                if (option.getHandleListener() != null) {
2917                                        option.getHandleListener().endParse(false);
2918                                }
2919                                ErrorInfo errorInfo = new ErrorInfo();
2920                                errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
2921                                if (e.getMessage() == null) {
2922                                        if (e.getStackTrace() != null && e.getStackTrace().length > 0) {
2923                                                errorInfo
2924                                                                .setErrorMessage(e.getClass().getSimpleName() + ": " + e.getStackTrace()[0].toString());
2925                                        } else {
2926                                                errorInfo.setErrorMessage(e.getClass().getSimpleName());
2927                                        }
2928                                } else {
2929                                        errorInfo.setErrorMessage(e.getClass().getSimpleName() + ": " + e.getMessage());
2930                                }
2931                                errorInfo.fillInfo(this);
2932                                errorInfos.add(errorInfo);
2933                                return;
2934                        }
2935
2936                        
2937                        TSQLResolver2 resolver = sqlparser.getResolver2();
2938                        if (resolver != null && option.getVendor() == EDbVendor.dbvbigquery) {
2939                                ScopeBuildResult buildResult = resolver.getScopeBuildResult();
2940                                List<TObjectName> columns = buildResult.getAllColumnReferences();
2941                                for (TObjectName col : columns) {
2942                                        structObjectMap.putIfAbsent(col.getSourceTable(), new TObjectNameList());
2943                                        structObjectMap.get(col.getSourceTable()).addObjectName(col);
2944                                }
2945                        }
2946                        
2947                        if (option.getHandleListener() != null) {
2948                                option.getHandleListener().startAnalyzeDataFlow(sqlparser);
2949                        }
2950
2951                        for (int i = 0; i < sqlparser.getSqlstatements().size(); i++) {
2952                                if (option.getHandleListener() != null && option.getHandleListener().isCanceled()) {
2953                                        break;
2954                                }
2955
2956                                TCustomSqlStatement stmt = sqlparser.getSqlstatements().get(i);
2957                                if (stmt.getErrorCount() == 0) {
2958                                        if (stmt.getParentStmt() == null) {
2959                                                modelManager.collectSqlHash(stmt);
2960                                                if (stmt instanceof TUseDatabase || stmt instanceof TUseSchema
2961                                                                || stmt instanceof TCreateTableSqlStatement
2962                                                                || stmt instanceof TCreateExternalDataSourceStmt || stmt instanceof TCreateStageStmt
2963                                                                || stmt instanceof TMssqlCreateType
2964                                                                || stmt instanceof TMssqlDeclare
2965                                                                || stmt instanceof TPlsqlCreateType_Placeholder
2966                                                                || stmt instanceof TPlsqlCreateType
2967                                                                || stmt instanceof TPlsqlTableTypeDefStmt
2968                                                                || (stmt instanceof TCreateFunctionStmt && hasDb2ReturnStmt((TCreateFunctionStmt)stmt))
2969                                                                || (stmt instanceof TMssqlCreateFunction
2970                                                                                && (((TMssqlCreateFunction) stmt).getReturnTableDefinitions() != null
2971                                                                                                || ((TMssqlCreateFunction) stmt).getReturnStmt() != null))) {
2972                                                        boolean listen = false;
2973                                                        if (option.getHandleListener() != null && !accessedStatements.contains(stmt)) {
2974                                                                option.getHandleListener().startAnalyzeStatment(stmt);
2975                                                                listen = true;
2976                                                        }
2977                                                        analyzeCustomSqlStmt(stmt);
2978                                                        if (listen && option.getHandleListener() != null) {
2979                                                                option.getHandleListener().endAnalyzeStatment(stmt);
2980                                                        }
2981                                                }
2982                                        }
2983                                }
2984                        }
2985
2986                        for (int i = 0; i < sqlparser.sqlstatements.size(); i++) {
2987                                if (option.getHandleListener() != null && option.getHandleListener().isCanceled()) {
2988                                        break;
2989                                }
2990
2991                                TCustomSqlStatement stmt = sqlparser.getSqlstatements().get(i);
2992                                if (stmt.getErrorCount() == 0) {
2993                                        if (stmt.getParentStmt() == null) {
2994                                                if (stmt instanceof TUseDatabase 
2995                                                                || stmt instanceof TUseSchema
2996                                                                || stmt instanceof TCreateViewSqlStatement 
2997                                                                || stmt instanceof TCreateSynonymStmt 
2998                                                                || stmt instanceof TStoredProcedureSqlStatement) {
2999                                                        boolean listen = false;
3000                                                        if (option.getHandleListener() != null && !accessedStatements.contains(stmt)) {
3001                                                                option.getHandleListener().startAnalyzeStatment(stmt);
3002                                                                listen = true;
3003                                                        }
3004                                                        if (stmt instanceof TUseDatabase || stmt instanceof TUseSchema) {
3005                                                                analyzeCustomSqlStmt(stmt);
3006                                                        } else if (stmt instanceof TCreateViewSqlStatement) {
3007                                                                TCreateViewSqlStatement view = (TCreateViewSqlStatement) stmt;
3008                                                                if(view.getViewName()!=null) {
3009                                                                        viewDDLMap.put(DlineageUtil.getTableFullName(view.getViewName().toString()), view);
3010                                                                }
3011                                                        } else if (stmt instanceof TCreateSynonymStmt) {
3012                                                                TCreateSynonymStmt synonym = (TCreateSynonymStmt) stmt;
3013                                                                if(synonym.getSynonymName()!=null) {
3014                                                                        viewDDLMap.put(DlineageUtil.getTableFullName(synonym.getSynonymName().toString()), synonym);
3015                                                                }
3016                                                        } else if (stmt instanceof TStoredProcedureSqlStatement) {
3017                                                                TStoredProcedureSqlStatement procedure = (TStoredProcedureSqlStatement) stmt;
3018                                                                if(procedure.getStoredProcedureName() == null) {
3019                                                                        continue;
3020                                                                }
3021                                                                procedureDDLMap.put(DlineageUtil.getProcedureNameWithArgNum(procedure), procedure);
3022                                                        }
3023                                                        if (listen && option.getHandleListener() != null) {
3024                                                                option.getHandleListener().endAnalyzeStatment(stmt);
3025                                                        }
3026                                                }
3027                                        }
3028                                }
3029                        }
3030
3031                        for (int i = 0; i < sqlparser.sqlstatements.size(); i++) {
3032                                if (option.getHandleListener() != null && option.getHandleListener().isCanceled()) {
3033                                        break;
3034                                }
3035
3036                                TCustomSqlStatement stmt = sqlparser.getSqlstatements().get(i);
3037                                if (stmt.getErrorCount() == 0) {
3038                                        if (stmt.getParentStmt() == null) {
3039                                                if (stmt instanceof TUseDatabase 
3040                                                                || stmt instanceof TUseSchema
3041                                                                || stmt instanceof TCreateViewSqlStatement 
3042                                                                || stmt instanceof TCreateSynonymStmt) {
3043                                                        boolean listen = false;
3044                                                        if (option.getHandleListener() != null && !accessedStatements.contains(stmt)) {
3045                                                                option.getHandleListener().startAnalyzeStatment(stmt);
3046                                                                listen = true;
3047                                                        }
3048                                                        analyzeCustomSqlStmt(stmt);
3049                                                        if (listen && option.getHandleListener() != null) {
3050                                                                option.getHandleListener().endAnalyzeStatment(stmt);
3051                                                        }
3052                                                }
3053                                        }
3054                                }
3055                        }
3056
3057                        for (int i = 0; i < sqlparser.sqlstatements.size(); i++) {
3058                                if (option.getHandleListener() != null && option.getHandleListener().isCanceled()) {
3059                                        break;
3060                                }
3061
3062                                TCustomSqlStatement stmt = sqlparser.getSqlstatements().get(i);
3063                                if (stmt.getErrorCount() == 0) {
3064                                        if (stmt.getParentStmt() == null) {
3065                                                if (stmt instanceof TUseDatabase || stmt instanceof TUseSchema
3066                                                                || stmt instanceof TStoredProcedureSqlStatement) {
3067                                                        boolean listen = false;
3068                                                        if (option.getHandleListener() != null && !accessedStatements.contains(stmt)) {
3069                                                                option.getHandleListener().startAnalyzeStatment(stmt);
3070                                                                listen = true;
3071                                                        }
3072                                                        if (stmt instanceof TPlsqlCreateTrigger)
3073                                                                continue;
3074                                                        analyzeCustomSqlStmt(stmt);
3075                                                        if (listen && option.getHandleListener() != null) {
3076                                                                option.getHandleListener().endAnalyzeStatment(stmt);
3077                                                        }
3078                                                }
3079                                        }
3080                                }
3081                        }
3082
3083                        for (int i = 0; i < sqlparser.sqlstatements.size(); i++) {
3084                                if (option.getHandleListener() != null && option.getHandleListener().isCanceled()) {
3085                                        break;
3086                                }
3087
3088                                TCustomSqlStatement stmt = sqlparser.getSqlstatements().get(i);
3089
3090                                if (option.isIgnoreTopSelect()) {
3091                                        if ((stmt instanceof TSelectSqlStatement && ((TSelectSqlStatement)stmt).getIntoClause() == null && ((TSelectSqlStatement)stmt).getIntoTableClause() == null ) || stmt instanceof TRedshiftDeclare
3092                                                        || stmt instanceof TRedshiftDeclare) {
3093                                                continue;
3094                                        }
3095                                }
3096
3097                                if (stmt.getErrorCount() == 0) {
3098                                        if (stmt.getParentStmt() == null) {
3099                                                if (!(stmt instanceof TCreateViewSqlStatement) && !(stmt instanceof TCreateStageStmt)
3100                                                                && !(stmt instanceof TCreateExternalDataSourceStmt)
3101                                                                && !(stmt instanceof TCreateViewSqlStatement) && !(stmt instanceof TMssqlDeclare)
3102                                                                && !(stmt instanceof TMssqlCreateFunction
3103                                                                                && ((TMssqlCreateFunction) stmt).getReturnTableDefinitions() != null)) {
3104                                                        boolean listen = false;
3105                                                        if (option.getHandleListener() != null && !accessedStatements.contains(stmt)) {
3106                                                                option.getHandleListener().startAnalyzeStatment(stmt);
3107                                                                listen = true;
3108                                                        }
3109                                                        analyzeCustomSqlStmt(stmt);
3110                                                        if (listen && option.getHandleListener() != null) {
3111                                                                option.getHandleListener().endAnalyzeStatment(stmt);
3112                                                        }
3113                                                }
3114                                        }
3115                                }
3116                        }
3117
3118                        if (option.getHandleListener() != null) {
3119                                option.getHandleListener().endAnalyzeDataFlow(sqlparser);
3120                        }
3121
3122                        // Finalize pipelined function stitching after all passes
3123                        if (!modelManager.getPendingPipelinedCallSites().isEmpty() && sqlparser.getSqlstatements().size() > 0 && pipelinedAnalyzer != null) {
3124                                TCustomSqlStatement lastStmt = sqlparser.getSqlstatements().get(sqlparser.getSqlstatements().size() - 1);
3125                                stmtStack.push(lastStmt);
3126                                try {
3127                                        pipelinedAnalyzer.stitchPendingCallSites();
3128                                } catch (Exception ex) {
3129                                        // Don't let pipelined stitching failure break main flow
3130                                } finally {
3131                                        stmtStack.pop();
3132                                }
3133                        }
3134                } catch (Throwable e) {
3135                        logger.error("analyze sql failed.", e);
3136                        ErrorInfo errorInfo = new ErrorInfo();
3137                        errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
3138                        if (e.getMessage() == null) {
3139                                if (e.getStackTrace() != null && e.getStackTrace().length > 0) {
3140                                        errorInfo.setErrorMessage(e.getClass().getSimpleName() + ": " + e.getStackTrace()[0].toString());
3141                                } else {
3142                                        errorInfo.setErrorMessage(e.getClass().getSimpleName());
3143                                }
3144                        } else {
3145                                errorInfo.setErrorMessage(e.getClass().getSimpleName() + ": " + e.getMessage());
3146                        }
3147                        errorInfo.fillInfo(this);
3148                        errorInfos.add(errorInfo);
3149                }
3150
3151        }
3152
3153        private boolean hasDb2ReturnStmt(TCreateFunctionStmt stmt) {
3154                if (stmt.getReturnStmt() != null) {
3155                        return true;
3156                }
3157                if (stmt.getBodyStatements() != null) {
3158                        for (int i = 0; i < stmt.getBodyStatements().size(); i++) {
3159                                if(stmt.getBodyStatements().get(i) instanceof TDb2ReturnStmt) {
3160                                        return true;
3161                                }
3162                        }
3163                }
3164                return false;
3165        }
3166
3167        private void analyzeCustomSqlStmt(TCustomSqlStatement stmt) {
3168                if (!accessedStatements.contains(stmt)) {
3169                        accessedStatements.add(stmt);
3170                } else if (!(stmt instanceof TUseDatabase || stmt instanceof TUseSchema)) {
3171                        return;
3172                }
3173
3174                ArrayList<TSyntaxError> errors = stmt.getSyntaxHints();
3175                if (errors != null && !errors.isEmpty()) {
3176                        for (int i = 0; i < errors.size(); i++) {
3177                                TSyntaxError error = errors.get(i);
3178                                ErrorInfo errorInfo = new ErrorInfo();
3179                                errorInfo.setErrorType(ErrorInfo.SYNTAX_HINT);
3180                                errorInfo.setErrorMessage(getErrorMessage(error, ErrorInfo.SYNTAX_HINT));
3181                                errorInfo.setStartPosition(new Pair3<Long, Long, String>(error.lineNo, error.columnNo,
3182                                                ModelBindingManager.getGlobalHash()));
3183                                String[] segments = error.tokentext.split("\n");
3184                                if (segments.length == 1) {
3185                                        errorInfo.setEndPosition(new Pair3<Long, Long, String>(error.lineNo,
3186                                                        error.columnNo + error.tokentext.length(), ModelBindingManager.getGlobalHash()));
3187                                } else {
3188                                        errorInfo.setEndPosition(new Pair3<Long, Long, String>(error.lineNo + segments.length - 1,
3189                                                        (long) segments[segments.length - 1].length() + 1, ModelBindingManager.getGlobalHash()));
3190                                }
3191                                errorInfo.fillInfo(this);
3192                                errorInfos.add(errorInfo);
3193                        }
3194                }
3195
3196                if (option.getAnalyzeMode() == AnalyzeMode.dynamic) {
3197                        if (!(stmt instanceof TStoredProcedureSqlStatement
3198                                        || stmt instanceof TExecuteSqlStatement
3199                                        || stmt instanceof TMssqlExecute
3200                                        || stmt instanceof TExecImmeStmt)) {
3201                                return;
3202                        }
3203                }
3204
3205                if(DlineageUtil.getTopStmt(stmt) == stmt){
3206                        modelManager.collectSqlHash(stmt);
3207                }
3208
3209                try {
3210                        if (stmt instanceof TUseDatabase) {
3211                                if (((TUseDatabase) stmt).getDatabaseName() != null) {
3212                                        ModelBindingManager.setGlobalDatabase(((TUseDatabase) stmt).getDatabaseName().toString());
3213                                }
3214                        } else if (stmt instanceof TUseSchema) {
3215                                if (((TUseSchema) stmt).getSchemaName() != null) {
3216                                        String schemaName = ((TUseSchema) stmt).getSchemaName().toString();
3217                                        List<String> splits = SQLUtil.parseNames(schemaName);
3218                                        if (splits.size() == 1) {
3219                                                ModelBindingManager.setGlobalSchema(schemaName);
3220                                        } else if (splits.size() > 1) {
3221                                                ModelBindingManager.setGlobalSchema(splits.get(splits.size() - 1));
3222                                                ModelBindingManager.setGlobalDatabase(splits.get(splits.size() - 2));
3223                                        }
3224                                }
3225                        } else if (stmt instanceof TPlsqlRecordTypeDefStmt) {
3226                                this.stmtStack.push(stmt);
3227                                this.analyzePlsqlRecordTypeDefStmt((TPlsqlRecordTypeDefStmt) stmt);
3228                                this.stmtStack.pop();
3229                        } else if (stmt instanceof TPlsqlCreateType_Placeholder) {
3230                                this.stmtStack.push(stmt);
3231                                TPlsqlCreateType_Placeholder placeholder = (TPlsqlCreateType_Placeholder) stmt;
3232                                if (placeholder.getObjectStatement() != null && pipelinedAnalyzer != null) {
3233                                        this.pipelinedAnalyzer.indexObjectType(placeholder.getObjectStatement());
3234                                }
3235                                if (placeholder.getNestedTableStatement() != null) {
3236                                        if (pipelinedAnalyzer != null) {
3237                                                this.pipelinedAnalyzer.indexCollectionType(placeholder.getNestedTableStatement());
3238                                        }
3239                                        this.analyzePlsqlTableTypeDefStmt(placeholder.getNestedTableStatement());
3240                                }
3241                                this.stmtStack.pop();
3242                        } else if (stmt instanceof TPlsqlCreateType) {
3243                                this.stmtStack.push(stmt);
3244                                if (pipelinedAnalyzer != null) {
3245                                        this.pipelinedAnalyzer.indexObjectType((TPlsqlCreateType) stmt);
3246                                }
3247                                this.stmtStack.pop();
3248                        } else if (stmt instanceof TPlsqlTableTypeDefStmt) {
3249                                this.stmtStack.push(stmt);
3250                                this.analyzePlsqlTableTypeDefStmt((TPlsqlTableTypeDefStmt) stmt);
3251                                if (pipelinedAnalyzer != null) {
3252                                        this.pipelinedAnalyzer.indexCollectionType((TPlsqlTableTypeDefStmt) stmt);
3253                                }
3254                                this.stmtStack.pop();
3255                        } else if (stmt instanceof TStoredProcedureSqlStatement) {
3256                                this.stmtStack.push(stmt);
3257                                this.analyzeStoredProcedureStmt((TStoredProcedureSqlStatement) stmt);
3258                                this.stmtStack.pop();
3259                        } else if (stmt instanceof TCreateTableSqlStatement) {
3260                                stmtStack.push(stmt);
3261                                analyzeCreateTableStmt((TCreateTableSqlStatement) stmt);
3262                                stmtStack.pop();
3263                        } else if (stmt instanceof TCreateStageStmt) {
3264                                stmtStack.push(stmt);
3265                                analyzeCreateStageStmt((TCreateStageStmt) stmt);
3266                                stmtStack.pop();
3267                        } else if (stmt instanceof TCreateExternalDataSourceStmt) {
3268                                stmtStack.push(stmt);
3269                                analyzeCreateExternalDataSourceStmt((TCreateExternalDataSourceStmt) stmt);
3270                                stmtStack.pop();
3271                        } else if (stmt instanceof TCreateStreamStmt) {
3272                                stmtStack.push(stmt);
3273                                analyzeCreateStreamStmt((TCreateStreamStmt) stmt);
3274                                stmtStack.pop();
3275                        } else if (stmt instanceof TSelectSqlStatement) {
3276                                analyzeSelectStmt((TSelectSqlStatement) stmt);
3277                        } else if (stmt instanceof TDropTableSqlStatement) {
3278                                stmtStack.push(stmt);
3279                                analyzeDropTableStmt((TDropTableSqlStatement) stmt);
3280                                stmtStack.pop();
3281                        } else if (stmt instanceof TTruncateStatement) {
3282                                stmtStack.push(stmt);
3283                                analyzeTruncateTableStmt((TTruncateStatement) stmt);
3284                                stmtStack.pop();
3285                        } else if (stmt instanceof TCreateMaterializedSqlStatement) {
3286                                stmtStack.push(stmt);
3287                                TCreateMaterializedSqlStatement view = (TCreateMaterializedSqlStatement) stmt;
3288                                analyzeCreateViewStmt(view, view.getSubquery(), view.getViewAliasClause(), view.getViewName());
3289                                stmtStack.pop();
3290                        } else if (stmt instanceof TCreateViewSqlStatement) {
3291                                stmtStack.push(stmt);
3292                                TCreateViewSqlStatement view = (TCreateViewSqlStatement) stmt;
3293                                analyzeCreateViewStmt(view, view.getSubquery(), view.getViewAliasClause(), view.getViewName());
3294                                stmtStack.pop();
3295                        } else if(stmt instanceof TDb2SqlVariableDeclaration){
3296                                stmtStack.push(stmt);
3297                                analyzeDb2Declare((TDb2SqlVariableDeclaration)stmt);
3298                                stmtStack.pop();
3299                        } else if (stmt instanceof TMssqlCreateType) {
3300                                stmtStack.push(stmt);
3301                                TMssqlCreateType createType = (TMssqlCreateType) stmt;
3302                                analyzeMssqlCreateType(createType);
3303                                stmtStack.pop();
3304                        } else if (stmt instanceof TMssqlDeclare) {
3305                                stmtStack.push(stmt);
3306                                TMssqlDeclare declare = (TMssqlDeclare) stmt;
3307                                analyzeMssqlDeclare(declare);
3308                                stmtStack.pop();
3309                        } else if (stmt instanceof TInsertSqlStatement) {
3310                                stmtStack.push(stmt);
3311                                TInsertSqlStatement insert = (TInsertSqlStatement)stmt;
3312                                analyzeInsertStmt(insert);
3313                                if(insert.getMultiInsertStatements()!=null) {
3314                                        for(int i=0;i<insert.getMultiInsertStatements().size();i++) {
3315                                                analyzeInsertStmt(insert.getMultiInsertStatements().get(i));
3316                                        }
3317                                }
3318                                stmtStack.pop();
3319                        } else if (stmt instanceof TRedshiftCopy) {
3320                                stmtStack.push(stmt);
3321                                analyzeRedshiftCopyStmt((TRedshiftCopy) stmt);
3322                                stmtStack.pop();
3323                        } else if (stmt instanceof TSnowflakeCopyIntoStmt) {
3324                                stmtStack.push(stmt);
3325                                analyzeCopyIntoStmt((TSnowflakeCopyIntoStmt) stmt);
3326                                stmtStack.pop();
3327                        } else if (stmt instanceof TUnloadStmt) {
3328                                stmtStack.push(stmt);
3329                                analyzeUnloadStmt((TUnloadStmt) stmt);
3330                                stmtStack.pop();
3331                        } else if (stmt instanceof TUpdateSqlStatement) {
3332                                stmtStack.push(stmt);
3333                                analyzeUpdateStmt((TUpdateSqlStatement) stmt);
3334                                stmtStack.pop();
3335                        } else if (stmt instanceof TMergeSqlStatement) {
3336                                stmtStack.push(stmt);
3337                                analyzeMergeStmt((TMergeSqlStatement) stmt);
3338                                stmtStack.pop();
3339                        } else if (stmt instanceof TDeleteSqlStatement) {
3340                                stmtStack.push(stmt);
3341                                analyzeDeleteStmt((TDeleteSqlStatement) stmt);
3342                                stmtStack.pop();
3343                        } else if (stmt instanceof TCursorDeclStmt) {
3344                                stmtStack.push(stmt);
3345                                analyzeCursorDeclStmt((TCursorDeclStmt) stmt);
3346                                stmtStack.pop();
3347                        } else if (stmt instanceof TFetchStmt) {
3348                                stmtStack.push(stmt);
3349                                analyzeFetchStmt((TFetchStmt) stmt);
3350                                stmtStack.pop();
3351                        } else if (stmt instanceof TMssqlFetch) {
3352                                stmtStack.push(stmt);
3353                                analyzeFetchStmt((TMssqlFetch) stmt);
3354                                stmtStack.pop();
3355                        } else if (stmt instanceof TForStmt) {
3356                                stmtStack.push(stmt);
3357                                analyzeForStmt((TForStmt) stmt);
3358                                stmtStack.pop();
3359                        } else if (stmt instanceof TOpenforStmt) {
3360                                stmtStack.push(stmt);
3361                                analyzeOpenForStmt((TOpenforStmt) stmt);
3362                                stmtStack.pop();
3363                        } else if (stmt instanceof TLoopStmt) {
3364                                stmtStack.push(stmt);
3365                                analyzeLoopStmt((TLoopStmt) stmt);
3366                                stmtStack.pop();
3367                        } else if (stmt instanceof TAssignStmt) {
3368                                stmtStack.push(stmt);
3369                                analyzeAssignStmt((TAssignStmt) stmt);
3370                                stmtStack.pop();
3371                        } else if (stmt instanceof TSetStmt) {
3372                                stmtStack.push(stmt);
3373                                analyzeSetStmt((TSetStmt) stmt);
3374                                stmtStack.pop();
3375                        } else if (stmt instanceof TMssqlSet) {
3376                                stmtStack.push(stmt);
3377                                analyzeMssqlSetStmt((TMssqlSet) stmt);
3378                                stmtStack.pop();
3379                        } else if (stmt instanceof TVarDeclStmt) {
3380                                stmtStack.push(stmt);
3381                                analyzeVarDeclStmt((TVarDeclStmt) stmt);
3382                                stmtStack.pop();
3383                        } else if (stmt instanceof TCreateDatabaseSqlStatement) {
3384                                stmtStack.push(stmt);
3385                                analyzeCloneDatabaseStmt((TCreateDatabaseSqlStatement) stmt);
3386                                stmtStack.pop();
3387                        } else if (stmt instanceof TCreateSchemaSqlStatement) {
3388                                stmtStack.push(stmt);
3389                                analyzeCloneSchemaStmt((TCreateSchemaSqlStatement) stmt);
3390                                stmtStack.pop();
3391                        } else if (stmt instanceof TAlterTableStatement) {
3392                                stmtStack.push(stmt);
3393                                analyzeAlterTableStmt((TAlterTableStatement) stmt);
3394                                stmtStack.pop();
3395                        } else if (stmt instanceof TAlterViewStatement) {
3396                                stmtStack.push(stmt);
3397                                analyzeAlterViewStmt((TAlterViewStatement) stmt);
3398                                stmtStack.pop();
3399                        } else if (stmt instanceof TRenameStmt) {
3400                                stmtStack.push(stmt);
3401                                analyzeRenameStmt((TRenameStmt) stmt);
3402                                stmtStack.pop();
3403                        } else if (stmt instanceof TCreateSynonymStmt) {
3404                                stmtStack.push(stmt);
3405                                analyzeCreateSynonymStmt((TCreateSynonymStmt) stmt);
3406                                stmtStack.pop();
3407                        } else if (stmt instanceof TLoadDataStmt) {
3408                                stmtStack.push(stmt);
3409                                analyzeLoadDataStmt((TLoadDataStmt) stmt);
3410                                stmtStack.pop();
3411                        } else if (stmt instanceof THiveLoad) {
3412                                stmtStack.push(stmt);
3413                                analyzeHiveLoadStmt((THiveLoad) stmt);
3414                                stmtStack.pop();
3415                        } else if (stmt instanceof TDb2ReturnStmt) {
3416                                stmtStack.push(stmt);
3417                                analyzeDb2ReturnStmt((TDb2ReturnStmt) stmt);
3418                                stmtStack.pop();
3419                        } else if (stmt instanceof TReturnStmt) {
3420                                stmtStack.push(stmt);
3421                                analyzeReturnStmt((TReturnStmt) stmt);
3422                                stmtStack.pop();
3423                        } else if (stmt instanceof TMssqlReturn) {
3424                                stmtStack.push(stmt);
3425                                analyzeMssqlReturnStmt((TMssqlReturn) stmt);
3426                                stmtStack.pop();
3427                        } else if (stmt instanceof TExecuteSqlStatement) {
3428                                String sqlText = ((TExecuteSqlStatement) stmt).getPreparedSqlText();
3429                                if(sqlText == null) {
3430                                        sqlText = ((TExecuteSqlStatement) stmt).getSqlText();
3431                                }
3432                                if (sqlText != null) {
3433                                        modelManager.collectDynamicSqlHash(stmt);
3434                                        TGSqlParser sqlparser = new TGSqlParser(option.getVendor());
3435                                        sqlparser.sqltext = SQLUtil.trimColumnStringQuote(sqlText);
3436                                        int result = sqlparser.parse();
3437                                        if (result != 0) {
3438                                                errors = sqlparser.getSyntaxErrors();
3439                                                if (errors != null && !errors.isEmpty()) {
3440                                                        for (int i = 0; i < errors.size(); i++) {
3441                                                                TSyntaxError error = errors.get(i);
3442                                                                ErrorInfo errorInfo = new ErrorInfo();
3443                                                                errorInfo.setErrorType(ErrorInfo.SYNTAX_ERROR);
3444                                                                errorInfo.setErrorMessage(getErrorMessage(error, ErrorInfo.SYNTAX_ERROR));
3445                                                                errorInfo.setStartPosition(new Pair3<Long, Long, String>(error.lineNo, error.columnNo,
3446                                                                                ModelBindingManager.getGlobalHash()));
3447                                                                String[] segments = error.tokentext.split("\n");
3448                                                                if (segments.length == 1) {
3449                                                                        errorInfo.setEndPosition(new Pair3<Long, Long, String>(error.lineNo,
3450                                                                                        error.columnNo + error.tokentext.length(),
3451                                                                                        ModelBindingManager.getGlobalHash()));
3452                                                                } else {
3453                                                                        errorInfo.setEndPosition(
3454                                                                                        new Pair3<Long, Long, String>(error.lineNo + segments.length - 1,
3455                                                                                                        (long) segments[segments.length - 1].length() + 1,
3456                                                                                                        ModelBindingManager.getGlobalHash()));
3457                                                                }
3458                                                                errorInfo.fillInfo(this);
3459                                                                errorInfos.add(errorInfo);
3460                                                        }
3461                                                }
3462                                        } else if (sqlparser.sqlstatements != null) {
3463                                                for (int i = 0; i < sqlparser.sqlstatements.size(); i++) {
3464                                                        analyzeCustomSqlStmt(sqlparser.sqlstatements.get(i));
3465                                                }
3466                                        }
3467                                }
3468                                else if (((TExecuteSqlStatement) stmt).getStmtString() != null) {
3469                                        modelManager.collectDynamicSqlHash(stmt);
3470                                }
3471                        } else if (stmt instanceof TMssqlExecute) {
3472                                TMssqlExecute executeStmt = (TMssqlExecute)stmt;
3473                                if (executeStmt.getSqlText() != null) {
3474                                        modelManager.collectDynamicSqlHash(stmt);
3475                                        TGSqlParser sqlparser = new TGSqlParser(option.getVendor());
3476                                        sqlparser.sqltext = ((TMssqlExecute) stmt).getSqlText();
3477                                        int relsBeforeDynamic = modelManager.getRelations().length;
3478                                        // A data-driven argument (variable / expression) is at most PARTIALLY folded:
3479                                        // GSP keeps the literal pieces but drops the unresolved trailing @var, leaving a
3480                                        // legitimately truncated fragment (e.g. "... WHERE " with no predicate -> end of
3481                                        // input). A parse failure on such text says nothing about the real SQL, so we must
3482                                        // not surface it as a unit SYNTAX_ERROR — that would discard the whole procedure's
3483                                        // lineage even though the outer parse succeeded. Only a true compile-time literal
3484                                        // EXEC('...') surfaces inner parse failures (mirrors the EXECUTE IMMEDIATE
3485                                        // isDynamicSQLPartial() guard below).
3486                                        boolean literalArg = isLiteralDynamicArg(executeStmt);
3487                                        int result = sqlparser.parse();
3488                                        if (result != 0) {
3489                                                errors = literalArg ? sqlparser.getSyntaxErrors() : null;
3490                                                if (errors != null && !errors.isEmpty()) {
3491                                                        for (int i = 0; i < errors.size(); i++) {
3492                                                                TSyntaxError error = errors.get(i);
3493                                                                ErrorInfo errorInfo = new ErrorInfo();
3494                                                                errorInfo.setErrorType(ErrorInfo.SYNTAX_ERROR);
3495                                                                errorInfo.setErrorMessage(getErrorMessage(error, ErrorInfo.SYNTAX_ERROR));
3496                                                                errorInfo.setStartPosition(new Pair3<Long, Long, String>(error.lineNo, error.columnNo,
3497                                                                                ModelBindingManager.getGlobalHash()));
3498                                                                String[] segments = error.tokentext.split("\n");
3499                                                                if (segments.length == 1) {
3500                                                                        errorInfo.setEndPosition(new Pair3<Long, Long, String>(error.lineNo,
3501                                                                                        error.columnNo + error.tokentext.length(),
3502                                                                                        ModelBindingManager.getGlobalHash()));
3503                                                                } else {
3504                                                                        errorInfo.setEndPosition(
3505                                                                                        new Pair3<Long, Long, String>(error.lineNo + segments.length - 1,
3506                                                                                                        (long) segments[segments.length - 1].length() + 1,
3507                                                                                                        ModelBindingManager.getGlobalHash()));
3508                                                                }
3509                                                                errorInfo.fillInfo(this);
3510                                                                errorInfos.add(errorInfo);
3511                                                        }
3512                                                }
3513                                        } else if (sqlparser.sqlstatements != null) {
3514                                                for (int i = 0; i < sqlparser.sqlstatements.size(); i++) {
3515                                                        analyzeCustomSqlStmt(sqlparser.sqlstatements.get(i));
3516                                                }
3517                                        }
3518                                        // getSqlText()!=null means GSP folded the dynamic text. RESOLVED only when the argument
3519                                        // was a compile-time literal AND it parsed; a variable/expression argument is data-driven
3520                                        // (often only partially folded, e.g. an unresolved @parameter left in) -> UNRESOLVED.
3521                                        int[] dynCounts = classifyDynamicSiteLineage(relsBeforeDynamic);
3522                                        if (!literalArg) {
3523                                                recordDynamicSqlSite(stmt, dynamicKindOf(executeStmt),
3524                                                                DynamicSqlSite.Status.UNRESOLVED, dynamicReasonOf(executeStmt), dynCounts[0], dynCounts[1]);
3525                                        } else {
3526                                                recordDynamicSqlSite(stmt, dynamicKindOf(executeStmt),
3527                                                                result != 0 ? DynamicSqlSite.Status.PARSE_ERROR : DynamicSqlSite.Status.RESOLVED,
3528                                                                result != 0 ? "inner dynamic SQL failed to parse" : null, dynCounts[0], dynCounts[1]);
3529                                        }
3530                                } else if (executeStmt.getModuleName() != null) {
3531                                        // getSqlText()==null here: a folded literal would have set sqlText. So this is either an
3532                                        // opaque sp_executesql (data-driven argument), a dynamic proc name (EXEC @var), or an
3533                                        // ordinary static procedure call. Record the dynamic ones; leave static EXEC proc alone.
3534                                        if (isSpExecutesql(executeStmt.getModuleName())) {
3535                                                recordDynamicSqlSite(stmt, DynamicSqlSite.Kind.SP_EXECUTESQL,
3536                                                                DynamicSqlSite.Status.UNRESOLVED, dynamicReasonOf(executeStmt));
3537                                        } else if (executeStmt.getModuleName().toString().startsWith("@")) {
3538                                                recordDynamicSqlSite(stmt, DynamicSqlSite.Kind.OTHER,
3539                                                                DynamicSqlSite.Status.UNRESOLVED, "EXEC target procedure name is a runtime variable");
3540                                        }
3541                                        stmtStack.push(stmt);
3542                                        analyzeMssqlExecute(executeStmt);
3543                                        stmtStack.pop();
3544                                } else if (executeStmt.getExecType() == TBaseType.metExecStringCmd) {
3545                                        // EXEC(<expr>) whose string was not folded: runtime variable / expression, or an inline
3546                                        // literal the analyzer did not fold (no lineage produced) -> honestly UNRESOLVED.
3547                                        recordDynamicSqlSite(stmt, DynamicSqlSite.Kind.EXEC_STRING,
3548                                                        DynamicSqlSite.Status.UNRESOLVED, dynamicReasonOf(executeStmt));
3549                                }
3550                        } else if (stmt instanceof TExecImmeStmt) {
3551                                TExecImmeStmt execImmeStmt = (TExecImmeStmt) stmt;
3552                                modelManager.collectDynamicSqlHash(stmt);
3553                                synchronized (DataFlowAnalyzer.class) {
3554                                        TStatementList stmts = execImmeStmt.getDynamicStatements();
3555                                        if (stmts != null && stmts.size() > 0) {
3556                                                for (int i = 0; i < stmts.size(); i++) {
3557                                                        analyzeCustomSqlStmt(stmts.get(i));
3558                                                }
3559                                        }
3560
3561                                        // Only re-parse the raw dynamic SQL when getDynamicStatements()
3562                                        // produced nothing. getDynamicStatements() already parses and
3563                                        // analyzes the fragment with coordinates remapped back to the
3564                                        // original file position; re-parsing it here with a fresh parser
3565                                        // would analyze the same SQL a second time and emit duplicate
3566                                        // hints/lineage carrying fragment-relative (un-remapped)
3567                                        // coordinates. This fallback still covers the case where
3568                                        // getDynamicStatements() failed (e.g. dynamic SQL syntax error),
3569                                        // reporting the parse error below.
3570                                        String dynamicSql = (stmts == null || stmts.size() == 0) ? execImmeStmt.getDynamicSQL() : null;
3571                                        if (!SQLUtil.isEmpty(dynamicSql)) {
3572                                                TGSqlParser sqlparser = new TGSqlParser(option.getVendor());
3573                                                sqlparser.sqltext = dynamicSql;
3574                                                int result = sqlparser.parse();
3575                                                        // This fresh parse runs the dynamic SQL un-padded, so its
3576                                                        // coordinates are relative to the fragment (first char at
3577                                                        // line 1). Shift them back to the position the fragment
3578                                                        // occupies in the original file, mirroring the shift
3579                                                        // TExecImmeStmt.getDynamicStatements() applies; otherwise
3580                                                        // fallback parse errors for EXECUTE IMMEDIATE text located
3581                                                        // later in the file would report fragment-relative lines.
3582                                                        if (execImmeStmt.getDynamicStringExpr() != null
3583                                                                        && execImmeStmt.getDynamicStringExpr().getPlainTextLineNo() != -1) {
3584                                                                int deltaLine = (int) execImmeStmt.getDynamicStringExpr().getPlainTextLineNo() - 1;
3585                                                                int deltaColumn = (int) execImmeStmt.getDynamicStringExpr().getPlainTextColumnNo();
3586                                                                if ((deltaLine != 0 || deltaColumn != 0) && sqlparser.getSyntaxErrors() != null) {
3587                                                                        for (int ei = 0; ei < sqlparser.getSyntaxErrors().size(); ei++) {
3588                                                                                TSyntaxError dynErr = sqlparser.getSyntaxErrors().get(ei);
3589                                                                                if (dynErr == null) continue;
3590                                                                                boolean onFirstLine = (dynErr.lineNo == 1);
3591                                                                                dynErr.lineNo += deltaLine;
3592                                                                                if (onFirstLine) dynErr.columnNo += deltaColumn;
3593                                                                        }
3594                                                                }
3595                                                        }
3596                                                if (result != 0) {
3597                                                        // a partially resolved value contains placeholder identifiers
3598                                                        // for unknowable parts; a parse failure on such text says
3599                                                        // nothing about the real SQL, so don't report it
3600                                                        errors = execImmeStmt.isDynamicSQLPartial() ? null : sqlparser.getSyntaxErrors();
3601                                                        if (errors != null && !errors.isEmpty()) {
3602                                                                for (int i = 0; i < errors.size(); i++) {
3603                                                                        TSyntaxError error = errors.get(i);
3604                                                                        ErrorInfo errorInfo = new ErrorInfo();
3605                                                                        errorInfo.setErrorType(ErrorInfo.SYNTAX_ERROR);
3606                                                                        errorInfo.setErrorMessage(getErrorMessage(error, ErrorInfo.SYNTAX_ERROR));
3607                                                                        errorInfo.setStartPosition(new Pair3<Long, Long, String>(error.lineNo, error.columnNo,
3608                                                                                        ModelBindingManager.getGlobalHash()));
3609                                                                        String[] segments = error.tokentext.split("\n");
3610                                                                        if (segments.length == 1) {
3611                                                                                errorInfo.setEndPosition(new Pair3<Long, Long, String>(error.lineNo,
3612                                                                                                error.columnNo + error.tokentext.length(),
3613                                                                                                ModelBindingManager.getGlobalHash()));
3614                                                                        } else {
3615                                                                                errorInfo.setEndPosition(
3616                                                                                                new Pair3<Long, Long, String>(error.lineNo + segments.length - 1,
3617                                                                                                                (long) segments[segments.length - 1].length() + 1,
3618                                                                                                                ModelBindingManager.getGlobalHash()));
3619                                                                        }
3620                                                                        errorInfo.fillInfo(this);
3621                                                                        errorInfos.add(errorInfo);
3622                                                                }
3623                                                        }
3624                                                } else if (sqlparser.sqlstatements != null) {
3625                                                        for (int i = 0; i < sqlparser.sqlstatements.size(); i++) {
3626                                                                analyzeCustomSqlStmt(sqlparser.sqlstatements.get(i));
3627                                                        }
3628                                                }
3629                                        }
3630                                }
3631                        } else if (stmt instanceof TCallStatement) {
3632                                TCallStatement callStmt = (TCallStatement)stmt;
3633                                stmtStack.push(stmt);
3634                                analyzeCallStmt(callStmt);
3635                                stmtStack.pop();
3636                        } else if (stmt instanceof TDb2CallStmt) {
3637                                TDb2CallStmt db2CallStmt = (TDb2CallStmt) stmt;
3638                                stmtStack.push(stmt);
3639                                analyzeDb2CallStmt(db2CallStmt);
3640                                stmtStack.pop();
3641                        } else if (stmt instanceof TBasicStmt) {
3642                                TBasicStmt oracleBasicStmt = (TBasicStmt) stmt;
3643                                stmtStack.push(stmt);
3644                                analyzeOracleBasicStmt(oracleBasicStmt);
3645                                stmtStack.pop();
3646                        } else if (stmt instanceof TIfStmt) {
3647                                TIfStmt ifStmt = (TIfStmt) stmt;
3648                                stmtStack.push(stmt);
3649                                analyzeIfStmt(ifStmt);
3650                                stmtStack.pop();
3651                        } else if (stmt instanceof TElsifStmt) {
3652                                TElsifStmt elsIfStmt = (TElsifStmt) stmt;
3653                                stmtStack.push(stmt);
3654                                analyzeElsIfStmt(elsIfStmt);
3655                                stmtStack.pop();
3656                        } else if (stmt.getStatements() != null && stmt.getStatements().size() > 0) {
3657                                for (int i = 0; i < stmt.getStatements().size(); i++) {
3658                                        analyzeCustomSqlStmt(stmt.getStatements().get(i));
3659                                }
3660                        } else if (stmt instanceof TCreateIndexSqlStatement) {
3661                                stmtStack.push(stmt);
3662                                analyzeCreateIndexStageStmt((TCreateIndexSqlStatement) stmt);
3663                                stmtStack.pop();
3664                        } else if (stmt instanceof gudusoft.gsqlparser.stmt.mdx.TMdxSelect) {
3665                                stmtStack.push(stmt);
3666                                analyzeMdxSelectStmt((gudusoft.gsqlparser.stmt.mdx.TMdxSelect) stmt);
3667                                stmtStack.pop();
3668                        } else if (stmt instanceof TPowerQueryDocumentStmt) {
3669                                stmtStack.push(stmt);
3670                                analyzePowerQueryDocumentStmt((TPowerQueryDocumentStmt) stmt);
3671                                stmtStack.pop();
3672                        }
3673                } catch (Exception e) {
3674            StringBuffer errorMessage = new StringBuffer();
3675            errorMessage.append("analyze sql stmt failed, ");
3676            if (stmt.getStartToken() != null) {
3677                errorMessage.append("line: " + stmt.getStartToken().lineNo + ", column: " + stmt.getStartToken().columnNo).append(", ");
3678            }
3679            if (stmt.getGsqlparser() != null && !SQLUtil.isEmpty(stmt.getGsqlparser().sqlfilename)) {
3680                errorMessage.append("file: "+ stmt.getGsqlparser().sqlfilename).append(", ");
3681            }
3682            if (stmt.toString() != null) {
3683                errorMessage.append("sql:\n" + stmt.toString());
3684            }
3685            logger.error(errorMessage.toString(), e);
3686                        ErrorInfo errorInfo = new ErrorInfo();
3687                        errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
3688                        if (e.getMessage() == null) {
3689                                if (e.getStackTrace() != null && e.getStackTrace().length > 0) {
3690                                        errorInfo.setErrorMessage(e.getClass().getSimpleName() + ": " + e.getStackTrace()[0].toString());
3691                                } else {
3692                                        errorInfo.setErrorMessage(e.getClass().getSimpleName());
3693                                }
3694                        } else {
3695                                errorInfo.setErrorMessage(e.getClass().getSimpleName() + ": " + e.getMessage());
3696                        }
3697                        errorInfo.setStartPosition(new Pair3<Long, Long, String>(stmt.getStartToken().lineNo,
3698                                        stmt.getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
3699                        String[] segments = stmt.getEndToken().getAstext().split("\n", -1);
3700                        if (segments.length == 1) {
3701                                errorInfo.setEndPosition(new Pair3<Long, Long, String>(stmt.getEndToken().lineNo,
3702                                                stmt.getEndToken().columnNo + stmt.getEndToken().getAstext().length(),
3703                                                ModelBindingManager.getGlobalHash()));
3704                        } else {
3705                                errorInfo.setEndPosition(new Pair3<Long, Long, String>(stmt.getEndToken().lineNo + segments.length - 1,
3706                                                (long) segments[segments.length - 1].length() + 1, ModelBindingManager.getGlobalHash()));
3707                        }
3708                        errorInfo.fillInfo(this);
3709                        errorInfos.add(errorInfo);
3710                }
3711        }
3712
3713        private void analyzePlsqlRecordTypeDefStmt(TPlsqlRecordTypeDefStmt stmt) {
3714                TObjectName typeName = stmt.getTypeName();
3715                Variable variable = modelFactory.createVariable(typeName);
3716                variable.setSubType(SubType.record_type);
3717        
3718                if (stmt.getFieldDeclarations() != null) {
3719                        for (int i = 0; i < stmt.getFieldDeclarations().size(); i++) {
3720                                TParameterDeclaration param = stmt.getFieldDeclarations().getParameterDeclarationItem(i);
3721                                String dataTypeName = param.getDataType().getDataTypeName();
3722                                TObjectName columnName = param.getParameterName();      
3723                                TableColumn variableProperty = modelFactory.createTableColumn(variable, columnName, true);
3724                                if(dataTypeName.indexOf(".")!=-1) {
3725                                        String tableName = dataTypeName.substring(0, dataTypeName.lastIndexOf("."));
3726                                        Table table = modelFactory.createTableByName(tableName, true);
3727                                        if(table!=null) {
3728                                                TableColumn tableColumn = modelFactory.createInsertTableColumn(table, dataTypeName);
3729                                                if (tableColumn != null) {
3730                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
3731                                                        relation.setEffectType(EffectType.rowtype);
3732                                                        relation.setTarget(new TableColumnRelationshipElement(variableProperty));
3733                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
3734                                                }
3735                                        }
3736                                }
3737                        }
3738                        variable.setDetermined(true);
3739                }
3740                else {
3741                        TObjectName starColumn = new TObjectName();
3742                        starColumn.setString("*");
3743                }
3744        }
3745        
3746        private void analyzePlsqlTableTypeDefStmt(TPlsqlTableTypeDefStmt stmt) {
3747                TTypeName typeName = stmt.getElementDataType();
3748                if (typeName != null && typeName.toString().toUpperCase().indexOf("ROWTYPE") != -1) {
3749                        Variable cursorVariable = modelFactory.createVariable(stmt.getTypeName());
3750                        cursorVariable.setSubType(SubType.record_type);
3751
3752                        Table variableTable = modelFactory.createTableByName(typeName.getDataTypeName(), false);
3753                        if(!variableTable.isCreateTable()) {
3754                                TObjectName starColumn1 = new TObjectName();
3755                                starColumn1.setString("*");
3756                                TableColumn variableTableStarColumn = modelFactory.createTableColumn(variableTable, starColumn1, true);
3757                                variableTableStarColumn.setShowStar(false);
3758                                variableTableStarColumn.setExpandStar(true);
3759
3760                                TObjectName starColumn = new TObjectName();
3761                                starColumn.setString("*");
3762                                TableColumn variableProperty = modelFactory.createTableColumn(cursorVariable, starColumn, true);
3763                                variableProperty.setShowStar(false);
3764                                variableProperty.setExpandStar(true);
3765
3766                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
3767                                dataflowRelation.setEffectType(EffectType.rowtype);
3768                                dataflowRelation.addSource(new TableColumnRelationshipElement(variableTableStarColumn));
3769                                dataflowRelation.setTarget(new TableColumnRelationshipElement(variableProperty));
3770                        } else {
3771                                for (TableColumn sourceColumn : variableTable.getColumns()) {
3772                                        String columnName = sourceColumn.getName();
3773                                        TObjectName targetColumn = new TObjectName();
3774                                        targetColumn.setString(columnName);
3775                                        TableColumn variableProperty = modelFactory.createTableColumn(cursorVariable, targetColumn, true);
3776                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
3777                                        dataflowRelation.setEffectType(EffectType.rowtype);
3778                                        dataflowRelation.addSource(new TableColumnRelationshipElement(sourceColumn));
3779                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(variableProperty));
3780                                }
3781                        }
3782                } 
3783        }
3784
3785        private void analyzeMssqlCreateType(TMssqlCreateType createType) {
3786
3787        }
3788
3789        private void analyzeMssqlExecute(TMssqlExecute executeStmt) {
3790                if (executeStmt.getModuleName() != null) {
3791                        TObjectName module = executeStmt.getModuleName();
3792                        if(module.toString().toLowerCase().endsWith("sp_rename")) {
3793                                String oldTableName = SQLUtil.trimColumnStringQuote(executeStmt.getParameters().getExecParameter(0).toString());        
3794                                Table oldNameTableModel = modelFactory.createTableByName(oldTableName, true);
3795                                List<String> oldTableNames = SQLUtil.parseNames(oldNameTableModel.getName());
3796                                TObjectName oldStarColumn = new TObjectName();
3797                                oldStarColumn.setString("*");
3798                                TableColumn oldTableStarColumn = modelFactory.createTableColumn(oldNameTableModel, oldStarColumn, true);
3799                                
3800                                String newTableName = SQLUtil.trimColumnStringQuote(executeStmt.getParameters().getExecParameter(1).toString());
3801                                List<String> newTableNames = SQLUtil.parseNames(newTableName);
3802                                if (oldTableNames.size() > newTableNames.size()) {
3803                                        for (int i = oldTableNames.size() - newTableNames.size() - 1; i >= 0; i--) {
3804                                                newTableName = (oldTableNames.get(i) + ".") + newTableName;
3805                                        }
3806                                }
3807
3808                                Table newNameTableModel = modelFactory.createTableByName(newTableName, true);
3809                                TObjectName newStarColumn = new TObjectName();
3810                                newStarColumn.setString("*");
3811                                TableColumn newTableStarColumn = modelFactory.createTableColumn(newNameTableModel, newStarColumn, true);
3812                                
3813                                Process process = modelFactory.createProcess(executeStmt);
3814                                newNameTableModel.addProcess(process);
3815                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
3816                                relation.setEffectType(EffectType.rename_table);
3817                                relation.setTarget(new TableColumnRelationshipElement(newTableStarColumn));
3818                                relation.addSource(new TableColumnRelationshipElement(oldTableStarColumn));
3819                                relation.setProcess(process);
3820                                
3821                                if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) {
3822                                        oldTableStarColumn.setShowStar(false);
3823                                        relation.setShowStarRelation(false);
3824                                }
3825
3826                                if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) {
3827                                        newTableStarColumn.setShowStar(false);
3828                                        relation.setShowStarRelation(false);
3829                                }
3830                        }
3831                        else if(module.toString().toLowerCase().endsWith("sp_executesql")) {
3832                                String sql = SQLUtil.trimColumnStringQuote(executeStmt.getParameters().getExecParameter(0).toString()); 
3833                                if(sql.startsWith("N")) {
3834                                        sql =  SQLUtil.trimColumnStringQuote(sql.substring(1));
3835                                }
3836                                executeDynamicSql(sql);
3837                        }
3838                        else {
3839                                int argumentSize = executeStmt.getParameters() == null ? 0 : executeStmt.getParameters().size();
3840                                String procedureNameWithArgSize = module.toString() + "(" + argumentSize + ")";
3841                                if (argumentSize <= 0  || !DlineageUtil.supportFunctionOverride(option.getVendor())) {
3842                                        procedureNameWithArgSize = module.toString();
3843                                }
3844                                if (procedureDDLMap.containsKey(procedureNameWithArgSize)) {
3845                                        analyzeCustomSqlStmt(procedureDDLMap.get(procedureNameWithArgSize));
3846                                }
3847                                Procedure procedure = modelFactory.createProcedureByName(module, executeStmt.getParameters() == null ? 0 : executeStmt.getParameters().size());
3848                                String procedureParent = getProcedureParentName(executeStmt);
3849                                if (procedureParent != null) {
3850                                        Procedure caller = modelManager
3851                                                        .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
3852                                        if (caller != null) {
3853                                                CallRelationship callRelation = modelFactory.createCallRelation();
3854                                                callRelation.setCallObject(executeStmt);
3855                                                callRelation.setTarget(new ProcedureRelationshipElement(caller));
3856                                                callRelation.addSource(new ProcedureRelationshipElement(procedure));
3857                                                if(isBuiltInFunctionName(module) || isKeyword(module)){
3858                                                        callRelation.setBuiltIn(true);
3859                                                }
3860                                        }
3861                                }
3862
3863                                if (procedure.getArguments() != null) {
3864                                        for (int i = 0; i < procedure.getArguments().size(); i++) {
3865                                                Argument argument = procedure.getArguments().get(i);
3866                                                Variable variable = modelFactory.createVariable(procedure, argument.getName(), false);
3867                                                if (variable != null) {
3868                                                        if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
3869                                                                Transform transform = new Transform();
3870                                                                transform.setType(Transform.FUNCTION);
3871                                                                transform.setCode(module);
3872                                                                variable.getColumns().get(0).setTransform(transform);
3873                                                        }
3874                                                        Process process = modelFactory.createProcess(executeStmt);
3875                                                        variable.addProcess(process);
3876                                                        analyzeFunctionArgumentsDataFlowRelation(variable.getColumns().get(0), executeStmt, argument.getName(), i, process);
3877                                                }
3878                                        }
3879                                }
3880                        }
3881                }
3882        }
3883
3884        private void analyzeDropTableStmt(TDropTableSqlStatement stmt) {
3885                TTable dropTable = stmt.getTargetTable();
3886                if(dropTable == null) {
3887                        return;
3888                }
3889                Table tableModel = modelManager.getTableByName(DlineageUtil.getTableFullName(dropTable.getTableName().toString()));
3890                if(tableModel!=null) {
3891                        modelManager.dropTable(tableModel);
3892                }
3893                
3894                if(option.getAnalyzeMode() == AnalyzeMode.crud) {
3895                        tableModel = modelFactory.createTable(dropTable);
3896                        CrudRelationship crudRelationship = modelFactory.createCrudRelation();
3897                        crudRelationship.setTarget(new TableRelationshipElement(tableModel));
3898                        crudRelationship.setEffectType(EffectType.drop_table);
3899                }
3900        }
3901        
3902        private void analyzeTruncateTableStmt(TTruncateStatement stmt) {
3903                if(option.getAnalyzeMode() == AnalyzeMode.crud) {
3904                        TObjectName table = stmt.getTableName();
3905                        Table tableModel = modelFactory.createTableByName(table);
3906                        CrudRelationship crudRelationship = modelFactory.createCrudRelation();
3907                        crudRelationship.setTarget(new TableRelationshipElement(tableModel));
3908                        crudRelationship.setEffectType(EffectType.truncate_table);
3909                }
3910        }
3911
3912        private void analyzeCallStmt(TCallStatement callStmt) {
3913                if (callStmt.getRoutineExpr() != null && callStmt.getRoutineExpr().getFunctionCall() != null) {
3914
3915                        TFunctionCall functionCall = callStmt.getRoutineExpr().getFunctionCall();
3916                        Procedure callee = modelManager.getProcedureByName(
3917                                        DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
3918                        if (callee == null && procedureDDLMap.containsKey(DlineageUtil.getFunctionNameWithArgNum(functionCall))) {
3919                                analyzeCustomSqlStmt(procedureDDLMap.get(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
3920                                callee = modelManager.getProcedureByName(DlineageUtil
3921                                                .getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
3922                        }
3923                        if (callee != null) {
3924                                String procedureParent = getProcedureParentName(callStmt);
3925                                if (procedureParent != null) {
3926                                        Procedure caller = modelManager
3927                                                        .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
3928                                        if (caller != null) {
3929                                                CallRelationship callRelation = modelFactory.createCallRelation();
3930                                                callRelation.setCallObject(callStmt);
3931                                                callRelation.setTarget(new ProcedureRelationshipElement(caller));
3932                                                callRelation.addSource(new ProcedureRelationshipElement(callee));
3933                                                if (isBuiltInFunctionName(functionCall.getFunctionName())
3934                                                                || isKeyword(functionCall.getFunctionName())) {
3935                                                        callRelation.setBuiltIn(true);
3936                                                }
3937                                        }
3938                                }
3939                                if (callee.getArguments() != null) {
3940                                        for (int i = 0; i < callee.getArguments().size(); i++) {
3941                                                Argument argument = callee.getArguments().get(i);
3942                                                Variable variable = modelFactory.createVariable(callee, argument.getName(), false);
3943                                                if (variable != null) {
3944                                                        if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
3945                                                                Transform transform = new Transform();
3946                                                                transform.setType(Transform.FUNCTION);
3947                                                                transform.setCode(functionCall);
3948                                                                variable.getColumns().get(0).setTransform(transform);
3949                                                        }
3950                                                        Process process = modelFactory.createProcess(callStmt);
3951                                                        variable.addProcess(process);
3952                                                        analyzeFunctionArgumentsDataFlowRelation(variable.getColumns().get(0), functionCall, i,
3953                                                                        process);
3954                                                }
3955                                        }
3956                                }
3957                        } else {
3958                                Function function = (Function) createFunction(functionCall);
3959                                String procedureParent = getProcedureParentName(callStmt);
3960                                if (procedureParent != null) {
3961                                        Procedure caller = modelManager
3962                                                        .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
3963                                        if (caller != null) {
3964                                                CallRelationship callRelation = modelFactory.createCallRelation();
3965                                                callRelation.setCallObject(callStmt);
3966                                                callRelation.setTarget(new ProcedureRelationshipElement(caller));
3967                                                callRelation.addSource(new FunctionRelationshipElement(function));
3968                                                if (isBuiltInFunctionName(functionCall.getFunctionName())
3969                                                                || isKeyword(functionCall.getFunctionName())) {
3970                                                        callRelation.setBuiltIn(true);
3971                                                }
3972                                        }
3973                                }
3974                        }
3975                }
3976                else if (callStmt.getRoutineName() != null) {
3977                        TObjectName function = callStmt.getRoutineName();
3978                        String functionName = function.toString();
3979                        Procedure callee = modelManager.getProcedureByName(
3980                                        DlineageUtil.getIdentifierNormalTableName(functionName));
3981                        if (callee == null && procedureDDLMap.containsKey(functionName)) {
3982                                analyzeCustomSqlStmt(procedureDDLMap.get(functionName));
3983                                callee = modelManager.getProcedureByName(functionName);
3984                        }
3985                        if (callee != null) {
3986                                String procedureParent = getProcedureParentName(callStmt);
3987                                if (procedureParent != null) {
3988                                        Procedure caller = modelManager
3989                                                        .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
3990                                        if (caller != null) {
3991                                                CallRelationship callRelation = modelFactory.createCallRelation();
3992                                                callRelation.setCallObject(callStmt);
3993                                                callRelation.setTarget(new ProcedureRelationshipElement(caller));
3994                                                callRelation.addSource(new ProcedureRelationshipElement(callee));
3995                                                if (isBuiltInFunctionName(function)
3996                                                                || isKeyword(function)) {
3997                                                        callRelation.setBuiltIn(true);
3998                                                }
3999                                        }
4000                                }
4001                                if (callee.getArguments() != null) {
4002                                        for (int i = 0; i < callee.getArguments().size(); i++) {
4003                                                Argument argument = callee.getArguments().get(i);
4004                                                Variable variable = modelFactory.createVariable(callee, argument.getName(), false);
4005                                                if (variable != null) {
4006                                                        if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
4007                                                                Transform transform = new Transform();
4008                                                                transform.setType(Transform.FUNCTION);
4009                                                                transform.setCode(callStmt);
4010                                                                variable.getColumns().get(0).setTransform(transform);
4011                                                        }
4012                                                        Process process = modelFactory.createProcess(callStmt);
4013                                                        variable.addProcess(process);
4014                                                        analyzeFunctionArgumentsDataFlowRelation(variable.getColumns().get(0), callStmt, i,
4015                                                                        process);
4016                                                }
4017                                        }
4018                                }
4019                        }
4020                }
4021        }
4022
4023        /**
4024         * DB2's CALL node is {@link TDb2CallStmt} (a sibling of {@link TCallStatement},
4025         * not a subclass), so the generic call dispatch never reaches it. Resolve its
4026         * caller/callee and emit the call relationship exactly as the
4027         * {@link #analyzeCallStmt} getRoutineName path does, using the DB2 accessors
4028         * getProcedureName()/getParameters().
4029         */
4030        private void analyzeDb2CallStmt(TDb2CallStmt callStmt) {
4031                if (callStmt.getProcedureName() == null) {
4032                        return;
4033                }
4034                TObjectName function = callStmt.getProcedureName();
4035                String functionName = function.toString();
4036                // DB2 supports procedure overloading by arity, so overloadable procedures
4037                // register under a "name(argCount)" key (see getFunctionNameWithArgNum).
4038                // Resolve by name+argCount first to bind the correct overload, then fall
4039                // back to the plain name for the single-definition / non-overloaded case.
4040                String nameWithArgNum = functionName;
4041                if (callStmt.getParameters() != null
4042                                && DlineageUtil.supportFunctionOverride(ModelBindingManager.getGlobalVendor())) {
4043                        nameWithArgNum = functionName + "(" + callStmt.getParameters().size() + ")";
4044                }
4045                Procedure callee = modelManager.getProcedureByName(
4046                                DlineageUtil.getIdentifierNormalTableName(nameWithArgNum));
4047                if (callee == null && !nameWithArgNum.equals(functionName)) {
4048                        callee = modelManager.getProcedureByName(
4049                                        DlineageUtil.getIdentifierNormalTableName(functionName));
4050                }
4051                if (callee == null && procedureDDLMap.containsKey(nameWithArgNum)) {
4052                        analyzeCustomSqlStmt(procedureDDLMap.get(nameWithArgNum));
4053                        callee = modelManager.getProcedureByName(nameWithArgNum);
4054                }
4055                if (callee == null && procedureDDLMap.containsKey(functionName)) {
4056                        analyzeCustomSqlStmt(procedureDDLMap.get(functionName));
4057                        callee = modelManager.getProcedureByName(functionName);
4058                }
4059                if (callee != null) {
4060                        String procedureParent = getProcedureParentName(callStmt);
4061                        if (procedureParent != null) {
4062                                Procedure caller = modelManager
4063                                                .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
4064                                if (caller != null) {
4065                                        CallRelationship callRelation = modelFactory.createCallRelation();
4066                                        callRelation.setCallObject(callStmt);
4067                                        callRelation.setTarget(new ProcedureRelationshipElement(caller));
4068                                        callRelation.addSource(new ProcedureRelationshipElement(callee));
4069                                        if (isBuiltInFunctionName(function) || isKeyword(function)) {
4070                                                callRelation.setBuiltIn(true);
4071                                        }
4072                                }
4073                        }
4074                        if (callee.getArguments() != null) {
4075                                for (int i = 0; i < callee.getArguments().size(); i++) {
4076                                        Argument argument = callee.getArguments().get(i);
4077                                        Variable variable = modelFactory.createVariable(callee, argument.getName(), false);
4078                                        if (variable != null) {
4079                                                if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
4080                                                        Transform transform = new Transform();
4081                                                        transform.setType(Transform.FUNCTION);
4082                                                        transform.setCode(callStmt);
4083                                                        variable.getColumns().get(0).setTransform(transform);
4084                                                }
4085                                                Process process = modelFactory.createProcess(callStmt);
4086                                                variable.addProcess(process);
4087                                                analyzeFunctionArgumentsDataFlowRelation(variable.getColumns().get(0), callStmt, i,
4088                                                                process);
4089                                        }
4090                                }
4091                        }
4092                }
4093        }
4094
4095        private boolean analyzeCustomFunctionCall(TFunctionCall functionCall) {
4096                Procedure callee = modelManager.getProcedureByName(
4097                                DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
4098                if(callee == null && procedureDDLMap.containsKey(DlineageUtil.getFunctionNameWithArgNum(functionCall))) {
4099                        analyzeCustomSqlStmt(procedureDDLMap.get(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
4100                        callee = modelManager.getProcedureByName(
4101                                        DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
4102                }
4103                if (callee != null) {
4104                        String procedureParent = getProcedureParentName(stmtStack.peek());
4105                        if (procedureParent != null) {
4106                                Procedure caller = modelManager
4107                                                .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
4108                                if (caller != null) {
4109                                        CallRelationship callRelation = modelFactory.createCallRelation();
4110                                        callRelation.setCallObject(functionCall);
4111                                        callRelation.setTarget(new ProcedureRelationshipElement(caller));
4112                                        callRelation.addSource(new ProcedureRelationshipElement(callee));
4113                                        if(isBuiltInFunctionName(functionCall.getFunctionName()) || isKeyword(functionCall.getFunctionName())){
4114                                                callRelation.setBuiltIn(true);
4115                                        }
4116                                }
4117                        }
4118                        if (callee.getArguments() != null) {
4119                                for (int i = 0; i < callee.getArguments().size(); i++) {
4120                                        Argument argument = callee.getArguments().get(i);
4121                                        Variable variable = modelFactory.createVariable(callee, argument.getName(), false);
4122                                        if(variable!=null) {
4123                                                if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
4124                                                        Transform transform = new Transform();
4125                                                        transform.setType(Transform.FUNCTION);
4126                                                        transform.setCode(functionCall);
4127                                                        variable.getColumns().get(0).setTransform(transform);
4128                                                }
4129                                                Process process = modelFactory.createProcess(functionCall);
4130                                                variable.addProcess(process);
4131                                                analyzeFunctionArgumentsDataFlowRelation(variable.getColumns().get(0), functionCall, i, process);
4132                                        }
4133                                }
4134                        }
4135                        return true;
4136                }
4137                return false;
4138        }
4139
4140        private void analyzeOracleBasicStmt(TBasicStmt oracleBasicStmt) {
4141                if (oracleBasicStmt.getExpr() == null || oracleBasicStmt.getExpr().getFunctionCall() == null) {
4142                        return;
4143                }
4144
4145                TFunctionCall functionCall = oracleBasicStmt.getExpr().getFunctionCall();
4146                Procedure callee = modelManager.getProcedureByName(
4147                                DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
4148                if(callee == null && procedureDDLMap.containsKey(DlineageUtil.getFunctionNameWithArgNum(functionCall))) {
4149                        analyzeCustomSqlStmt(procedureDDLMap.get(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
4150                        callee = modelManager.getProcedureByName(
4151                                        DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
4152                }
4153                if (callee != null) {
4154                        String procedureParent = getProcedureParentName(oracleBasicStmt);
4155                        if (procedureParent != null) {
4156                                Procedure caller = modelManager
4157                                                .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
4158                                if (caller != null) {
4159                                        CallRelationship callRelation = modelFactory.createCallRelation();
4160                                        callRelation.setCallObject(functionCall);
4161                                        callRelation.setTarget(new ProcedureRelationshipElement(caller));
4162                                        callRelation.addSource(new ProcedureRelationshipElement(callee));
4163                                        if (functionChecker.isOraclePredefinedPackageFunction(functionCall.getFunctionName().toString())
4164                                                        || (isBuiltInFunctionName(functionCall.getFunctionName())
4165                                                        || isKeyword(functionCall.getFunctionName()))) {
4166                                                callRelation.setBuiltIn(true);
4167                                        }
4168                                }
4169                        }
4170                        if (callee.getArguments() != null) {
4171                                for (int i = 0; i < callee.getArguments().size(); i++) {
4172                                        Argument argument = callee.getArguments().get(i);
4173                                        Variable variable = modelFactory.createVariable(callee, argument.getName(), false);
4174                                        if(variable!=null) {
4175                                                if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
4176                                                        Transform transform = new Transform();
4177                                                        transform.setType(Transform.FUNCTION);
4178                                                        transform.setCode(functionCall);
4179                                                        variable.getColumns().get(0).setTransform(transform);
4180                                                }
4181                                                Process process = modelFactory.createProcess(functionCall);
4182                                                variable.addProcess(process);
4183                                                analyzeFunctionArgumentsDataFlowRelation(variable.getColumns().get(0), functionCall, i, process);
4184                                        }
4185                                }
4186                        }
4187                } else {
4188                        Function function = modelFactory.createFunction(functionCall);
4189                        String procedureParent = getProcedureParentName(oracleBasicStmt);
4190                        if (procedureParent != null) {
4191                                Procedure caller = modelManager
4192                                                .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
4193                                if (caller != null) {
4194                                        CallRelationship callRelation = modelFactory.createCallRelation();
4195                                        callRelation.setCallObject(functionCall);
4196                                        callRelation.setTarget(new ProcedureRelationshipElement(caller));
4197                                        callRelation.addSource(new FunctionRelationshipElement(function));
4198                                        if (functionChecker.isOraclePredefinedPackageFunction(functionCall.getFunctionName().toString())
4199                                                        || (isBuiltInFunctionName(functionCall.getFunctionName()) || isKeyword(functionCall.getFunctionName()))) {
4200                                                callRelation.setBuiltIn(true);
4201                                        }
4202                                }
4203                        }
4204                }
4205        }
4206
4207        private void analyzeIfStmt(TIfStmt ifStmt) {
4208                if (ifStmt.getCondition() != null) {
4209                        columnsInExpr visitor = new columnsInExpr();
4210                        ifStmt.getCondition().inOrderTraverse(visitor);
4211                        List<TParseTreeNode> functions = visitor.getFunctions();
4212
4213                        if (functions != null && !functions.isEmpty()) {
4214                                for (int i = 0; i < functions.size(); i++) {
4215                                        if (!(functions.get(i) instanceof TFunctionCall))
4216                                                continue;
4217                                        TFunctionCall functionCall = (TFunctionCall) functions.get(i);
4218                                        Procedure callee = modelManager.getProcedureByName(DlineageUtil
4219                                                        .getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
4220                                        if(callee == null && procedureDDLMap.containsKey(DlineageUtil.getFunctionNameWithArgNum(functionCall))) {
4221                                                analyzeCustomSqlStmt(procedureDDLMap.get(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
4222                                                callee = modelManager.getProcedureByName(
4223                                                                DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
4224                                        }
4225                                        if (callee != null) {
4226                                                String procedureParent = getProcedureParentName(ifStmt);
4227                                                if (procedureParent != null) {
4228                                                        Procedure caller = modelManager
4229                                                                        .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
4230                                                        if (caller != null) {
4231                                                                CallRelationship callRelation = modelFactory.createCallRelation();
4232                                                                callRelation.setCallObject(functionCall);
4233                                                                callRelation.setTarget(new ProcedureRelationshipElement(caller));
4234                                                                callRelation.addSource(new ProcedureRelationshipElement(callee));
4235                                                                if (isBuiltInFunctionName(functionCall.getFunctionName()) || isKeyword(functionCall.getFunctionName())) {
4236                                                                        callRelation.setBuiltIn(true);
4237                                                                }
4238                                                        }
4239                                                }
4240                                                if (callee.getArguments() != null) {
4241                                                        for (int j = 0; j < callee.getArguments().size(); j++) {
4242                                                                Argument argument = callee.getArguments().get(j);
4243                                                                Variable variable = modelFactory.createVariable(callee, argument.getName(), false);
4244                                                                if(variable!=null) {
4245                                                                        if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
4246                                                                                Transform transform = new Transform();
4247                                                                                transform.setType(Transform.FUNCTION);
4248                                                                                transform.setCode(functionCall);
4249                                                                                variable.getColumns().get(0).setTransform(transform);
4250                                                                        }
4251                                                                        Process process = modelFactory.createProcess(functionCall);
4252                                                                        variable.addProcess(process);
4253                                                                        analyzeFunctionArgumentsDataFlowRelation(variable.getColumns().get(0), functionCall, j, process);
4254                                                                }
4255                                                        }
4256                                                }
4257                                        } else {
4258                                                Function function = modelFactory.createFunction(functionCall);
4259                                                String procedureParent = getProcedureParentName(ifStmt);
4260                                                if (procedureParent != null) {
4261                                                        Procedure caller = modelManager
4262                                                                        .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
4263                                                        if (caller != null) {
4264                                                                CallRelationship callRelation = modelFactory.createCallRelation();
4265                                                                callRelation.setCallObject(functionCall);
4266                                                                callRelation.setTarget(new ProcedureRelationshipElement(caller));
4267                                                                callRelation.addSource(new FunctionRelationshipElement(function));
4268                                                                if (isBuiltInFunctionName(functionCall.getFunctionName()) || isKeyword(functionCall.getFunctionName())) {
4269                                                                        callRelation.setBuiltIn(true);
4270                                                                }
4271                                                        }
4272                                                }
4273                                        }
4274                                }
4275                        }
4276                }
4277
4278                if (ifStmt.getThenStatements() != null) {
4279                        for (int i = 0; i < ifStmt.getThenStatements().size(); ++i) {
4280                                analyzeCustomSqlStmt(ifStmt.getThenStatements().get(i));
4281                                ResultSet returnResult = modelFactory.createResultSet(ifStmt.getThenStatements().get(i), false);
4282                                if(returnResult!=null){
4283                                        for(ResultColumn resultColumn: returnResult.getColumns()){
4284                                                analyzeFilterCondition(resultColumn, ifStmt.getCondition(), null, null, EffectType.function);
4285                                        }
4286                                }
4287                        }
4288                }
4289
4290                if (ifStmt.getElseifStatements() != null) {
4291                        for (int i = 0; i < ifStmt.getElseifStatements().size(); ++i) {
4292                                analyzeCustomSqlStmt(ifStmt.getElseifStatements().get(i));
4293                        }
4294                }
4295
4296                if (ifStmt.getElseStatements() != null) {
4297                        for (int i = 0; i < ifStmt.getElseStatements().size(); ++i) {
4298                                analyzeCustomSqlStmt(ifStmt.getElseStatements().get(i));
4299                        }
4300                }
4301        }
4302
4303        private void analyzeElsIfStmt(TElsifStmt elsIfStmt) {
4304                if (elsIfStmt.getCondition() != null) {
4305                        columnsInExpr visitor = new columnsInExpr();
4306                        elsIfStmt.getCondition().inOrderTraverse(visitor);
4307                        List<TParseTreeNode> functions = visitor.getFunctions();
4308
4309                        if (functions != null && !functions.isEmpty()) {
4310                                for (int i = 0; i < functions.size(); i++) {
4311                                        if (!(functions.get(i) instanceof TFunctionCall))
4312                                                continue;
4313                                        TFunctionCall functionCall = (TFunctionCall) functions.get(i);
4314                                        Procedure callee = modelManager.getProcedureByName(DlineageUtil
4315                                                        .getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
4316                                        if(callee == null && procedureDDLMap.containsKey(DlineageUtil.getFunctionNameWithArgNum(functionCall))) {
4317                                                analyzeCustomSqlStmt(procedureDDLMap.get(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
4318                                                callee = modelManager.getProcedureByName(
4319                                                                DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
4320                                        }
4321                                        if (callee != null) {
4322                                                String procedureParent = getProcedureParentName(elsIfStmt);
4323                                                if (procedureParent != null) {
4324                                                        Procedure caller = modelManager
4325                                                                        .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
4326                                                        if (caller != null) {
4327                                                                CallRelationship callRelation = modelFactory.createCallRelation();
4328                                                                callRelation.setCallObject(functionCall);
4329                                                                callRelation.setTarget(new ProcedureRelationshipElement(caller));
4330                                                                callRelation.addSource(new ProcedureRelationshipElement(callee));
4331                                                                if(isBuiltInFunctionName(functionCall.getFunctionName()) || isKeyword(functionCall.getFunctionName())){
4332                                                                        callRelation.setBuiltIn(true);
4333                                                                }
4334                                                        }
4335                                                }
4336                                                if (callee.getArguments() != null) {
4337                                                        for (int j = 0; j < callee.getArguments().size(); j++) {
4338                                                                Argument argument = callee.getArguments().get(j);
4339                                                                Variable variable = modelFactory.createVariable(callee, argument.getName(), false);
4340                                                                if(variable!=null) {
4341                                                                        if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
4342                                                                                Transform transform = new Transform();
4343                                                                                transform.setType(Transform.FUNCTION);
4344                                                                                transform.setCode(functionCall);
4345                                                                                variable.getColumns().get(0).setTransform(transform);
4346                                                                        }
4347                                                                        Process process = modelFactory.createProcess(functionCall);
4348                                                                        variable.addProcess(process);
4349                                                                        analyzeFunctionArgumentsDataFlowRelation(variable.getColumns().get(0), functionCall, j, process);
4350                                                                }
4351                                                        }
4352                                                }
4353                                        } else {
4354                                                Function function = modelFactory.createFunction(functionCall);
4355                                                String procedureParent = getProcedureParentName(elsIfStmt);
4356                                                if (procedureParent != null) {
4357                                                        Procedure caller = modelManager
4358                                                                        .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
4359                                                        if (caller != null) {
4360                                                                CallRelationship callRelation = modelFactory.createCallRelation();
4361                                                                callRelation.setCallObject(functionCall);
4362                                                                callRelation.setTarget(new ProcedureRelationshipElement(caller));
4363                                                                callRelation.addSource(new FunctionRelationshipElement(function));
4364                                                                if(isBuiltInFunctionName(functionCall.getFunctionName()) || isKeyword(functionCall.getFunctionName())){
4365                                                                        callRelation.setBuiltIn(true);
4366                                                                }
4367                                                        }
4368                                                }
4369                                        }
4370                                }
4371                        }
4372                }
4373
4374                if (elsIfStmt.getThenStatements() != null) {
4375                        for (int i = 0; i < elsIfStmt.getThenStatements().size(); ++i) {
4376                                ResultSet returnResult = modelFactory.createResultSet(elsIfStmt.getThenStatements().get(i), false);
4377                                if (returnResult != null) {
4378                                        for (ResultColumn resultColumn : returnResult.getColumns()) {
4379                                                analyzeFilterCondition(resultColumn, elsIfStmt.getCondition(), null, null, EffectType.function);
4380                                        }
4381                                }
4382                                analyzeCustomSqlStmt(elsIfStmt.getThenStatements().get(i));
4383                        }
4384                }
4385        }
4386
4387        private void analyzeCloneTableStmt(TCreateTableSqlStatement stmt) {
4388                if (stmt.getCloneSourceTable() != null) {
4389                        Table sourceTable = modelFactory.createTableByName(stmt.getCloneSourceTable());
4390                        Table cloneTable = modelFactory.createTableByName(stmt.getTableName());
4391                        // Clone creates the target. Parse fact, set regardless of source
4392                        // resolution (the gated setFromDDL below stays as-is). See
4393                        // dlineage-authoritative-endpoint-classification.md.
4394                        cloneTable.setCreatedInSql(true);
4395                        cloneTable.setEndpointIntroduction(EndpointIntroduction.CLONE_TABLE);
4396                        Process process = modelFactory.createProcess(stmt);
4397                        cloneTable.addProcess(process);
4398
4399                        if (sourceTable.isDetermined()) {
4400                                for (int k = 0; k < sourceTable.getColumns().size(); k++) {
4401                                        TableColumn sourceColumn = sourceTable.getColumns().get(k);
4402                                        TObjectName objectName = new TObjectName();
4403                                        objectName.setString(sourceColumn.getName());
4404                                        TableColumn tableColumn = modelFactory.createTableColumn(cloneTable, objectName, true);
4405                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
4406                                        dataflowRelation.setEffectType(EffectType.clone_table);
4407                                        dataflowRelation
4408                                                        .addSource(new TableColumnRelationshipElement(sourceColumn));
4409                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(tableColumn));
4410                                        dataflowRelation.setProcess(process);
4411                                }
4412                                cloneTable.setDetermined(true);
4413                                cloneTable.setFromDDL(true);
4414                        } else {
4415                                TObjectName sourceName = new TObjectName();
4416                                sourceName.setString("*");
4417                                TableColumn sourceTableColumn = modelFactory.createTableColumn(sourceTable, sourceName, false);
4418                                TObjectName targetName = new TObjectName();
4419                                targetName.setString("*");
4420                                TableColumn targetTableColumn = modelFactory.createTableColumn(cloneTable, targetName, false);
4421                                if(sourceTableColumn!=null && targetTableColumn!=null) {
4422                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
4423                                        dataflowRelation.setEffectType(EffectType.clone_table);
4424                                        dataflowRelation
4425                                                        .addSource(new TableColumnRelationshipElement(sourceTableColumn));
4426                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(targetTableColumn));
4427                                        dataflowRelation.setProcess(process);
4428                                }
4429                        }
4430                }
4431        }
4432
4433        private void analyzeCloneDatabaseStmt(TCreateDatabaseSqlStatement stmt) {
4434                if (stmt.getCloneSourceDb() != null) {
4435                        Database sourceDatabase = modelFactory.createDatabase(stmt.getCloneSourceDb());
4436                        Database cloneDatabase = modelFactory.createDatabase(stmt.getDatabaseName());
4437                        Process process = modelFactory.createProcess(stmt);
4438                        cloneDatabase.addProcess(process);
4439                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
4440                        relation.setEffectType(EffectType.clone_database);
4441                        relation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(cloneDatabase.getRelationRows()));
4442                        relation.addSource(
4443                                        new RelationRowsRelationshipElement<TableRelationRows>(sourceDatabase.getRelationRows()));
4444                        relation.setProcess(process);
4445                }
4446        }
4447
4448        private void analyzeCloneSchemaStmt(TCreateSchemaSqlStatement stmt) {
4449                if (stmt.getCloneSourceSchema() != null) {
4450                        Schema sourceSchema = modelFactory.createSchema(stmt.getCloneSourceSchema());
4451                        Schema cloneSchema = modelFactory.createSchema(stmt.getSchemaName());
4452                        Process process = modelFactory.createProcess(stmt);
4453                        cloneSchema.addProcess(process);
4454                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
4455                        relation.setEffectType(EffectType.clone_schema);
4456                        relation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(cloneSchema.getRelationRows()));
4457                        relation.addSource(new RelationRowsRelationshipElement<TableRelationRows>(sourceSchema.getRelationRows()));
4458                        relation.setProcess(process);
4459                }
4460        }
4461
4462        /**
4463         * Record one dynamic-SQL site. Span is the statement extent (1-based, end-exclusive), tagged with
4464         * the per-statement file hash, mirroring the {@link ErrorInfo} position convention. When the
4465         * {@code reportDynamicSqlSitesAsErrors} option is on, non-RESOLVED sites are also mirrored into an
4466         * {@link ErrorInfo} so legacy {@code getErrorMessages()} consumers can see them.
4467         */
4468        private void recordDynamicSqlSite(TCustomSqlStatement stmt, DynamicSqlSite.Kind kind,
4469                        DynamicSqlSite.Status status, String reason) {
4470                recordDynamicSqlSite(stmt, kind, status, reason, 0, 0);
4471        }
4472
4473        private void recordDynamicSqlSite(TCustomSqlStatement stmt, DynamicSqlSite.Kind kind,
4474                        DynamicSqlSite.Status status, String reason, int resolvedCount, int partialCount) {
4475                if (stmt == null || stmt.getStartToken() == null || stmt.getEndToken() == null) {
4476                        return;
4477                }
4478                Pair3<Long, Long, String> start = new Pair3<Long, Long, String>(stmt.getStartToken().lineNo,
4479                                stmt.getStartToken().columnNo, ModelBindingManager.getGlobalHash());
4480                Pair3<Long, Long, String> end;
4481                String[] segments = stmt.getEndToken().getAstext().split("\n", -1);
4482                if (segments.length == 1) {
4483                        end = new Pair3<Long, Long, String>(stmt.getEndToken().lineNo,
4484                                        stmt.getEndToken().columnNo + stmt.getEndToken().getAstext().length(),
4485                                        ModelBindingManager.getGlobalHash());
4486                } else {
4487                        end = new Pair3<Long, Long, String>(stmt.getEndToken().lineNo + segments.length - 1,
4488                                        (long) segments[segments.length - 1].length() + 1, ModelBindingManager.getGlobalHash());
4489                }
4490                dynamicSqlSites.add(new DynamicSqlSite(kind, status, reason, start, end, resolvedCount, partialCount));
4491
4492                if (status != DynamicSqlSite.Status.RESOLVED && option != null
4493                                && option.isReportDynamicSqlSitesAsErrors()) {
4494                        ErrorInfo errorInfo = new ErrorInfo();
4495                        errorInfo.setErrorType(ErrorInfo.DYNAMIC_SQL_UNRESOLVED);
4496                        errorInfo.setErrorMessage(reason != null ? reason : "dynamic SQL not resolved");
4497                        errorInfo.setStartPosition(start);
4498                        errorInfo.setEndPosition(end);
4499                        errorInfo.fillInfo(this);
4500                        errorInfos.add(errorInfo);
4501                }
4502        }
4503
4504        /**
4505         * Classify the lineage edges a folded dynamic site produced. {@code relsBefore} is the relation
4506         * snapshot taken immediately before the folded SQL was analyzed; the tail of the current relation set
4507         * is this site's delta (relationHolder is insertion-ordered). counts[0]=resolved (edge into a real
4508         * target column), counts[1]=partial (edge into a T-SQL variable / placeholder target). Edges whose
4509         * target is an intermediate result-set are ignored — only final-target edges are counted.
4510         */
4511        private int[] classifyDynamicSiteLineage(int beforeCount) {
4512                int resolved = 0;
4513                int partial = 0;
4514                Relationship[] after = modelManager.getRelations();
4515                for (int i = beforeCount; i < after.length; i++) {
4516                        Relationship r = after[i];
4517                        if (r == null || !(r.getTarget() instanceof TableColumnRelationshipElement)) {
4518                                continue;
4519                        }
4520                        TableColumn targetColumn = ((TableColumnRelationshipElement) r.getTarget()).getElement();
4521                        Table targetTable = targetColumn == null ? null : targetColumn.getTable();
4522                        if (isUnresolvedTargetTable(targetTable)) {
4523                                partial++;
4524                        } else {
4525                                resolved++;
4526                        }
4527                }
4528                return new int[] { resolved, partial };
4529        }
4530
4531        /**
4532         * A folded dynamic site's target is "unresolved" when it is a T-SQL variable / placeholder rather
4533         * than a real object. When GSP only partially folds (e.g. {@code EXEC sp_executesql @sql} →
4534         * {@code INSERT INTO @p SELECT ...}), the re-parsed target is a table literally named {@code @p};
4535         * its {@code isVariable()} flag is not set, so the reliable signal is a name starting with '@'.
4536         */
4537        private boolean isUnresolvedTargetTable(Table table) {
4538                if (table == null) {
4539                        return true;
4540                }
4541                if (table.isVariable() || table.isPseudo()) {
4542                        return true;
4543                }
4544                String name = table.getName();
4545                if (name != null) {
4546                        name = name.replace("[", "").replace("]", "").replace("\"", "").trim();
4547                        if (name.startsWith("@")) {
4548                                return true;
4549                        }
4550                }
4551                return false;
4552        }
4553
4554        /** True when a module name resolves to sp_executesql, ignoring schema qualifier and brackets/quotes. */
4555        private boolean isSpExecutesql(TObjectName moduleName) {
4556                if (moduleName == null) {
4557                        return false;
4558                }
4559                String s = moduleName.toString();
4560                int dot = s.lastIndexOf('.');
4561                if (dot >= 0) {
4562                        s = s.substring(dot + 1);
4563                }
4564                s = s.replace("[", "").replace("]", "").replace("\"", "").replace("`", "").trim();
4565                return "sp_executesql".equalsIgnoreCase(s);
4566        }
4567
4568        /** Kind of a {@link TMssqlExecute} dynamic site: sp_executesql vs the EXEC(string) family. */
4569        private DynamicSqlSite.Kind dynamicKindOf(TMssqlExecute e) {
4570                if (e.getExecType() == TBaseType.metExecStringCmd) {
4571                        return DynamicSqlSite.Kind.EXEC_STRING;
4572                }
4573                if (isSpExecutesql(e.getModuleName())) {
4574                        return DynamicSqlSite.Kind.SP_EXECUTESQL;
4575                }
4576                return DynamicSqlSite.Kind.OTHER;
4577        }
4578
4579        /** The dynamic-SQL argument expression of a {@link TMssqlExecute}, or null. */
4580        private TExpression dynamicArgOf(TMssqlExecute e) {
4581                if (e.getExecType() == TBaseType.metExecStringCmd) {
4582                        if (e.getStringValues() != null && e.getStringValues().size() > 0) {
4583                                return e.getStringValues().getExpression(0);
4584                        }
4585                        return null;
4586                }
4587                if (e.getParameters() != null && e.getParameters().size() > 0) {
4588                        TExecParameter p = e.getParameters().getExecParameter(0);
4589                        if (p != null) {
4590                                return p.getParameterValue();
4591                        }
4592                }
4593                return null;
4594        }
4595
4596        /**
4597         * True only when the dynamic-SQL argument is a compile-time string literal. A variable or
4598         * expression argument is data-driven: even when GSP partially folds it (e.g. {@code @sql} →
4599         * {@code INSERT INTO @t SELECT ...} with an unresolved parameter left in), the site is NOT
4600         * fully resolved, so it must not be reported RESOLVED.
4601         */
4602        private boolean isLiteralDynamicArg(TMssqlExecute e) {
4603                TExpression arg = dynamicArgOf(e);
4604                return arg != null && arg.getExpressionType() == EExpressionType.simple_constant_t;
4605        }
4606
4607        /** Refine the UNRESOLVED reason by inspecting the dynamic argument expression. */
4608        private String dynamicReasonOf(TMssqlExecute e) {
4609                TExpression arg = dynamicArgOf(e);
4610                if (arg == null) {
4611                        return "dynamic SQL target could not be resolved";
4612                }
4613                if (arg.getExpressionType() == EExpressionType.simple_object_name_t && arg.getObjectOperand() != null
4614                                && arg.getObjectOperand().getDbObjectType() == EDbObjectType.variable) {
4615                        return "argument is a runtime variable";
4616                }
4617                if (arg.getExpressionType() == EExpressionType.simple_constant_t) {
4618                        return "inline literal not folded; no lineage produced";
4619                }
4620                return "argument is a runtime expression or partially-folded variable";
4621        }
4622
4623        private void executeDynamicSql(String sql) {
4624                TGSqlParser sqlparser = new TGSqlParser(option.getVendor());
4625                sqlparser.sqltext = sql;
4626                int result = sqlparser.parse();
4627                if (result == 0) {
4628                        for (int i = 0; i < sqlparser.sqlstatements.size(); i++) {
4629                                analyzeCustomSqlStmt(sqlparser.sqlstatements.get(i));
4630                        }
4631                }
4632        }
4633
4634        private void extractSnowflakeSQLFromProcedure(TCreateProcedureStmt procedure) {
4635                Map<String, String> argMap = new LinkedHashMap<String, String>();
4636                if (procedure.getParameterDeclarations() != null) {
4637                        for (int i = 0; i < procedure.getParameterDeclarations().size(); i++) {
4638                                TParameterDeclaration def = procedure.getParameterDeclarations().getParameterDeclarationItem(i);
4639                                argMap.put(def.getParameterName().toString(), def.getDataType().getDataTypeName());
4640                        }
4641                }
4642                StringBuilder buffer = new StringBuilder();
4643                buffer.append("(function(");
4644                String[] args = argMap.keySet().toArray(new String[0]);
4645                for (int i = 0; i < args.length; i++) {
4646                        buffer.append(args[i].toUpperCase());
4647                        if (i < args.length - 1) {
4648                                buffer.append(",");
4649                        }
4650                }
4651                buffer.append("){\n");
4652
4653                int start = -1;
4654                int end = -1;
4655                boolean dollar = false;
4656                boolean quote = false;
4657                if (procedure.getRoutineBody().indexOf("$$") != -1) {
4658                        start = procedure.getRoutineBody().indexOf("$$") + 2;
4659                        end = procedure.getRoutineBody().lastIndexOf("$$") - 1;
4660                        dollar = true;
4661                } else if (procedure.getRoutineBody().indexOf("'") != -1) {
4662                        start = procedure.getRoutineBody().indexOf("'") + 1;
4663                        end = procedure.getRoutineBody().lastIndexOf("'");
4664                        quote = true;
4665                }
4666                String body = procedure.getRoutineBody().substring(start, end);
4667                if (dollar && body.indexOf("`") != -1) {
4668                        Pattern pattern = Pattern.compile("`.+?`", Pattern.CASE_INSENSITIVE | Pattern.DOTALL);
4669                        Matcher matcher = pattern.matcher(body);
4670                        StringBuffer replaceBuffer = new StringBuffer();
4671                        while (matcher.find()) {
4672                                String condition = matcher.group().replace("\r\n", "\n").replace("'", "\\\\'")
4673                                                .replace("\n", "\\\\n'\n+'").replace("`", "'").replace("$", "RDS_CHAR_DOLLAR");
4674                                matcher.appendReplacement(replaceBuffer, condition);
4675                        }
4676                        matcher.appendTail(replaceBuffer);
4677                        body = replaceBuffer.toString().replace("RDS_CHAR_DOLLAR", "$");
4678                }
4679                if (quote && body.indexOf("'") != -1) {
4680                        body = body.replace("''", "'");
4681                }
4682                buffer.append(body);
4683                buffer.append("})(");
4684                for (int i = 0; i < args.length; i++) {
4685                        String type = argMap.get(args[i]);
4686                        if (type.equalsIgnoreCase("VARCHAR")) {
4687                                buffer.append("'pseudo'");
4688                        } else if (type.equalsIgnoreCase("STRING")) {
4689                                buffer.append("'pseudo'");
4690                        } else if (type.equalsIgnoreCase("CHAR")) {
4691                                buffer.append("'pseudo'");
4692                        } else if (type.equalsIgnoreCase("CHARACTER")) {
4693                                buffer.append("'pseudo'");
4694                        } else if (type.equalsIgnoreCase("TEXT")) {
4695                                buffer.append("'pseudo'");
4696                        } else if (type.equalsIgnoreCase("BINARY")) {
4697                                buffer.append("'pseudo'");
4698                        } else if (type.equalsIgnoreCase("VARBINARY")) {
4699                                buffer.append("'pseudo'");
4700                        } else if (type.equalsIgnoreCase("BOOLEAN")) {
4701                                buffer.append(true);
4702                        } else if (type.equalsIgnoreCase("FLOAT")) {
4703                                buffer.append("1.0");
4704                        } else if (type.equalsIgnoreCase("FLOAT4")) {
4705                                buffer.append("1.0");
4706                        } else if (type.equalsIgnoreCase("FLOAT8")) {
4707                                buffer.append("1.0");
4708                        } else if (type.equalsIgnoreCase("DOUBLE")) {
4709                                buffer.append("1.0");
4710                        } else if (type.equalsIgnoreCase("DOUBLE PRECISION")) {
4711                                buffer.append("1.0");
4712                        } else if (type.equalsIgnoreCase("REAL")) {
4713                                buffer.append("1.0");
4714                        } else if (type.equalsIgnoreCase("NUMBER")) {
4715                                buffer.append("1.0");
4716                        } else if (type.equalsIgnoreCase("DECIMAL")) {
4717                                buffer.append("1.0");
4718                        } else if (type.equalsIgnoreCase("NUMERIC")) {
4719                                buffer.append("1.0");
4720                        } else if (type.equalsIgnoreCase("INT")) {
4721                                buffer.append("1");
4722                        } else if (type.equalsIgnoreCase("INTEGER")) {
4723                                buffer.append("1");
4724                        } else if (type.equalsIgnoreCase("BIGINT")) {
4725                                buffer.append("1");
4726                        } else if (type.equalsIgnoreCase("SMALLINT")) {
4727                                buffer.append("1");
4728                        } else if (type.equalsIgnoreCase("DATE")) {
4729                                buffer.append("new Date()");
4730                        } else if (type.equalsIgnoreCase("DATETIME")) {
4731                                buffer.append("new Date()");
4732                        } else if (type.equalsIgnoreCase("TIME")) {
4733                                buffer.append("new Date()");
4734                        } else if (type.equalsIgnoreCase("TIMESTAMP")) {
4735                                buffer.append("new Date()");
4736                        } else if (type.equalsIgnoreCase("TIMESTAMP_LTZ")) {
4737                                buffer.append("new Date()");
4738                        } else if (type.equalsIgnoreCase("TIMESTAMP_NTZ")) {
4739                                buffer.append("new Date()");
4740                        } else if (type.equalsIgnoreCase("TIMESTAMP_TZ")) {
4741                                buffer.append("new Date()");
4742                        } else if (type.equalsIgnoreCase("VARIANT")) {
4743                                buffer.append("{}");
4744                        } else if (type.equalsIgnoreCase("OBJECT")) {
4745                                buffer.append("{}");
4746                        } else if (type.equalsIgnoreCase("ARRAY")) {
4747                                buffer.append("[]");
4748                        } else if (type.equalsIgnoreCase("GEOGRAPHY")) {
4749                                buffer.append("{}");
4750                        }
4751                        if (i < args.length - 1) {
4752                                buffer.append(",");
4753                        }
4754                }
4755                buffer.append(");");
4756
4757                try {
4758                        ScriptEngineManager scriptEngineManager = new ScriptEngineManager();
4759                        ScriptEngine nashorn = scriptEngineManager.getEngineByName("nashorn");
4760                        nashorn.put("analyzer", this);
4761                        nashorn.eval(new InputStreamReader(
4762                                        getClass().getResourceAsStream("/gudusoft/gsqlparser/parser/snowflake/snowflake.js")));
4763                        nashorn.eval(new StringReader(buffer.toString()));
4764                } catch (ScriptException e) {
4765                        TGSqlParser sqlparser = new TGSqlParser(option.getVendor());
4766                        sqlparser.sqltext = body;
4767                        int result = sqlparser.parse();
4768                        if (result == 0) {
4769                                for (int i = 0; i < sqlparser.sqlstatements.size(); i++) {
4770                                        analyzeCustomSqlStmt(sqlparser.sqlstatements.get(i));
4771                                }
4772                                return;
4773                        }
4774                        ErrorInfo errorInfo = new ErrorInfo();
4775                        errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
4776                        errorInfo.setErrorMessage("Invoke script error: " + e.getMessage());
4777                        errorInfo.setStartPosition(new Pair3<Long, Long, String>(procedure.getStartToken().lineNo,
4778                                        procedure.getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
4779                        errorInfo.setEndPosition(new Pair3<Long, Long, String>(procedure.getEndToken().lineNo,
4780                                        procedure.getEndToken().columnNo + procedure.getEndToken().getAstext().length(),
4781                                        ModelBindingManager.getGlobalHash()));
4782                        errorInfos.add(errorInfo);
4783                }
4784        }
4785
4786        private void analyzeHiveLoadStmt(THiveLoad stmt) {
4787                if (stmt.getPath() != null && stmt.getTable() != null) {
4788                        Table uriFile = modelFactory.createTableByName(stmt.getPath(), true);
4789                        uriFile.setPath(true);
4790                        uriFile.setCreateTable(true);
4791                        TObjectName fileUri = new TObjectName();
4792                        fileUri.setString("uri=" + stmt.getPath());
4793                        TableColumn fileUriColumn = modelFactory.createFileUri(uriFile, fileUri);
4794
4795                        Table tableModel = modelFactory.createTable(stmt.getTable());
4796                        Process process = modelFactory.createProcess(stmt);
4797                        tableModel.addProcess(process);
4798
4799                        TPartitionExtensionClause p = stmt.getTable().getPartitionExtensionClause();
4800                        if (p.getKeyValues() != null && p.getKeyValues().size() > 0) {
4801                                for (int i = 0; i < p.getKeyValues().size(); i++) {
4802                                        TExpression expression = p.getKeyValues().getExpression(i);
4803                                        if (expression.getLeftOperand().getExpressionType() == EExpressionType.simple_object_name_t) {
4804                                                modelFactory.createTableColumn(tableModel, expression.getLeftOperand().getObjectOperand(),
4805                                                                true);
4806                                        }
4807                                }
4808                        }
4809
4810                        for (int j = 0; j < tableModel.getColumns().size(); j++) {
4811                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
4812                                relation.addSource(new TableColumnRelationshipElement(fileUriColumn));
4813                                relation.setTarget(new TableColumnRelationshipElement(tableModel.getColumns().get(j)));
4814                                relation.setProcess(process);
4815                        }
4816                }
4817        }
4818        
4819        private void analyzeLoadDataStmt(TLoadDataStmt stmt) {
4820                
4821        }
4822
4823        private void analyzeUnloadStmt(TUnloadStmt unloadStmt) {
4824                if (unloadStmt.getSelectSqlStatement() != null && unloadStmt.getS3() != null) {
4825
4826                        Table uriFile = modelFactory.createTableByName(unloadStmt.getS3(), true);
4827                        uriFile.setPath(true);
4828                        uriFile.setCreateTable(true);
4829                        TObjectName fileUri = new TObjectName();
4830                        fileUri.setString("uri=" + unloadStmt.getS3());
4831                        TableColumn fileUriColumn = modelFactory.createFileUri(uriFile, fileUri);
4832
4833                        Process process = modelFactory.createProcess(unloadStmt);
4834                        uriFile.addProcess(process);
4835
4836                        TCustomSqlStatement stmt = unloadStmt.getSelectSqlStatement();
4837                        analyzeCustomSqlStmt(stmt);
4838                        if (stmt instanceof TSelectSqlStatement) {
4839                                TSelectSqlStatement select = (TSelectSqlStatement) stmt;
4840                                ResultSet resultSetModel = (ResultSet) modelManager.getModel(select);
4841                                if (resultSetModel != null) {
4842                                        for (int j = 0; j < resultSetModel.getColumns().size(); j++) {
4843                                                ResultColumn resultColumn = resultSetModel.getColumns().get(j);
4844                                                if (resultColumn.hasStarLinkColumn()
4845                                                                && resultColumn.getStarLinkColumnNames().size() > 0) {
4846                                                        for (int k = 0; k < resultColumn.getStarLinkColumnNames().size(); k++) {
4847                                                                ResultColumn expandStarColumn = modelFactory.createResultColumn(resultSetModel,
4848                                                                                resultColumn.getStarLinkColumnName(k), false);
4849                                                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
4850                                                                dataflowRelation.setEffectType(EffectType.unload);
4851                                                                dataflowRelation
4852                                                                                .addSource(new ResultColumnRelationshipElement(expandStarColumn));
4853                                                                dataflowRelation.setTarget(new TableColumnRelationshipElement(fileUriColumn));
4854                                                                dataflowRelation.setProcess(process);
4855                                                        }
4856                                                }
4857                                                if (!resultColumn.hasStarLinkColumn() || resultColumn.isShowStar()) {
4858                                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
4859                                                        dataflowRelation.setEffectType(EffectType.unload);
4860                                                        dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn));
4861                                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(fileUriColumn));
4862                                                        dataflowRelation.setProcess(process);
4863                                                }
4864                                        }
4865                                }
4866                        }
4867                }
4868
4869        }
4870
4871        private void analyzeCopyIntoStmt(TSnowflakeCopyIntoStmt stmt) {
4872                if (stmt.getTableName() != null) {
4873                        if (stmt.getStageLocation() != null) {
4874                                Table intoTable = modelManager
4875                                                .getTableByName(DlineageUtil.getTableFullName(stmt.getTableName().toString()));
4876                                if (intoTable == null) {
4877                                        intoTable = modelFactory.createTableByName(stmt.getTableName(), false);
4878                                        TObjectName starColumn = new TObjectName();
4879                                        starColumn.setString("*");
4880                                        TableColumn column = modelFactory.createTableColumn(intoTable, starColumn, false);
4881                                        if (column != null) {
4882                                                column.setExpandStar(false);
4883                                                column.setPseduo(true);
4884                                        }
4885                                }
4886                                Process process = modelFactory.createProcess(stmt);
4887                                intoTable.addProcess(process);
4888
4889                                TObjectName stageName = stmt.getStageLocation().getStageName();
4890                                if (stageName == null || stmt.getStageLocation().getTableName() != null) {
4891                                        stageName = stmt.getStageLocation().getTableName();
4892                                }
4893                                if (stageName != null) {
4894                                        String stageFullName = DlineageUtil.getTableFullName(stageName.toString());
4895                                        Table stage = modelManager.getTableByName(stageFullName);
4896                                        if (stage == null) {
4897                                                stage = modelFactory.createStage(stageName);
4898                                                stage.setCreateTable(true);
4899                                                stage.setStage(true);
4900                                                String stagePath = stmt.getStageLocation().getPath() == null ? null
4901                                                                : stmt.getStageLocation().getPath().toString();
4902                                                if (stagePath != null) {
4903                                                        stage.setLocation(stagePath);
4904                                                        TObjectName location = new TObjectName();
4905                                                        location.setString(stagePath);
4906                                                        modelFactory.createStageLocation(stage, location);
4907                                                } else {
4908                                                        stage.setLocation("unknownPath");
4909                                                        TObjectName location = new TObjectName();
4910                                                        location.setString("unknownPath");
4911                                                        modelFactory.createStageLocation(stage, location);
4912                                                }
4913                                        }
4914
4915                                        if (stage != null && intoTable != null) {
4916                                                if (intoTable != null && !intoTable.getColumns().isEmpty() && !stage.getColumns().isEmpty()) {
4917                                                        for (int i = 0; i < intoTable.getColumns().size(); i++) {
4918                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
4919                                                                relation.addSource(new TableColumnRelationshipElement(stage.getColumns().get(0)));
4920                                                                relation.setTarget(new TableColumnRelationshipElement(intoTable.getColumns().get(i)));
4921                                                                relation.setProcess(process);
4922                                                        }
4923                                                }
4924                                        }
4925                                } else if (stmt.getStageLocation().getExternalLocation() != null) {
4926                                        Table pathModel = modelFactory.createTableByName(stmt.getStageLocation().getExternalLocation(),
4927                                                        true);
4928                                        pathModel.setPath(true);
4929                                        pathModel.setCreateTable(true);
4930                                        TableColumn fileUriColumn = modelFactory.createFileUri(pathModel,
4931                                                        stmt.getStageLocation().getExternalLocation());
4932                                        if (intoTable != null) {
4933                                                if (intoTable != null && !intoTable.getColumns().isEmpty()) {
4934                                                        for (int i = 0; i < intoTable.getColumns().size(); i++) {
4935                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
4936                                                                relation.addSource(new TableColumnRelationshipElement(fileUriColumn));
4937                                                                relation.setTarget(new TableColumnRelationshipElement(intoTable.getColumns().get(i)));
4938                                                                relation.setProcess(process);
4939                                                        }
4940                                                }
4941                                        }
4942                                }
4943                        }
4944                        else if (stmt.getSubQuery() != null) {
4945                                analyzeSelectStmt(stmt.getSubQuery());
4946                                
4947                                Table intoTable = modelManager
4948                                                .getTableByName(DlineageUtil.getTableFullName(stmt.getTableName().toString()));
4949                                if (intoTable == null) {
4950                                        intoTable = modelFactory.createTableByName(stmt.getTableName(), false);
4951                                        TObjectName starColumn = new TObjectName();
4952                                        starColumn.setString("*");
4953                                        TableColumn column = modelFactory.createTableColumn(intoTable, starColumn, false);
4954                                        if (column != null) {
4955                                                column.setExpandStar(true);
4956                                                column.setPseduo(true);
4957                                        }
4958
4959                                        Process process = modelFactory.createProcess(stmt);
4960                                        intoTable.addProcess(process);
4961
4962                                        ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt.getSubQuery());
4963                                        if (resultSetModel != null && column != null) {
4964                                                for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
4965                                                        ResultColumn resultColumn = resultSetModel.getColumns().get(i);
4966                                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
4967                                                        dataflowRelation.setEffectType(EffectType.copy);
4968                                                        dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn));
4969                                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(column));
4970                                                        dataflowRelation.setProcess(process);
4971                                                }
4972                                        }
4973                                }
4974                        }
4975                }
4976        }
4977
4978        private void analyzeRedshiftCopyStmt(TRedshiftCopy stmt) {
4979                if (stmt.getTableName() != null && stmt.getFromSource() != null) {
4980
4981                        Table intoTable = modelManager
4982                                        .getTableByName(DlineageUtil.getTableFullName(stmt.getTableName().toString()));
4983                        if (intoTable == null) {
4984                                intoTable = modelFactory.createTableByName(stmt.getTableName(), false);
4985                                if (stmt.getColumnList() == null || stmt.getColumnList().size() == 0) {
4986                                        TObjectName starColumn = new TObjectName();
4987                                        starColumn.setString("*");
4988                                        TableColumn column = modelFactory.createTableColumn(intoTable, starColumn, false);
4989                                        if (column != null) {
4990                                                column.setExpandStar(false);
4991                                                column.setPseduo(true);
4992                                        }
4993                                } else {
4994                                        for (TObjectName columnName : stmt.getColumnList()) {
4995                                                modelFactory.createTableColumn(intoTable, columnName, true);
4996                                        }
4997                                }
4998                        }
4999                        Process process = modelFactory.createProcess(stmt);
5000                        intoTable.addProcess(process);
5001
5002                        if (stmt.getFromSource() != null) {
5003                                Table pathModel = modelFactory.createTableByName(stmt.getFromSource(), true);
5004                                pathModel.setPath(true);
5005                                pathModel.setCreateTable(true);
5006                                TObjectName fileUri = new TObjectName();
5007                                fileUri.setString(stmt.getFromSource());
5008                                TableColumn fileUriColumn = modelFactory.createFileUri(pathModel, fileUri);
5009                                if (intoTable != null) {
5010                                        if (intoTable != null && !intoTable.getColumns().isEmpty()) {
5011                                                for (int i = 0; i < intoTable.getColumns().size(); i++) {
5012                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
5013                                                        relation.addSource(new TableColumnRelationshipElement(fileUriColumn));
5014                                                        relation.setTarget(new TableColumnRelationshipElement(intoTable.getColumns().get(i)));
5015                                                        relation.setProcess(process);
5016                                                }
5017                                        }
5018                                }
5019                        }
5020                }
5021        }
5022
5023        private void analyzeCreateIndexExpressionOperand(TExpression expr, Table tableModel){
5024                if(expr == null) return;
5025                Deque<TExpression> stack = new ArrayDeque<>();
5026                stack.push(expr);
5027                while (!stack.isEmpty()) {
5028                        TExpression current = stack.pop();
5029                        if (current == null) continue;
5030                        TObjectName columnObj = current.getObjectOperand();
5031                        if (columnObj != null) {
5032                                TableColumn tableConstraint = modelFactory.createTableColumn(tableModel, columnObj, true);
5033                                tableConstraint.setIndexKey(true);
5034                        } else {
5035                                if (current.getRightOperand() != null) {
5036                                        stack.push(current.getRightOperand());
5037                                }
5038                                if (current.getLeftOperand() != null) {
5039                                        stack.push(current.getLeftOperand());
5040                                }
5041                        }
5042                }
5043        }
5044        private void analyzeCreateIndexStageStmt(TCreateIndexSqlStatement stmt) {
5045        if(stmt.getTableName() == null){
5046            return;
5047        }
5048                Table tableModel = modelFactory.createTableByName(stmt.getTableName());
5049                TOrderByItemList columns = stmt.getColumnNameList();
5050        if(columns!=null) {
5051            for (int i = 0; i < columns.size(); i++) {
5052                TExpression expr = columns.getOrderByItem(i).getSortKey();
5053                analyzeCreateIndexExpressionOperand(expr, tableModel);
5054            }
5055        }
5056        }
5057
5058        private void analyzeCreateSynonymStmt(TCreateSynonymStmt stmt) {
5059                TObjectName sourceTableName = stmt.getForName();
5060                if(sourceTableName == null) {
5061                        return;
5062                }
5063                TCustomSqlStatement createView = viewDDLMap
5064                                .get(DlineageUtil.getTableFullName(sourceTableName.toString()));
5065                if (createView != null) {
5066                        analyzeCustomSqlStmt(createView);
5067                }
5068                Table sourceTableModel = modelFactory.createTableByName(sourceTableName);
5069                Process process = modelFactory.createProcess(stmt);
5070                sourceTableModel.addProcess(process);
5071                if(stmt.getSynonymName()!=null) {
5072                        Table synonymTableModel = modelFactory.createTableByName(stmt.getSynonymName());
5073                        synonymTableModel.setSubType(SubType.synonym);
5074                        if(sourceTableModel.isCreateTable()) {
5075                                synonymTableModel.setCreateTable(true);
5076                                for(TableColumn sourceTableColumn: sourceTableModel.getColumns()) {
5077                                        TObjectName columnName = new TObjectName();
5078                                        columnName.setString(sourceTableColumn.getName());
5079                                        TableColumn synonymTableColumn = new TableColumn(synonymTableModel, columnName);
5080                                        synonymTableModel.addColumn(synonymTableColumn);
5081                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
5082                                        relation.setEffectType(EffectType.create_synonym);
5083                                        relation.setTarget(new TableColumnRelationshipElement(synonymTableColumn));
5084                                        relation.addSource(new TableColumnRelationshipElement(sourceTableColumn));
5085                                        relation.setProcess(process);
5086                                }
5087                        }
5088                        else {
5089                                TObjectName synonymStarColumn = new TObjectName();
5090                                synonymStarColumn.setString("*");
5091                                TableColumn synonymTableStarColumn = modelFactory.createTableColumn(synonymTableModel,
5092                                                synonymStarColumn, true);
5093                                TObjectName sourceStarColumn = new TObjectName();
5094                                sourceStarColumn.setString("*");
5095                                TableColumn sourceTableStarColumn = modelFactory.createTableColumn(sourceTableModel, sourceStarColumn, true);
5096                                
5097                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
5098                                relation.setEffectType(EffectType.create_synonym);
5099                                relation.setTarget(new TableColumnRelationshipElement(synonymTableStarColumn));
5100                                relation.addSource(new TableColumnRelationshipElement(sourceTableStarColumn));
5101                                relation.setProcess(process);
5102                        }
5103                }
5104        }
5105
5106        /**
5107         * For every Table model that maps to a {@link TSQLSynonyms} in the current
5108         * {@link #sqlenv}, materialize the synonym's base table as a separate Table
5109         * model and emit per-column {@code fdd}/{@code synonym} relations from the
5110         * synonym's columns to the base table's matching columns. Lets lineage flow
5111         * through synonyms defined in the metadata SQLEnv (e.g. loaded via
5112         * {@code TDDLSQLEnv} or sqlflow metadata) the same way it does for
5113         * {@code CREATE SYNONYM} DDL inside the analyzed script.
5114         */
5115        private void materializeSqlEnvSynonyms() {
5116                if (sqlenv == null) {
5117                        return;
5118                }
5119                List<Table> snapshot = new ArrayList<Table>(modelManager.getTablesByName());
5120
5121                Map<String, TSQLSynonyms> synonymCache = new HashMap<>();
5122                List<TSQLCatalog> catalogs = sqlenv.getCatalogList();
5123                if (catalogs != null) {
5124                        for (TSQLCatalog catalog : catalogs) {
5125                                if (catalog == null) {
5126                                        continue;
5127                                }
5128                                List<TSQLSchema> schemas = catalog.getSchemaList();
5129                                if (schemas == null) {
5130                                        continue;
5131                                }
5132                                for (TSQLSchema schema : schemas) {
5133                                        if (schema == null) {
5134                                                continue;
5135                                        }
5136                                        List<TSQLSchemaObject> schemaObjects = schema.getSchemaObjectList();
5137                                        if (schemaObjects == null || schemaObjects.isEmpty()) {
5138                                                continue;
5139                                        }
5140                                        for (TSQLSchemaObject schemaObject : schemaObjects) {
5141                                                if (schemaObject instanceof TSQLSynonyms) {
5142                                                        TSQLSynonyms synonym = (TSQLSynonyms) schemaObject;
5143                                                        String qualifiedName = synonym.getQualifiedName();
5144                                                        String normalizedName = DlineageUtil.getIdentifierNormalTableName(qualifiedName);
5145                                                        synonymCache.put(normalizedName, synonym);
5146                                                }
5147                                        }
5148                                }
5149                        }
5150                }
5151
5152                for (Table synonymTableModel : snapshot) {
5153                        if (synonymTableModel == null) {
5154                                continue;
5155                        }
5156                        if (SubType.synonym.equals(synonymTableModel.getSubType())) {
5157                                continue;
5158                        }
5159                        String qualifiedName = ModelFactory.getQualifiedTableName(synonymTableModel);
5160                        String normalizedName = DlineageUtil.getIdentifierNormalTableName(qualifiedName);
5161                        TSQLSynonyms synonym = synonymCache.get(normalizedName);
5162                        if (synonym == null) {
5163                                continue;
5164                        }
5165
5166                        String baseSqlTableQualifiedName = synonym.getBaseTableQualifiedName();
5167                        if (baseSqlTableQualifiedName == null) {
5168                                continue;
5169                        }
5170
5171                        synonymTableModel.setSubType(SubType.synonym);
5172
5173                        TObjectName baseObjectName = new TObjectName();
5174                        baseObjectName.setString(baseSqlTableQualifiedName);
5175                        Table baseTableModel = modelFactory.createTableByName(baseObjectName);
5176                        if (baseTableModel == null || baseTableModel == synonymTableModel) {
5177                                continue;
5178                        }
5179
5180                        for (TableColumn synonymColumn : new ArrayList<TableColumn>(synonymTableModel.getColumns())) {
5181                                String colName = synonymColumn.getName();
5182                                if (SQLUtil.isEmpty(colName) || "*".equals(colName)) {
5183                                        continue;
5184                                }
5185                                TableColumn baseColumn = null;
5186                                for (TableColumn bc : baseTableModel.getColumns()) {
5187                                        if (bc.getName() != null && bc.getName().equalsIgnoreCase(colName)) {
5188                                                baseColumn = bc;
5189                                                break;
5190                                        }
5191                                }
5192                                if (baseColumn == null) {
5193                                        TObjectName columnName = new TObjectName();
5194                                        columnName.setString(colName);
5195                                        baseColumn = new TableColumn(baseTableModel, columnName);
5196                                        baseTableModel.addColumn(baseColumn);
5197                                }
5198                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
5199                                relation.setEffectType(EffectType.synonym);
5200                                relation.setTarget(new TableColumnRelationshipElement(synonymColumn));
5201                                relation.addSource(new TableColumnRelationshipElement(baseColumn));
5202                        }
5203                }
5204        }
5205
5206        private void analyzeRenameStmt(TRenameStmt stmt) {
5207                TObjectName oldTableName = stmt.getOldName();
5208                TObjectName newTableName = stmt.getNewName();
5209                
5210                Table oldNameTableModel = modelFactory.createTableByName(oldTableName);
5211                TObjectName oldStarColumn = new TObjectName();
5212                oldStarColumn.setString("*");
5213                TableColumn oldTableStarColumn = modelFactory.createTableColumn(oldNameTableModel, oldStarColumn, true);
5214                
5215                Table newNameTableModel = modelFactory.createTableByName(newTableName);
5216                TObjectName newStarColumn = new TObjectName();
5217                newStarColumn.setString("*");
5218                TableColumn newTableStarColumn = modelFactory.createTableColumn(newNameTableModel, newStarColumn, true);
5219                
5220                Process process = modelFactory.createProcess(stmt);
5221                newNameTableModel.addProcess(process);
5222                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
5223                relation.setEffectType( EffectType.rename_table);
5224                relation.setTarget(new TableColumnRelationshipElement(newTableStarColumn));
5225                relation.addSource(new TableColumnRelationshipElement(oldTableStarColumn));
5226                relation.setProcess(process);
5227                
5228                if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) {
5229                        oldTableStarColumn.setShowStar(false);
5230                        relation.setShowStarRelation(false);
5231                }
5232
5233                if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) {
5234                        newTableStarColumn.setShowStar(false);
5235                        relation.setShowStarRelation(false);
5236                }
5237        }
5238        
5239        private void analyzeAlterTableStmt(TAlterTableStatement stmt) {
5240                TTable oldNameTable = stmt.getTargetTable();
5241                if (oldNameTable == null) {
5242                        return;
5243                }
5244
5245                Table oldNameTableModel = modelFactory.createTable(oldNameTable);
5246                TObjectName oldStarColumn = new TObjectName();
5247                oldStarColumn.setString("*");
5248                TableColumn oldTableStarColumn = modelFactory.createTableColumn(oldNameTableModel, oldStarColumn, true);
5249
5250                for (int i = 0; stmt.getAlterTableOptionList() != null && i < stmt.getAlterTableOptionList().size(); i++) {
5251                        TAlterTableOption option = stmt.getAlterTableOptionList().getAlterTableOption(i);
5252                        if (option.getOptionType() == EAlterTableOptionType.RenameTable
5253                                        || option.getOptionType() == EAlterTableOptionType.swapWith) {
5254                                TObjectName newTableName = option.getNewTableName();
5255                                Stack<TParseTreeNode> list = newTableName.getStartToken().getNodesStartFromThisToken();
5256                                boolean containsTable = false;
5257                                for (int j = 0; j < list.size(); j++) {
5258                                        if (list.get(j) instanceof TTable) {
5259                                                TTable newTableTable = (TTable) list.get(j);
5260                                                Table newNameTableModel = modelFactory.createTable(newTableTable);
5261                                                newNameTableModel.setStarStmt("rename_table");
5262
5263                                                TObjectName newStarColumn = new TObjectName();
5264                                                newStarColumn.setString("*");
5265                                                TableColumn newTableStarColumn = modelFactory.createTableColumn(newNameTableModel,
5266                                                                newStarColumn, true);
5267
5268                                                Process process = modelFactory.createProcess(stmt);
5269                                                newNameTableModel.addProcess(process);
5270                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
5271                                                relation.setEffectType(
5272                                                                option.getOptionType() == EAlterTableOptionType.RenameTable ? EffectType.rename_table
5273                                                                                : EffectType.swap_table);                                               
5274                                                if (option.getOptionType() == EAlterTableOptionType.RenameTable) {
5275                                                        relation.setTarget(new TableColumnRelationshipElement(newTableStarColumn));
5276                                                        relation.addSource(new TableColumnRelationshipElement(oldTableStarColumn));
5277                                                } else if (option.getOptionType() == EAlterTableOptionType.swapWith) {
5278                                                        relation.setTarget(new TableColumnRelationshipElement(oldTableStarColumn));
5279                                                        relation.addSource(new TableColumnRelationshipElement(newTableStarColumn));
5280                                                }
5281                                                relation.setProcess(process);
5282                                                containsTable = true;
5283
5284                                                if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) {
5285                                                        oldTableStarColumn.setShowStar(false);
5286                                                        relation.setShowStarRelation(false);
5287                                                }
5288
5289                                                if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) {
5290                                                        newTableStarColumn.setShowStar(false);
5291                                                        relation.setShowStarRelation(false);
5292                                                }
5293                                        }
5294                                }
5295                                if (!containsTable) {
5296                                        Table newNameTableModel = modelFactory.createTableByName(newTableName);
5297                                        newNameTableModel.setStarStmt("rename_table");
5298                                        TObjectName newStarColumn = new TObjectName();
5299                                        newStarColumn.setString("*");
5300                                        TableColumn newTableStarColumn = modelFactory.createTableColumn(newNameTableModel, newStarColumn,
5301                                                        true);
5302
5303                                        Process process = modelFactory.createProcess(stmt);
5304                                        newNameTableModel.addProcess(process);
5305                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
5306                                        relation.setEffectType(
5307                                                        option.getOptionType() == EAlterTableOptionType.RenameTable ? EffectType.rename_table
5308                                                                        : EffectType.swap_table);
5309                                        if (option.getOptionType() == EAlterTableOptionType.RenameTable) {
5310                                                relation.setTarget(new TableColumnRelationshipElement(newTableStarColumn));
5311                                                relation.addSource(new TableColumnRelationshipElement(oldTableStarColumn));
5312                                        } else if (option.getOptionType() == EAlterTableOptionType.swapWith) {
5313                                                relation.setTarget(new TableColumnRelationshipElement(oldTableStarColumn));
5314                                                relation.addSource(new TableColumnRelationshipElement(newTableStarColumn));
5315                                        }
5316                                        relation.setProcess(process);
5317                                        if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) {
5318                                                oldTableStarColumn.setShowStar(false);
5319                                                relation.setShowStarRelation(false);
5320                                        }
5321
5322                                        if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) {
5323                                                newTableStarColumn.setShowStar(false);
5324                                                relation.setShowStarRelation(false);
5325                                        }
5326
5327                                }
5328                        }
5329                        else if (option.getOptionType() == EAlterTableOptionType.appendFrom) {
5330                                TObjectName newTableName = option.getSourceTableName();
5331                                Stack<TParseTreeNode> list = newTableName.getStartToken().getNodesStartFromThisToken();
5332                                boolean containsTable = false;
5333                                for (int j = 0; j < list.size(); j++) {
5334                                        if (list.get(j) instanceof TTable) {
5335                                                TTable newTableTable = (TTable) list.get(j);
5336                                                Table newNameTableModel = modelFactory.createTable(newTableTable);
5337                                                newNameTableModel.setStarStmt("append_from");
5338
5339                                                TObjectName newStarColumn = new TObjectName();
5340                                                newStarColumn.setString("*");
5341                                                TableColumn newTableStarColumn = modelFactory.createTableColumn(newNameTableModel,
5342                                                                newStarColumn, true);
5343
5344                                                Process process = modelFactory.createProcess(stmt);
5345                                                newNameTableModel.addProcess(process);
5346                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
5347                                                relation.setEffectType(EffectType.append_from);
5348                                                relation.setTarget(new TableColumnRelationshipElement(oldTableStarColumn));
5349                                                relation.addSource(new TableColumnRelationshipElement(newTableStarColumn));
5350                                                relation.setProcess(process);
5351                                                containsTable = true;
5352
5353                                                if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) {
5354                                                        oldTableStarColumn.setShowStar(false);
5355                                                        relation.setShowStarRelation(false);
5356                                                }
5357
5358                                                if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) {
5359                                                        newTableStarColumn.setShowStar(false);
5360                                                        relation.setShowStarRelation(false);
5361                                                }
5362                                        }
5363                                }
5364                                if (!containsTable) {
5365                                        Table newNameTableModel = modelFactory.createTableByName(newTableName);
5366                                        newNameTableModel.setStarStmt("append_from");
5367                                        TObjectName newStarColumn = new TObjectName();
5368                                        newStarColumn.setString("*");
5369                                        TableColumn newTableStarColumn = modelFactory.createTableColumn(newNameTableModel, newStarColumn,
5370                                                        true);
5371
5372                                        Process process = modelFactory.createProcess(stmt);
5373                                        newNameTableModel.addProcess(process);
5374                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
5375                                        relation.setEffectType(EffectType.append_from);
5376                                        relation.setTarget(new TableColumnRelationshipElement(oldTableStarColumn));
5377                                        relation.addSource(new TableColumnRelationshipElement(newTableStarColumn));
5378                                        relation.setProcess(process);
5379                                        if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) {
5380                                                oldTableStarColumn.setShowStar(false);
5381                                                relation.setShowStarRelation(false);
5382                                        }
5383
5384                                        if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) {
5385                                                newTableStarColumn.setShowStar(false);
5386                                                relation.setShowStarRelation(false);
5387                                        }
5388
5389                                }
5390                        }
5391                        else if (option.getOptionType() == EAlterTableOptionType.exchangePartition) {
5392                                TObjectName newTableName = option.getNewTableName();
5393                                Stack<TParseTreeNode> list = newTableName.getStartToken().getNodesStartFromThisToken();
5394                                boolean containsTable = false;
5395                                for (int j = 0; j < list.size(); j++) {
5396                                        if (list.get(j) instanceof TTable) {
5397                                                TTable newTableTable = (TTable) list.get(j);
5398                                                Table newNameTableModel = modelFactory.createTable(newTableTable);
5399                                                newNameTableModel.setStarStmt("exchange_partition");
5400
5401                                                TObjectName newStarColumn = new TObjectName();
5402                                                newStarColumn.setString("*");
5403                                                TableColumn newTableStarColumn = modelFactory.createTableColumn(newNameTableModel,
5404                                                                newStarColumn, true);
5405
5406                                                Process process = modelFactory.createProcess(stmt);
5407                                                newNameTableModel.addProcess(process);
5408                                                {
5409                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
5410                                                        relation.setEffectType(EffectType.exchange_partition);
5411                                                        relation.setTarget(new TableColumnRelationshipElement(oldTableStarColumn));
5412                                                        relation.addSource(new TableColumnRelationshipElement(newTableStarColumn));
5413                                                        relation.setProcess(process);
5414                                                        if (option.getPartitionName() != null) {
5415                                                                relation.setPartition(option.getPartitionName().toString());
5416                                                        }
5417                                                        if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) {
5418                                                                oldTableStarColumn.setShowStar(false);
5419                                                                relation.setShowStarRelation(false);
5420                                                        }
5421
5422                                                        if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) {
5423                                                                newTableStarColumn.setShowStar(false);
5424                                                                relation.setShowStarRelation(false);
5425                                                        }
5426                                                }
5427                                                {
5428                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
5429                                                        relation.setEffectType(EffectType.exchange_partition);
5430                                                        relation.setTarget(new TableColumnRelationshipElement(newTableStarColumn));
5431                                                        relation.addSource(new TableColumnRelationshipElement(oldTableStarColumn));
5432                                                        relation.setProcess(process);
5433                                                        if (option.getPartitionName() != null) {
5434                                                                relation.setPartition(option.getPartitionName().toString());
5435                                                        }
5436                                                        if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) {
5437                                                                oldTableStarColumn.setShowStar(false);
5438                                                                relation.setShowStarRelation(false);
5439                                                        }
5440
5441                                                        if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) {
5442                                                                newTableStarColumn.setShowStar(false);
5443                                                                relation.setShowStarRelation(false);
5444                                                        }
5445                                                }
5446                                                containsTable = true;   
5447                                        }
5448                                }
5449                                if (!containsTable) {
5450                                        Table newNameTableModel = modelFactory.createTableByName(newTableName);
5451                                        newNameTableModel.setStarStmt("exchange_partition");
5452                                        TObjectName newStarColumn = new TObjectName();
5453                                        newStarColumn.setString("*");
5454                                        TableColumn newTableStarColumn = modelFactory.createTableColumn(newNameTableModel, newStarColumn,
5455                                                        true);
5456
5457                                        Process process = modelFactory.createProcess(stmt);
5458                                        newNameTableModel.addProcess(process);
5459                                        {
5460                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
5461                                                relation.setEffectType(EffectType.exchange_partition);
5462                                                relation.setTarget(new TableColumnRelationshipElement(oldTableStarColumn));
5463                                                relation.addSource(new TableColumnRelationshipElement(newTableStarColumn));
5464                                                if (option.getPartitionName() != null) {
5465                                                        relation.setPartition(option.getPartitionName().toString());
5466                                                }
5467                                                relation.setProcess(process);
5468                                                if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) {
5469                                                        oldTableStarColumn.setShowStar(false);
5470                                                        relation.setShowStarRelation(false);
5471                                                }
5472
5473                                                if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) {
5474                                                        newTableStarColumn.setShowStar(false);
5475                                                        relation.setShowStarRelation(false);
5476                                                }
5477                                        }
5478                                        {
5479                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
5480                                                relation.setEffectType(EffectType.exchange_partition);
5481                                                relation.setTarget(new TableColumnRelationshipElement(newTableStarColumn));
5482                                                relation.addSource(new TableColumnRelationshipElement(oldTableStarColumn));
5483                                                if (option.getPartitionName() != null) {
5484                                                        relation.setPartition(option.getPartitionName().toString());
5485                                                }
5486                                                relation.setProcess(process);
5487                                                if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) {
5488                                                        oldTableStarColumn.setShowStar(false);
5489                                                        relation.setShowStarRelation(false);
5490                                                }
5491
5492                                                if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) {
5493                                                        newTableStarColumn.setShowStar(false);
5494                                                        relation.setShowStarRelation(false);
5495                                                }
5496                                        }
5497                                }
5498                        }
5499                        else if(option.getOptionType() == EAlterTableOptionType.setLocation) {
5500                                TObjectName location = option.getTableLocation();
5501                                Process process = modelFactory.createProcess(stmt);
5502                                process.setType("Set Table Location");
5503                                oldNameTableModel.addProcess(process);
5504                                Table uriFile = modelFactory.createTableByName(location, true);
5505                                uriFile.setPath(true);
5506                                for (int j = 0; j < oldNameTableModel.getColumns().size(); j++) {
5507                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
5508                                        TObjectName fileUri = new TObjectName();
5509                                        fileUri.setString("*");
5510                                        TableColumn fileUriColumn = modelFactory.createFileUri(uriFile, fileUri);
5511                                        relation.addSource(new TableColumnRelationshipElement(fileUriColumn));
5512                                        relation.setTarget(new TableColumnRelationshipElement(oldNameTableModel.getColumns().get(j)));
5513                                        relation.setProcess(process);
5514                                }
5515                        }
5516                        else if (option.getOptionType() == EAlterTableOptionType.AddColumn
5517                                        || option.getOptionType() == EAlterTableOptionType.addColumnIfNotExists) {
5518                                if (option.getColumnDefinitionList() != null) {
5519                                        for (TColumnDefinition column : option.getColumnDefinitionList()) {
5520                                                if (column != null && column.getColumnName() != null) {
5521                                                        TableColumn tableColumn = modelFactory.createTableColumn(oldNameTableModel, column.getColumnName(), true);
5522                                                        if(this.option.getAnalyzeMode() == AnalyzeMode.crud) {
5523                                                                CrudRelationship crudRelationship = modelFactory.createCrudRelation();
5524                                                                crudRelationship.setTarget(new TableColumnRelationshipElement(tableColumn));
5525                                                                crudRelationship.setEffectType(EffectType.add_table_column);
5526                                                        }
5527                                                }
5528                                        }
5529                                }
5530                        }
5531                        else if (option.getOptionType() == EAlterTableOptionType.DropColumn && this.option.getAnalyzeMode() == AnalyzeMode.crud) {
5532                                if (option.getColumnNameList() != null) {
5533                                        for (TObjectName column : option.getColumnNameList()) {
5534                                                TableColumn tableColumn = modelFactory.createTableColumn(oldNameTableModel, column, true);
5535                                                CrudRelationship crudRelationship = modelFactory.createCrudRelation();
5536                                                crudRelationship.setTarget(new TableColumnRelationshipElement(tableColumn));
5537                                                crudRelationship.setEffectType(EffectType.drop_table_column);
5538                                        }
5539                                }
5540                        }
5541                        else if(option.getOptionType() == EAlterTableOptionType.AddConstraint || option.getOptionType() == EAlterTableOptionType.AddConstraintFK
5542                                        || option.getOptionType() == EAlterTableOptionType.AddConstraintPK || option.getOptionType() == EAlterTableOptionType.AddConstraintUnique
5543                                        || option.getOptionType() == EAlterTableOptionType.AddConstraintIndex){
5544                                if (option.getTableConstraint() != null) {
5545                                        TConstraint alertTableConstraint = option.getTableConstraint();
5546                                        TPTNodeList<TColumnWithSortOrder> keyNames = alertTableConstraint.getColumnList();
5547                                        if (keyNames == null) {
5548                                                continue;
5549                                        }
5550                                        for (int k = 0; k < keyNames.size(); k++) {
5551                                                TObjectName keyName = keyNames.getElement(k).getColumnName();
5552                                                TObjectName referencedTableName = alertTableConstraint.getReferencedObject();
5553                                                Table tableModel = modelFactory.createTableByName(stmt.getTableName());
5554                                                TableColumn tableConstraint = modelFactory.createTableColumn(tableModel, keyName, true);
5555                                                if(alertTableConstraint.getConstraint_type() == EConstraintType.primary_key){
5556                                                        tableConstraint.setPrimaryKey(true);
5557                                                }
5558                                                else if(alertTableConstraint.getConstraint_type() == EConstraintType.table_index){
5559                                                        tableConstraint.setIndexKey(true);
5560                                                }
5561                                                else if(alertTableConstraint.getConstraint_type() == EConstraintType.unique){
5562                                                        tableConstraint.setUnqiueKey(true);
5563                                                }
5564                                                else if (alertTableConstraint.getConstraint_type() == EConstraintType.foreign_key) {
5565                                                        tableConstraint.setForeignKey(true);
5566                                                        Table referencedTable = modelManager.getTableByName(DlineageUtil.getTableFullName(referencedTableName.toString()));
5567                                                        if (referencedTable == null) {
5568                                                                referencedTable = modelFactory.createTableByName(referencedTableName);
5569                                                        }
5570                                                        TObjectNameList referencedTableColumns = alertTableConstraint.getReferencedColumnList();
5571                                                        if (referencedTableColumns != null) {
5572                                                                for (int j = 0; j < referencedTableColumns.size(); j++) {
5573                                                                        TableColumn tableColumn = modelFactory.createTableColumn(referencedTable,
5574                                                                                        referencedTableColumns.getObjectName(j), false);
5575                                                                        if (tableColumn != null) {
5576                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
5577                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
5578                                                                                relation.setTarget(new TableColumnRelationshipElement(tableConstraint));
5579                                                                                relation.setEffectType(EffectType.foreign_key);
5580                                                                                Process process = modelFactory.createProcess(stmt);
5581                                                                                relation.setProcess(process);
5582                                                                                if(this.option.isShowERDiagram()){
5583                                                                                        ERRelationship erRelation = modelFactory.createERRelation();
5584                                                                                        erRelation.addSource(new TableColumnRelationshipElement(tableColumn));
5585                                                                                        erRelation.setTarget(new TableColumnRelationshipElement(tableConstraint));
5586                                                                                }
5587                                                                        }
5588                                                                }
5589                                                        }
5590                                                        else{
5591                                                                TableColumn tableColumn = modelFactory.createTableColumn(referencedTable, keyName, false);
5592                                                                if (tableColumn != null) {
5593                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
5594                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
5595                                                                        relation.setTarget(new TableColumnRelationshipElement(tableConstraint));
5596                                                                        relation.setEffectType(EffectType.foreign_key);
5597                                                                        if(this.option.isShowERDiagram()){
5598                                                                                ERRelationship erRelation = modelFactory.createERRelation();
5599                                                                                erRelation.addSource(new TableColumnRelationshipElement(tableColumn));
5600                                                                                erRelation.setTarget(new TableColumnRelationshipElement(tableConstraint));
5601                                                                        }
5602                                                                }
5603                                                        }
5604                                                }
5605                                        }
5606                                }
5607                                else if (option.getConstraintList() != null) {
5608                                        for(int iCons=0; iCons<option.getConstraintList().size(); iCons++){
5609                                                TConstraint alertTableConstraint = option.getConstraintList().getConstraint(iCons);
5610                                                TPTNodeList<TColumnWithSortOrder> keyNames = alertTableConstraint.getColumnList();
5611                                                if(keyNames != null){
5612                                                        for (int k = 0; k < keyNames.size(); k++) {
5613                                                                TObjectName keyName = keyNames.getElement(k).getColumnName();
5614                                                                TObjectName referencedTableName = alertTableConstraint.getReferencedObject();
5615                                                                Table tableModel = modelFactory.createTableByName(stmt.getTableName());
5616                                                                TableColumn tableConstraint = modelFactory.createTableColumn(tableModel, keyName, true);
5617                                                                if(alertTableConstraint.getConstraint_type() == EConstraintType.primary_key){
5618                                                                        tableConstraint.setPrimaryKey(true);
5619                                                                }
5620                                                                else if(alertTableConstraint.getConstraint_type() == EConstraintType.table_index){
5621                                                                        tableConstraint.setIndexKey(true);
5622                                                                }
5623                                                                else if(alertTableConstraint.getConstraint_type() == EConstraintType.unique){
5624                                                                        tableConstraint.setUnqiueKey(true);
5625                                                                }
5626                                                                else if (alertTableConstraint.getConstraint_type() == EConstraintType.foreign_key) {
5627                                                                        tableConstraint.setForeignKey(true);
5628                                                                        Table referencedTable = modelManager.getTableByName(DlineageUtil.getTableFullName(referencedTableName.toString()));
5629                                                                        if (referencedTable == null) {
5630                                                                                referencedTable = modelFactory.createTableByName(referencedTableName);
5631                                                                        }
5632                                                                        TObjectNameList referencedTableColumns = alertTableConstraint.getReferencedColumnList();
5633                                                                        if (referencedTableColumns != null) {
5634                                                                                for (int j = 0; j < referencedTableColumns.size(); j++) {
5635                                                                                        TableColumn tableColumn = modelFactory.createTableColumn(referencedTable,
5636                                                                                                        referencedTableColumns.getObjectName(j), false);
5637                                                                                        if (tableColumn != null) {
5638                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
5639                                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
5640                                                                                                relation.setTarget(new TableColumnRelationshipElement(tableConstraint));
5641                                                                                                relation.setEffectType(EffectType.foreign_key);
5642                                                                                                Process process = modelFactory.createProcess(stmt);
5643                                                                                                relation.setProcess(process);
5644                                                                                                if(this.option.isShowERDiagram()){
5645                                                                                                        ERRelationship erRelation = modelFactory.createERRelation();
5646                                                                                                        erRelation.addSource(new TableColumnRelationshipElement(tableColumn));
5647                                                                                                        erRelation.setTarget(new TableColumnRelationshipElement(tableConstraint));
5648                                                                                                }
5649                                                                                        }
5650                                                                                }
5651                                                                        }
5652                                                                        else{
5653                                                                                TableColumn tableColumn = modelFactory.createTableColumn(referencedTable, keyName, false);
5654                                                                                if (tableColumn != null) {
5655                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
5656                                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
5657                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableConstraint));
5658                                                                                        relation.setEffectType(EffectType.foreign_key);
5659                                                                                        Process process = modelFactory.createProcess(stmt);
5660                                                                                        relation.setProcess(process);
5661                                                                                        if(this.option.isShowERDiagram()){
5662                                                                                                ERRelationship erRelation = modelFactory.createERRelation();
5663                                                                                                erRelation.addSource(new TableColumnRelationshipElement(tableColumn));
5664                                                                                                erRelation.setTarget(new TableColumnRelationshipElement(tableConstraint));
5665                                                                                        }
5666                                                                                }
5667                                                                        }
5668                                                                }
5669                                                        }
5670                                                }
5671                                        }
5672                                }
5673                                else if (option.getIndexCols() != null){
5674                                        TPTNodeList<TColumnWithSortOrder> keyNames = option.getIndexCols();
5675                                        for (int k = 0; k < keyNames.size(); k++) {
5676                                                TObjectName keyName = keyNames.getElement(k).getColumnName();
5677                                                Table tableModel = modelFactory.createTableByName(stmt.getTableName());
5678                                                TableColumn tableConstraint = modelFactory.createTableColumn(tableModel, keyName, true);
5679                                                if(option.getOptionType() == EAlterTableOptionType.AddConstraintPK){
5680                                                        tableConstraint.setPrimaryKey(true);
5681                                                }
5682                                                else if(option.getOptionType() == EAlterTableOptionType.AddConstraintIndex){
5683                                                        tableConstraint.setIndexKey(true);
5684                                                }
5685                                                else if(option.getOptionType() == EAlterTableOptionType.AddConstraintUnique){
5686                                                        tableConstraint.setUnqiueKey(true);
5687                                                }
5688                                                else if (option.getOptionType() == EAlterTableOptionType.AddConstraintFK) {
5689                                                        TObjectName referencedTableName = option.getReferencedObjectName();
5690                                                        tableConstraint.setForeignKey(true);
5691                                                        Table referencedTable = modelManager.getTableByName(DlineageUtil.getTableFullName(referencedTableName.toString()));
5692                                                        if (referencedTable == null) {
5693                                                                referencedTable = modelFactory.createTableByName(referencedTableName);
5694                                                        }
5695                                                        TObjectNameList referencedTableColumns = option.getReferencedColumnList();
5696                                                        if (referencedTableColumns != null) {
5697                                                                for (int j = 0; j < referencedTableColumns.size(); j++) {
5698                                                                        TableColumn tableColumn = modelFactory.createTableColumn(referencedTable,
5699                                                                                        referencedTableColumns.getObjectName(j), false);
5700                                                                        if (tableColumn != null) {
5701                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
5702                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
5703                                                                                relation.setTarget(new TableColumnRelationshipElement(tableConstraint));
5704                                                                                relation.setEffectType(EffectType.foreign_key);
5705                                                                                Process process = modelFactory.createProcess(stmt);
5706                                                                                relation.setProcess(process);
5707                                                                                if(this.option.isShowERDiagram()){
5708                                                                                        ERRelationship erRelation = modelFactory.createERRelation();
5709                                                                                        erRelation.addSource(new TableColumnRelationshipElement(tableColumn));
5710                                                                                        erRelation.setTarget(new TableColumnRelationshipElement(tableConstraint));
5711                                                                                }
5712                                                                        }
5713                                                                }
5714                                                        }
5715                                                        else{
5716                                                                TableColumn tableColumn = modelFactory.createTableColumn(referencedTable, keyName, false);
5717                                                                if (tableColumn != null) {
5718                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
5719                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
5720                                                                        relation.setTarget(new TableColumnRelationshipElement(tableConstraint));
5721                                                                        relation.setEffectType(EffectType.foreign_key);
5722                                                                        Process process = modelFactory.createProcess(stmt);
5723                                                                        relation.setProcess(process);
5724                                                                        if(this.option.isShowERDiagram()){
5725                                                                                ERRelationship erRelation = modelFactory.createERRelation();
5726                                                                                erRelation.addSource(new TableColumnRelationshipElement(tableColumn));
5727                                                                                erRelation.setTarget(new TableColumnRelationshipElement(tableConstraint));
5728                                                                        }
5729                                                                }
5730                                                        }
5731                                                }
5732                                        }
5733                                }
5734                        }
5735                }
5736        }
5737
5738        private void analyzeAlterViewStmt(TAlterViewStatement stmt) {
5739                if (stmt.getAlterViewOption() == EAlterViewOption.asSelect) {
5740                        analyzeCreateViewStmt(stmt, stmt.getSelectSqlStatement(), null, stmt.getViewName());
5741                } else {
5742                        throw new UnsupportedOperationException("Can't handle this alter view statement case, alter option = "+ stmt.getAlterViewOption().name());
5743                }
5744        }
5745        
5746        private void analyzeDeleteStmt(TDeleteSqlStatement stmt) {
5747                TTable table = stmt.getTargetTable();
5748                if (table == null)
5749                        return;
5750                
5751                if (table.getCTE() != null) {
5752                        table = table.getCTE().getSubquery().getTables().getTable(0);
5753                } else if (table.getLinkTable() != null && table.getLinkTable().getSubquery() != null) {
5754                        table = table.getLinkTable().getSubquery().getTables().getTable(0);
5755                } else if (table.getSubquery() != null) {
5756                        table = table.getSubquery().getTables().getTable(0);
5757                }
5758                
5759                Table tableModel = modelFactory.createTable(table);
5760                if (getTableLinkedColumns(table) != null && getTableLinkedColumns(table).size() > 0) {
5761                        for (int j = 0; j < getTableLinkedColumns(table).size(); j++) {
5762                                TObjectName object = getTableLinkedColumns(table).getObjectName(j);
5763
5764                                if (object.getDbObjectType() == EDbObjectType.variable) {
5765                                        continue;
5766                                }
5767
5768                                if (object.getColumnNameOnly().startsWith("@")
5769                                                && (option.getVendor() == EDbVendor.dbvmssql || option.getVendor() == EDbVendor.dbvazuresql)) {
5770                                        continue;
5771                                }
5772
5773                                if (object.getColumnNameOnly().startsWith(":")
5774                                                && (option.getVendor() == EDbVendor.dbvhana || option.getVendor() == EDbVendor.dbvteradata)) {
5775                                        continue;
5776                                }
5777
5778                                if (!isBuiltInFunctionName(object)) {
5779                                        if (object.getSourceTable() == null || object.getSourceTable() == table) {
5780                                                modelFactory.createTableColumn(tableModel, object, false);
5781                                        }
5782                                }
5783                        }
5784                }
5785
5786                if(option.getAnalyzeMode() == AnalyzeMode.crud) {
5787                        CrudRelationship crudRelationship = modelFactory.createCrudRelation();
5788                        crudRelationship.setTarget(new TableRelationshipElement(tableModel));
5789                        crudRelationship.setEffectType(EffectType.delete);
5790                }
5791                
5792                if (stmt.getWhereClause() != null && stmt.getWhereClause().getCondition() != null) {
5793                        analyzeFilterCondition(null, stmt.getWhereClause().getCondition(), null, JoinClauseType.where,
5794                                        EffectType.delete);
5795                }
5796        }
5797
5798        private TObjectName getProcedureName(TStoredProcedureSqlStatement stmt) {
5799                if (stmt instanceof TTeradataCreateProcedure) {
5800                        return ((TTeradataCreateProcedure) stmt).getProcedureName();
5801                }
5802                return stmt.getStoredProcedureName();
5803        }
5804
5805        private void analyzePlsqlCreatePackage(TPlsqlCreatePackage stmt) {
5806                TObjectName procedureName = getProcedureName(stmt);
5807                OraclePackage oraclePackage;
5808                if (procedureName != null) {
5809                        if (this.modelManager.getOraclePackageByName(
5810                                        DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getProcedureNameWithArgs(stmt))) == null) {
5811                                oraclePackage = this.modelFactory.createOraclePackage(stmt);
5812
5813                                if (stmt.getParameterDeclarations() != null) {
5814                                        TParameterDeclarationList parameters = stmt.getParameterDeclarations();
5815
5816                                        for (int i = 0; i < parameters.size(); ++i) {
5817                                                TParameterDeclaration parameter = parameters.getParameterDeclarationItem(i);
5818                                                if (parameter.getParameterName() != null) {
5819                                                        this.modelFactory.createProcedureArgument(oraclePackage, parameter, i + 1);
5820                                                } else if (parameter.getDataType() != null) {
5821                                                        this.modelFactory.createProcedureArgument(oraclePackage, parameter, i + 1);
5822                                                }
5823                                        }
5824                                }
5825
5826                                ModelBindingManager.setGlobalOraclePackage(oraclePackage);
5827                        } else {
5828                                ModelBindingManager.setGlobalOraclePackage(this.modelManager.getOraclePackageByName(
5829                                                DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getProcedureNameWithArgs(stmt))));
5830                        }
5831                        
5832                        try {
5833                                if (stmt.getDeclareStatements() != null) {
5834                                        for (int i = 0; i < stmt.getDeclareStatements().size(); ++i) {
5835                                                analyzeCustomSqlStmt(stmt.getDeclareStatements().get(i));
5836                                        }
5837                                }
5838                        } finally {
5839                                ModelBindingManager.removeGlobalOraclePackage();
5840                        }
5841                }
5842        }
5843
5844        private void analyzeStoredProcedureStmt(TStoredProcedureSqlStatement stmt) {
5845
5846                if (stmt instanceof TPlsqlCreatePackage) {
5847                        analyzePlsqlCreatePackage((TPlsqlCreatePackage) stmt);
5848                        return;
5849                }
5850                
5851                ModelBindingManager.setGlobalProcedure(stmt);
5852
5853                try {
5854                        Procedure procedure = null;
5855        
5856                        TObjectName procedureName = getProcedureName(stmt);
5857
5858                        // Honor procedure-exclusion patterns/names for procedures defined in actual SQL
5859                        // (CREATE PROCEDURE). Previously exclusion was only applied to procedures loaded
5860                        // from metadata JSON; a CREATE PROCEDURE parsed from SQL still produced its
5861                        // procedure node plus all of its inner-statement processes and relationships
5862                        // regardless of the exclusion configuration (MantisBT 4533).
5863                        //
5864                        // analyzeStoredProcedureStmt also handles functions and triggers (every dialect's
5865                        // CREATE FUNCTION / CREATE TRIGGER class extends TStoredProcedureSqlStatement), which
5866                        // are NOT procedures and must never be dropped by procedure exclusion. Gate on the
5867                        // statement type rather than enumerating classes: across all dialects the procedure
5868                        // statement types are exactly those whose ESqlStatementType name contains "procedure"
5869                        // (e.g. sstcreateprocedure, sstoraclecreateprocedure, sstmssqlcreateprocedure),
5870                        // while function/trigger types contain "function"/"trigger".
5871                        boolean isProcedureStmt = stmt.sqlstatementtype != null
5872                                        && stmt.sqlstatementtype.name().toLowerCase().contains("procedure");
5873                        if (procedureName != null && isProcedureStmt
5874                                        && DlineageUtil.isProcedureExcluded(procedureName.toString())) {
5875                                return;
5876                        }
5877
5878                        if (procedureName != null) {
5879                                procedure = this.modelFactory.createProcedure(stmt);
5880                                if (procedure != null) {
5881                                        modelManager.bindModel(stmt, procedure);
5882                                }
5883                                if (ModelBindingManager.getGlobalOraclePackage() != null) {
5884                                        ModelBindingManager.getGlobalOraclePackage().addProcedure(procedure);
5885                                        procedure.setParentPackage(ModelBindingManager.getGlobalOraclePackage());
5886                                }
5887                                if (stmt.getParameterDeclarations() != null) {
5888                                        TParameterDeclarationList parameters = stmt.getParameterDeclarations();
5889        
5890                                        for (int i = 0; i < parameters.size(); ++i) {
5891                                                TParameterDeclaration parameter = parameters.getParameterDeclarationItem(i);
5892                                                Argument argument = null;
5893                                                TObjectName argumentName = null;
5894                                                if (parameter.getParameterName() != null) {
5895                                                        argument = this.modelFactory.createProcedureArgument(procedure, parameter, i + 1);
5896                                                        argumentName = parameter.getParameterName();
5897                                                } else if (parameter.getDataType() != null) {
5898                                                        argument = this.modelFactory.createProcedureArgument(procedure, parameter, i + 1);
5899                                                }
5900                                                
5901                                                if (argument != null) {
5902                                                        if (argumentName == null) {
5903                                                                argumentName = new TObjectName();
5904                                                                argumentName.setString(argument.getName());
5905                                                        }
5906                                                        Variable variable = modelFactory.createVariable(argument.getName());
5907                                                        if (argument.getMode() == EParameterMode.in || argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
5908                                                                variable.setSubType(SubType.of(argument.getMode().name()));
5909                                                        } else {
5910                                                                variable.setSubType(SubType.argument);
5911                                                        }
5912                                                        if(isSimpleDataType(argument.getDataType())){
5913                                                                modelFactory.createTableColumn(variable, argumentName, true);
5914                                                        }
5915                                                        else {
5916                                                                TObjectName variableProperties = new TObjectName();
5917                                                                variableProperties.setString("*");
5918                                                                modelFactory.createTableColumn(variable, variableProperties, true);
5919                                                        }
5920                                                }
5921                                        }
5922                                }
5923                        }
5924        
5925                        if (stmt instanceof TCreateTriggerStmt) {
5926                                TCreateTriggerStmt trigger = (TCreateTriggerStmt) stmt;
5927        
5928                                if (trigger.getFunctionCall() != null) {
5929                                        modelFactory.createProcedureFromFunctionCall(trigger.getFunctionCall());
5930                                }
5931        
5932                                if (trigger.getTables() != null) {
5933                                        for (int i = 0; i < trigger.getTables().size(); i++) {
5934                                                Table tableModel = this.modelFactory.createTriggerOnTable(trigger.getTables().getTable(i));
5935                                        }
5936                                }
5937                        }
5938        
5939                        if (stmt instanceof TPlsqlCreateTrigger
5940                                        && ((TPlsqlCreateTrigger) stmt).getTriggeringClause().getEventClause() instanceof TDmlEventClause) {
5941                                TPlsqlCreateTrigger trigger = (TPlsqlCreateTrigger) stmt;
5942                                TDmlEventClause clause = (TDmlEventClause) ((TPlsqlCreateTrigger) stmt).getTriggeringClause()
5943                                                .getEventClause();
5944                                Table sourceTable = modelFactory.createTableByName(clause.getTableName());
5945        
5946                                for (TTriggerEventItem item : clause.getEventItems()) {
5947                                        if (item instanceof TDmlEventItem) {
5948                                                if (((TDmlEventItem) item).getColumnList() != null) {
5949                                                        for (TObjectName column : ((TDmlEventItem) item).getColumnList()) {
5950                                                                modelFactory.createTableColumn(sourceTable, column, true);
5951                                                        }
5952                                                }
5953                                        }
5954                                }
5955        
5956                                for (TCustomSqlStatement subStmt : ((TPlsqlCreateTrigger) stmt).getStatements()) {
5957                                        if (!(subStmt instanceof TCommonBlock))
5958                                                continue;
5959                                        for (TCustomSqlStatement blockSubStmt : ((TCommonBlock) subStmt).getStatements()) {
5960                                                if (!(blockSubStmt instanceof TBasicStmt))
5961                                                        continue;
5962                                                TBasicStmt basicStmt = (TBasicStmt) blockSubStmt;
5963                                                TExpression expression = basicStmt.getExpr();
5964                                                if (expression != null && expression.getExpressionType() == EExpressionType.function_t) {
5965                                                        Procedure targetProcedure = modelManager.getProcedureByName(DlineageUtil
5966                                                                        .getTableFullName(expression.getFunctionCall().getFunctionName().toString()));
5967                                                        if (targetProcedure == null) {
5968                                                                targetProcedure = modelManager
5969                                                                                .getProcedureByName(DlineageUtil.getTableFullName(procedure.getSchema() + "."
5970                                                                                                + expression.getFunctionCall().getFunctionName().toString()));
5971                                                        }
5972                                                        if (targetProcedure != null && expression.getFunctionCall().getArgs() != null && expression
5973                                                                        .getFunctionCall().getArgs().size() == targetProcedure.getArguments().size()) {
5974                                                                for (int j = 0; j < expression.getFunctionCall().getArgs().size(); j++) {
5975                                                                        TExpression columnExpr = expression.getFunctionCall().getArgs().getExpression(j);
5976                                                                        if (columnExpr.getExpressionType() == EExpressionType.simple_object_name_t) {
5977                                                                                TObjectName columnObject = columnExpr.getObjectOperand();
5978                                                                                if (columnObject.toString().indexOf(":") != -1) {
5979                                                                                        TableColumn tableColumn = modelFactory.createTableColumn(sourceTable,
5980                                                                                                        columnObject, true);
5981                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
5982                                                                                        relation.setEffectType(EffectType.trigger);
5983                                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
5984                                                                                        relation.setTarget(
5985                                                                                                        new ArgumentRelationshipElement(targetProcedure.getArguments().get(j)));
5986                                                                                        Process process = modelFactory.createProcess(stmt);
5987                                                                                        relation.setProcess(process);
5988                                                                                }
5989                                                                        }
5990                                                                }
5991                                                        }
5992                                                }
5993                                        }
5994                                }
5995                        }
5996        
5997                        if (stmt instanceof TMssqlCreateFunction) {
5998                                TMssqlCreateFunction createFunction = (TMssqlCreateFunction) stmt;
5999                                if (createFunction.getReturnTableVaraible() != null && createFunction.getReturnTableDefinitions() != null) {
6000                                        Variable tableModel = this.modelFactory.createVariable(createFunction.getReturnTableVaraible());
6001                                        tableModel.setVariable(true);
6002                                        tableModel.setCreateTable(true);
6003                                        String procedureParent = createFunction.getFunctionName().toString();
6004                                        if (procedureParent != null) {
6005                                                tableModel.setParent(procedureParent);
6006                                        }
6007                                        modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent), tableModel);
6008        
6009                                        if (createFunction.getReturnTableDefinitions() != null) {
6010                                                for (int j = 0; j < createFunction.getReturnTableDefinitions().size(); j++) {
6011                                                        TTableElement tableElement = createFunction.getReturnTableDefinitions().getTableElement(j);
6012                                                        TColumnDefinition column = tableElement.getColumnDefinition();
6013                                                        if (column != null && column.getColumnName() != null) {
6014                                                                modelFactory.createTableColumn(tableModel, column.getColumnName(), true);
6015                                                        }
6016                                                }
6017                                        }
6018                                }
6019        
6020                                if (createFunction.getReturnStmt() != null && createFunction.getReturnStmt().getSubquery() != null) {
6021                                        String procedureParent = createFunction.getFunctionName().toString();
6022                                        analyzeSelectStmt(createFunction.getReturnStmt().getSubquery());
6023                                        ResultSet resultSetModel = (ResultSet) modelManager
6024                                                        .getModel(createFunction.getReturnStmt().getSubquery());
6025                                        modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent),
6026                                                        resultSetModel);
6027                                }
6028                        } else if (stmt instanceof TCreateFunctionStmt) {
6029                                TCreateFunctionStmt createFunction = (TCreateFunctionStmt) stmt;
6030                                if (createFunction.getReturnDataType() != null
6031                                                && createFunction.getReturnDataType().getColumnDefList() != null) {
6032                                        Table tableModel = this.modelFactory.createTableByName(createFunction.getFunctionName(), true);
6033                                        tableModel.setCreateTable(true);
6034                                        String procedureParent = createFunction.getFunctionName().toString();
6035                                        if (procedureParent != null) {
6036                                                tableModel.setParent(procedureParent);
6037                                        }
6038        
6039                                        modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent), tableModel);
6040                                        for (int j = 0; j < createFunction.getReturnDataType().getColumnDefList().size(); j++) {
6041                                                TColumnDefinition column = createFunction.getReturnDataType().getColumnDefList().getColumn(j);
6042                                                if (column != null && column.getColumnName() != null) {
6043                                                        modelFactory.createTableColumn(tableModel, column.getColumnName(), true);
6044                                                }
6045                                        }
6046        
6047                                        if (createFunction.getSqlQuery() != null) {
6048                                                analyzeSelectStmt(createFunction.getSqlQuery());
6049                                                ResultSet resultSetModel = (ResultSet) modelManager.getModel(createFunction.getSqlQuery());
6050                                                if (resultSetModel != null) {
6051                                                        for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
6052                                                                ResultColumn resultColumn = resultSetModel.getColumns().get(i);
6053                                                                for (int j = 0; j < tableModel.getColumns().size(); j++) {
6054                                                                        TableColumn tableColumn = tableModel.getColumns().get(j);
6055                                                                        if (DlineageUtil.compareColumnIdentifier(getColumnName(resultColumn.getName()),
6056                                                                                        DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
6057                                                                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
6058                                                                                dataflowRelation.setEffectType(EffectType.select);
6059                                                                                dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn));
6060                                                                                dataflowRelation.setTarget(new TableColumnRelationshipElement(tableColumn));
6061                                                                        }
6062                                                                }
6063                                                        }
6064                                                }
6065                                        }
6066                                }
6067                                
6068                                if (createFunction.getReturnStmt() != null && createFunction.getReturnStmt().getSubquery() != null) {
6069                                        String procedureParent = createFunction.getFunctionName().toString();
6070                                        analyzeSelectStmt(createFunction.getReturnStmt().getSubquery());
6071                                        ResultSet resultSetModel = (ResultSet) modelManager
6072                                                        .getModel(createFunction.getReturnStmt().getSubquery());
6073                                        modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent),
6074                                                        resultSetModel);
6075                                }
6076                                
6077                                if (createFunction.getReturnStmt() != null && createFunction.getReturnStmt().getReturnExpr() != null) {
6078                                        TExpression returnExpression =  createFunction.getReturnStmt().getReturnExpr();
6079                                        ResultSet returnResult = modelFactory.createResultSet(createFunction.getReturnStmt(), false);
6080                                        ResultColumn resultColumn = modelFactory.createResultColumn(returnResult, returnExpression);
6081                                        
6082                                        TExpression expression = createFunction.getReturnStmt().getReturnExpr();
6083                                        analyzeResultColumnExpressionRelation(resultColumn, expression);
6084        
6085                                        String procedureParent = createFunction.getFunctionName().toString();
6086                                        modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent),
6087                                                        returnResult);
6088                                }
6089                        }
6090        
6091                        if (stmt instanceof TCreateProcedureStmt) {
6092                                TCreateProcedureStmt createProcedure = (TCreateProcedureStmt) stmt;
6093                                if (EDbVendor.dbvsnowflake == option.getVendor() && createProcedure.getRoutineBodyInConstant() != null) {
6094                                        extractSnowflakeSQLFromProcedure(createProcedure);
6095                                }
6096                        }
6097                                
6098                        if (stmt.getStatements().size() > 0) {
6099                                for (int i = 0; i < stmt.getStatements().size(); ++i) {
6100                                        this.analyzeCustomSqlStmt(stmt.getStatements().get(i));
6101                                }
6102                        }
6103
6104                        if (stmt.getBodyStatements().size() > 0) {
6105                                for (int i = 0; i < stmt.getBodyStatements().size(); ++i) {
6106                                        this.analyzeCustomSqlStmt(stmt.getBodyStatements().get(i));
6107                                }
6108                        }
6109                        
6110                        // Detect pipelined functions and build signatures
6111                        if (stmt instanceof TPlsqlCreateFunction && pipelinedAnalyzer != null) {
6112                                try {
6113                                        pipelinedAnalyzer.analyzePipelinedFunction((TPlsqlCreateFunction) stmt);
6114                                } catch (Exception e) {
6115                                        // Don't let pipelined analysis failure break main flow
6116                                }
6117                        }
6118
6119                        if (procedure != null && !getLastSelectStmt(stmt).isEmpty()) {
6120                                for (TSelectSqlStatement select : getLastSelectStmt(stmt)) {
6121                                        ResultSet resultSet = (ResultSet) modelManager.getModel(select);
6122                                        modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedure.getName()),
6123                                                        resultSet);
6124                                }
6125//                              List<Argument> outArgs = new ArrayList<Argument>();
6126//                              for (int i = 0; i < procedure.getArguments().size(); i++) {
6127//                                      Argument argument = procedure.getArguments().get(i);
6128//                                      if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
6129//                                              outArgs.add(argument);
6130//                                      }
6131//                              }
6132//
6133//                              if (resultSet != null && resultSet.getColumns().size() == outArgs.size()) {
6134//                                      for (int i = 0; i < outArgs.size(); i++) {
6135//                                              Argument argument = outArgs.get(i);
6136//                                              Variable variable = modelFactory.createVariable(argument.getName(), false);
6137//                                              if (variable != null) {
6138//                                                      DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
6139//                                                      dataflowRelation.setEffectType(EffectType.output);
6140//                                                      dataflowRelation
6141//                                                                      .addSource(new ResultColumnRelationshipElement(resultSet.getColumns().get(i)));
6142//                                                      dataflowRelation
6143//                                                                      .setTarget(new TableColumnRelationshipElement(variable.getColumns().get(0)));
6144//                                              }
6145//                                      }
6146//
6147//                              }
6148                        }
6149
6150                        if (stmt instanceof TCreateFunctionStmt && ((TCreateFunctionStmt)stmt).getSqlExpression()!=null) {
6151                                TCreateFunctionStmt createFunction = (TCreateFunctionStmt) stmt;
6152                                TExpression returnExpression = createFunction.getSqlExpression();
6153
6154                                ResultSet returnResult = modelFactory.createResultSet(stmt, false);
6155                                ResultColumn resultColumn = modelFactory.createResultColumn(returnResult, returnExpression);
6156
6157                                columnsInExpr visitor = new columnsInExpr();
6158                                returnExpression.inOrderTraverse(visitor);
6159
6160                                List<TObjectName> objectNames = visitor.getObjectNames();
6161                                List<TParseTreeNode> functions = visitor.getFunctions();
6162                                List<TParseTreeNode> constants = visitor.getConstants();
6163                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
6164
6165                                if (functions != null && !functions.isEmpty()) {
6166                                        analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function);
6167                                }
6168                                if (subquerys != null && !subquerys.isEmpty()) {
6169                                        analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.select);
6170                                }
6171                                if (objectNames != null && !objectNames.isEmpty()) {
6172                                        analyzeDataFlowRelation(resultColumn, objectNames, EffectType.select, functions);
6173                                }
6174                                if (constants != null && !constants.isEmpty()) {
6175                                        analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.select, functions);
6176                                }
6177
6178                                String procedureParent = SQLUtil.trimColumnStringQuote(getProcedureParentName(stmt));
6179                                modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent),
6180                                                returnResult);
6181                        }
6182                } finally {
6183                        ModelBindingManager.removeGlobalProcedure();
6184                }
6185
6186        }
6187
6188        private boolean isSimpleDataType(TTypeName dataType) {
6189                if (dataType.getDataType() == EDataType.variant_t) {
6190                        return false;
6191                }
6192                if (dataType.getDataType() == EDataType.cursor_t) {
6193                        return false;
6194                }
6195                if (dataType.getDataType() == EDataType.generic_t) {
6196                        return false;
6197                }
6198                if (dataType.getDataType() == EDataType.unknown_t) {
6199                        return false;
6200                }
6201                if (dataType.getDataType() == EDataType.sql_variant_t) {
6202                        return false;
6203                }
6204                if (dataType.getDataType() == EDataType.table_t) {
6205                        return false;
6206                }
6207                if (dataType.getDataType() == EDataType.raw_t) {
6208                        return false;
6209                }
6210                if (dataType.getDataType() == EDataType.resultset_t) {
6211                        return false;
6212                }
6213                if (dataType.getDataType() == EDataType.row_t) {
6214                        return false;
6215                }
6216                if (dataType.getDataType() == EDataType.map_t) {
6217                        return false;
6218                }
6219                if (dataType.getDataType() == EDataType.anyType_t) {
6220                        return false;
6221                }
6222                if (dataType.getDataType() == EDataType.struct_t) {
6223                        return false;
6224                }
6225                if (dataType.getDataType() == EDataType.structType_t) {
6226                        return false;
6227                }
6228                if (dataType.getDataType() == EDataType.mapType_t) {
6229                        return false;
6230                }
6231                return true;
6232        }
6233
6234        protected void analyzeResultColumnExpressionRelation(Object resultColumn, TExpression expression) {
6235                columnsInExpr visitor = new columnsInExpr();
6236                expression.inOrderTraverse(visitor);
6237                List<TObjectName> objectNames = visitor.getObjectNames();
6238                List<TParseTreeNode> functions = visitor.getFunctions();
6239                List<TParseTreeNode> constants = visitor.getConstants();
6240                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
6241
6242                if (functions != null && !functions.isEmpty()) {
6243                        analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function);
6244                }
6245                if (subquerys != null && !subquerys.isEmpty()) {
6246                        analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.select);
6247                }
6248                if (objectNames != null && !objectNames.isEmpty()) {
6249                        analyzeDataFlowRelation(resultColumn, objectNames, EffectType.select, functions);
6250                }
6251                if (constants != null && !constants.isEmpty()) {
6252                        analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.select, functions);
6253                }
6254        }
6255
6256        private void analyzeDb2ReturnStmt(TDb2ReturnStmt stmt) {
6257                if (stmt.getReturnExpr() != null) {
6258                        TExpression returnExpression = stmt.getReturnExpr();
6259                        ResultSet returnResult = modelFactory.createResultSet(stmt, true);
6260                        ResultColumn resultColumn = modelFactory.createResultColumn(returnResult, returnExpression);
6261                        
6262                        columnsInExpr visitor = new columnsInExpr();
6263                        stmt.getReturnExpr().inOrderTraverse(visitor);
6264                        
6265                        List<TObjectName> objectNames = visitor.getObjectNames();
6266                        List<TParseTreeNode> functions = visitor.getFunctions();
6267                        List<TParseTreeNode> constants = visitor.getConstants();
6268                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
6269
6270                        if (functions != null && !functions.isEmpty()) {
6271                                analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function);
6272                        }
6273                        if (subquerys != null && !subquerys.isEmpty()) {
6274                                analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.select);
6275                        }
6276                        if (objectNames != null && !objectNames.isEmpty()) {
6277                                analyzeDataFlowRelation(resultColumn, objectNames, EffectType.select, functions);
6278                        }
6279                        if (constants != null && !constants.isEmpty()) {
6280                                analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.select, functions);
6281                        }
6282
6283                        String procedureParent = SQLUtil.trimColumnStringQuote(getProcedureParentName(stmt));
6284                        if (procedureParent != null) {
6285                                modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent),
6286                                                returnResult);
6287                        }
6288                }
6289        }
6290
6291        private void analyzeReturnStmt(TReturnStmt stmt) {
6292                if (stmt.getResultColumnList() != null) {
6293                        ResultSet returnResult = modelFactory.createResultSet(stmt, true);
6294                        for (TResultColumn column : stmt.getResultColumnList()) {
6295                                ResultColumn resultColumn = modelFactory.createResultColumn(returnResult, column);
6296
6297                                columnsInExpr visitor = new columnsInExpr();
6298                                column.getExpr().inOrderTraverse(visitor);
6299
6300                                List<TObjectName> objectNames = visitor.getObjectNames();
6301                                List<TParseTreeNode> functions = visitor.getFunctions();
6302                                List<TParseTreeNode> constants = visitor.getConstants();
6303                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
6304
6305                                if (functions != null && !functions.isEmpty()) {
6306                                        analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function);
6307                                }
6308                                if (subquerys != null && !subquerys.isEmpty()) {
6309                                        analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.select);
6310                                }
6311                                if (objectNames != null && !objectNames.isEmpty()) {
6312                                        analyzeDataFlowRelation(resultColumn, objectNames, EffectType.select, functions);
6313                                }
6314                                if (constants != null && !constants.isEmpty()) {
6315                                        analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.select, functions);
6316                                }
6317
6318                        }
6319
6320                        String procedureParent = SQLUtil.trimColumnStringQuote(getProcedureParentName(stmt));
6321                        modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent),
6322                                        returnResult);
6323                }
6324                else if(stmt.getExpression()!=null){
6325                        TExpression returnExpression = stmt.getExpression();
6326                        ResultSet returnResult = modelFactory.createResultSet(stmt, true);
6327                        
6328                        columnsInExpr visitor = null;
6329                        List<TSelectSqlStatement> subquerys = null;
6330                        
6331                        if (returnExpression.getFunctionCall() != null
6332                                        && returnExpression.getFunctionCall().getFunctionName().toString().equalsIgnoreCase("table")
6333                                        && returnExpression.getFunctionCall().getArgs() != null
6334                                        && returnExpression.getFunctionCall().getArgs().size()>0) {
6335                                visitor = new columnsInExpr();
6336                                returnExpression.getFunctionCall().getArgs().getExpression(0).inOrderTraverse(visitor);
6337                                subquerys = visitor.getSubquerys();
6338                                if (subquerys != null && !subquerys.isEmpty()) {
6339                                        analyzeSelectStmt(subquerys.get(0));
6340                                        ResultSet resultSet = (ResultSet) modelManager.getModel(subquerys.get(0));
6341                                        if (resultSet != null && resultSet.getColumns() != null) {
6342                                                for (ResultColumn column : resultSet.getColumns()) {
6343                                                        TObjectName columnName = new TObjectName();
6344                                                        columnName.setString(column.getName());
6345                                                        ResultColumn resultColumn = modelFactory.createResultColumn(returnResult, columnName);
6346                                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
6347                                                        dataflowRelation.setEffectType(EffectType.select);
6348                                                        dataflowRelation.addSource(new ResultColumnRelationshipElement(column));
6349                                                        dataflowRelation.setTarget(new ResultColumnRelationshipElement(resultColumn));
6350                                                }
6351                                        }
6352                                        returnResult.setDetermined(resultSet.isDetermined());
6353                                        String procedureParent = SQLUtil.trimColumnStringQuote(getProcedureParentName(stmt));
6354                                        modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent),
6355                                                        returnResult);
6356                                        return;
6357                                }
6358                        }
6359                        
6360                        ResultColumn resultColumn = null;
6361                        if (returnExpression.getExpressionType() == EExpressionType.simple_object_name_t) {
6362                                TObjectName columnName = new TObjectName();
6363                                columnName.setString("*");
6364                                resultColumn = modelFactory.createResultColumn(returnResult, columnName);
6365                        }
6366                        else {
6367                                resultColumn = modelFactory.createResultColumn(returnResult, returnExpression);
6368                        }
6369
6370                        visitor = new columnsInExpr();
6371                        stmt.getExpression().inOrderTraverse(visitor);
6372
6373                        List<TObjectName> objectNames = visitor.getObjectNames();
6374                        List<TParseTreeNode> functions = visitor.getFunctions();
6375                        List<TParseTreeNode> constants = visitor.getConstants();
6376                        subquerys = visitor.getSubquerys();
6377
6378                        if (functions != null && !functions.isEmpty()) {
6379                                analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function);
6380                        }
6381                        if (subquerys != null && !subquerys.isEmpty()) {
6382                                analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.select);
6383                        }
6384                        if (objectNames != null && !objectNames.isEmpty()) {
6385                                DataFlowRelationship relation = analyzeDataFlowRelation(resultColumn, objectNames, EffectType.select, functions);
6386                                //如果variable对应的不是一个复杂结构，则resultColumn不要设置为*
6387                                if (relation != null && relation.getTarget().getElement() == resultColumn && relation.getSources().size() == 1) {
6388                                        Object column = relation.getSources().iterator().next().getElement();
6389                                        boolean star = true;
6390                                        if (column instanceof TableColumn && ((TableColumn) column).getName().indexOf("*") == -1) {
6391                                                star = false;
6392                                        }
6393                                        if (column instanceof ResultColumn && ((ResultColumn) column).getName().indexOf("*") == -1) {
6394                                                star = false;
6395                                        }
6396                                        if (!star && returnExpression.getExpressionType() == EExpressionType.simple_object_name_t) {
6397                                                resultColumn = modelFactory.createResultColumn(returnResult, returnExpression.getObjectOperand());
6398                                                returnResult.getColumns().clear();
6399                                                returnResult.addColumn(resultColumn);
6400                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
6401                                        }
6402                                }
6403                        }
6404                        if (constants != null && !constants.isEmpty()) {
6405                                analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.select, functions);
6406                        }
6407
6408                        String procedureParent = SQLUtil.trimColumnStringQuote(getProcedureParentName(stmt));
6409                        if (procedureParent != null) {
6410                                modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent),
6411                                                returnResult);
6412                        }
6413                }
6414        }
6415
6416        private void analyzeMssqlReturnStmt(TMssqlReturn stmt) {
6417                if (stmt.getResultColumnList() != null) {
6418                        ResultSet returnResult = modelFactory.createResultSet(stmt, true);
6419                        for (TResultColumn column : stmt.getResultColumnList()) {
6420                                ResultColumn resultColumn = modelFactory.createResultColumn(returnResult, column);
6421
6422                                columnsInExpr visitor = new columnsInExpr();
6423                                column.getExpr().inOrderTraverse(visitor);
6424
6425                                List<TObjectName> objectNames = visitor.getObjectNames();
6426                                List<TParseTreeNode> functions = visitor.getFunctions();
6427                                List<TParseTreeNode> constants = visitor.getConstants();
6428                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
6429
6430                                if (functions != null && !functions.isEmpty()) {
6431                                        analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function);
6432                                }
6433                                if (subquerys != null && !subquerys.isEmpty()) {
6434                                        analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.select);
6435                                }
6436                                if (objectNames != null && !objectNames.isEmpty()) {
6437                                        analyzeDataFlowRelation(resultColumn, objectNames, EffectType.select, functions);
6438                                }
6439                                if (constants != null && !constants.isEmpty()) {
6440                                        analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.select, functions);
6441                                }
6442
6443                        }
6444
6445                        String procedureParent = SQLUtil.trimColumnStringQuote(getProcedureParentName(stmt));
6446                        modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent),
6447                                        returnResult);
6448                }
6449                else if(stmt.getReturnExpr()!=null){
6450                        TExpression returnExpression = stmt.getReturnExpr();
6451                        ResultSet returnResult = modelFactory.createResultSet(stmt, true);
6452                        
6453                        if (returnExpression.getSubQuery() == null) {
6454                                ResultColumn resultColumn = modelFactory.createResultColumn(returnResult, returnExpression);
6455                                columnsInExpr visitor = new columnsInExpr();
6456                                stmt.getReturnExpr().inOrderTraverse(visitor);
6457
6458                                List<TObjectName> objectNames = visitor.getObjectNames();
6459                                List<TParseTreeNode> functions = visitor.getFunctions();
6460                                List<TParseTreeNode> constants = visitor.getConstants();
6461                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
6462
6463                                if (functions != null && !functions.isEmpty()) {
6464                                        analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function);
6465                                }
6466                                if (subquerys != null && !subquerys.isEmpty()) {
6467                                        analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.select);
6468                                }
6469                                if (objectNames != null && !objectNames.isEmpty()) {
6470                                        analyzeDataFlowRelation(resultColumn, objectNames, EffectType.select, functions);
6471                                }
6472                                if (constants != null && !constants.isEmpty()) {
6473                                        analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.select, functions);
6474                                }
6475                        }
6476                        else {
6477                                analyzeSelectStmt(returnExpression.getSubQuery());
6478                                ResultSet subResultSet = (ResultSet)modelManager.getModel(returnExpression.getSubQuery());
6479                                if (subResultSet != null) {
6480                                        for (ResultColumn subResultColumn : subResultSet.getColumns()) {
6481                                                TObjectName objectName = new TObjectName();
6482                                                objectName.setString(subResultColumn.getName());
6483                                                ResultColumn resultColumn = modelFactory.createResultColumn(returnResult, objectName);
6484                                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
6485                                                dataflowRelation.setEffectType(EffectType.select);
6486                                                dataflowRelation.addSource(new ResultColumnRelationshipElement(subResultColumn));
6487                                                dataflowRelation.setTarget(new ResultColumnRelationshipElement(resultColumn));
6488
6489                                        }
6490                                        
6491                                        if(subResultSet.getRelationRows().hasRelation()) {
6492                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
6493                                                impactRelation.setEffectType(EffectType.select);
6494                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
6495                                                                subResultSet.getRelationRows()));
6496                                                impactRelation.setTarget(
6497                                                                new RelationRowsRelationshipElement<ResultSetRelationRows>(returnResult.getRelationRows()));
6498                                        }
6499                                }
6500                        }
6501                        
6502                        String procedureParent = SQLUtil.trimColumnStringQuote(getProcedureParentName(stmt));
6503                        if (procedureParent != null) {
6504                                modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent),
6505                                                returnResult);
6506                        }
6507                }
6508        }
6509        
6510        private void analyzeFetchStmt(TFetchStmt stmt) {
6511                if (stmt.getVariableNames() != null) {
6512                        for (int i = 0; i < stmt.getVariableNames().size(); i++) {
6513                                TExpression variableExpression = stmt.getVariableNames().getExpression(i);
6514                                if (variableExpression.getExpressionType() == EExpressionType.simple_object_name_t) {
6515                                        TObjectName columnObject = variableExpression.getObjectOperand();
6516                                        if (columnObject.getDbObjectType() == EDbObjectType.variable) {
6517                                                continue;
6518                                        }
6519
6520                                        if (columnObject.getColumnNameOnly().startsWith("@") && (option.getVendor() == EDbVendor.dbvmssql
6521                                                        || option.getVendor() == EDbVendor.dbvazuresql)) {
6522                                                continue;
6523                                        }
6524
6525                                        if (columnObject.getColumnNameOnly().startsWith(":") && (option.getVendor() == EDbVendor.dbvhana
6526                                                        || option.getVendor() == EDbVendor.dbvteradata)) {
6527                                                continue;
6528                                        }
6529
6530                                        Variable cursorVariable = modelFactory.createVariable(columnObject);
6531                                        cursorVariable.setSubType(SubType.record);
6532                                        if (cursorVariable.isDetermined()) {
6533                                                if (stmt.getCursorName() != null) {
6534                                                        String procedureName = DlineageUtil.getProcedureParentName(stmt);
6535                                                        String variableString = stmt.getCursorName().toString();
6536                                                        if (variableString.startsWith(":")) {
6537                                                                variableString = variableString.substring(variableString.indexOf(":") + 1);
6538                                                        }
6539                                                        if (!SQLUtil.isEmpty(procedureName)) {
6540                                                                variableString = procedureName + "."
6541                                                                                + SQLUtil.getIdentifierNormalTableName(variableString);
6542                                                        }
6543                                                        Table cursor = modelManager.getTableByName(DlineageUtil.getTableFullName(variableString));
6544                                                        if (cursor != null) {
6545                                                                for (TableColumn variableProperty : cursorVariable.getColumns()) {
6546                                                                        boolean flag = false;
6547                                                                        for (TableColumn cursorColumn : cursor.getColumns()) {
6548                                                                                if (getColumnName(cursorColumn.getName())
6549                                                                                                .equalsIgnoreCase(variableProperty.getName())) {
6550                                                                                        DataFlowRelationship dataflowRelation = modelFactory
6551                                                                                                        .createDataFlowRelation();
6552                                                                                        dataflowRelation.setEffectType(EffectType.cursor);
6553                                                                                        dataflowRelation
6554                                                                                                        .addSource(new TableColumnRelationshipElement(cursorColumn));
6555                                                                                        dataflowRelation
6556                                                                                                        .setTarget(new TableColumnRelationshipElement(variableProperty));
6557                                                                                        flag = true;
6558                                                                                        break;
6559                                                                                }
6560                                                                        }
6561
6562                                                                        if (!flag) {
6563                                                                                if (cursor.getColumns().size() == stmt.getVariableNames().size()) {
6564                                                                                        DataFlowRelationship dataflowRelation = modelFactory
6565                                                                                                        .createDataFlowRelation();
6566                                                                                        dataflowRelation.setEffectType(EffectType.cursor);
6567                                                                                        dataflowRelation
6568                                                                                                        .setTarget(new TableColumnRelationshipElement(variableProperty));
6569                                                                                        dataflowRelation.addSource(
6570                                                                                                        new TableColumnRelationshipElement(cursor.getColumns().get(i)));
6571                                                                                }
6572                                                                                else {
6573                                                                                        for (int j = 0; j < cursor.getColumns().size(); j++) {
6574                                                                                                DataFlowRelationship dataflowRelation = modelFactory
6575                                                                                                                .createDataFlowRelation();
6576                                                                                                dataflowRelation.setEffectType(EffectType.cursor);
6577                                                                                                if (stmt.getVariableNames().size() == 1) {
6578                                                                                                        dataflowRelation.addSource(new TableColumnRelationshipElement(
6579                                                                                                                        cursor.getColumns().get(j)));
6580                                                                                                } else {
6581                                                                                                        dataflowRelation.addSource(new TableColumnRelationshipElement(
6582                                                                                                                        cursor.getColumns().get(j), i));
6583                                                                                                }
6584                                                                                                dataflowRelation.setTarget(
6585                                                                                                                new TableColumnRelationshipElement(variableProperty));
6586                                                                                        }
6587                                                                                }
6588                                                                        }
6589                                                                }
6590                                                        }
6591                                                }
6592                                                
6593                                        } else {
6594                                                TableColumn variableProperty = null;
6595                                                if (stmt.getVariableNames().size() == 1) {
6596                                                        if (cursorVariable.getColumns() == null || cursorVariable.getColumns().isEmpty()) {
6597                                                                TObjectName starColumn = new TObjectName();
6598                                                                starColumn.setString("*");
6599                                                                variableProperty = modelFactory.createTableColumn(cursorVariable, starColumn, true);
6600                                                        } else {
6601                                                                variableProperty = cursorVariable.getColumns().get(0);
6602                                                        }
6603                                                } else {
6604                                                        variableProperty = modelFactory.createTableColumn(cursorVariable, columnObject, true);
6605                                                }
6606
6607                                                if (stmt.getCursorName() != null) {
6608                                                        String procedureName = DlineageUtil.getProcedureParentName(stmt);
6609                                                        String variableString = stmt.getCursorName().toString();
6610                                                        if (variableString.startsWith(":")) {
6611                                                                variableString = variableString.substring(variableString.indexOf(":") + 1);
6612                                                        }
6613                                                        if (!SQLUtil.isEmpty(procedureName)) {
6614                                                                variableString = procedureName + "."
6615                                                                                + SQLUtil.getIdentifierNormalTableName(variableString);
6616                                                        }
6617                                                        Table cursor = modelManager.getTableByName(DlineageUtil.getTableFullName(variableString));
6618                                                        if (cursor != null) {
6619                                                                for (int j = 0; j < cursor.getColumns().size(); j++) {
6620                                                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
6621                                                                        dataflowRelation.setEffectType(EffectType.cursor);
6622                                                                        if (stmt.getVariableNames().size() == 1) {
6623                                                                                dataflowRelation.addSource(
6624                                                                                                new TableColumnRelationshipElement(cursor.getColumns().get(j)));
6625                                                                        } else {
6626                                                                                dataflowRelation.addSource(
6627                                                                                                new TableColumnRelationshipElement(cursor.getColumns().get(j), i));
6628                                                                        }
6629                                                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(variableProperty));
6630                                                                }
6631                                                        }
6632                                                }
6633                                        }
6634                                }
6635                        }
6636                }
6637        }
6638
6639        private void analyzeFetchStmt(TMssqlFetch stmt) {
6640                if (stmt.getVariableNames() != null) {
6641                        for (int i = 0; i < stmt.getVariableNames().size(); i++) {
6642                                TObjectName columnObject = stmt.getVariableNames().getObjectName(i);
6643                                Variable cursorVariable = modelFactory.createVariable(columnObject);
6644                                cursorVariable.setCreateTable(true);
6645                                cursorVariable.setSubType(SubType.record);
6646                                TableColumn variableProperty = null;
6647                                if (stmt.getVariableNames().size() == 1) {
6648                                        if (cursorVariable.getColumns() == null || cursorVariable.getColumns().isEmpty()) {
6649                                                TObjectName starColumn = new TObjectName();
6650                                                starColumn.setString("*");
6651                                                variableProperty = modelFactory.createTableColumn(cursorVariable, starColumn, true);
6652                                        } else {
6653                                                variableProperty = cursorVariable.getColumns().get(0);
6654                                        }
6655                                } else {
6656                                        variableProperty = modelFactory.createTableColumn(cursorVariable, columnObject, true);
6657                                }
6658
6659                                if (stmt.getCursorName() != null) {
6660                                        String procedureName = DlineageUtil.getProcedureParentName(stmt);
6661                                        String variableString = stmt.getCursorName().toString();
6662                                        if (variableString.startsWith(":")) {
6663                                                variableString = variableString.substring(variableString.indexOf(":") + 1);
6664                                        }
6665                                        if (!SQLUtil.isEmpty(procedureName)) {
6666                                                variableString = procedureName + "." + SQLUtil.getIdentifierNormalTableName(variableString);
6667                                        }
6668                                        Table cursor = modelManager
6669                                                        .getTableByName(DlineageUtil.getTableFullName(variableString));
6670                                        if (cursor != null) {
6671                                                for (int j = 0; j < cursor.getColumns().size(); j++) {
6672                                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
6673                                                        dataflowRelation.setEffectType(EffectType.cursor);
6674                                                        if (stmt.getVariableNames().size() == 1) {
6675                                                                dataflowRelation
6676                                                                                .addSource(new TableColumnRelationshipElement(cursor.getColumns().get(j)));
6677                                                        } else {
6678                                                                dataflowRelation
6679                                                                                .addSource(new TableColumnRelationshipElement(cursor.getColumns().get(j), i));
6680                                                        }
6681                                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(variableProperty));
6682                                                }
6683                                        }
6684                                }
6685                        }
6686
6687                }
6688        }
6689
6690        private void analyzeLoopStmt(TLoopStmt stmt) {
6691
6692                if (stmt.getCursorName() != null && stmt.getIndexName() != null) {
6693                        modelManager.bindCursorIndex(stmt.getIndexName(), stmt.getCursorName());
6694                }
6695
6696                if (stmt.getRecordName() != null && stmt.getSubquery() != null) {
6697                        Variable cursorTempTable = modelFactory.createCursor(stmt);
6698                        cursorTempTable.setVariable(true);
6699                        cursorTempTable.setSubType(SubType.cursor);
6700                        modelManager.bindCursorModel(stmt, cursorTempTable);
6701                        analyzeSelectStmt(stmt.getSubquery());
6702
6703                        TableColumn cursorColumn = null;
6704                        if (cursorTempTable.getColumns() == null || cursorTempTable.getColumns().isEmpty()) {
6705                                TObjectName starColumn = new TObjectName();
6706                                starColumn.setString("*");
6707                                cursorColumn = modelFactory.createTableColumn(cursorTempTable, starColumn, true);
6708                        } else {
6709                                cursorColumn = cursorTempTable.getColumns().get(0);
6710                        }
6711
6712
6713                        ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt.getSubquery());
6714                        if (resultSetModel != null) {
6715                                for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
6716                                        ResultColumn resultColumn = resultSetModel.getColumns().get(i);
6717                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
6718                                        dataflowRelation.setEffectType(EffectType.cursor);
6719                                        dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn));
6720                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(cursorColumn));
6721                                }
6722                        }
6723                }
6724
6725                for (int i = 0; i < stmt.getStatements().size(); i++) {
6726                        analyzeCustomSqlStmt(stmt.getStatements().get(i));
6727                }
6728        }
6729
6730        private void analyzeForStmt(TForStmt stmt) {
6731                if (stmt.getSubquery() == null) {
6732                        return;
6733                }
6734
6735                Variable cursorTempTable = modelFactory.createCursor(stmt);
6736                cursorTempTable.setVariable(true);
6737                cursorTempTable.setSubType(SubType.cursor);
6738                cursorTempTable.setCreateTable(true);
6739                modelManager.bindCursorModel(stmt, cursorTempTable);
6740                analyzeSelectStmt(stmt.getSubquery());
6741
6742                TableColumn cursorColumn = null;
6743                if (cursorTempTable.getColumns() == null || cursorTempTable.getColumns().isEmpty()) {
6744                        TObjectName starColumn = new TObjectName();
6745                        starColumn.setString("*");
6746                        cursorColumn = modelFactory.createTableColumn(cursorTempTable, starColumn, true);
6747                } else {
6748                        cursorColumn = cursorTempTable.getColumns().get(0);
6749                }
6750
6751
6752                ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt.getSubquery());
6753                if (resultSetModel != null) {
6754                        for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
6755                                ResultColumn resultColumn = resultSetModel.getColumns().get(i);
6756                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
6757                                dataflowRelation.setEffectType(EffectType.cursor);
6758                                dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn));
6759                                dataflowRelation.setTarget(new TableColumnRelationshipElement(cursorColumn));
6760                        }
6761                }
6762
6763                if (stmt.getStatements() != null && stmt.getStatements().size() > 0) {
6764                        for (int i = 0; i < stmt.getStatements().size(); i++) {
6765                                analyzeCustomSqlStmt(stmt.getStatements().get(i));
6766                        }
6767                }
6768        }
6769
6770        private void analyzeVarDeclStmt(TVarDeclStmt stmt) {
6771                TTypeName typeName = stmt.getDataType();
6772                if (typeName != null && typeName.toString().toUpperCase().indexOf("ROWTYPE") != -1) {
6773                        Variable cursorVariable = modelFactory.createVariable(stmt.getElementName());
6774                        cursorVariable.setSubType(SubType.record_type);
6775
6776                        Table variableTable = modelFactory.createTableByName(typeName.getDataTypeName(), false);
6777                        if(!variableTable.isCreateTable()) {
6778                                TObjectName starColumn1 = new TObjectName();
6779                                starColumn1.setString("*");
6780                                TableColumn variableTableStarColumn = modelFactory.createTableColumn(variableTable, starColumn1, true);
6781                                variableTableStarColumn.setShowStar(false);
6782                                variableTableStarColumn.setExpandStar(true);
6783
6784                                TObjectName starColumn = new TObjectName();
6785                                starColumn.setString("*");
6786                                TableColumn variableProperty = modelFactory.createTableColumn(cursorVariable, starColumn, true);
6787                                variableProperty.setShowStar(false);
6788                                variableProperty.setExpandStar(true);
6789
6790                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
6791                                dataflowRelation.setEffectType(EffectType.rowtype);
6792                                dataflowRelation.addSource(new TableColumnRelationshipElement(variableTableStarColumn));
6793                                dataflowRelation.setTarget(new TableColumnRelationshipElement(variableProperty));
6794                        } else {
6795                                for (TableColumn sourceColumn : variableTable.getColumns()) {
6796                                        String columnName = sourceColumn.getName();
6797                                        TObjectName targetColumn = new TObjectName();
6798                                        targetColumn.setString(columnName);
6799                                        TableColumn variableProperty = modelFactory.createTableColumn(cursorVariable, targetColumn, true);
6800                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
6801                                        dataflowRelation.setEffectType(EffectType.rowtype);
6802                                        dataflowRelation.addSource(new TableColumnRelationshipElement(sourceColumn));
6803                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(variableProperty));
6804                                }
6805                        }
6806                } else if (stmt.getElementName() != null) {
6807                        Variable variable = modelFactory.createVariable(stmt.getElementName());
6808                        variable.setCreateTable(true);
6809                        variable.setSubType(SubType.record);
6810                        TableColumn tableColumn = null;
6811                        if (stmt.getDataType() != null && (modelFactory.createVariable(stmt.getDataType().getDataTypeName(), false)!=null || isCursorType(stmt.getDataType().getDataTypeName()))) {
6812                                Variable cursorVariable = modelFactory.createVariable(stmt.getDataType().getDataTypeName(), false);
6813                                if (cursorVariable != null) {
6814                                        if (cursorVariable.getSubType() == SubType.record_type) {
6815                                                variable.setSubType(SubType.record_type);
6816                                        }
6817                                        if(cursorVariable.isDetermined()) {
6818                                                for (int k = 0; k < cursorVariable.getColumns().size(); k++) {
6819                                                        TableColumn sourceColumn = cursorVariable.getColumns().get(k);
6820                                                        TObjectName objectName = new TObjectName();
6821                                                        objectName.setString(sourceColumn.getName());
6822                                                        tableColumn = modelFactory.createTableColumn(variable, objectName, true);
6823                                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
6824                                                        dataflowRelation.setEffectType(EffectType.cursor);
6825                                                        dataflowRelation
6826                                                                        .addSource(new TableColumnRelationshipElement(sourceColumn));
6827                                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(tableColumn));
6828                                                }
6829                                                variable.setDetermined(true);
6830                                                return;
6831                                        }
6832                                        else {
6833                                                TObjectName objectName = new TObjectName();
6834                                                objectName.setString("*");
6835                                                tableColumn = modelFactory.createTableColumn(variable, objectName, true);
6836                                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
6837                                                dataflowRelation.setEffectType(EffectType.cursor);
6838                                                dataflowRelation
6839                                                                .addSource(new TableColumnRelationshipElement(cursorVariable.getColumns().get(0)));
6840                                                dataflowRelation.setTarget(new TableColumnRelationshipElement(tableColumn));
6841                                        }
6842                                }
6843                                else {
6844                                        TObjectName objectName = new TObjectName();
6845                                        objectName.setString("*");
6846                                        tableColumn = modelFactory.createTableColumn(variable, objectName, true);                                       
6847                                }
6848                        } else {
6849                                tableColumn = modelFactory.createTableColumn(variable, stmt.getElementName(), true);
6850                                tableColumn.setVariant(true);
6851                        }
6852
6853                        if (stmt.getDefaultValue() != null) {
6854                                columnsInExpr visitor = new columnsInExpr();
6855                                stmt.getDefaultValue().inOrderTraverse(visitor);
6856                                List<TObjectName> objectNames = visitor.getObjectNames();
6857                                List<TParseTreeNode> functions = visitor.getFunctions();
6858                                List<TParseTreeNode> constants = visitor.getConstants();
6859                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
6860
6861                                if (functions != null && !functions.isEmpty()) {
6862                                        analyzeFunctionDataFlowRelation(tableColumn, functions, EffectType.function);
6863                                }
6864                                if (subquerys != null && !subquerys.isEmpty()) {
6865                                        analyzeSubqueryDataFlowRelation(tableColumn, subquerys, EffectType.select);
6866                                }
6867                                if (objectNames != null && !objectNames.isEmpty()) {
6868                                        analyzeDataFlowRelation(tableColumn, objectNames, EffectType.select, functions);
6869                                }
6870                                if (constants != null && !constants.isEmpty()) {
6871                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select, functions);
6872                                }
6873                        }
6874                        
6875                        
6876                }
6877        }
6878
6879        private boolean isCursorType(String dataTypeName) {
6880                if (dataTypeName != null && dataTypeName.toLowerCase().contains("cursor")) {
6881                        return true;
6882                }
6883                return false;
6884        }
6885
6886        private void analyzeSetStmt(TSetStmt stmt) {
6887                TExpression right = stmt.getVariableValue();
6888                TObjectName columnObject = stmt.getVariableName();
6889                if (columnObject != null) {
6890                        TableColumn tableColumn = null;
6891                        Variable tableModel;
6892                        if (columnObject.toString().indexOf(".") != -1) {
6893                                List<String> splits = SQLUtil.parseNames(columnObject.toString());
6894                                tableModel = modelFactory.createVariable(splits.get(splits.size() - 2));
6895                        } else {
6896                                tableModel = modelFactory.createVariable(columnObject);
6897                        }
6898                        tableModel.setCreateTable(true);
6899                        tableModel.setSubType(SubType.record);
6900
6901                        if (tableModel.getColumns() == null || tableModel.getColumns().isEmpty()) {
6902                                tableColumn = modelFactory.createTableColumn(tableModel, columnObject, true);
6903                        } else {
6904                                tableColumn = tableModel.getColumns().get(0);
6905                        }
6906
6907                        if (tableColumn != null && right!=null) {
6908                                columnsInExpr visitor = new columnsInExpr();
6909                                right.inOrderTraverse(visitor);
6910                                List<TObjectName> objectNames = visitor.getObjectNames();
6911                                List<TParseTreeNode> functions = visitor.getFunctions();
6912                                List<TParseTreeNode> constants = visitor.getConstants();
6913                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
6914
6915                                if (functions != null && !functions.isEmpty()) {
6916                                        analyzeFunctionDataFlowRelation(tableColumn, functions, EffectType.function);
6917                                }
6918                                if (subquerys != null && !subquerys.isEmpty()) {
6919                                        analyzeSubqueryDataFlowRelation(tableColumn, subquerys, EffectType.select);
6920                                }
6921                                if (objectNames != null && !objectNames.isEmpty()) {
6922                                        analyzeDataFlowRelation(tableColumn, objectNames, EffectType.select, functions);
6923                                }
6924                                if (constants != null && !constants.isEmpty()) {
6925                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select, functions);
6926                                }
6927                                
6928                                if(columnObject.toString().equalsIgnoreCase("search_path") && !constants.isEmpty()) {
6929                                        ModelBindingManager.setGlobalSchema(constants.get(0).toString());
6930                                }
6931                        }
6932                }
6933
6934                if(stmt.getAssignments()!=null){
6935                        for (int i = 0; i < stmt.getAssignments().size(); i++) {
6936                                TSetAssignment assignStmt = stmt.getAssignments().getElement(i);
6937                                analyzeSetAssignmentStmt(assignStmt);
6938                        }
6939                }
6940        }
6941
6942        private void analyzeSetAssignmentStmt(TSetAssignment stmt) {
6943                TExpression right = stmt.getParameterValue();
6944                TObjectName columnObject = stmt.getParameterName();
6945                if (columnObject != null) {
6946                        TableColumn tableColumn = null;
6947                        Variable tableModel;
6948                        if (columnObject.toString().indexOf(".") != -1) {
6949                                List<String> splits = SQLUtil.parseNames(columnObject.toString());
6950                                tableModel = modelFactory.createVariable(splits.get(splits.size() - 2));
6951                        } else {
6952                                tableModel = modelFactory.createVariable(columnObject);
6953                        }
6954                        tableModel.setCreateTable(true);
6955                        tableModel.setSubType(SubType.record);
6956                        if (tableModel.getColumns() == null || tableModel.getColumns().isEmpty()) {
6957                                tableColumn = modelFactory.createTableColumn(tableModel, columnObject, true);
6958                        } else {
6959                                tableColumn = tableModel.getColumns().get(0);
6960                        }
6961
6962                        if (tableColumn != null && right!=null) {
6963                                columnsInExpr visitor = new columnsInExpr();
6964                                right.inOrderTraverse(visitor);
6965                                List<TObjectName> objectNames = visitor.getObjectNames();
6966                                List<TParseTreeNode> functions = visitor.getFunctions();
6967                                List<TParseTreeNode> constants = visitor.getConstants();
6968                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
6969
6970                                if (functions != null && !functions.isEmpty()) {
6971                                        analyzeFunctionDataFlowRelation(tableColumn, functions, EffectType.function);
6972                                }
6973                                if (subquerys != null && !subquerys.isEmpty()) {
6974                                        analyzeSubqueryDataFlowRelation(tableColumn, subquerys, EffectType.select);
6975                                }
6976                                if (objectNames != null && !objectNames.isEmpty()) {
6977                                        analyzeDataFlowRelation(tableColumn, objectNames, EffectType.select, functions);
6978                                }
6979                                if (constants != null && !constants.isEmpty()) {
6980                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select, functions);
6981                                }
6982
6983                                if(columnObject.toString().equalsIgnoreCase("search_path") && !constants.isEmpty()) {
6984                                        ModelBindingManager.setGlobalSchema(constants.get(0).toString());
6985                                }
6986                        }
6987                }
6988        }
6989
6990        private void analyzeMssqlSetStmt(TMssqlSet stmt) {
6991                TExpression right = stmt.getVarExpr();
6992                TObjectName columnObject = stmt.getVarName();
6993                if (columnObject != null) {
6994                        TableColumn tableColumn = null;
6995                        Variable tableModel;
6996                        if (columnObject.toString().indexOf(".") != -1) {
6997                                List<String> splits = SQLUtil.parseNames(columnObject.toString());
6998                                tableModel = modelFactory.createVariable(splits.get(splits.size() - 2));
6999                        } else {
7000                                tableModel = modelFactory.createVariable(columnObject);
7001                        }
7002                        tableModel.setCreateTable(true);
7003                        tableModel.setSubType(SubType.record);
7004                        if (tableModel.getColumns() == null || tableModel.getColumns().isEmpty()) {
7005                                tableColumn = modelFactory.createTableColumn(tableModel, columnObject, true);
7006                        }
7007                        else {
7008                                tableColumn = tableModel.getColumns().get(0);
7009                        }
7010
7011                        if (tableColumn != null && right!=null) {
7012                                columnsInExpr visitor = new columnsInExpr();
7013                                right.inOrderTraverse(visitor);
7014                                List<TObjectName> objectNames = visitor.getObjectNames();
7015                                List<TParseTreeNode> functions = visitor.getFunctions();
7016                                List<TParseTreeNode> constants = visitor.getConstants();
7017                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
7018
7019                                if (functions != null && !functions.isEmpty()) {
7020                                        analyzeFunctionDataFlowRelation(tableColumn, functions, EffectType.function);
7021                                }
7022                                if (subquerys != null && !subquerys.isEmpty()) {
7023                                        analyzeSubqueryDataFlowRelation(tableColumn, subquerys, EffectType.select);
7024                                }
7025                                if (objectNames != null && !objectNames.isEmpty()) {
7026                                        analyzeDataFlowRelation(tableColumn, objectNames, EffectType.select, functions);
7027                                }
7028                                if (constants != null && !constants.isEmpty()) {
7029                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select, functions);
7030                                }
7031
7032                                if(columnObject.toString().equalsIgnoreCase("search_path") && !constants.isEmpty()) {
7033                                        ModelBindingManager.setGlobalSchema(constants.get(0).toString());
7034                                }
7035                        }
7036                        else if(tableColumn!=null && stmt.getSubquery()!=null){
7037                                analyzeCustomSqlStmt(stmt.getSubquery());
7038                                analyzeSubqueryDataFlowRelation(tableColumn, Arrays.asList(stmt.getSubquery()), EffectType.select);
7039                        }
7040                }
7041        }
7042
7043        private void analyzeAssignStmt(TAssignStmt stmt) {
7044                TExpression left = stmt.getLeft();
7045                TExpression right = stmt.getExpression();
7046                TObjectName columnObject = null;
7047                if (left == null) {
7048                        columnObject = stmt.getVariableName();
7049                } else if (left.getExpressionType() == EExpressionType.simple_object_name_t) {
7050                        columnObject = left.getObjectOperand();
7051                }
7052                if (columnObject != null) {
7053                        TableColumn tableColumn = null;
7054                        if (columnObject.getDbObjectType() == EDbObjectType.variable || stmt.getVariableName() != null) {
7055                                Variable tableModel;
7056                                if (columnObject.toString().indexOf(".") != -1) {
7057                                        List<String> splits = SQLUtil.parseNames(columnObject.toString());
7058                                        tableModel = modelFactory.createVariable(splits.get(splits.size() - 2));
7059                                } else {
7060                                        tableModel = modelFactory.createVariable(columnObject);
7061                                }
7062                                tableModel.setCreateTable(true);
7063                                tableModel.setSubType(SubType.record);
7064                                if (tableModel.getColumns() == null || tableModel.getColumns().isEmpty()) {
7065                                        tableColumn = modelFactory.createTableColumn(tableModel, columnObject, true);
7066                                } else {
7067                                        tableColumn = tableModel.getColumns().get(0);
7068                                }
7069                        } else {
7070                                List<String> splits = SQLUtil.parseNames(columnObject.toString());
7071                                if (splits.size() > 1) {
7072                                        Table tableModel = modelManager
7073                                                        .getTableByName(DlineageUtil.getTableFullName(splits.get(splits.size() - 2)));
7074                                        if (tableModel == null) {
7075                                                String procedureName = DlineageUtil.getProcedureParentName(stmt);
7076                                                String variableString = splits.get(splits.size() - 2).toString();
7077                                                if (variableString.startsWith(":")) {
7078                                                        variableString = variableString.substring(variableString.indexOf(":") + 1);
7079                                                }
7080                                                if (!SQLUtil.isEmpty(procedureName)) {
7081                                                        variableString = procedureName + "." + SQLUtil.getIdentifierNormalTableName(variableString);
7082                                                }
7083                                                tableModel = modelManager.getTableByName(DlineageUtil.getTableFullName(variableString));
7084                                        }
7085                                        if (tableModel != null) {
7086                                                tableColumn = modelFactory.createTableColumn(tableModel, columnObject, true);
7087                                        }
7088                                }
7089                        }
7090
7091                        if (tableColumn != null && right != null) {
7092                                Transform transform = new Transform();
7093                                transform.setType(Transform.EXPRESSION);
7094                                transform.setCode(right);
7095                                tableColumn.setTransform(transform);;
7096                                columnsInExpr visitor = new columnsInExpr();
7097                                right.inOrderTraverse(visitor);
7098                                List<TObjectName> objectNames = visitor.getObjectNames();
7099                                List<TParseTreeNode> functions = visitor.getFunctions();
7100                                List<TParseTreeNode> constants = visitor.getConstants();
7101                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
7102
7103                                if (functions != null && !functions.isEmpty()) {
7104                                        analyzeFunctionDataFlowRelation(tableColumn, functions, EffectType.function);
7105                                }
7106                                if (subquerys != null && !subquerys.isEmpty()) {
7107                                        analyzeSubqueryDataFlowRelation(tableColumn, subquerys, EffectType.select);
7108                                }
7109                                if (objectNames != null && !objectNames.isEmpty()) {
7110                                        analyzeDataFlowRelation(tableColumn, objectNames, EffectType.select, functions);
7111                                }
7112                                if (constants != null && !constants.isEmpty()) {
7113                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select, functions);
7114                                }
7115                        }
7116                }
7117        }
7118
7119        private void analyzeOpenForStmt(TOpenforStmt stmt) {
7120                if (stmt.getSubquery() == null) {
7121                        return;
7122                }
7123
7124                Variable cursorTempTable = modelFactory.createCursor(stmt);
7125                cursorTempTable.setVariable(true);
7126                cursorTempTable.setSubType(SubType.cursor);
7127                modelManager.bindCursorModel(stmt, cursorTempTable);
7128                analyzeSelectStmt(stmt.getSubquery());
7129
7130                TableColumn cursorColumn = null;
7131                if (cursorTempTable.getColumns() == null || cursorTempTable.getColumns().isEmpty()) {
7132                        TObjectName starColumn = new TObjectName();
7133                        starColumn.setString("*");
7134                        cursorColumn = modelFactory.createTableColumn(cursorTempTable, starColumn, true);
7135                } else {
7136                        cursorColumn = cursorTempTable.getColumns().get(0);
7137                }
7138
7139                ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt.getSubquery());
7140                if (resultSetModel != null) {
7141                        for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
7142                                ResultColumn resultColumn = resultSetModel.getColumns().get(i);
7143                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
7144                                dataflowRelation.setEffectType(EffectType.cursor);
7145                                dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn));
7146                                dataflowRelation.setTarget(new TableColumnRelationshipElement(cursorColumn));
7147                        }
7148                }
7149        }
7150
7151        private void analyzeCursorDeclStmt(TCursorDeclStmt stmt) {
7152                if (stmt.getSubquery() == null) {
7153                        return;
7154                }
7155
7156                Variable cursorTempTable = modelFactory.createCursor(stmt);
7157                cursorTempTable.setVariable(true);
7158                cursorTempTable.setSubType(SubType.cursor);
7159                modelManager.bindCursorModel(stmt, cursorTempTable);
7160                analyzeSelectStmt(stmt.getSubquery());
7161                
7162                ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt.getSubquery());
7163                if(resultSetModel!=null && resultSetModel.isDetermined()) {
7164                        for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
7165                                ResultColumn resultColumn = resultSetModel.getColumns().get(i);
7166                                TObjectName columnName = new TObjectName();
7167                                columnName.setString(resultColumn.getName());
7168                                TableColumn cursorColumn = modelFactory.createTableColumn(cursorTempTable, columnName, true);           
7169                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
7170                                dataflowRelation.setEffectType(EffectType.cursor);
7171                                dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn));
7172                                dataflowRelation.setTarget(new TableColumnRelationshipElement(cursorColumn));
7173                        }
7174                }
7175                else {
7176                        TableColumn cursorColumn = null;
7177                        if (cursorTempTable.getColumns() == null || cursorTempTable.getColumns().isEmpty()) {
7178                                TObjectName starColumn = new TObjectName();
7179                                starColumn.setString("*");
7180                                cursorColumn = modelFactory.createTableColumn(cursorTempTable, starColumn, true);
7181                        } else {
7182                                cursorColumn = cursorTempTable.getColumns().get(0);
7183                        }
7184                        if (resultSetModel != null) {
7185                                for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
7186                                        ResultColumn resultColumn = resultSetModel.getColumns().get(i);
7187                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
7188                                        dataflowRelation.setEffectType(EffectType.cursor);
7189                                        dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn));
7190                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(cursorColumn));
7191                                }
7192                        }
7193                }
7194                
7195        }
7196
7197        private void analyzeDb2Declare(TDb2SqlVariableDeclaration stmt) {
7198                TDeclareVariableList variables = stmt.getVariables();
7199                if (variables == null) {
7200                        return;
7201                }
7202                for (int i = 0; i < variables.size(); i++) {
7203                        TDeclareVariable variable = variables.getDeclareVariable(i);
7204                        if (variable.getTableTypeDefinitions() != null && variable.getTableTypeDefinitions().size() > 0) {
7205
7206
7207                                TObjectName tableName = variable.getVariableName();
7208                                TTableElementList columns = variable.getTableTypeDefinitions();
7209
7210                                Table tableModel = modelFactory.createTableByName(tableName, true);
7211                                tableModel.setCreateTable(true);
7212                                String procedureParent = getProcedureParentName(stmt);
7213                                if (procedureParent != null) {
7214                                        tableModel.setParent(procedureParent);
7215                                }
7216                                modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent), tableModel);
7217
7218                                for (int j = 0; j < columns.size(); j++) {
7219                                        TTableElement tableElement = columns.getTableElement(j);
7220                                        TColumnDefinition column = tableElement.getColumnDefinition();
7221                                        if (column != null && column.getColumnName() != null) {
7222                                                modelFactory.createTableColumn(tableModel, column.getColumnName(), true);
7223                                        }
7224                                }
7225                        } else if (variable.getVariableName() != null) {
7226                                Variable cursorVariable = modelFactory.createVariable(variable.getVariableName());
7227                                cursorVariable.setCreateTable(true);
7228                                cursorVariable.setSubType(SubType.record);
7229                                if(variable.getDatatype()!=null && isSimpleDataType(variable.getDatatype())){
7230                                        TableColumn variableProperty = modelFactory.createTableColumn(cursorVariable,
7231                                                        variable.getVariableName(), true);
7232                                }
7233                                else {
7234                                        TObjectName variableProperties = new TObjectName();
7235                                        variableProperties.setString("*");
7236                                        TableColumn variableProperty = modelFactory.createTableColumn(cursorVariable,
7237                                                        variableProperties, true);
7238                                }
7239                        }
7240                }
7241        }
7242
7243        private void analyzeMssqlDeclare(TMssqlDeclare stmt) {
7244                if (stmt.getDeclareType() == EDeclareType.variable) {
7245                        TDeclareVariableList variables = stmt.getVariables();
7246                        if (variables == null) {
7247                                return;
7248                        }
7249                        for (int i = 0; i < variables.size(); i++) {
7250                                TDeclareVariable variable = variables.getDeclareVariable(i);
7251                                if (variable.getTableTypeDefinitions() != null && variable.getTableTypeDefinitions().size() > 0) {
7252
7253
7254                                        TObjectName tableName = variable.getVariableName();
7255                                        TTableElementList columns = variable.getTableTypeDefinitions();
7256
7257                                        Variable tableModel = modelFactory.createVariable(tableName);
7258                                        tableModel.setVariable(true);
7259                    tableModel.setCreateTable(true);
7260                                        String procedureParent = getProcedureParentName(stmt);
7261                                        if (procedureParent != null) {
7262                                                tableModel.setParent(procedureParent);
7263                                        }
7264                                        modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent), tableModel);
7265
7266                                        for (int j = 0; j < columns.size(); j++) {
7267                                                TTableElement tableElement = columns.getTableElement(j);
7268                                                TColumnDefinition column = tableElement.getColumnDefinition();
7269                                                if (column != null && column.getColumnName() != null) {
7270                                                        modelFactory.createTableColumn(tableModel, column.getColumnName(), true);
7271                                                }
7272                                        }
7273                                } else if (variable.getVariableName() != null) {
7274                                        Variable cursorVariable = modelFactory.createVariable(variable.getVariableName());
7275                                        cursorVariable.setCreateTable(true);
7276                                        cursorVariable.setSubType(SubType.record);
7277                                        TableColumn variableProperty = null;
7278                                        if (variable.getDatatype() != null && isSimpleDataType(variable.getDatatype())) {
7279                                                variableProperty = modelFactory.createTableColumn(cursorVariable, variable.getVariableName(),
7280                                                                true);
7281                                        } else {
7282                                                TObjectName variableProperties = new TObjectName();
7283                                                variableProperties.setString("*");
7284                                                variableProperty = modelFactory.createTableColumn(cursorVariable, variableProperties, true);
7285                                        }
7286                                        
7287                                        if (variable.getDefaultValue() != null && variable.getDefaultValue().getSubQuery() != null) {
7288                                                analyzeSelectStmt(variable.getDefaultValue().getSubQuery());
7289                                                ResultSet resultSetModel = (ResultSet) modelManager
7290                                                                .getModel(variable.getDefaultValue().getSubQuery());
7291                                                if (variableProperty != null && resultSetModel != null) {
7292                                                        for (ResultColumn column : resultSetModel.getColumns()) {
7293                                                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
7294                                                                dataflowRelation.setEffectType(EffectType.select);
7295                                                                dataflowRelation.addSource(new ResultColumnRelationshipElement(column));
7296                                                                dataflowRelation.setTarget(new TableColumnRelationshipElement(variableProperty));
7297                                                        }
7298                                                }
7299                                        }
7300                                }
7301                        }
7302                } else if (stmt.getDeclareType() == EDeclareType.cursor) {
7303                        Variable cursorTempTable = modelFactory.createCursor(stmt);
7304                        cursorTempTable.setVariable(true);
7305                        cursorTempTable.setSubType(SubType.cursor);
7306                        modelManager.bindCursorModel(stmt, cursorTempTable);
7307                        analyzeSelectStmt(stmt.getSubquery());
7308                        ResultSet resultSetModel = (ResultSet)modelManager.getModel(stmt.getSubquery());
7309                        if (resultSetModel != null && resultSetModel.isDetermined()) {
7310                                for(ResultColumn resultColumn: resultSetModel.getColumns()){
7311                                        TObjectName starColumn = new TObjectName();
7312                                        starColumn.setString(resultColumn.getName());
7313                                        TableColumn cursorColumn = modelFactory.createTableColumn(cursorTempTable, starColumn, true);
7314                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
7315                                        dataflowRelation.setEffectType(EffectType.cursor);
7316                                        dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn));
7317                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(cursorColumn));
7318                                }
7319                        }
7320                        else {
7321                                TableColumn cursorColumn = null;
7322                                if (cursorTempTable.getColumns() == null || cursorTempTable.getColumns().isEmpty()) {
7323                                        TObjectName starColumn = new TObjectName();
7324                                        starColumn.setString("*");
7325                                        cursorColumn = modelFactory.createTableColumn(cursorTempTable, starColumn, true);
7326                                        cursorColumn.setShowStar(false);
7327                                        cursorColumn.setExpandStar(true);
7328                                } else {
7329                                        cursorColumn = cursorTempTable.getColumns().get(0);
7330                                }
7331
7332                                if (resultSetModel != null) {
7333                                        for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
7334                                                ResultColumn resultColumn = resultSetModel.getColumns().get(i);
7335                                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
7336                                                dataflowRelation.setEffectType(EffectType.cursor);
7337                                                dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn));
7338                                                dataflowRelation.setTarget(new TableColumnRelationshipElement(cursorColumn));
7339                                        }
7340                                }
7341                        }
7342                }
7343        }
7344
7345
7346        private boolean analyzeMssqlJsonDeclare(TMssqlDeclare stmt, String jsonName, Table jsonTable) {
7347                TDeclareVariableList variables = stmt.getVariables();
7348                if (variables == null) {
7349                        return false;
7350                }
7351                for (int i = 0; i < variables.size(); i++) {
7352                        TDeclareVariable variable = variables.getDeclareVariable(i);
7353                        TObjectName variableName = variable.getVariableName();
7354                        if (DlineageUtil.getIdentifierNormalTableName(variableName.toString())
7355                                        .equals(DlineageUtil.getIdentifierNormalTableName(jsonName))) {
7356                                if (variable.getDefaultValue() != null) {
7357                                        Table variableTable = modelFactory.createJsonVariable(variableName);
7358                                        variableTable.setVariable(true);
7359                                        variableTable.setSubType(SubType.scalar);
7360                                        variableTable.setCreateTable(true);
7361                                        TableColumn property = modelFactory.createVariableProperty(variableTable, variable);
7362
7363                                        for (int j = 0; j < jsonTable.getColumns().size(); j++) {
7364                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7365                                                relation.setEffectType(EffectType.select);
7366                                                relation.setTarget(new TableColumnRelationshipElement(jsonTable.getColumns().get(j)));
7367                                                relation.addSource(new TableColumnRelationshipElement(property));
7368                                        }
7369                                }
7370                                return true;
7371                        }
7372                }
7373                return false;
7374        }
7375
7376        private void analyzeCreateTableStmt(TCreateTableSqlStatement stmt) {
7377                if (stmt.getCloneSourceTable() != null) {
7378                        analyzeCloneTableStmt(stmt);
7379                        return;
7380                }
7381
7382                TTable table = stmt.getTargetTable();
7383
7384                boolean hasDefinition = false;
7385
7386                if (stmt.getColumnList() != null && stmt.getColumnList().size() > 0) {
7387                        hasDefinition = true;
7388                }
7389
7390                if (table != null) {
7391                        Table tableModel = modelFactory.createTableFromCreateDDL(table, hasDefinition || (stmt.getSubQuery() == null && hasDefinition)
7392                                        || (stmt.getSubQuery()!=null && stmt.getSubQuery().getSetOperatorType() == ESetOperatorType.none && stmt.getSubQuery().getResultColumnList().toString().indexOf("*") == -1));
7393                        // Authoritative create-effect classification (parse fact, NOT gated on
7394                        // isDetermined() like fromDDL below). CTAS when there is a defining query,
7395                        // plain DDL otherwise. See dlineage-authoritative-endpoint-classification.md.
7396                        tableModel.setCreatedInSql(true);
7397                        tableModel.setEndpointIntroduction(stmt.getSubQuery() != null
7398                                        ? EndpointIntroduction.CTAS : EndpointIntroduction.CREATE_TABLE_DDL);
7399                        if (stmt.isExternal()) {
7400                                tableModel.setExternal(true);
7401                        }
7402
7403                        if (stmt.getSubQuery() != null) {
7404                                Process process = modelFactory.createProcess(stmt);
7405                                tableModel.addProcess(process);
7406                        }
7407
7408                        String procedureParent = getProcedureParentName(stmt);
7409                        if (procedureParent != null) {
7410                                tableModel.setParent(procedureParent);
7411                        }
7412
7413                        if (stmt.isUsingTemplate()) {
7414                                // Snowflake CREATE TABLE ... USING TEMPLATE <query>: the query only
7415                                // infers the column definitions (e.g. ARRAY_AGG(OBJECT_CONSTRUCT(*))
7416                                // over INFER_SCHEMA); it does NOT populate the table. Analyze it so its
7417                                // own sources (the stage / INFER_SCHEMA) still resolve, but do not
7418                                // project its result columns onto the target table or emit CTAS-style
7419                                // data-flow edges that would misrepresent it as data population.
7420                                if (stmt.getSubQuery() != null) {
7421                                        analyzeSelectStmt(stmt.getSubQuery());
7422                                }
7423                                return;
7424                        }
7425
7426                        if (hasDefinition) {
7427                                if(stmt.getSubQuery()!=null) {
7428                                        TSelectSqlStatement subquery = stmt.getSubQuery();
7429                                        analyzeSelectStmt(subquery);
7430                                        Process process = modelFactory.createProcess(stmt);
7431                                        ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt.getSubQuery());
7432                                        if(resultSetModel.isDetermined()){
7433                                                tableModel.setFromDDL(true);
7434                                        }
7435                                        if (resultSetModel != null) {
7436                                                int resultSetSize = resultSetModel.getColumns().size();
7437                                                int stmtColumnSize = stmt.getColumnList().size();
7438                                                int j = 0;
7439                                                int tableColumnSize = stmtColumnSize;
7440                                                if (resultSetModel.isDetermined() && resultSetSize > tableColumnSize) {
7441                                                        tableColumnSize = resultSetSize;
7442                                                }
7443                                                for (int i = 0; i < tableColumnSize && j < resultSetSize; i++) {
7444                                                        ResultColumn resultColumn = resultSetModel.getColumns().get(j);
7445                                                        if (i < stmtColumnSize) {
7446                                                                TObjectName alias = stmt.getColumnList().getColumn(i).getColumnName();
7447                                                                
7448                                                                if (!resultSetModel.getColumns().get(j).getName().contains("*")) {
7449                                                                        j++;
7450                                                                } else {
7451                                                                        if (resultSetSize - j == stmt.getColumnList().size() - i) {
7452                                                                                j++;
7453                                                                        }
7454                                                                }
7455
7456                                                                if (alias != null) {
7457                                                                        TableColumn viewColumn = modelFactory.createTableColumn(tableModel, alias, true);
7458                                                                        if (resultColumn != null) {
7459                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7460                                                                                relation.setEffectType(EffectType.create_table);
7461                                                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
7462                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
7463                                                                                relation.setProcess(process);
7464                                                                        }
7465                                                                } else if (resultColumn.getColumnObject() instanceof TObjectName) {
7466                                                                        TableColumn viewColumn = modelFactory.createTableColumn(tableModel,
7467                                                                                        (TObjectName) resultColumn.getColumnObject(), true);
7468                                                                        if (resultColumn != null) {
7469                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7470                                                                                relation.setEffectType(EffectType.create_table);
7471                                                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
7472                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
7473                                                                                relation.setProcess(process);
7474                                                                        }
7475                                                                } else if (resultColumn.getColumnObject() instanceof TResultColumn) {
7476                                                                        TableColumn viewColumn = modelFactory.createTableColumn(tableModel,
7477                                                                                        ((TResultColumn) resultColumn.getColumnObject()).getFieldAttr(), true);
7478                                                                        ResultColumn column = (ResultColumn) modelManager
7479                                                                                        .getModel(resultColumn.getColumnObject());
7480                                                                        if (column != null && !column.getStarLinkColumns().isEmpty()) {
7481                                                                                viewColumn.bindStarLinkColumns(column.getStarLinkColumns());
7482                                                                        }
7483                                                                        if (resultColumn != null) {
7484                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7485                                                                                relation.setEffectType(EffectType.create_table);
7486                                                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
7487                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
7488                                                                                relation.setProcess(process);
7489                                                                        }
7490                                                                }
7491                                                        }
7492                                                        else if(resultSetModel.isDetermined()){
7493                                                                TObjectName tableName = new TObjectName();
7494                                                                tableName.setString(resultColumn.getName());
7495                                                                TableColumn viewColumn = modelFactory.createTableColumn(tableModel, tableName, true);
7496                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7497                                                                relation.setEffectType(EffectType.create_table);
7498                                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
7499                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
7500                                                                relation.setProcess(process);
7501                                                                j++;
7502                                                        }
7503                                                }
7504                                                if (resultSetModel != null && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
7505                                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
7506                                                        impactRelation.setEffectType(EffectType.create_table);
7507                                                        impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
7508                                                                        resultSetModel.getRelationRows()));
7509                                                        impactRelation.setTarget(
7510                                                                        new RelationRowsRelationshipElement<TableRelationRows>(tableModel.getRelationRows()));
7511                                                }
7512                                        }
7513
7514                                        if (subquery.getResultColumnList() == null && subquery.getValueClause() != null
7515                                                        && subquery.getValueClause().getValueRows().size() == stmt.getColumnList().size()) {
7516                                                for (int i = 0; i < stmt.getColumnList().size(); i++) {
7517                                                        TObjectName alias = stmt.getColumnList().getColumn(i).getColumnName();
7518
7519                                                        if (alias != null) {
7520                                                                TableColumn viewColumn = modelFactory.createTableColumn(tableModel, alias, true);
7521
7522                                                                TExpression expression = subquery.getValueClause().getValueRows().getValueRowItem(i).getExpr();
7523
7524                                                                columnsInExpr visitor = new columnsInExpr();
7525                                                                expression.inOrderTraverse(visitor);
7526                                                                List<TObjectName> objectNames = visitor.getObjectNames();
7527                                                                List<TParseTreeNode> functions = visitor.getFunctions();
7528
7529                                                                if (functions != null && !functions.isEmpty()) {
7530                                                                        analyzeFunctionDataFlowRelation(viewColumn, functions, EffectType.select);
7531
7532                                                                }
7533
7534                                                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
7535                                                                if (subquerys != null && !subquerys.isEmpty()) {
7536                                                                        analyzeSubqueryDataFlowRelation(viewColumn, subquerys, EffectType.select);
7537                                                                }
7538
7539                                                                analyzeDataFlowRelation(viewColumn, objectNames, EffectType.select, functions);
7540                                                                List<TParseTreeNode> constants = visitor.getConstants();
7541                                                                analyzeConstantDataFlowRelation(viewColumn, constants, EffectType.select, functions);
7542                                                        }
7543                                                }
7544                                        }
7545                                        
7546                                        return;
7547                                }
7548                                else {
7549                                        for (int i = 0; i < stmt.getColumnList().size(); i++) {
7550                                                TColumnDefinition column = stmt.getColumnList().getColumn(i);
7551                                                if (column.getDatatype() != null && column.getDatatype().getTypeOfList() != null
7552                                                                && column.getDatatype().getTypeOfList().getColumnDefList() != null) {
7553                                                        for (int j = 0; j < column.getDatatype().getTypeOfList().getColumnDefList().size(); j++) {
7554                                                                TObjectName columnName = new TObjectName();
7555                                                                if (column.getDatatype().getDataType() == EDataType.array_t) {
7556//                                                                      columnName.setString(column.getColumnName().getColumnNameOnly() + ".array."
7557//                                                                                      + column.getDatatype().getTypeOfList().getColumnDefList().getColumn(j)
7558//                                                                                                      .getColumnName().getColumnNameOnly());
7559                                                                        columnName.setString(column.getColumnName().getColumnNameOnly() + "."
7560                                                                                        + column.getDatatype().getTypeOfList().getColumnDefList().getColumn(j)
7561                                                                                        .getColumnName().getColumnNameOnly());
7562                                                                } else {
7563                                                                        columnName.setString(column.getColumnName().getColumnNameOnly() + "."
7564                                                                                        + column.getDatatype().getTypeOfList().getColumnDefList().getColumn(j)
7565                                                                                                        .getColumnName().getColumnNameOnly());
7566                                                                }
7567                                                                TableColumn tableColumn = modelFactory.createTableColumn(tableModel, columnName,
7568                                                                                hasDefinition);
7569                                                                tableColumn.setStruct(true);
7570                                                                tableColumn.setColumnIndex(i);
7571                                                                appendTableColumnToSQLEnv(tableModel, tableColumn);
7572                                                                
7573                                                                if(option.getAnalyzeMode() == AnalyzeMode.crud) {
7574                                                                        CrudRelationship crudRelationship = modelFactory.createCrudRelation();
7575                                                                        crudRelationship.setTarget(new TableColumnRelationshipElement(tableColumn));
7576                                                                        crudRelationship.setEffectType(EffectType.create_table);
7577                                                                }
7578                                                        }
7579                                                        continue;
7580                                                }
7581                                                if (column.getDatatype() != null && column.getDatatype().getColumnDefList() != null) {
7582                                                        Stack<TColumnDefinition> columnPaths = new Stack<TColumnDefinition>();
7583                                                        flattenStructColumns(hasDefinition, tableModel, column, columnPaths, i);
7584                                                        continue;
7585                                                }
7586                                                TableColumn tableColumn = modelFactory.createTableColumn(tableModel, column.getColumnName(),
7587                                                                hasDefinition);
7588                                                if (tableColumn == null) {
7589                                                        continue;
7590                                                }
7591                                                
7592                                                if(option.getAnalyzeMode() == AnalyzeMode.crud) {
7593                                                        CrudRelationship crudRelationship = modelFactory.createCrudRelation();
7594                                                        crudRelationship.setTarget(new TableColumnRelationshipElement(tableColumn));
7595                                                        crudRelationship.setEffectType(EffectType.create_table);
7596                                                }
7597                                                
7598                                                if (column.getDatatype() != null && column.getDatatype().getDataType() == EDataType.variant_t) {
7599                                                        if (tableColumn != null) {
7600                                                                tableColumn.setVariant(true);
7601                                                        }
7602                                                }
7603                                                if (column.getDatatype() != null) {
7604                                                        String dType = column.getDatatype().getDataTypeName();
7605                                                        if ((!SQLUtil.isEmpty(dType)) && (dType.indexOf("_") > 0)) {
7606                                                                dType = dType.split("_")[0];
7607                                                        }
7608                                                        tableColumn.setDataType(dType);
7609                                                }
7610
7611                                                // Inline (column-level) PK / FK constraint FLAGS. The
7612                                                // table-level pass below (stmt.getTableConstraints())
7613                                                // never sees inline constraints, so set the flags here.
7614                                                // Without this, "id INTEGER PRIMARY KEY" or
7615                                                // "fk INTEGER REFERENCES t (c)" produced no
7616                                                // isPrimaryKey()/isForeignKey() flag, unlike the
7617                                                // equivalent table-level CONSTRAINT form.
7618                                                // EConstraintType.reference is the inline REFERENCES
7619                                                // form; foreign_key is the (rare) inline FOREIGN KEY.
7620                                                // Only flags are set here (no model objects allocated)
7621                                                // so column ids are unchanged; the FK relationship
7622                                                // edges are emitted in a deferred pass after the column
7623                                                // loop (see "inline FK relationships" below), matching
7624                                                // the table-level pass ordering so ids stay stable.
7625                                                TConstraintList inlineConstraints = column.getConstraints();
7626                                                if (inlineConstraints != null) {
7627                                                        for (int c = 0; c < inlineConstraints.size(); c++) {
7628                                                                EConstraintType inlineType = inlineConstraints.getConstraint(c)
7629                                                                                .getConstraint_type();
7630                                                                if (inlineType == EConstraintType.primary_key) {
7631                                                                        tableColumn.setPrimaryKey(true);
7632                                                                } else if (inlineType == EConstraintType.foreign_key
7633                                                                                || inlineType == EConstraintType.reference) {
7634                                                                        tableColumn.setForeignKey(true);
7635                                                                }
7636                                                        }
7637                                                }
7638
7639                                                appendTableColumnToSQLEnv(tableModel, tableColumn);
7640                                        }
7641                                }
7642                        }
7643
7644                        if (stmt.getExternalTableOption("DATA_SOURCE") != null) {
7645                                String dataSourceName = stmt.getExternalTableOption("DATA_SOURCE");
7646                                Table dataSource = modelManager.getTableByName(DlineageUtil.getTableFullName(dataSourceName));
7647                                if (dataSource != null) {
7648                                        TableColumn dataSourceColumn = dataSource.getColumns().get(0);
7649                                        for (int i = 0; i < tableModel.getColumns().size(); i++) {
7650                                                TableColumn column = tableModel.getColumns().get(i);
7651                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7652                                                relation.setTarget(new TableColumnRelationshipElement(column));
7653                                                relation.addSource(new TableColumnRelationshipElement(dataSourceColumn));
7654
7655                                                appendTableColumnToSQLEnv(tableModel, column);
7656                                        }
7657                                }
7658                        }
7659
7660                        if (stmt.getSubQuery() != null) {
7661
7662                                analyzeSelectStmt(stmt.getSubQuery());
7663
7664                                ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt.getSubQuery());
7665                                if (resultSetModel != null && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
7666                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
7667                                        impactRelation.setEffectType(EffectType.create_table);
7668                                        impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
7669                                                        resultSetModel.getRelationRows()));
7670                                        impactRelation.setTarget(
7671                                                        new RelationRowsRelationshipElement<TableRelationRows>(tableModel.getRelationRows()));
7672                                }
7673                        }
7674
7675                        if (stmt.getSubQuery() != null && !stmt.getSubQuery().isCombinedQuery()) {
7676                                SelectResultSet resultSetModel = (SelectResultSet) modelManager
7677                                                .getModel(stmt.getSubQuery().getResultColumnList());
7678                                if(resultSetModel.isDetermined()){
7679                                        tableModel.setDetermined(true);
7680                                        tableModel.setFromDDL(true);
7681                                }
7682                                for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
7683                                        ResultColumn resultColumn = resultSetModel.getColumns().get(i);
7684                                        if (resultSetModel.isDetermined() && resultColumn.getName().endsWith("*")) {
7685                                                continue;
7686                                        }
7687
7688                                        if (resultColumn.getColumnObject() instanceof TResultColumn) {
7689                                                TResultColumn columnObject = (TResultColumn) resultColumn.getColumnObject();
7690
7691                                                TAliasClause alias = columnObject.getAliasClause();
7692                                                if (alias != null && alias.getAliasName() != null) {
7693                                                        TableColumn tableColumn = null;
7694                                                        if (!hasDefinition) {
7695                                                                tableColumn = modelFactory.createTableColumn(tableModel, alias.getAliasName(),
7696                                                                                !hasDefinition);
7697                                                        } else {
7698                                                                tableColumn = tableModel.getColumns().get(i);
7699                                                        }
7700
7701                                                        if (!tableColumn.getName().endsWith("*") && tableModel.isDetermined()) {
7702                                                                appendTableColumnToSQLEnv(tableModel, tableColumn);
7703                                                        }
7704
7705                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7706                                                        relation.setEffectType(EffectType.create_table);
7707                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
7708                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
7709                                                        Process process = modelFactory.createProcess(stmt);
7710                                                        relation.setProcess(process);
7711                                                } else if (columnObject.getFieldAttr() != null || (columnObject.getExpr()!=null && columnObject.getExpr().getExpressionType() == EExpressionType.typecast_t)) {
7712                                                        TableColumn tableColumn = null;
7713                                                        TObjectName columnObj = columnObject.getFieldAttr();
7714                                                        if(columnObj == null) {
7715                                                                columnObj = columnObject.getExpr().getLeftOperand().getObjectOperand();
7716                                                        }
7717                                                        if ((columnObj == null || columnObj.toString().endsWith("*")) && !resultColumn.getName().endsWith("*")) {
7718                                                                columnObj = new TObjectName();
7719                                                                columnObj.setString(resultColumn.getName());
7720                                                        }
7721
7722                                                        if(columnObj == null){
7723                                                                logger.info("Can't handle column " + resultColumn.getName());
7724                                                                continue;
7725                                                        }
7726
7727                                                        if (!hasDefinition) {
7728                                                                tableColumn = modelFactory.createTableColumn(tableModel, columnObj,
7729                                                                                !hasDefinition);
7730                                                                if (tableColumn == null) {
7731                                                                        if (tableModel.getColumns().isEmpty()) {
7732                                                                                logger.info("Add table " + tableModel.getName() + " column " + columnObj.toString() + " failed");
7733                                                                        }
7734                                                                        else if (resultColumn.getName().endsWith("*")) {
7735                                                                                for (int j = 0; j < tableModel.getColumns().size(); j++) {
7736                                                                                        tableColumn = tableModel.getColumns().get(j);
7737                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7738                                                                                        relation.setEffectType(EffectType.create_table);
7739                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
7740                                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
7741                                                                                        if (tableColumn.getName().endsWith("*")
7742                                                                                                        && resultColumn.getName().endsWith("*")) {
7743                                                                                                tableModel.setStarStmt("create_table");
7744                                                                                        }
7745                                                                                        Process process = modelFactory.createProcess(stmt);
7746                                                                                        relation.setProcess(process);
7747                                                                                }
7748                                                                        }
7749                                                                        continue;
7750                                                                }
7751                                                                if (!tableColumn.getName().endsWith("*") && tableModel.isDetermined()) {
7752                                                                        appendTableColumnToSQLEnv(tableModel, tableColumn);
7753                                                                }
7754
7755                                                                Object model = modelManager
7756                                                                                .getModel(resultColumn.getColumnObject());
7757                                                                if (model instanceof ResultColumn) {
7758                                                                        ResultColumn column = (ResultColumn) model;
7759                                                                        if (tableColumn.getName().endsWith("*") && column != null
7760                                                                                        && !column.getStarLinkColumns().isEmpty()) {
7761                                                                                tableColumn.bindStarLinkColumns(column.getStarLinkColumns());
7762                                                                        }
7763
7764                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7765                                                                        relation.setEffectType(EffectType.create_table);
7766                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
7767                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
7768                                                                        if (tableColumn.getName().endsWith("*") && resultColumn.getName().endsWith("*")) {
7769                                                                                tableModel.setStarStmt("create_table");
7770                                                                        }
7771                                                                        Process process = modelFactory.createProcess(stmt);
7772                                                                        relation.setProcess(process);
7773                                                                }
7774                                                                else if(model instanceof LinkedHashMap) {
7775                                                                        String columnName = getColumnNameOnly(resultColumn.getName());
7776                                                                        LinkedHashMap<String, ResultColumn> resultColumns =  (LinkedHashMap<String, ResultColumn>)model;
7777                                                                        if (columnObj.toString().endsWith("*")) {
7778                                                                                for (String key : resultColumns.keySet()) {
7779                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel, resultColumns.get(key).getName());
7780                                                                                        DataFlowRelationship relation = modelFactory
7781                                                                                                        .createDataFlowRelation();
7782                                                                                        relation.setEffectType(EffectType.create_table);
7783                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
7784                                                                                        relation.addSource(
7785                                                                                                        new ResultColumnRelationshipElement(resultColumns.get(key)));
7786                                                                                        Process process = modelFactory.createProcess(stmt);
7787                                                                                        relation.setProcess(process);
7788                                                                                }
7789                                                                        } else if (resultColumns.containsKey(columnName)) {
7790                                                                                ResultColumn column = resultColumns.get(columnName);
7791                                                                                tableColumn = modelFactory.createInsertTableColumn(tableModel, column.getName());
7792                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7793                                                                                relation.setEffectType(EffectType.create_table);
7794                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
7795                                                                                relation.addSource(new ResultColumnRelationshipElement(column));
7796                                                                                Process process = modelFactory.createProcess(stmt);
7797                                                                                relation.setProcess(process);
7798                                                                        }
7799                                                                }
7800                                                        } else {
7801                                                                if (resultColumn.getName().endsWith("*")) {
7802                                                                        for (int j = 0; j < tableModel.getColumns().size(); j++) {
7803                                                                                tableColumn = tableModel.getColumns().get(j);
7804                                                                                ResultColumn column = (ResultColumn) modelManager
7805                                                                                                .getModel(resultColumn.getColumnObject());
7806                                                                                if (tableColumn.getName().endsWith("*") && column != null
7807                                                                                                && !column.getStarLinkColumns().isEmpty()) {
7808                                                                                        tableColumn.bindStarLinkColumns(column.getStarLinkColumns());
7809                                                                                }
7810
7811                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7812                                                                                relation.setEffectType(EffectType.create_table);
7813                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
7814                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
7815                                                                                if (tableColumn.getName().endsWith("*")
7816                                                                                                && resultColumn.getName().endsWith("*")) {
7817                                                                                        tableModel.setStarStmt("create_table");
7818                                                                                        ;
7819                                                                                }
7820                                                                                Process process = modelFactory.createProcess(stmt);
7821                                                                                relation.setProcess(process);
7822                                                                        }
7823                                                                } else {
7824                                                                        tableColumn = tableModel.getColumns().get(i);
7825                                                                        Object model = modelManager
7826                                                                                        .getModel(resultColumn.getColumnObject());
7827                                                                        String columnName = getColumnNameOnly(resultColumn.getName());
7828                                                                        if(model instanceof LinkedHashMap) {
7829                                                                                LinkedHashMap<String, ResultColumn> resultColumns =  (LinkedHashMap<String, ResultColumn>)model;
7830                                                                                if (resultColumns.size() == tableModel.getColumns().size()) {
7831                                                                                        int j = 0;
7832                                                                                        for (String key : resultColumns.keySet()) {
7833                                                                                                if (j == i) {
7834                                                                                                        ResultColumn column = resultColumns.get(key);
7835                                                                                                        DataFlowRelationship relation = modelFactory
7836                                                                                                                        .createDataFlowRelation();
7837                                                                                                        relation.setEffectType(EffectType.create_table);
7838                                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
7839                                                                                                        relation.addSource(new ResultColumnRelationshipElement(column));
7840                                                                                                        Process process = modelFactory.createProcess(stmt);
7841                                                                                                        relation.setProcess(process);
7842                                                                                                }
7843                                                                                                j++;
7844                                                                                        }
7845                                                                                } else if (resultColumns.containsKey(columnName)) {
7846                                                                                        ResultColumn column = resultColumns.get(columnName);
7847                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel, column.getName());
7848                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7849                                                                                        relation.setEffectType(EffectType.create_table);
7850                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
7851                                                                                        relation.addSource(new ResultColumnRelationshipElement(column));
7852                                                                                        Process process = modelFactory.createProcess(stmt);
7853                                                                                        relation.setProcess(process);
7854                                                                                } else {
7855                                                                                        throw new UnsupportedOperationException("Can't handle this star case.");
7856                                                                                }
7857                                                                        }
7858                                                                        else if (model instanceof ResultColumn) {
7859                                                                                ResultColumn column = (ResultColumn) modelManager
7860                                                                                                .getModel(resultColumn.getColumnObject());
7861                                                                                if (tableColumn.getName().endsWith("*") && column != null
7862                                                                                                && !column.getStarLinkColumns().isEmpty()) {
7863                                                                                        tableColumn.bindStarLinkColumns(column.getStarLinkColumns());
7864                                                                                }
7865
7866                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7867                                                                                relation.setEffectType(EffectType.create_table);
7868                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
7869                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
7870                                                                                if (tableColumn.getName().endsWith("*")
7871                                                                                                && resultColumn.getName().endsWith("*")) {
7872                                                                                        tableModel.setStarStmt("create_table");
7873                                                                                }
7874                                                                                Process process = modelFactory.createProcess(stmt);
7875                                                                                relation.setProcess(process);
7876                                                                        }
7877                                                                }
7878                                                        }
7879                                                } else {
7880                                                        TableColumn tableColumn = null;
7881                                                        if (!hasDefinition) {
7882                                                                TObjectName columnName = new TObjectName();
7883                                                                columnName.setString(resultColumn.getColumnObject().toString());
7884                                                                tableColumn = modelFactory.createTableColumn(tableModel, columnName, !hasDefinition);
7885                                                                if(tableColumn == null){
7886                                                                        continue;
7887                                                                }
7888                                                        } else {
7889                                                                tableColumn = tableModel.getColumns().get(i);
7890                                                        }
7891                                                        ResultColumn column = (ResultColumn) modelManager.getModel(resultColumn.getColumnObject());
7892                                                        if (column != null && !column.getStarLinkColumns().isEmpty()) {
7893                                                                tableColumn.bindStarLinkColumns(column.getStarLinkColumns());
7894                                                        }
7895
7896                                                        if (!tableColumn.getName().endsWith("*") && tableModel.isDetermined()) {
7897                                                                appendTableColumnToSQLEnv(tableModel, tableColumn);
7898                                                        }
7899
7900                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7901                                                        relation.setEffectType(EffectType.create_table);
7902                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
7903                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
7904                                                        Process process = modelFactory.createProcess(stmt);
7905                                                        relation.setProcess(process);
7906                                                }
7907                                        } else if (resultColumn.getColumnObject() instanceof TObjectName) {
7908                                                TableColumn tableColumn = null;
7909                                                if (!hasDefinition) {
7910                                                        tableColumn = modelFactory.createTableColumn(tableModel,
7911                                                                        (TObjectName) resultColumn.getColumnObject(), !hasDefinition);
7912                                                } else {
7913                                                        tableColumn = tableModel.getColumns().get(i);
7914                                                }
7915
7916                                                if (!tableColumn.getName().endsWith("*") && tableModel.isDetermined()) {
7917                                                        appendTableColumnToSQLEnv(tableModel, tableColumn);
7918                                                }
7919
7920                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7921                                                relation.setEffectType(EffectType.create_table);
7922                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
7923                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
7924                                                Process process = modelFactory.createProcess(stmt);
7925                                                relation.setProcess(process);
7926                                        }
7927                                }
7928                        } else if (stmt.getSubQuery() != null) {
7929                                SelectSetResultSet resultSetModel = (SelectSetResultSet) modelManager.getModel(stmt.getSubQuery());
7930                                if(resultSetModel.isDetermined()){
7931                                        tableModel.setDetermined(true);
7932                                        tableModel.setFromDDL(true);
7933                                }
7934                                for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
7935                                        ResultColumn resultColumn = resultSetModel.getColumns().get(i);
7936                                        if (resultColumn.getColumnObject() instanceof TResultColumn) {
7937                                                TResultColumn columnObject = (TResultColumn) resultColumn.getColumnObject();
7938
7939                                                TAliasClause alias = columnObject.getAliasClause();
7940                                                if (alias != null && alias.getAliasName() != null) {
7941                                                        TableColumn tableColumn = null;
7942                                                        if (!hasDefinition) {
7943                                                                tableColumn = modelFactory.createTableColumn(tableModel, alias.getAliasName(),
7944                                                                                !hasDefinition);
7945                                                                if (tableColumn == null) {
7946                                                                        continue;
7947                                                                }
7948                                                        } else {
7949                                                                tableColumn = tableModel.getColumns().get(i);
7950                                                        }
7951
7952                                                        if (!tableColumn.getName().endsWith("*") && tableModel.isDetermined()) {
7953                                                                appendTableColumnToSQLEnv(tableModel, tableColumn);
7954                                                        }
7955
7956                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7957                                                        relation.setEffectType(EffectType.create_table);
7958                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
7959                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
7960                                                        Process process = modelFactory.createProcess(stmt);
7961                                                        relation.setProcess(process);
7962                                                } else if (columnObject.getFieldAttr() != null || (columnObject.getExpr()!=null && columnObject.getExpr().getExpressionType() == EExpressionType.typecast_t)) {
7963                                                        TableColumn tableColumn = null;
7964                                                        TObjectName columnObj = columnObject.getFieldAttr();
7965                                                        if(columnObj == null) {
7966                                                                columnObj = columnObject.getExpr().getLeftOperand().getObjectOperand();
7967                                                        }
7968                                                        if (!hasDefinition) {
7969                                                                tableColumn = modelFactory.createTableColumn(tableModel, columnObj,
7970                                                                                !hasDefinition);
7971                                                                if (tableColumn == null) {
7972                                                                        continue;
7973                                                                }
7974                                                        } else {
7975                                                                tableColumn = tableModel.getColumns().get(i);
7976                                                        }
7977                                                        ResultColumn column = (ResultColumn) modelManager.getModel(resultColumn.getColumnObject());
7978                                                        if (column != null && !column.getStarLinkColumns().isEmpty()) {
7979                                                                tableColumn.bindStarLinkColumns(column.getStarLinkColumns());
7980                                                        }
7981
7982                                                        if (!tableColumn.getName().endsWith("*") && tableModel.isDetermined()) {
7983                                                                appendTableColumnToSQLEnv(tableModel, tableColumn);
7984                                                        }
7985
7986                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7987                                                        relation.setEffectType(EffectType.create_table);
7988                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
7989                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
7990                                                        Process process = modelFactory.createProcess(stmt);
7991                                                        relation.setProcess(process);
7992                                                } else {
7993                                                        ErrorInfo errorInfo = new ErrorInfo();
7994                                                        errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
7995                                                        errorInfo.setErrorMessage("Can't handle the table column " + columnObject.toString());
7996                                                        errorInfo.setStartPosition(new Pair3<Long, Long, String>(
7997                                                                        columnObject.getStartToken().lineNo, columnObject.getStartToken().columnNo,
7998                                                                        ModelBindingManager.getGlobalHash()));
7999                                                        errorInfo.setEndPosition(new Pair3<Long, Long, String>(columnObject.getEndToken().lineNo,
8000                                                                        columnObject.getEndToken().columnNo + columnObject.getEndToken().getAstext().length(),
8001                                                                        ModelBindingManager.getGlobalHash()));
8002                                                        errorInfos.add(errorInfo);
8003                                                        continue;
8004                                                }
8005                                        } else if (resultColumn.getColumnObject() instanceof TObjectName) {
8006                                                TableColumn tableColumn = null;
8007                                                if (!hasDefinition) {
8008                                                        tableColumn = modelFactory.createTableColumn(tableModel,
8009                                                                        (TObjectName) resultColumn.getColumnObject(), !hasDefinition);
8010                                                } else {
8011                                                        tableColumn = tableModel.getColumns().get(i);
8012                                                }
8013
8014                                                if (!tableColumn.getName().endsWith("*") && tableModel.isDetermined()) {
8015                                                        appendTableColumnToSQLEnv(tableModel, tableColumn);
8016                                                }
8017
8018                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8019                                                relation.setEffectType(EffectType.create_table);
8020                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
8021                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
8022                                                Process process = modelFactory.createProcess(stmt);
8023                                                relation.setProcess(process);
8024                                        }
8025                                }
8026                        } else if (stmt.getLikeTableName() != null) {
8027                                Table likeTableModel = modelFactory.createTableByName(stmt.getLikeTableName());
8028
8029                                if(likeTableModel.isCreateTable()){
8030                                        for(TableColumn column: likeTableModel.getColumns()){
8031                                                TObjectName tableColumn = new TObjectName();
8032                                                tableColumn.setString(column.getName());
8033                                                TableColumn createTableColumn = modelFactory.createTableColumn(tableModel, tableColumn, true);
8034                                                createTableColumn.setPrimaryKey(column.getPrimaryKey());
8035                                                createTableColumn.setForeignKey(column.getForeignKey());
8036                                                createTableColumn.setIndexKey(column.getIndexKey());
8037                                                createTableColumn.setUnqiueKey(column.getUnqiueKey());
8038                                                createTableColumn.setDataType(column.getDataType());
8039                                        }
8040                                        tableModel.setCreateTable(true);
8041                                }
8042
8043//                              Process process = modelFactory.createProcess(stmt);
8044//                              process.setType("Like Table");
8045//                              tableModel.addProcess(process);
8046//
8047//
8048//                              DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8049//                              relation.setEffectType(EffectType.like_table);
8050//                              relation.setTarget(
8051//                                              new RelationRowsRelationshipElement<TableRelationRows>(tableModel.getRelationRows()));
8052//                              relation.addSource(
8053//                                              new RelationRowsRelationshipElement<TableRelationRows>(likeTableModel.getRelationRows()));
8054//                              relation.setProcess(process);
8055                        }
8056
8057                        if (stmt.getStageLocation() != null && stmt.getStageLocation().getStageName() != null) {
8058                                tableModel
8059                                                .setLocation(DlineageUtil.getTableFullName(stmt.getStageLocation().getStageName().toString()));
8060                                Process process = modelFactory.createProcess(stmt);
8061                                process.setType("Create External Table");
8062                                tableModel.addProcess(process);
8063                                Table stage = modelManager.getTableByName(DlineageUtil.getTableFullName(tableModel.getLocation()));
8064                                if (stage == null) {
8065                                        stage = modelManager.getTableByName(
8066                                                        DlineageUtil.getTableFullName(stmt.getStageLocation().toString().replaceFirst("@", "")));
8067                                }
8068                                if (stage == null) {
8069                                        stage = modelFactory.createTableByName(stmt.getStageLocation().toString().replaceFirst("@", ""),
8070                                                        false);
8071                                        stage.setCreateTable(false);
8072                                        stage.setStage(true);
8073                                        if (stmt.getStageLocation().getPath() != null) {
8074                                                stage.setLocation(stmt.getStageLocation().getPath().toString());
8075                                                TObjectName location = new TObjectName();
8076                                                location.setString(stmt.getStageLocation().getPath().toString());
8077                                                modelFactory.createStageLocation(stage, location);
8078                                        } else if (stmt.getRegex_pattern() != null) {
8079                                                stage.setLocation(stmt.getRegex_pattern());
8080                                                TObjectName location = new TObjectName();
8081                                                location.setString(stmt.getRegex_pattern());
8082                                                modelFactory.createStageLocation(stage, location);
8083                                        } else {
8084                                                stage.setLocation("unknownPath");
8085                                                TObjectName location = new TObjectName();
8086                                                location.setString("unknownPath");
8087                                                modelFactory.createStageLocation(stage, location);
8088                                        }
8089                                }
8090                                if (stage != null && !stage.getColumns().isEmpty()) {
8091                                        if (!tableModel.getColumns().isEmpty()) {
8092                                                for (int i = 0; i < tableModel.getColumns().size(); i++) {
8093                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8094                                                        relation.addSource(new TableColumnRelationshipElement(stage.getColumns().get(0)));
8095                                                        relation.setTarget(new TableColumnRelationshipElement(tableModel.getColumns().get(i)));
8096                                                        relation.setProcess(process);
8097                                                }
8098                                        } else {
8099                                                TObjectName starColumn = new TObjectName();
8100                                                starColumn.setString("*");
8101                                                TableColumn tableColumn = modelFactory.createTableColumn(tableModel, starColumn, true);
8102                                                tableColumn.setExpandStar(false);
8103                                                tableColumn.setShowStar(true);
8104                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8105                                                relation.addSource(new TableColumnRelationshipElement(stage.getColumns().get(0)));
8106                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
8107                                                relation.setProcess(process);
8108                                        }
8109                                }
8110                        } else if (stmt.getTableOptions() != null && !stmt.getTableOptions().isEmpty()) {
8111                                for (int i = 0; i < stmt.getTableOptions().size(); i++) {
8112                                        TCreateTableOption createTableOption = stmt.getTableOptions().get(i);
8113                                        if (createTableOption.getCreateTableOptionType() != ECreateTableOption.etoBigQueryExternal)
8114                                                continue;
8115                                        List<String> uris = createTableOption.getUris();
8116                                        if (uris == null || uris.isEmpty())
8117                                                continue;
8118                                        Process process = modelFactory.createProcess(stmt);
8119                                        process.setType("Create External Table");
8120                                        tableModel.addProcess(process);
8121                                        if (tableModel.getColumns().isEmpty()) {
8122                                                TObjectName starColumn = new TObjectName();
8123                                                starColumn.setString("*");
8124                                                TableColumn tableColumn = modelFactory.createTableColumn(tableModel, starColumn, true);
8125                                                tableColumn.setExpandStar(false);
8126                                                tableColumn.setShowStar(true);
8127                                        }
8128                                        for (String uri : uris) {
8129                                                Table uriFile = modelFactory.createTableByName(uri, true);
8130                                                uriFile.setPath(true);
8131                                                uriFile.setFileFormat(createTableOption.getFormat());
8132                                                for (int j = 0; j < tableModel.getColumns().size(); j++) {
8133                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8134                                                        if (stmt.getColumnList() != null) {
8135                                                                TObjectName fileUri = new TObjectName();
8136                                                                fileUri.setString(tableModel.getColumns().get(j).getColumnObject().toString());
8137                                                                TableColumn fileUriColumn = modelFactory.createFileUri(uriFile, fileUri);
8138                                                                relation.addSource(new TableColumnRelationshipElement(fileUriColumn));
8139                                                        } else {
8140                                                                TObjectName fileUri = new TObjectName();
8141                                                                fileUri.setString("*");
8142                                                                TableColumn fileUriColumn = modelFactory.createFileUri(uriFile, fileUri);
8143                                                                relation.addSource(new TableColumnRelationshipElement(fileUriColumn));
8144                                                        }
8145                                                        relation.setTarget(new TableColumnRelationshipElement(tableModel.getColumns().get(j)));
8146                                                        relation.setProcess(process);
8147                                                }
8148                                                if (tableModel.getColumns().isEmpty()) {
8149                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8150                                                        TObjectName fileUri = new TObjectName();
8151                                                        fileUri.setString("*");
8152                                                        TableColumn fileUriColumn = modelFactory.createFileUri(uriFile, fileUri);
8153                                                        relation.addSource(new TableColumnRelationshipElement(fileUriColumn));
8154                                                        relation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(
8155                                                                        tableModel.getRelationRows()));
8156                                                        relation.setProcess(process);
8157                                                }
8158                                        }
8159                                }
8160                        } else if (stmt.getSubQuery() == null && stmt.getTableLocation() != null) {
8161                                Process process = modelFactory.createProcess(stmt);
8162                                process.setType("Create External Table");
8163                                tableModel.addProcess(process);
8164                                Table uriFile = modelFactory.createTableByName(stmt.getTableLocation(), true);
8165                                uriFile.setPath(true);
8166                                for (int j = 0; j < tableModel.getColumns().size(); j++) {
8167                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8168                                        if (stmt.getColumnList() != null) {
8169                                                TObjectName fileUri = new TObjectName();
8170                                                fileUri.setString(tableModel.getColumns().get(j).getColumnObject().toString());
8171                                                TableColumn fileUriColumn = modelFactory.createFileUri(uriFile, fileUri);
8172                                                relation.addSource(new TableColumnRelationshipElement(fileUriColumn));
8173                                        } else {
8174                                                TObjectName fileUri = new TObjectName();
8175                                                fileUri.setString("*");
8176                                                TableColumn fileUriColumn = modelFactory.createFileUri(uriFile, fileUri);
8177                                                relation.addSource(new TableColumnRelationshipElement(fileUriColumn));
8178                                        }
8179                                        relation.setTarget(new TableColumnRelationshipElement(tableModel.getColumns().get(j)));
8180                                        relation.setProcess(process);
8181                                }
8182                        }
8183
8184                        // Inline (column-level) FK relationships. Emitted here, after the
8185                        // column loop, rather than inline in it, so the referenced-column /
8186                        // relation / process model objects allocate their ids AFTER all
8187                        // column ids (matching the table-level pass below) and column ids
8188                        // stay stable. Mirrors the table-level FK emission: source =
8189                        // referenced column, target = this FK column, effect = foreign_key,
8190                        // plus the optional ER relationship. EConstraintType.reference is
8191                        // the inline REFERENCES form; foreign_key the inline FOREIGN KEY.
8192                        if (stmt.getColumnList() != null) {
8193                                for (int i = 0; i < stmt.getColumnList().size(); i++) {
8194                                        TColumnDefinition fkColumnDef = stmt.getColumnList().getColumn(i);
8195                                        TConstraintList fkConstraints = fkColumnDef.getConstraints();
8196                                        if (fkConstraints == null) {
8197                                                continue;
8198                                        }
8199                                        for (int c = 0; c < fkConstraints.size(); c++) {
8200                                                TConstraint inlineConstraint = fkConstraints.getConstraint(c);
8201                                                EConstraintType inlineType = inlineConstraint.getConstraint_type();
8202                                                if (inlineType != EConstraintType.foreign_key
8203                                                                && inlineType != EConstraintType.reference) {
8204                                                        continue;
8205                                                }
8206                                                TObjectName referencedTableName = inlineConstraint.getReferencedObject();
8207                                                TObjectNameList referencedTableColumns = inlineConstraint.getReferencedColumnList();
8208                                                if (referencedTableName == null || referencedTableColumns == null) {
8209                                                        continue;
8210                                                }
8211                                                TableColumn fkColumn = modelFactory.createTableColumn(tableModel,
8212                                                                fkColumnDef.getColumnName(), true);
8213                                                if (fkColumn == null) {
8214                                                        continue;
8215                                                }
8216                                                Table referencedTable = modelManager.getTableByName(
8217                                                                DlineageUtil.getTableFullName(referencedTableName.toString()));
8218                                                if (referencedTable == null) {
8219                                                        referencedTable = modelFactory.createTableByName(referencedTableName);
8220                                                }
8221                                                for (int j = 0; j < referencedTableColumns.size(); j++) {
8222                                                        TableColumn referencedColumn = modelFactory.createTableColumn(referencedTable,
8223                                                                        referencedTableColumns.getObjectName(j), false);
8224                                                        if (referencedColumn != null) {
8225                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8226                                                                relation.addSource(new TableColumnRelationshipElement(referencedColumn));
8227                                                                relation.setTarget(new TableColumnRelationshipElement(fkColumn));
8228                                                                relation.setEffectType(EffectType.foreign_key);
8229                                                                Process process = modelFactory.createProcess(stmt);
8230                                                                relation.setProcess(process);
8231                                                                if (this.option.isShowERDiagram()) {
8232                                                                        ERRelationship erRelation = modelFactory.createERRelation();
8233                                                                        erRelation.addSource(new TableColumnRelationshipElement(referencedColumn));
8234                                                                        erRelation.setTarget(new TableColumnRelationshipElement(fkColumn));
8235                                                                }
8236                                                        }
8237                                                }
8238                                        }
8239                                }
8240                        }
8241
8242                        if (stmt.getTableConstraints() != null && stmt.getTableConstraints().size() > 0) {
8243                                for (int i = 0; i < stmt.getTableConstraints().size(); i++) {
8244                                        TConstraint createTableConstraint = stmt.getTableConstraints().getConstraint(i);
8245                                        TPTNodeList<TColumnWithSortOrder> keyNames = createTableConstraint.getColumnList();
8246                                        if (keyNames != null) {
8247                                                for (int k = 0; k < keyNames.size(); k++) {
8248                                                        TObjectName keyName = keyNames.getElement(k).getColumnName();
8249                                                        // Skip functional indexes (expression-based indexes) where columnName is null
8250                                                        if (keyName == null) {
8251                                                                continue;
8252                                                        }
8253                                                        TObjectName referencedTableName = createTableConstraint.getReferencedObject();
8254                                                        TObjectNameList referencedTableColumns = createTableConstraint.getReferencedColumnList();
8255
8256                                                        TableColumn tableConstraint = modelFactory.createTableColumn(tableModel, keyName, true);
8257                                                        if (createTableConstraint.getConstraint_type() == EConstraintType.primary_key) {
8258                                                                tableConstraint.setPrimaryKey(true);
8259                                                        } else if (createTableConstraint.getConstraint_type() == EConstraintType.table_index) {
8260                                                                tableConstraint.setIndexKey(true);
8261                                                        } else if (createTableConstraint.getConstraint_type() == EConstraintType.unique) {
8262                                                                tableConstraint.setUnqiueKey(true);
8263                                                        } else if (createTableConstraint.getConstraint_type() == EConstraintType.foreign_key) {
8264                                                                tableConstraint.setForeignKey(true);
8265                                                                Table referencedTable = modelManager
8266                                                                                .getTableByName(DlineageUtil.getTableFullName(referencedTableName.toString()));
8267                                                                if (referencedTable == null) {
8268                                                                        referencedTable = modelFactory.createTableByName(referencedTableName);
8269                                                                }
8270
8271                                                                if (referencedTableColumns != null) {
8272                                                                        for (int j = 0; j < referencedTableColumns.size(); j++) {
8273                                                                                TableColumn tableColumn = modelFactory.createTableColumn(referencedTable,
8274                                                                                                referencedTableColumns.getObjectName(j), false);
8275                                                                                if (tableColumn != null) {
8276                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8277                                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
8278                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableConstraint));
8279                                                                                        relation.setEffectType(EffectType.foreign_key);
8280                                                                                        Process process = modelFactory.createProcess(stmt);
8281                                                                                        relation.setProcess(process);
8282                                                                                        if (this.option.isShowERDiagram()) {
8283                                                                                                ERRelationship erRelation = modelFactory.createERRelation();
8284                                                                                                erRelation.addSource(new TableColumnRelationshipElement(tableColumn));
8285                                                                                                erRelation
8286                                                                                                                .setTarget(new TableColumnRelationshipElement(tableConstraint));
8287                                                                                        }
8288                                                                                }
8289                                                                        }
8290                                                                }
8291                                                        }
8292                                                }
8293                                        }
8294                                }
8295                        }
8296
8297                        if (stmt.getHiveTablePartition() != null && stmt.getHiveTablePartition().getColumnDefList() != null) {
8298                                for (int i = 0; i < stmt.getHiveTablePartition().getColumnDefList().size(); i++) {
8299                                        TColumnDefinition column = stmt.getHiveTablePartition().getColumnDefList().getColumn(i);
8300                                        modelFactory.createTableColumn(tableModel, column.getColumnName(), true);
8301                                        appendTableColumnToSQLEnv(tableModel, column.getColumnName());
8302                                }
8303                        }
8304
8305                } else {
8306                        ErrorInfo errorInfo = new ErrorInfo();
8307                        errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
8308                        errorInfo.setErrorMessage("Can't get target table. CreateTableSqlStatement is " + stmt.toString());
8309                        errorInfo.setStartPosition(new Pair3<Long, Long, String>(stmt.getStartToken().lineNo,
8310                                        stmt.getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
8311                        errorInfo.setEndPosition(new Pair3<Long, Long, String>(stmt.getEndToken().lineNo,
8312                                        stmt.getEndToken().columnNo + stmt.getEndToken().getAstext().length(),
8313                                        ModelBindingManager.getGlobalHash()));
8314                        errorInfos.add(errorInfo);
8315                }
8316        }
8317
8318        protected void flattenStructColumns(boolean hasDefinition, Table tableModel, TColumnDefinition column,
8319                        Stack<TColumnDefinition> columnPaths, int index) {
8320                columnPaths.push(column);
8321                for (int j = 0; j < column.getDatatype().getColumnDefList().size(); j++) {
8322                        TColumnDefinition columnDefinition = column.getDatatype().getColumnDefList().getColumn(j);
8323                        if (columnDefinition.getDatatype().getColumnDefList() != null) {
8324                                flattenStructColumns(hasDefinition, tableModel, columnDefinition, columnPaths, index);
8325                        } else {
8326                                TObjectName columnName = new TObjectName();
8327                                columnName.setString(getColumnName(columnPaths, columnDefinition));
8328                                TableColumn tableColumn = modelFactory.createTableColumn(tableModel, columnName, hasDefinition);
8329                                tableColumn.setColumnIndex(index);
8330                                tableColumn.setStruct(true);
8331                                
8332                                if(option.getAnalyzeMode() == AnalyzeMode.crud) {
8333                                        CrudRelationship crudRelationship = modelFactory.createCrudRelation();
8334                                        crudRelationship.setTarget(new TableColumnRelationshipElement(tableColumn));
8335                                        crudRelationship.setEffectType(EffectType.create_table);
8336                                }
8337                                
8338                                appendTableColumnToSQLEnv(tableModel, tableColumn);
8339                        }
8340                }
8341                columnPaths.pop();
8342        }
8343
8344        private String getColumnName(Stack<TColumnDefinition> columnPaths, TColumnDefinition column) {
8345                StringBuilder buffer = new StringBuilder();
8346                Iterator<TColumnDefinition> iter = columnPaths.iterator();
8347                while(iter.hasNext()) {
8348                        buffer.append(iter.next().getColumnName().getColumnNameOnly()).append(".");
8349                }
8350                buffer.append(column.getColumnName().getColumnNameOnly());
8351                return buffer.toString();
8352        }
8353
8354        private void appendTableColumnToSQLEnv(Table tableModel, TableColumn tableColumn) {
8355                //tableModel如果非determined，请不要添加到sqlenv里
8356                if (sqlenv != null && tableColumn!=null) {
8357                        TSQLSchema schema = sqlenv.getSQLSchema(DlineageUtil.getTableSchema(tableModel), true);
8358                        if (schema != null) {
8359                                TSQLTable tempTable = schema.createTable(DlineageUtil.getSimpleTableName(tableModel.getName()));
8360                                if (tableColumn.hasStarLinkColumn()) {
8361                                        for (String column : tableColumn.getStarLinkColumnNames()) {
8362                                                tempTable.addColumn(DlineageUtil.getColumnNameOnly(column));
8363                                        }
8364                                } else {
8365                                        tempTable.addColumn(DlineageUtil.getColumnNameOnly(tableColumn.getName()));
8366                                }
8367                        }
8368                }
8369        }
8370
8371        private void appendTableColumnToSQLEnv(Table tableModel, TObjectName tableColumn) {
8372                if (sqlenv != null) {
8373                        TSQLSchema schema = sqlenv.getSQLSchema(DlineageUtil.getTableSchema(tableModel), true);
8374                        if (schema != null) {
8375                                TSQLTable tempTable = schema.createTable(DlineageUtil.getSimpleTableName(tableModel.getName()));
8376                                tempTable.addColumn(DlineageUtil.getColumnNameOnly(tableColumn.getColumnNameOnly()));
8377                        }
8378                }
8379        }
8380
8381        private void analyzeCreateStageStmt(TCreateStageStmt stmt) {
8382                TObjectName stageName = stmt.getStageName();
8383
8384                if (stageName != null) {
8385
8386                        Table tableModel = modelFactory.createStage(stageName);
8387                        tableModel.setCreateTable(true);
8388                        tableModel.setStage(true);
8389                        tableModel.setLocation(stmt.getExternalStageURL());
8390
8391                        Process process = modelFactory.createProcess(stmt);
8392                        tableModel.addProcess(process);
8393
8394                        TableColumn locationColumn = null;
8395                        if (stmt.getExternalStageURL() != null) {
8396                                TObjectName location = new TObjectName();
8397                                location.setString(stmt.getExternalStageURL());
8398                                locationColumn = modelFactory.createStageLocation(tableModel, location);
8399
8400                                Table pathModel = modelFactory.createTableByName(stmt.getExternalStageURL(), true);
8401                                pathModel.setPath(true);
8402                                pathModel.setCreateTable(true);
8403                                TObjectName fileUri = new TObjectName();
8404                                fileUri.setString(stmt.getExternalStageURL());
8405                                TableColumn fileUriColumn = modelFactory.createFileUri(pathModel, fileUri);
8406
8407                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8408                                relation.addSource(new TableColumnRelationshipElement(fileUriColumn));
8409                                relation.setTarget(new TableColumnRelationshipElement(locationColumn));
8410                                relation.setProcess(process);
8411
8412                        } else {
8413                                for (TCustomSqlStatement temp : stmt.getGsqlparser().getSqlstatements()) {
8414                                        if (temp instanceof TPutStmt) {
8415                                                TPutStmt put = (TPutStmt) temp;
8416                                                TStageLocation stageLocation = put.getStageLocation();
8417                                                if (stageLocation != null && stageLocation.getStageName() != null) {
8418                                                        Table stage = modelManager.getTableByName(
8419                                                                        DlineageUtil.getTableFullName(stageLocation.getStageName().toString()));
8420                                                        if (stage == tableModel) {
8421                                                                TObjectName location = new TObjectName();
8422                                                                if (!SQLUtil.isEmpty(put.getFileName())) {
8423                                                                        location.setString(put.getFileName());
8424                                                                        locationColumn = modelFactory.createStageLocation(tableModel, location);
8425                                                                }
8426                                                                break;
8427                                                        }
8428                                                }
8429                                        }
8430                                }
8431                        }
8432
8433                        String fileFormat = stmt.getFileFormatName();
8434                        if (fileFormat != null) {
8435                                for (TCustomSqlStatement temp : stmt.getGsqlparser().getSqlstatements()) {
8436                                        if (temp instanceof TCreateFileFormatStmt) {
8437                                                TCreateFileFormatStmt fileFormatStmt = (TCreateFileFormatStmt) temp;
8438                                                if (fileFormatStmt.getFileFormatName() != null
8439                                                                && fileFormatStmt.getFileFormatName().toString().equalsIgnoreCase(fileFormat)) {
8440                                                        tableModel.setFileType(fileFormatStmt.getTypeName());
8441                                                        break;
8442                                                }
8443                                        }
8444                                }
8445                        }
8446
8447                        String procedureParent = getProcedureParentName(stmt);
8448                        if (procedureParent != null) {
8449                                tableModel.setParent(procedureParent);
8450                        }
8451
8452                        if (locationColumn != null) {
8453                                List<Table> tables = modelManager.getTablesByName();
8454                                if (tables != null) {
8455                                        for (Table referTable : tables) {
8456                                                if (referTable.getLocation() != null && referTable.getLocation()
8457                                                                .equals(DlineageUtil.getIdentifierNormalTableName(tableModel.getName()))) {
8458                                                        for (int i = 0; i < referTable.getColumns().size(); i++) {
8459                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8460                                                                relation.addSource(new TableColumnRelationshipElement(locationColumn));
8461                                                                relation.setTarget(new TableColumnRelationshipElement(referTable.getColumns().get(i)));
8462                                                        }
8463                                                }
8464                                        }
8465                                }
8466                        }
8467                } else {
8468                        ErrorInfo errorInfo = new ErrorInfo();
8469                        errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
8470                        errorInfo.setErrorMessage("Can't get target table. CreateStageStmt is " + stmt.toString());
8471                        errorInfo.setStartPosition(new Pair3<Long, Long, String>(stmt.getStartToken().lineNo,
8472                                        stmt.getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
8473                        errorInfo.setEndPosition(new Pair3<Long, Long, String>(stmt.getEndToken().lineNo,
8474                                        stmt.getEndToken().columnNo + stmt.getEndToken().getAstext().length(),
8475                                        ModelBindingManager.getGlobalHash()));
8476                        errorInfo.fillInfo(this);
8477                        errorInfos.add(errorInfo);
8478                }
8479        }
8480
8481        private void analyzeCreateExternalDataSourceStmt(TCreateExternalDataSourceStmt stmt) {
8482                TObjectName dataSourceName = stmt.getDataSourceName();
8483                String locationUrl = stmt.getOption("LOCATION");
8484                if (dataSourceName != null && locationUrl != null) {
8485                        Table tableModel = modelFactory.createDataSource(dataSourceName);
8486                        tableModel.setCreateTable(true);
8487                        tableModel.setDataSource(true);
8488                        tableModel.setLocation(locationUrl);
8489
8490                        Process process = modelFactory.createProcess(stmt);
8491                        tableModel.addProcess(process);
8492
8493                        TObjectName location = new TObjectName();
8494                        location.setString(locationUrl);
8495                        modelFactory.createTableColumn(tableModel, location, true);
8496
8497                        String procedureParent = getProcedureParentName(stmt);
8498                        if (procedureParent != null) {
8499                                tableModel.setParent(procedureParent);
8500                        }
8501                } else {
8502                        ErrorInfo errorInfo = new ErrorInfo();
8503                        errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
8504                        errorInfo.setErrorMessage("Can't get target table. CreateExternalDataSourceStmt is " + stmt.toString());
8505                        errorInfo.setStartPosition(new Pair3<Long, Long, String>(stmt.getStartToken().lineNo,
8506                                        stmt.getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
8507                        errorInfo.setEndPosition(new Pair3<Long, Long, String>(stmt.getEndToken().lineNo,
8508                                        stmt.getEndToken().columnNo + stmt.getEndToken().getAstext().length(),
8509                                        ModelBindingManager.getGlobalHash()));
8510                        errorInfo.fillInfo(this);
8511                        errorInfos.add(errorInfo);
8512                }
8513        }
8514
8515        private void analyzeCreateStreamStmt(TCreateStreamStmt stmt) {
8516                TObjectName streamName = stmt.getStreamName();
8517
8518                if (streamName != null) {
8519                        Table tableModel = modelFactory.createTableByName(stmt.getTableName());
8520                        tableModel.setCreateTable(true);
8521
8522                        Table streamModel = modelFactory.createStream(streamName);
8523                        streamModel.setCreateTable(true);
8524                        streamModel.setStream(true);
8525
8526                        Process process = modelFactory.createProcess(stmt);
8527                        tableModel.addProcess(process);
8528
8529                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8530                        relation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(streamModel.getRelationRows()));
8531                        relation.addSource(new RelationRowsRelationshipElement<TableRelationRows>(tableModel.getRelationRows()));
8532                        relation.setProcess(process);
8533                }
8534        }
8535
8536        private String getProcedureParentName(TCustomSqlStatement stmt) {
8537                if (stmt instanceof TStoredProcedureSqlStatement) {
8538                        if (((TStoredProcedureSqlStatement) stmt).getStoredProcedureName() != null) {
8539                                return ((TStoredProcedureSqlStatement) stmt).getStoredProcedureName().toString();
8540                        }
8541                }
8542
8543                stmt = stmt.getParentStmt();
8544                if (stmt == null)
8545                        return null;
8546
8547        if (stmt instanceof TCommonBlock) {
8548                        if(((TCommonBlock) stmt).getBlockBody().getParentObjectName() instanceof TStoredProcedureSqlStatement) {
8549                                stmt = (TStoredProcedureSqlStatement)((TCommonBlock) stmt).getBlockBody().getParentObjectName();
8550                        }
8551                }
8552
8553                if (stmt instanceof TStoredProcedureSqlStatement) {
8554                        if (((TStoredProcedureSqlStatement) stmt).getStoredProcedureName() != null) {
8555                                return ((TStoredProcedureSqlStatement) stmt).getStoredProcedureName().toString();
8556                        }
8557                }
8558                if (stmt instanceof TTeradataCreateProcedure) {
8559                        if (((TTeradataCreateProcedure) stmt).getProcedureName() != null) {
8560                                return ((TTeradataCreateProcedure) stmt).getProcedureName().toString();
8561                        }
8562                }
8563
8564                return getProcedureParentName(stmt);
8565        }
8566
8567        private TStoredProcedureSqlStatement getProcedureParent(TCustomSqlStatement stmt) {
8568                if (stmt instanceof TStoredProcedureSqlStatement) {
8569                        return (TStoredProcedureSqlStatement) stmt;
8570                }
8571
8572                stmt = stmt.getParentStmt();
8573                if (stmt == null)
8574                        return null;
8575
8576                if (stmt instanceof TCommonBlock) {
8577                        if (((TCommonBlock) stmt).getBlockBody().getParentObjectName() instanceof TStoredProcedureSqlStatement) {
8578                                stmt = (TStoredProcedureSqlStatement) ((TCommonBlock) stmt).getBlockBody().getParentObjectName();
8579                        }
8580                }
8581
8582                if (stmt instanceof TStoredProcedureSqlStatement) {
8583                        return ((TStoredProcedureSqlStatement) stmt);
8584                }
8585                if (stmt instanceof TTeradataCreateProcedure) {
8586                        return ((TTeradataCreateProcedure) stmt);
8587                }
8588
8589                return getProcedureParent(stmt);
8590        }
8591
8592        private void analyzeMergeStmt(TMergeSqlStatement stmt) {
8593                Object tableModel;
8594                Process process;
8595                if (stmt.getUsingTable() != null) {
8596                        TTable table = stmt.getTargetTable();
8597                        if(table.getSubquery()!=null) {
8598                                tableModel = modelFactory.createQueryTable(table);
8599                                analyzeSelectStmt(table.getSubquery());
8600                                process = modelFactory.createProcess(stmt);
8601                        }
8602                        else {
8603                                tableModel = modelFactory.createTable(table);
8604                                process = modelFactory.createProcess(stmt);
8605                                ((Table)tableModel).addProcess(process);
8606                        }
8607                        
8608                        for(TTable item: stmt.tables) {
8609                                // Skip subqueries and CTE references: a CTE feeding USING is resolved
8610                                // through its subquery (see the getCTE() branch below), so materializing
8611                                // it here as a physical base table leaves a spurious table named after
8612                                // the CTE in the lineage output. (MantisBT #4493)
8613                                if(item.getSubquery()!=null || item.getCTE()!=null) {
8614                                        continue;
8615                                }
8616                                Table tableItemModel = modelFactory.createTable(item);                          
8617                                if (tableItemModel.getColumns() == null || tableItemModel.getColumns().isEmpty()) {
8618                                        tableItemModel.addColumnsFromSQLEnv();
8619                                }
8620                        }
8621
8622                        if (stmt.getUsingTable().getSubquery() != null) {
8623                                QueryTable queryTable = modelFactory.createQueryTable(stmt.getUsingTable());
8624                                analyzeSelectStmt(stmt.getUsingTable().getSubquery());
8625
8626                                ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt.getUsingTable().getSubquery());
8627                                
8628                                if (queryTable != null && resultSetModel != null && queryTable != resultSetModel) {
8629                                        if (queryTable.getColumns().size() == resultSetModel.getColumns().size()) {
8630                                                for (int i = 0; i < queryTable.getColumns().size(); i++) {
8631                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8632                                                        relation.setEffectType(EffectType.select);
8633                                                        relation.setTarget(new ResultColumnRelationshipElement(queryTable.getColumns().get(i)));
8634                                                        relation.addSource(new ResultColumnRelationshipElement(resultSetModel.getColumns().get(i)));
8635                                                        relation.setProcess(process);
8636                                                }
8637                                        }
8638                                }
8639                                
8640                                if (resultSetModel != null && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
8641                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
8642                                        impactRelation.setEffectType(EffectType.merge);
8643                                        impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
8644                                                        resultSetModel.getRelationRows()));
8645                                        if (tableModel instanceof Table) {
8646                                                impactRelation.setTarget(
8647                                                                new RelationRowsRelationshipElement<TableRelationRows>(((Table)tableModel).getRelationRows()));
8648                                        }
8649                                        else {
8650                                                impactRelation.setTarget(
8651                                                                new RelationRowsRelationshipElement<ResultSetRelationRows>(((ResultSet)tableModel).getRelationRows()));
8652                                        }
8653                                }
8654                        } else if (stmt.getUsingTable().getCTE() != null) {
8655                                // USING references a CTE defined in the WITH clause attached to the
8656                                // MERGE. Mirror the SELECT-side CTE handling so the CTE subquery (and
8657                                // the base tables inside it) are analyzed and linked through to the
8658                                // merge source; otherwise the CTE is treated as an opaque base table
8659                                // and the tables feeding it are dropped from the lineage. (MantisBT #4493)
8660                                QueryTable queryTable = modelFactory.createQueryTable(stmt.getUsingTable());
8661
8662                                TObjectNameList cteColumns = stmt.getUsingTable().getCTE().getColumnList();
8663                                if (cteColumns != null) {
8664                                        for (int j = 0; j < cteColumns.size(); j++) {
8665                                                modelFactory.createResultColumn(queryTable, cteColumns.getObjectName(j));
8666                                        }
8667                                }
8668
8669                                TSelectSqlStatement subquery = stmt.getUsingTable().getCTE().getSubquery();
8670                                if (subquery != null && !stmtStack.contains(subquery)) {
8671                                        analyzeSelectStmt(subquery);
8672
8673                                        ResultSet resultSetModel = (ResultSet) modelManager.getModel(subquery);
8674
8675                                        if (resultSetModel != null && resultSetModel != queryTable) {
8676                                                if (subquery.getSetOperatorType() != ESetOperatorType.none) {
8677                                                        SelectSetResultSet selectSetResultSetModel = (SelectSetResultSet) modelManager
8678                                                                        .getModel(subquery);
8679                                                        for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) {
8680                                                                ResultColumn sourceColumn = selectSetResultSetModel.getColumns().get(j);
8681                                                                ResultColumn targetColumn = null;
8682                                                                if (cteColumns != null && j < queryTable.getColumns().size()) {
8683                                                                        targetColumn = queryTable.getColumns().get(j);
8684                                                                } else {
8685                                                                        targetColumn = modelFactory.createSelectSetResultColumn(queryTable, sourceColumn);
8686                                                                }
8687                                                                for (Set<TObjectName> starLinkColumns : sourceColumn.getStarLinkColumns().values()) {
8688                                                                        for (TObjectName starLinkColumn : starLinkColumns) {
8689                                                                                targetColumn.bindStarLinkColumn(starLinkColumn);
8690                                                                        }
8691                                                                }
8692                                                                DataFlowRelationship cteRelation = modelFactory.createDataFlowRelation();
8693                                                                cteRelation.setEffectType(EffectType.select);
8694                                                                cteRelation.setTarget(new ResultColumnRelationshipElement(targetColumn));
8695                                                                cteRelation.addSource(new ResultColumnRelationshipElement(sourceColumn));
8696                                                                cteRelation.setProcess(process);
8697                                                        }
8698                                                } else {
8699                                                        for (int j = 0; j < resultSetModel.getColumns().size(); j++) {
8700                                                                ResultColumn sourceColumn = resultSetModel.getColumns().get(j);
8701                                                                ResultColumn targetColumn = null;
8702                                                                if (cteColumns != null && j < queryTable.getColumns().size()) {
8703                                                                        targetColumn = queryTable.getColumns().get(j);
8704                                                                } else {
8705                                                                        targetColumn = modelFactory.createSelectSetResultColumn(queryTable, sourceColumn);
8706                                                                }
8707                                                                for (TObjectName starLinkColumn : sourceColumn.getStarLinkColumnList()) {
8708                                                                        targetColumn.bindStarLinkColumn(starLinkColumn);
8709                                                                }
8710                                                                DataFlowRelationship cteRelation = modelFactory.createDataFlowRelation();
8711                                                                cteRelation.setEffectType(EffectType.select);
8712                                                                cteRelation.setTarget(new ResultColumnRelationshipElement(targetColumn));
8713                                                                cteRelation.addSource(new ResultColumnRelationshipElement(sourceColumn));
8714                                                                cteRelation.setProcess(process);
8715                                                        }
8716                                                }
8717                                        }
8718
8719                                        if (resultSetModel != null && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
8720                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
8721                                                impactRelation.setEffectType(EffectType.merge);
8722                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
8723                                                                resultSetModel.getRelationRows()));
8724                                                if (tableModel instanceof Table) {
8725                                                        impactRelation.setTarget(
8726                                                                        new RelationRowsRelationshipElement<TableRelationRows>(((Table) tableModel).getRelationRows()));
8727                                                } else {
8728                                                        impactRelation.setTarget(
8729                                                                        new RelationRowsRelationshipElement<ResultSetRelationRows>(((ResultSet) tableModel).getRelationRows()));
8730                                                }
8731                                        }
8732                                }
8733                        } else {
8734                                if (stmt.getUsingTable().getAliasClause() != null && stmt.getUsingTable().getAliasClause().getColumns() != null && stmt.getUsingTable().getValueClause().getRows()!=null) {
8735                                        Table usingTable = modelFactory.createTableFromCreateDDL(stmt.getUsingTable(), false, stmt.getUsingTable().getAliasName() + stmt.getUsingTable().getTableName());
8736                                        usingTable.setCreateTable(true);
8737                                        usingTable.setSubType(SubType.function);
8738                                        for (int z=0;z<stmt.getUsingTable().getAliasClause().getColumns().size();z++) {
8739                                                TObjectName columnName = stmt.getUsingTable().getAliasClause().getColumns().getObjectName(z);
8740                                                TableColumn tableColumn = modelFactory.createTableColumn(usingTable, columnName, true);
8741                                                TResultColumn resultColumn = stmt.getUsingTable().getValueClause().getRows().get(0).getResultColumn(z);
8742                                                modelManager.bindModel(resultColumn, tableColumn);
8743                                                analyzeResultColumnExpressionRelation(tableColumn, resultColumn.getExpr());                                             }
8744                                }
8745                                else {
8746                                        modelFactory.createTable(stmt.getUsingTable());
8747                                }
8748                        }
8749                
8750
8751                        if (stmt.getWhenClauses() != null && stmt.getWhenClauses().size() > 0) {
8752                                for (int i = 0; i < stmt.getWhenClauses().size(); i++) {
8753                                        TMergeWhenClause clause = stmt.getWhenClauses().getElement(i);
8754                                        if (clause.getCondition() != null) {
8755                                                analyzeFilterCondition(null, clause.getCondition(), null, null, EffectType.merge_when);
8756                                        }
8757                                        if (clause.getUpdateClause() != null) {
8758                                                TResultColumnList columns = clause.getUpdateClause().getUpdateColumnList();
8759                                                if (columns == null || columns.size() == 0)
8760                                                        continue;
8761
8762                                                ResultSet resultSet = modelFactory.createResultSet(clause.getUpdateClause(), false);
8763                                                createPseudoImpactRelation(stmt, resultSet, EffectType.merge_update);
8764
8765                                                for (int j = 0; j < columns.size(); j++) {
8766                                                        TResultColumn resultColumn = columns.getResultColumn(j);
8767                                                        if (resultColumn.getExpr().getLeftOperand()
8768                                                                        .getExpressionType() == EExpressionType.simple_object_name_t) {
8769                                                                TObjectName columnObject = resultColumn.getExpr().getLeftOperand().getObjectOperand();
8770
8771                                                                if (columnObject.getDbObjectType() == EDbObjectType.variable) {
8772                                                                        continue;
8773                                                                }
8774
8775                                                                if (columnObject.getColumnNameOnly().startsWith("@")
8776                                                                                && (option.getVendor() == EDbVendor.dbvmssql
8777                                                                                                || option.getVendor() == EDbVendor.dbvazuresql)) {
8778                                                                        continue;
8779                                                                }
8780
8781                                                                if (columnObject.getColumnNameOnly().startsWith(":")
8782                                                                                && (option.getVendor() == EDbVendor.dbvhana
8783                                                                                                || option.getVendor() == EDbVendor.dbvteradata)) {
8784                                                                        continue;
8785                                                                }
8786
8787                                                                ResultColumn updateColumn = modelFactory.createMergeResultColumn(resultSet,
8788                                                                                columnObject);
8789
8790                                                                TExpression valueExpression = resultColumn.getExpr().getRightOperand();
8791                                                                if (valueExpression == null)
8792                                                                        continue;
8793
8794                                                                columnsInExpr visitor = new columnsInExpr();
8795                                                                valueExpression.inOrderTraverse(visitor);
8796                                                                List<TObjectName> objectNames = visitor.getObjectNames();
8797                                                                List<TParseTreeNode> functions = visitor.getFunctions();
8798
8799                                                                if (functions != null && !functions.isEmpty()) {
8800                                                                        analyzeFunctionDataFlowRelation(updateColumn, functions, EffectType.merge_update);
8801                                                                }
8802
8803                                                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
8804                                                                if (subquerys != null && !subquerys.isEmpty()) {
8805                                                                        analyzeSubqueryDataFlowRelation(updateColumn, subquerys, EffectType.merge_update);
8806                                                                }
8807
8808                                                                analyzeDataFlowRelation(updateColumn, objectNames, EffectType.merge_update, functions);
8809
8810                                                                List<TParseTreeNode> constants = visitor.getConstants();
8811                                                                analyzeConstantDataFlowRelation(updateColumn, constants, EffectType.merge_update,
8812                                                                                functions);
8813
8814                                                                if (tableModel instanceof Table) {
8815                                                                        TableColumn tableColumn = modelFactory.createTableColumn((Table)tableModel, columnObject,
8816                                                                                        false);
8817
8818                                                                        if (tableColumn != null) {
8819                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8820                                                                                relation.setEffectType(EffectType.merge_update);
8821                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
8822                                                                                relation.addSource(new ResultColumnRelationshipElement(updateColumn));
8823                                                                                relation.setProcess(process);
8824                                                                        }
8825                                                                }
8826                                                                else {
8827                                                                        TTable targetTable = stmt.getTargetTable().getSubquery().getTables().getTable(0);
8828                                                                        if (targetTable != null && modelManager.getModel(targetTable) instanceof Table) {
8829                                                                                Table targetTableModel = (Table) modelManager.getModel(targetTable);
8830                                                                                TableColumn tableColumn = modelFactory
8831                                                                                                .createTableColumn((Table) targetTableModel, columnObject, false);
8832                                                                                if (tableColumn != null) {
8833                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8834                                                                                        relation.setEffectType(EffectType.merge_update);
8835                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
8836                                                                                        relation.addSource(new ResultColumnRelationshipElement(updateColumn));
8837                                                                                        relation.setProcess(process);
8838                                                                                }
8839                                                                        }
8840                                                                }
8841                                                        }
8842                                                }
8843                                        }
8844                                        if (clause.getInsertClause() != null && tableModel instanceof Table) {
8845                                                TExpression insertValue = clause.getInsertClause().getInsertValue();
8846                                                if (insertValue != null
8847                                                                && insertValue.getExpressionType() == EExpressionType.objectConstruct_t) {
8848                                                        ResultSet resultSet = modelFactory.createResultSet(clause.getInsertClause(), false);
8849
8850                                                        createPseudoImpactRelation(stmt, resultSet, EffectType.merge_insert);
8851
8852                                                        TObjectConstruct objectConstruct = insertValue.getObjectConstruct();
8853                                                        for (int z = 0; z < objectConstruct.getPairs().size(); z++) {
8854                                                                TPair pair = objectConstruct.getPairs().getElement(z);
8855
8856                                                                if (pair.getKeyName().getExpressionType() == EExpressionType.simple_constant_t) {
8857                                                                        TObjectName columnObject = new TObjectName();
8858                                                                        TConstant constant = pair.getKeyName().getConstantOperand();
8859                                                                        TSourceToken newSt = new TSourceToken(
8860                                                                                        constant.getValueToken().getTextWithoutQuoted());
8861                                                                        columnObject.setPartToken(newSt);
8862                                                                        columnObject.setSourceTable(stmt.getTargetTable());
8863                                                                        columnObject.setStartToken(constant.getStartToken());
8864                                                                        columnObject.setEndToken(constant.getEndToken());
8865
8866                                                                        ResultColumn insertColumn = modelFactory.createMergeResultColumn(resultSet,
8867                                                                                        columnObject);
8868
8869                                                                        TExpression valueExpression = pair.getKeyValue();
8870                                                                        if (valueExpression == null)
8871                                                                                continue;
8872
8873                                                                        columnsInExpr visitor = new columnsInExpr();
8874                                                                        valueExpression.inOrderTraverse(visitor);
8875                                                                        List<TObjectName> objectNames = visitor.getObjectNames();
8876                                                                        List<TParseTreeNode> functions = visitor.getFunctions();
8877
8878                                                                        if (functions != null && !functions.isEmpty()) {
8879                                                                                analyzeFunctionDataFlowRelation(insertColumn, functions,
8880                                                                                                EffectType.merge_insert);
8881                                                                        }
8882
8883                                                                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
8884                                                                        if (subquerys != null && !subquerys.isEmpty()) {
8885                                                                                analyzeSubqueryDataFlowRelation(insertColumn, subquerys,
8886                                                                                                EffectType.merge_insert);
8887                                                                        }
8888
8889                                                                        analyzeDataFlowRelation(insertColumn, objectNames, EffectType.merge_insert,
8890                                                                                        functions);
8891
8892                                                                        List<TParseTreeNode> constants = visitor.getConstants();
8893                                                                        analyzeConstantDataFlowRelation(insertColumn, constants, EffectType.merge_insert,
8894                                                                                        functions);
8895
8896                                                                        TableColumn tableColumn = modelFactory.createTableColumn((Table)tableModel,
8897                                                                                        columnObject, false);
8898
8899                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8900                                                                        relation.setEffectType(EffectType.merge_insert);
8901                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
8902                                                                        relation.addSource(new ResultColumnRelationshipElement(insertColumn));
8903                                                                        relation.setProcess(process);
8904                                                                }
8905                                                        }
8906                                                } else {
8907                                                        TObjectNameList columns = clause.getInsertClause().getColumnList();
8908                                                        TResultColumnList values = clause.getInsertClause().getValuelist();
8909                                                        if (values == null || values.size() == 0) {
8910                                                                if (clause.getInsertClause().toString().toLowerCase().indexOf("row") != -1) {
8911                                                                        if (stmt.getUsingTable().getSubquery() != null) {
8912                                                                                ResultSet sourceResultSet = modelFactory.createQueryTable(stmt.getUsingTable());
8913                                                                                TObjectName targetStarColumn = new TObjectName();
8914                                                                                targetStarColumn.setString("*");
8915                                                                                TableColumn targetTableColumn = modelFactory.createTableColumn((Table)tableModel,
8916                                                                                                targetStarColumn, true);
8917                                                                                if (sourceResultSet.getColumns() == null
8918                                                                                                || sourceResultSet.getColumns().isEmpty()) {
8919                                                                                        TObjectName sourceStarColumn = new TObjectName();
8920                                                                                        sourceStarColumn.setString("*");
8921                                                                                        modelFactory.createResultColumn(sourceResultSet, sourceStarColumn);
8922                                                                                }
8923                                                                                for (ResultColumn sourceColumn : sourceResultSet.getColumns()) {
8924                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8925                                                                                        relation.setEffectType(EffectType.merge_insert);
8926                                                                                        relation.setTarget(new TableColumnRelationshipElement(targetTableColumn));
8927                                                                                        relation.addSource(new ResultColumnRelationshipElement(sourceColumn));
8928                                                                                        relation.setProcess(process);
8929                                                                                }
8930                                                                        } else {
8931                                                                                Table sourceTable = modelFactory.createTable(stmt.getUsingTable());
8932                                                                                TObjectName sourceStarColumn = new TObjectName();
8933                                                                                sourceStarColumn.setString("*");
8934                                                                                TableColumn sourceTableColumn = modelFactory.createTableColumn(sourceTable,
8935                                                                                                sourceStarColumn, true);
8936                                                                                TObjectName targetStarColumn = new TObjectName();
8937                                                                                targetStarColumn.setString("*");
8938                                                                                TableColumn targetTableColumn = modelFactory.createTableColumn(((Table)tableModel),
8939                                                                                                targetStarColumn, true);
8940                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8941                                                                                relation.setEffectType(EffectType.merge_insert);
8942                                                                                relation.setTarget(new TableColumnRelationshipElement(targetTableColumn));
8943                                                                                relation.addSource(new TableColumnRelationshipElement(sourceTableColumn));
8944                                                                                relation.setProcess(process);
8945                                                                        }
8946
8947                                                                }
8948                                                                continue;
8949                                                        }
8950
8951                                                        List<TObjectName> tableColumns = new ArrayList<TObjectName>();
8952                                                        if (columns == null || columns.size() == 0) {
8953//                                                              if (!((Table)tableModel).getColumns().isEmpty()) {
8954//                                                                      for (int j = 0; j < ((Table)tableModel).getColumns().size(); j++) {
8955//                                                                              if (((Table)tableModel).getColumns().get(j).getColumnObject() == null) {
8956//                                                                                      continue;
8957//                                                                              }
8958//                                                                              tableColumns.add(((Table)tableModel).getColumns().get(j).getColumnObject());
8959//                                                                      }
8960//                                                              } else {
8961                                                                        for (int j = 0; j < values.size(); j++) {
8962                                                                                TResultColumn column = values.getResultColumn(j);
8963                                                                                if (column.getAliasClause() != null) {
8964                                                                                        tableColumns.add(column.getAliasClause().getAliasName());
8965                                                                                } else if (column.getFieldAttr() != null) {
8966                                                                                        tableColumns.add(column.getFieldAttr());
8967                                                                                } else {
8968                                                                                        TObjectName columnName = new TObjectName();
8969                                                                                        columnName.setString(column.toString());
8970                                                                                        tableColumns.add(columnName);
8971                                                                                }
8972//                                                                      }
8973                                                                }
8974                                                        } else {
8975                                                                for (int j = 0; j < columns.size(); j++) {
8976                                                                        tableColumns.add(columns.getObjectName(j));
8977                                                                }
8978                                                        }
8979
8980                                                        ResultSet resultSet = modelFactory.createResultSet(clause.getInsertClause(), false);
8981
8982                                                        createPseudoImpactRelation(stmt, resultSet, EffectType.merge_insert);
8983
8984                                                        for (int j = 0; j < tableColumns.size() && j < values.size(); j++) {
8985                                                                TObjectName columnObject = tableColumns.get(j);
8986
8987                                                                ResultColumn insertColumn = modelFactory.createMergeResultColumn(resultSet,
8988                                                                                columnObject);
8989
8990                                                                TExpression valueExpression = values.getResultColumn(j).getExpr();
8991                                                                if (valueExpression == null)
8992                                                                        continue;
8993
8994                                                                columnsInExpr visitor = new columnsInExpr();
8995                                                                valueExpression.inOrderTraverse(visitor);
8996                                                                List<TObjectName> objectNames = visitor.getObjectNames();
8997                                                                List<TParseTreeNode> functions = visitor.getFunctions();
8998
8999                                                                if (functions != null && !functions.isEmpty()) {
9000                                                                        analyzeFunctionDataFlowRelation(insertColumn, functions, EffectType.merge_insert);
9001                                                                }
9002
9003                                                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
9004                                                                if (subquerys != null && !subquerys.isEmpty()) {
9005                                                                        analyzeSubqueryDataFlowRelation(insertColumn, subquerys, EffectType.merge_insert);
9006                                                                }
9007
9008                                                                analyzeDataFlowRelation(insertColumn, objectNames, EffectType.merge_insert, functions);
9009
9010                                                                List<TParseTreeNode> constants = visitor.getConstants();
9011                                                                analyzeConstantDataFlowRelation(insertColumn, constants, EffectType.merge_insert,
9012                                                                                functions);
9013
9014                                                                TableColumn tableColumn = modelFactory.createTableColumn(((Table)tableModel), columnObject,
9015                                                                                false);
9016                                                                if(tableColumn == null) {
9017                                                                        if (((Table) tableModel).isCreateTable()) {
9018                                                                                tableColumn = ((Table) tableModel).getColumns().get(j);
9019                                                                        }
9020                                                                        else {
9021                                                                                continue;
9022                                                                        }
9023                                                                }
9024
9025                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9026                                                                relation.setEffectType(EffectType.merge_insert);
9027                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9028                                                                relation.addSource(new ResultColumnRelationshipElement(insertColumn));
9029                                                                relation.setProcess(process);
9030                                                        }
9031                                                }
9032                                        }
9033                                }
9034
9035                        }
9036
9037                        if (stmt.getCondition() != null) {
9038                                analyzeFilterCondition(null, stmt.getCondition(), null, JoinClauseType.on, EffectType.merge);
9039                        }
9040                }
9041        }
9042
9043        private List<TableColumn> bindInsertTableColumn(Table tableModel, TInsertIntoValue value, List<TObjectName> keyMap,
9044                        List<TResultColumn> valueMap) {
9045                List<TableColumn> tableColumns = new ArrayList<TableColumn>();
9046                if (value.getColumnList() != null) {
9047                        for (int z = 0; z < value.getColumnList().size(); z++) {
9048                                TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
9049                                                value.getColumnList().getObjectName(z));
9050                                tableColumns.add(tableColumn);
9051                                keyMap.add(tableColumn.getColumnObject());
9052                        }
9053                }
9054
9055                if (value.getTargetList() != null) {
9056                        for (int z = 0; z < value.getTargetList().size(); z++) {
9057                                TMultiTarget target = value.getTargetList().getMultiTarget(z);
9058                                TResultColumnList columns = target.getColumnList();
9059                                for (int i = 0; i < columns.size(); i++) {
9060                                        if (value.getColumnList() == null) {
9061                                                TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
9062                                                                columns.getResultColumn(i).getFieldAttr());
9063                                                tableColumns.add(tableColumn);
9064                                        }
9065                                        valueMap.add(columns.getResultColumn(i));
9066                                }
9067                        }
9068                }
9069
9070                return tableColumns;
9071        }
9072
9073        private TableColumn matchColumn(List<TableColumn> tableColumns, TableColumn targetColumn) {
9074                String columnName = targetColumn.getName();
9075                if (tableColumns == null) {
9076                        return null;
9077                }
9078                for (int i = 0; i < tableColumns.size(); i++) {
9079                        TableColumn column = tableColumns.get(i);
9080                        if (column.getColumnObject() == null) {
9081                                continue;
9082                        }
9083                        if(column.isStruct() && targetColumn.isStruct()) {
9084                                List<String> names = SQLUtil.parseNames(column.getName());
9085                                List<String> targetNames = SQLUtil
9086                                                .parseNames(targetColumn.getName());
9087                                if (!getColumnName(targetNames.get(0))
9088                                                .equals(getColumnName(names.get(0)))) {
9089                                        continue;
9090                                }
9091                        }
9092                        if (getColumnName(column.getColumnObject().toString()).equals(getColumnName(columnName))) {
9093                                return column;
9094                        }
9095                }
9096                return null;
9097        }
9098        
9099        private TableColumn matchColumn(List<TableColumn> tableColumns, TObjectName columnName) {
9100                if (tableColumns == null) {
9101                        return null;
9102                }
9103                for (int i = 0; i < tableColumns.size(); i++) {
9104                        TableColumn column = tableColumns.get(i);
9105                        if (column.getColumnObject() == null) {
9106                                continue;
9107                        }
9108                        if (DlineageUtil.getColumnName(column.getColumnObject()).equalsIgnoreCase(DlineageUtil.getColumnName(columnName)))
9109                                return column;
9110                }
9111                return null;
9112        }
9113
9114        private ResultColumn matchResultColumn(List<ResultColumn> resultColumns, ResultColumn resultColumn) {
9115                if (resultColumns == null) {
9116                        return null;
9117                }
9118
9119                TObjectName columnName = getObjectName(resultColumn);
9120                if (columnName == null) {
9121                        return null;
9122                }
9123
9124                for (int i = 0; i < resultColumns.size(); i++) {
9125                        ResultColumn column = resultColumns.get(i);
9126                        if (column.getAlias() != null
9127                                        && getColumnName(column.getAlias()).equalsIgnoreCase(getColumnName(columnName)))
9128                                return column;
9129                        if (column.getName() != null && getColumnName(column.getName()).equalsIgnoreCase(getColumnName(columnName)))
9130                                return column;
9131                        if (column.getName() != null && column.getName().endsWith("*")) {
9132                                if ("*".equals(column.getColumnObject().toString())) {
9133                                        return column;
9134                                } else {
9135                                        TObjectName columnObjectName = getObjectName(column);
9136                                        if (columnObjectName.getTableString() != null
9137                                                        && columnObjectName.getTableString().equals(getResultSetAlias(resultColumn))) {
9138                                                return column;
9139                                        }
9140                                }
9141                        }
9142                }
9143                return null;
9144        }
9145
9146        private String getResultSetAlias(ResultColumn resultColumn) {
9147                ResultSet resultSet = resultColumn.getResultSet();
9148                if (resultSet instanceof QueryTable) {
9149                        return ((QueryTable) resultSet).getAlias();
9150                }
9151                return null;
9152        }
9153
9154        private ResultColumn matchResultColumn(List<ResultColumn> resultColumns, TObjectName columnName) {
9155                if (resultColumns == null) {
9156                        return null;
9157                }
9158                for (int i = 0; i < resultColumns.size(); i++) {
9159                        ResultColumn column = resultColumns.get(i);
9160                        if (column.getAlias() != null
9161                                        && getColumnName(column.getAlias()).equalsIgnoreCase(getColumnName(columnName)))
9162                                return column;
9163                        if (column.getName() != null && getColumnName(column.getName()).equalsIgnoreCase(getColumnName(columnName)))
9164                                return column;
9165                        if (column.getName() != null && column.getName().endsWith("*")) {
9166                                if ("*".equals(column.getColumnObject().toString())) {
9167                                        return column;
9168                                } else {
9169                                        TObjectName columnObjectName = getObjectName(column);
9170                                        if (columnObjectName.getTableString() != null
9171                                                        && columnObjectName.getTableString().equals(columnName.getTableString())) {
9172                                                return column;
9173                                        }
9174                                }
9175                        }
9176                }
9177                return null;
9178        }
9179
9180        private void analyzeInsertStmt(TInsertSqlStatement stmt) {
9181                Map<Table, List<TObjectName>> insertTableKeyMap = new LinkedHashMap<Table, List<TObjectName>>();
9182                Map<Table, List<TResultColumn>> insertTableValueMap = new LinkedHashMap<Table, List<TResultColumn>>();
9183                Map<String, List<TableColumn>> tableColumnMap = new LinkedHashMap<String, List<TableColumn>>();
9184                List<Table> inserTables = new ArrayList<Table>();
9185                List<TExpression> expressions = new ArrayList<TExpression>();
9186                boolean hasInsertColumns = false;
9187                
9188                EffectType effectType = EffectType.insert;
9189                if(stmt.getInsertToken()!=null && stmt.getInsertToken().toString().toLowerCase().startsWith("replace")) {
9190                        effectType = EffectType.replace;
9191                }
9192                
9193                if (stmt.getInsertConditions() != null && stmt.getInsertConditions().size() > 0) {
9194                        for (int i = 0; i < stmt.getInsertConditions().size(); i++) {
9195                                TInsertCondition condition = stmt.getInsertConditions().getElement(i);
9196                                if (condition.getCondition() != null) {
9197                                        expressions.add(condition.getCondition());
9198                                }
9199                                for (int j = 0; j < condition.getInsertIntoValues().size(); j++) {
9200                                        TInsertIntoValue value = condition.getInsertIntoValues().getElement(j);
9201                                        TTable table = value.getTable();
9202                                        Table tableModel = modelFactory.createTable(table);
9203
9204                                        inserTables.add(tableModel);
9205                                        List<TObjectName> keyMap = new ArrayList<TObjectName>();
9206                                        List<TResultColumn> valueMap = new ArrayList<TResultColumn>();
9207                                        insertTableKeyMap.put(tableModel, keyMap);
9208                                        insertTableValueMap.put(tableModel, valueMap);
9209
9210                                        List<TableColumn> tableColumns = bindInsertTableColumn(tableModel, value, keyMap, valueMap);
9211                                        if (tableColumnMap.get(DlineageUtil.getIdentifierNormalTableName(table.getFullName())) == null
9212                                                        && !tableColumns.isEmpty()) {
9213                                                tableColumnMap.put(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName()),
9214                                                                tableColumns);
9215                                        }
9216
9217                                        // if (stmt.getSubQuery() != null)
9218                                        {
9219                                                Process process = modelFactory.createProcess(stmt);
9220                                                tableModel.addProcess(process);
9221                                        }
9222                                }
9223                        }
9224                        hasInsertColumns = true;
9225                } else if (stmt.getInsertIntoValues() != null && stmt.getInsertIntoValues().size() > 0) {
9226                        for (int i = 0; i < stmt.getInsertIntoValues().size(); i++) {
9227                                TInsertIntoValue value = stmt.getInsertIntoValues().getElement(i);
9228                                TTable table = value.getTable();
9229                                Table tableModel = modelFactory.createTable(table);
9230
9231                                inserTables.add(tableModel);
9232                                List<TObjectName> keyMap = new ArrayList<TObjectName>();
9233                                List<TResultColumn> valueMap = new ArrayList<TResultColumn>();
9234                                insertTableKeyMap.put(tableModel, keyMap);
9235                                insertTableValueMap.put(tableModel, valueMap);
9236
9237                                List<TableColumn> tableColumns = bindInsertTableColumn(tableModel, value, keyMap, valueMap);
9238                                if (tableColumnMap.get(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName())) == null && !tableColumns.isEmpty()) {
9239                                        tableColumnMap.put(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName()), tableColumns);
9240                                }
9241
9242                                // if (stmt.getSubQuery() != null)
9243                                {
9244                                        Process process = modelFactory.createProcess(stmt);
9245                                        tableModel.addProcess(process);
9246                                }
9247                        }
9248                        hasInsertColumns = true;
9249                } else if (stmt.getColumnList() != null && stmt.getColumnList().size() > 0) {
9250                        TTable table = stmt.getTargetTable();
9251                        Table tableModel = modelFactory.createTable(table);
9252
9253                        inserTables.add(tableModel);
9254                        List<TObjectName> keyMap = new ArrayList<TObjectName>();
9255                        insertTableKeyMap.put(tableModel, keyMap);
9256                        List<TableColumn> tableColumns = new ArrayList<TableColumn>();
9257                        for (int i = 0; i < stmt.getColumnList().size(); i++) {
9258                                TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
9259                                                stmt.getColumnList().getObjectName(i));
9260                                tableColumns.add(tableColumn);
9261                        }
9262                        if (tableColumnMap.get(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName())) == null && !tableColumns.isEmpty()) {
9263                                tableColumnMap.put(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName()), tableColumns);
9264                        }
9265
9266                        // if (stmt.getSubQuery() != null)
9267                        {
9268                                Process process = modelFactory.createProcess(stmt);
9269                                tableModel.addProcess(process);
9270                        }
9271                        hasInsertColumns = true;
9272                } else if (stmt.getOutputClause() != null && stmt.getOutputClause().getSelectItemList().size() > 0) {
9273                        TTable table = stmt.getTargetTable();
9274                        Table tableModel = modelFactory.createTable(table);
9275
9276                        inserTables.add(tableModel);
9277                        List<TObjectName> keyMap = new ArrayList<TObjectName>();
9278                        insertTableKeyMap.put(tableModel, keyMap);
9279                        List<TableColumn> tableColumns = new ArrayList<TableColumn>();
9280                        for (int i = 0; i < stmt.getOutputClause().getSelectItemList().size(); i++) {
9281                                TObjectName columnName = stmt.getOutputClause().getSelectItemList().getResultColumn(i).getFieldAttr();
9282                                if (columnName.getPseudoTableType() != EPseudoTableType.none) {
9283                                        // Phase 1 already swapped tokens; getColumnNameOnly() returns actual column name
9284                                        String column = columnName.getColumnNameOnly();
9285                                        columnName = new TObjectName();
9286                                        columnName.setString(column);
9287                                } else {
9288                                        String column = columnName.toString().toLowerCase();
9289                                        if ((column.startsWith("inserted.") || column.startsWith("deleted."))
9290                                                        && columnName.getPropertyToken() != null) {
9291                                                column = columnName.getPropertyToken().getAstext();
9292                                                columnName = new TObjectName();
9293                                                columnName.setString(column);
9294                                        }
9295                                }
9296                                TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel, columnName);
9297                                tableColumns.add(tableColumn);
9298                        }
9299                        if (tableColumnMap.get(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName())) == null && !tableColumns.isEmpty()) {
9300                                tableColumnMap.put(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName()), tableColumns);
9301                        }
9302
9303                        // if (stmt.getSubQuery() != null)
9304                        {
9305                                Process process = modelFactory.createProcess(stmt);
9306                                tableModel.addProcess(process);
9307                        }
9308                        hasInsertColumns = true;
9309                } else {
9310                        TTable table = stmt.getTargetTable();
9311                        Table tableModel;
9312                        if (table != null) {
9313                                tableModel = modelFactory.createTable(table);
9314                                // if (stmt.getSubQuery() != null)
9315                                {
9316                                        Process process = modelFactory.createProcess(stmt);
9317                                        tableModel.addProcess(process);
9318                                }
9319                                if (tableModel.getColumns() == null || tableModel.getColumns().isEmpty()) {
9320                                        tableModel.addColumnsFromSQLEnv();
9321                                }
9322                        } else if (stmt.getDirectoryName() != null) {
9323                                tableModel = modelFactory.createTableByName(stmt.getDirectoryName(), true);
9324                                tableModel.setPath(true);
9325                                tableModel.setCreateTable(true);
9326                                TObjectName fileUri = new TObjectName();
9327                                fileUri.setString("uri=" + stmt.getDirectoryName());
9328                                TableColumn tableColumn = modelFactory.createFileUri(tableModel, fileUri);
9329                                // if (stmt.getSubQuery() != null)
9330                                {
9331                                        Process process = modelFactory.createProcess(stmt);
9332                                        tableModel.addProcess(process);
9333                                }
9334                        } else {
9335                                ErrorInfo errorInfo = new ErrorInfo();
9336                                errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
9337                                errorInfo.setErrorMessage("Can't get target table. InsertSqlStatement is " + stmt.toString());
9338                                errorInfo.setStartPosition(new Pair3<Long, Long, String>(stmt.getStartToken().lineNo,
9339                                                stmt.getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
9340                                errorInfo.setEndPosition(new Pair3<Long, Long, String>(stmt.getEndToken().lineNo,
9341                                                stmt.getEndToken().columnNo + stmt.getEndToken().getAstext().length(),
9342                                                ModelBindingManager.getGlobalHash()));
9343                                errorInfo.fillInfo(this);
9344                                errorInfos.add(errorInfo);
9345                                return;
9346                        }
9347                        inserTables.add(tableModel);
9348                        if (table != null
9349                                        && tableColumnMap.get(DlineageUtil.getIdentifierNormalTableName(table.getFullName())) == null) {
9350                                if (tableModel.getColumns() != null && !tableModel.getColumns().isEmpty()) {
9351                                        tableColumnMap.put(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName()),
9352                                                        tableModel.getColumns());
9353                                } else {
9354                                        tableColumnMap.put(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName()), null);
9355                                }
9356                        }
9357                }
9358
9359                if (stmt.getSubQuery() != null) {
9360                        analyzeSelectStmt(stmt.getSubQuery());
9361                }
9362
9363                Iterator<Table> tableIter = inserTables.iterator();
9364                while (tableIter.hasNext()) {
9365                        Table tableModel = tableIter.next();
9366                        List<TableColumn> tableColumns = tableColumnMap.get(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName()));
9367                        List<TObjectName> keyMap = insertTableKeyMap.get(tableModel);
9368                        List<TResultColumn> valueMap = insertTableValueMap.get(tableModel);
9369                        boolean initColumn = (hasInsertColumns && tableColumns != null && !containStarColumn(tableColumns));
9370
9371                        if (stmt.getSubQuery() != null) {
9372
9373                                List<TSelectSqlStatement> subquerys = new ArrayList<TSelectSqlStatement>();
9374                                if (stmt.getSubQuery().getResultColumnList() != null || stmt.getSubQuery().getTransformClause() != null) {
9375                                        subquerys.add(stmt.getSubQuery());
9376                                } else if (stmt.getSubQuery().getValueClause() != null
9377                                                && stmt.getSubQuery().getValueClause().getRows() != null) {
9378                                        for (TResultColumnList resultColumnList : stmt.getSubQuery().getValueClause().getRows()) {
9379                                                for(TResultColumn resultColumn: resultColumnList) {
9380                                                        if(resultColumn.getExpr()!=null && resultColumn.getExpr().getSubQuery()!=null) {
9381                                                                analyzeSelectStmt(resultColumn.getExpr().getSubQuery());
9382                                                                subquerys.add(resultColumn.getExpr().getSubQuery());
9383                                                        }
9384                                                }
9385                                        }
9386                                }
9387
9388                                for(TSelectSqlStatement subquery: subquerys) {
9389                                        if ((tableModel.isCreateTable() && tableModel.getColumns() != null)
9390                                                        || (subquery.getSetOperatorType() == ESetOperatorType.none
9391                                                                        && stmt.getColumnList() != null && stmt.getColumnList().size() > 0)) {
9392
9393                                                ResultSet resultSetModel = null;
9394
9395                                                if (subquery != null) {
9396                                                        resultSetModel = (ResultSet) modelManager.getModel(subquery);
9397                                                }
9398
9399                                                TResultColumnList resultset = subquery.getResultColumnList();
9400                                                if (resultSetModel == null && resultset != null) {
9401                                                        resultSetModel = (ResultSet) modelManager.getModel(resultset);
9402                                                }
9403
9404                                                if (resultSetModel == null) {
9405                                                        ErrorInfo errorInfo = new ErrorInfo();
9406                                                        errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
9407                                                        errorInfo.setErrorMessage("Can't get resultset model");
9408                                                        errorInfo.setStartPosition(new Pair3<Long, Long, String>(resultset.getStartToken().lineNo,
9409                                                                        resultset.getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
9410                                                        errorInfo.setEndPosition(new Pair3<Long, Long, String>(resultset.getEndToken().lineNo,
9411                                                                        resultset.getEndToken().columnNo + resultset.getEndToken().getAstext().length(),
9412                                                                        ModelBindingManager.getGlobalHash()));
9413                                                        errorInfos.add(errorInfo);
9414                                                }
9415
9416                                                if (!resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
9417                                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
9418                                                        impactRelation.setEffectType(effectType);
9419                                                        impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
9420                                                                        resultSetModel.getRelationRows()));
9421                                                        impactRelation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(
9422                                                                        tableModel.getRelationRows()));
9423                                                }
9424
9425                                                int resultSetSize = resultSetModel.getColumns().size();
9426                                                int j = 0;
9427                                                int starIndex = 0;
9428                                                TObjectNameList items = stmt.getColumnList();
9429                                                List<String> itemNames = new ArrayList<String>();
9430                                                int starColumnCount = 0;
9431                                                for (ResultColumn item : resultSetModel.getColumns()) {
9432                                                        if (item.getName().endsWith("*")) {
9433                                                                starColumnCount += 1;
9434                                                        }
9435                                                }
9436                                                if (items != null) {
9437                                                        for (int i = 0; i < items.size() && j < resultSetSize; i++) {
9438                                                                TObjectName column = items.getObjectName(i);
9439
9440                                                                if (column.getDbObjectType() == EDbObjectType.variable) {
9441                                                                        continue;
9442                                                                }
9443
9444                                                                if (column.getColumnNameOnly().startsWith("@")
9445                                                                                && (option.getVendor() == EDbVendor.dbvmssql
9446                                                                                                || option.getVendor() == EDbVendor.dbvazuresql)) {
9447                                                                        continue;
9448                                                                }
9449
9450                                                                if (column.getColumnNameOnly().startsWith(":")
9451                                                                                && (option.getVendor() == EDbVendor.dbvhana
9452                                                                                                || option.getVendor() == EDbVendor.dbvteradata)) {
9453                                                                        continue;
9454                                                                }
9455
9456                                                                ResultColumn resultColumn = resultSetModel.getColumns().get(j);
9457                                                                if (!resultSetModel.getColumns().get(j).getName().contains("*")) {
9458                                                                        j++;
9459                                                                } else {
9460                                                                        starIndex++;
9461                                                                        if (resultSetSize - j == items.size() - i) {
9462                                                                                j++;
9463
9464                                                                        }
9465                                                                }
9466                                                                if (column != null) {
9467                                                                        TableColumn tableColumn;
9468                                                                        // if (!initColumn) {
9469                                                                        tableColumn = matchColumn(tableModel.getColumns(), column);
9470                                                                        if (tableColumn == null) {
9471                                                                                if (tableModel.isCreateTable() && !containStarColumn(tableModel.getColumns())) {
9472                                                                                        if (tableModel.getColumns().size() <= i) {
9473                                                                                                continue;
9474                                                                                        }
9475                                                                                        tableColumn = tableModel.getColumns().get(i);
9476                                                                                } else {
9477                                                                                        tableColumn = modelFactory.createTableColumn(tableModel, column, false);
9478                                                                                        if(tableColumn == null) {
9479                                                                                                continue;
9480                                                                                        }
9481                                                                                }
9482                                                                        }
9483//                                                      } else {
9484//                                                              tableColumn = matchColumn(tableColumns, column);
9485//                                                              if (tableColumn == null) {
9486//                                                                      continue;
9487//                                                              }
9488//                                                      }
9489                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9490                                                                        relation.setEffectType(effectType);
9491                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9492                                                                        if (resultColumn.hasStarLinkColumn()
9493                                                                                        && resultColumn.getStarLinkColumnNames().size() > starIndex - 1 && starColumnCount<=1) {
9494                                                                                boolean find = false;
9495                                                                                while (resultColumn.getStarLinkColumnNames().size() > starIndex - 1) {
9496                                                                                        TObjectName name = resultColumn.getStarLinkColumnName(starIndex - 1);
9497                                                                                        if (itemNames.contains(name.toString())) {
9498                                                                                                starIndex++;
9499                                                                                                continue;
9500                                                                                        }
9501                                                                                        ResultColumn expandStarColumn = modelFactory
9502                                                                                                        .createResultColumn(resultSetModel, name, false);
9503                                                                                        relation.addSource(new ResultColumnRelationshipElement(expandStarColumn));
9504                                                                                        itemNames.add(resultColumn.getName());
9505                                                                                        find = true;
9506                                                                                        break;
9507                                                                                }
9508                                                                                if (!find) {
9509                                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9510                                                                                        itemNames.add(resultColumn.getName());
9511                                                                                }
9512                                                                        } else {
9513                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9514                                                                                itemNames.add(resultColumn.getName());
9515                                                                        }
9516                                                                        Process process = modelFactory.createProcess(stmt);
9517                                                                        relation.setProcess(process);
9518                                                                }
9519                                                        }
9520                                                } else {
9521                                                        List<TableColumn> columns = tableModel.getColumns();
9522                                                        if (columns.size() == 1 && tableModel.isPath()) {
9523                                                                for(int i=0;i<resultSetSize;i++) {
9524                                                                        ResultColumn resultColumn = resultSetModel.getColumns().get(i);
9525                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9526                                                                        relation.setEffectType(effectType);
9527                                                                        relation.setTarget(new TableColumnRelationshipElement(tableModel.getColumns().get(0)));
9528                                                                        if (resultColumn.hasStarLinkColumn()
9529                                                                                        && resultColumn.getStarLinkColumnNames().size() > starIndex - 1) {
9530                                                                                ResultColumn expandStarColumn = modelFactory.createResultColumn(
9531                                                                                                resultSetModel, resultColumn.getStarLinkColumnName(starIndex - 1),
9532                                                                                                false);
9533                                                                                relation.addSource(new ResultColumnRelationshipElement(expandStarColumn));
9534                                                                        } else {
9535                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9536                                                                        }
9537                                                                        Process process = modelFactory.createProcess(stmt);
9538                                                                        relation.setProcess(process);
9539                                                                }
9540                                                        } else {
9541                                                                boolean fromStruct = false;
9542                                                                for (int i = 0; i < columns.size() && j < resultSetSize; i++) {
9543                                                                        String column = columns.get(i).getName();
9544                                                                        ResultColumn resultColumn = resultSetModel.getColumns().get(j);
9545                                                                        if (!resultColumn.getName().contains("*")) {
9546                                                                                if (resultColumn.getName().equals(resultColumn.getRefColumnName())
9547                                                                                                && resultColumn.getColumnObject().toString().endsWith("*")
9548                                                                                                && resultSetSize == 1) {
9549                                                                                        starIndex++;
9550                                                                                        if (resultSetSize - j == columns.size() - i) {
9551                                                                                                j++;
9552
9553                                                                                        }
9554                                                                                }
9555                                                                                else {
9556                                                                                        j++;
9557                                                                                }
9558                                                                        } else {
9559                                                                                starIndex++;
9560                                                                                if (resultSetSize - j == columns.size() - i) {
9561                                                                                        j++;
9562
9563                                                                                }
9564                                                                        }
9565                                                                        if (column != null) {
9566                                                                                TableColumn tableColumn;
9567                                                                                // if (!initColumn) {
9568                                                                                tableColumn = matchColumn(tableModel.getColumns(), columns.get(i));
9569                                                                                if (tableColumn == null) {
9570                                                                                        if (tableModel.isCreateTable()
9571                                                                                                        && !containStarColumn(tableModel.getColumns())) {
9572                                                                                                if (tableModel.getColumns().size() <= i) {
9573                                                                                                        continue;
9574                                                                                                }
9575                                                                                                tableColumn = tableModel.getColumns().get(i);
9576                                                                                        } else {
9577                                                                                                TObjectName columnName = new TObjectName();
9578                                                                                                columnName.setString(column);
9579                                                                                                tableColumn = modelFactory.createTableColumn(tableModel, columnName,
9580                                                                                                                false);
9581                                                                                        }
9582                                                                                }
9583                                                                                else if (!resultColumn.isStruct() && tableColumn.isStruct() && columns.size() != resultSetSize) {
9584                                                                                        j--;
9585                                                                                        fromStruct = true;
9586                                                                                }
9587                                                                                if(fromStruct && !tableColumn.isStruct()) {
9588                                                                                        fromStruct = false;
9589                                                                                        resultColumn = resultSetModel.getColumns().get(j);
9590                                                                                        j++;
9591                                                                                }
9592                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9593                                                                                relation.setEffectType(effectType);
9594                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9595                                                                                if (resultColumn.hasStarLinkColumn()
9596                                                                                                && resultColumn.getStarLinkColumnNames().size() > starIndex - 1) {
9597                                                                                        ResultColumn expandStarColumn = modelFactory.createResultColumn(
9598                                                                                                        resultSetModel, resultColumn.getStarLinkColumnName(starIndex - 1),
9599                                                                                                        false);
9600                                                                                        relation.addSource(new ResultColumnRelationshipElement(expandStarColumn));
9601                                                                                } else {
9602                                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn, starIndex - 1));
9603                                                                                }
9604                                                                                Process process = modelFactory.createProcess(stmt);
9605                                                                                relation.setProcess(process);
9606                                                                        }
9607                                                                }
9608                                                        }
9609                                                }
9610                                        } else if (!subquery.isCombinedQuery()) {
9611                                                SelectResultSet resultSetModel = (SelectResultSet) modelManager
9612                                                                .getModel(subquery.getResultColumnList() != null ? subquery.getResultColumnList()
9613                                                                                : subquery.getTransformClause());
9614
9615                                                if (resultSetModel != null && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
9616                                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
9617                                                        impactRelation.setEffectType(effectType);
9618                                                        impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
9619                                                                        resultSetModel.getRelationRows()));
9620                                                        impactRelation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(
9621                                                                        tableModel.getRelationRows()));
9622                                                }
9623
9624                                                List<ResultColumn> columnsSnapshot = new ArrayList<ResultColumn>();
9625                                                columnsSnapshot.addAll(resultSetModel.getColumns());
9626                                                if(resultSetModel.isDetermined() && stmt.getColumnList() == null) {
9627                                                        tableModel.setDetermined(true);
9628                                                }
9629                                                for (int i = 0; i < columnsSnapshot.size(); i++) {
9630                                                        ResultColumn resultColumn = columnsSnapshot.get(i);
9631                                                        if (resultColumn.getColumnObject() instanceof TObjectName) {
9632                                                                TableColumn tableColumn;
9633                                                                if (!initColumn) {
9634                                                                        if (tableModel.isCreateTable() && !containStarColumn(tableModel.getColumns())) {
9635                                                                                if (tableModel.getColumns().size() <= i) {
9636                                                                                        continue;
9637                                                                                }
9638                                                                                tableColumn = tableModel.getColumns().get(i);
9639                                                                        } else {
9640                                                                                tableColumn = modelFactory.createInsertTableColumn(tableModel,
9641                                                                                                (TObjectName) resultColumn.getColumnObject());
9642                                                                        }
9643                                                                        if (containStarColumn(tableColumns)) {
9644                                                                                getStarColumn(tableColumns)
9645                                                                                                .bindStarLinkColumn((TObjectName) resultColumn.getColumnObject());
9646                                                                        }
9647                                                                } else {
9648                                                                        TObjectName matchedColumnName = (TObjectName) resultColumn.getColumnObject();
9649                                                                        tableColumn = matchColumn(tableColumns, matchedColumnName);
9650                                                                        if (tableColumn == null) {
9651                                                                                if (!isEmptyCollection(valueMap)) {
9652                                                                                        int index = indexOfColumn(valueMap, matchedColumnName);
9653                                                                                        if (index != -1) {
9654                                                                                                if (!isEmptyCollection(keyMap) && index < keyMap.size()) {
9655                                                                                                        tableColumn = matchColumn(tableColumns, keyMap.get(index));
9656                                                                                                } else if (isEmptyCollection(keyMap) && index < tableColumns.size()) {
9657                                                                                                        tableColumn = tableColumns.get(index);
9658                                                                                                } else {
9659                                                                                                        continue;
9660                                                                                                }
9661                                                                                        } else {
9662                                                                                                continue;
9663                                                                                        }
9664                                                                                } else if (!isEmptyCollection(keyMap) && i < keyMap.size()) {
9665                                                                                        tableColumn = matchColumn(tableColumns, keyMap.get(i));
9666                                                                                } else if (isEmptyCollection(keyMap) && isEmptyCollection(valueMap)
9667                                                                                                && i < tableColumns.size()) {
9668                                                                                        tableColumn = tableColumns.get(i);
9669                                                                                } else {
9670                                                                                        continue;
9671                                                                                }
9672                                                                        }
9673                                                                }
9674
9675                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9676                                                                relation.setEffectType(effectType);
9677                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9678                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9679                                                                Process process = modelFactory.createProcess(stmt);
9680                                                                relation.setProcess(process);
9681                                                        } else {
9682                                                                TAliasClause alias = ((TResultColumn) resultColumn.getColumnObject()).getAliasClause();
9683                                                                if (alias != null && alias.getAliasName() != null) {
9684                                                                        TableColumn tableColumn;
9685                                                                        if (!initColumn) {
9686                                                                                if (tableModel.isCreateTable() && !containStarColumn(tableModel.getColumns())) {
9687                                                                                        if (tableModel.getColumns().size() <= i) {
9688                                                                                                if (tableModel.isPath()) {
9689                                                                                                        tableColumn = tableModel.getColumns().get(0);
9690                                                                                                } else {
9691                                                                                                        continue;
9692                                                                                                }
9693                                                                                        } else {
9694                                                                                                tableColumn = tableModel.getColumns().get(i);
9695                                                                                        }
9696                                                                                } else {
9697                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
9698                                                                                                        alias.getAliasName());
9699                                                                                        if (containStarColumn(resultSetModel)) {
9700                                                                                                tableColumn.notBindStarLinkColumn(true);
9701                                                                                        }
9702                                                                                }
9703                                                                                if (containStarColumn(tableColumns)) {
9704                                                                                        getStarColumn(tableColumns).bindStarLinkColumn(alias.getAliasName());
9705                                                                                }
9706                                                                        } else {
9707                                                                                TObjectName matchedColumnName = alias.getAliasName();
9708                                                                                tableColumn = matchColumn(tableColumns, matchedColumnName);
9709                                                                                if (tableColumn == null) {
9710                                                                                        if (!isEmptyCollection(valueMap)) {
9711                                                                                                int index = indexOfColumn(valueMap, matchedColumnName);
9712                                                                                                if (index != -1) {
9713                                                                                                        if (!isEmptyCollection(keyMap) && index < keyMap.size()) {
9714                                                                                                                tableColumn = matchColumn(tableColumns, keyMap.get(index));
9715                                                                                                        } else if (isEmptyCollection(keyMap)
9716                                                                                                                        && index < tableColumns.size()) {
9717                                                                                                                tableColumn = tableColumns.get(index);
9718                                                                                                        } else {
9719                                                                                                                continue;
9720                                                                                                        }
9721                                                                                                } else {
9722                                                                                                        continue;
9723                                                                                                }
9724                                                                                        } else if (!isEmptyCollection(keyMap) && i < keyMap.size()) {
9725                                                                                                tableColumn = matchColumn(tableColumns, keyMap.get(i));
9726                                                                                        } else {
9727                                                                                                tableColumn = modelFactory.createInsertTableColumn(tableModel,
9728                                                                                                                alias.getAliasName());
9729                                                                                        }
9730                                                                                }
9731
9732                                                                        }
9733
9734                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9735                                                                        relation.setEffectType(effectType);
9736                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9737                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9738                                                                        Process process = modelFactory.createProcess(stmt);
9739                                                                        relation.setProcess(process);
9740
9741                                                                } else if (((TResultColumn) resultColumn.getColumnObject()).getFieldAttr() != null) {
9742                                                                        TObjectName fieldAttr = ((TResultColumn) resultColumn.getColumnObject())
9743                                                                                        .getFieldAttr();
9744
9745                                                                        Object model = modelManager.getModel( resultColumn.getColumnObject());
9746                                                                        
9747                                                                        TableColumn tableColumn;
9748                                                                        if (!initColumn) {
9749                                                                                if (tableModel.isCreateTable() && !containStarColumn(tableModel.getColumns())) {
9750                                                                                        if (fieldAttr.toString().endsWith("*")) {
9751                                                                                                int starIndex = 0;
9752                                                                                                for (TableColumn column : tableModel.getColumns()) {
9753                                                                                                        starIndex++;
9754                                                                                                        DataFlowRelationship relation = modelFactory
9755                                                                                                                        .createDataFlowRelation();
9756                                                                                                        relation.setEffectType(effectType);
9757                                                                                                        relation.setTarget(new TableColumnRelationshipElement(column));
9758                                                                                                        if (resultColumn.getStarLinkColumnList().size() == tableModel
9759                                                                                                                        .getColumns().size()) {
9760                                                                                                                ResultColumn expandStarColumn = modelFactory.createResultColumn(
9761                                                                                                                                resultSetModel,
9762                                                                                                                                resultColumn.getStarLinkColumnList().get(starIndex - 1),
9763                                                                                                                                false);
9764                                                                                                                relation.addSource(
9765                                                                                                                                new ResultColumnRelationshipElement(expandStarColumn));
9766                                                                                                        } else {
9767                                                                                                                relation.addSource(
9768                                                                                                                                new ResultColumnRelationshipElement(resultColumn));
9769                                                                                                        }
9770                                                                                                        Process process = modelFactory.createProcess(stmt);
9771                                                                                                        relation.setProcess(process);
9772                                                                                                }
9773                                                                                                continue;
9774                                                                                        }
9775                                                                                        if (tableModel.getColumns().size() <= i) {
9776                                                                                                continue;
9777                                                                                        }
9778                                                                                        tableColumn = tableModel.getColumns().get(i);
9779                                                                                } else {
9780                                                                                        if(model instanceof LinkedHashMap) {
9781                                                                                                LinkedHashMap<String, ResultColumn> resultColumns = (LinkedHashMap<String, ResultColumn>)model;
9782                                                                                                for(String key: resultColumns.keySet()) {
9783                                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel, resultColumns.get(key).getName());
9784                                                                                                        DataFlowRelationship relation = modelFactory
9785                                                                                                                        .createDataFlowRelation();
9786                                                                                                        relation.setEffectType(effectType);
9787                                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9788                                                                                                        relation.addSource(
9789                                                                                                                        new ResultColumnRelationshipElement(resultColumns.get(key)));
9790                                                                                                        Process process = modelFactory.createProcess(stmt);
9791                                                                                                        relation.setProcess(process);
9792                                                                                                }
9793                                                                                                continue;
9794                                                                                        }
9795                                                                                        else {
9796                                                                                                tableColumn = modelFactory.createInsertTableColumn(tableModel, fieldAttr);
9797                                                                                        }
9798                                                                                }
9799                                                                        } else if (tableModel.isDetermined() && i < tableModel.getColumns().size()) {
9800                                                                                if (!isEmptyCollection(valueMap)) {
9801                                                                                        TObjectName matchedColumnName = fieldAttr;
9802                                                                                        int index = indexOfColumn(valueMap, matchedColumnName);
9803                                                                                        if (index != -1) {
9804                                                                                                if (!isEmptyCollection(keyMap) && index < keyMap.size()) {
9805                                                                                                        tableColumn = matchColumn(tableColumns, keyMap.get(index));
9806                                                                                                } else if (isEmptyCollection(keyMap)
9807                                                                                                                && index < tableColumns.size()) {
9808                                                                                                        tableColumn = tableColumns.get(index);
9809                                                                                                } else {
9810                                                                                                        continue;
9811                                                                                                }
9812                                                                                        } else {
9813                                                                                                continue;
9814                                                                                        }
9815                                                                                } else if (!isEmptyCollection(keyMap) && i < keyMap.size()) {
9816                                                                                        tableColumn = matchColumn(tableColumns, keyMap.get(i));
9817                                                                                } else {
9818                                                                                        tableColumn = tableModel.getColumns().get(i);
9819                                                                                }
9820                                                                        } else {
9821                                                                                TObjectName matchedColumnName = fieldAttr;
9822                                                                                tableColumn = matchColumn(tableColumns, matchedColumnName);
9823                                                                                if (tableColumn == null) {
9824                                                                                        if (!isEmptyCollection(valueMap)) {
9825                                                                                                int index = indexOfColumn(valueMap, matchedColumnName);
9826                                                                                                if (index != -1) {
9827                                                                                                        if (!isEmptyCollection(keyMap) && index < keyMap.size()) {
9828                                                                                                                tableColumn = matchColumn(tableColumns, keyMap.get(index));
9829                                                                                                        } else if (isEmptyCollection(keyMap)
9830                                                                                                                        && index < tableColumns.size()) {
9831                                                                                                                tableColumn = tableColumns.get(index);
9832                                                                                                        } else {
9833                                                                                                                continue;
9834                                                                                                        }
9835                                                                                                } else {
9836                                                                                                        continue;
9837                                                                                                }
9838                                                                                        } else if (!isEmptyCollection(keyMap) && i < keyMap.size()) {
9839                                                                                                tableColumn = matchColumn(tableColumns, keyMap.get(i));
9840                                                                                        } else {
9841                                                                                                tableColumn = modelFactory.createInsertTableColumn(tableModel,
9842                                                                                                                fieldAttr);
9843                                                                                        }
9844                                                                                }
9845                                                                        }
9846
9847                                                                        if (!"*".equals(getColumnName(tableColumn.getColumnObject()))
9848                                                                                        && "*".equals(getColumnName(fieldAttr))) {
9849                                                                                TObjectName columnObject = fieldAttr;
9850                                                                                TTable sourceTable = columnObject.getSourceTable();
9851                                                                                if (columnObject.getTableToken() != null && sourceTable != null) {
9852                                                                                        TObjectName[] columns = modelManager.getTableColumns(sourceTable);
9853                                                                                        for (int j = 0; j < columns.length; j++) {
9854                                                                                                TObjectName columnName = columns[j];
9855                                                                                                if (columnName == null || "*".equals(getColumnName(columnName))) {
9856                                                                                                        continue;
9857                                                                                                }
9858                                                                                                resultColumn.bindStarLinkColumn(columnName);
9859                                                                                        }
9860                                                                                } else {
9861                                                                                        TTableList tables = stmt.getTables();
9862                                                                                        for (int k = 0; k < tables.size(); k++) {
9863                                                                                                TTable tableElement = tables.getTable(k);
9864                                                                                                TObjectName[] columns = modelManager.getTableColumns(tableElement);
9865                                                                                                for (int j = 0; j < columns.length; j++) {
9866                                                                                                        TObjectName columnName = columns[j];
9867                                                                                                        if (columnName == null || "*".equals(getColumnName(columnName))) {
9868                                                                                                                continue;
9869                                                                                                        }
9870                                                                                                        resultColumn.bindStarLinkColumn(columnName);
9871                                                                                                }
9872                                                                                        }
9873                                                                                }
9874                                                                        }
9875
9876                                                                        if ("*".equals(getColumnName(tableColumn.getColumnObject())) && resultColumn != null
9877                                                                                        && !resultColumn.getStarLinkColumns().isEmpty()) {
9878                                                                                tableColumn.bindStarLinkColumns(resultColumn.getStarLinkColumns());
9879                                                                        }
9880
9881                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9882                                                                        relation.setEffectType(effectType);
9883                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9884                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9885
9886                                                                        if (tableColumn.getName().endsWith("*") && resultColumn.getName().endsWith("*")) {
9887                                                                                tableColumn.getTable().setStarStmt("insert");
9888                                                                        }
9889
9890                                                                        Process process = modelFactory.createProcess(stmt);
9891                                                                        relation.setProcess(process);
9892                                                                } else if (((TResultColumn) resultColumn.getColumnObject()).getExpr()
9893                                                                                .getExpressionType() == EExpressionType.simple_constant_t) {
9894                                                                        if (!initColumn) {
9895                                                                                TableColumn tableColumn;
9896                                                                                if (tableModel.isCreateTable() && !containStarColumn(tableModel.getColumns())) {
9897                                                                                        if (tableModel.getColumns().size() <= i) {
9898                                                                                                continue;
9899                                                                                        }
9900                                                                                        tableColumn = tableModel.getColumns().get(i);
9901                                                                                } else {
9902                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
9903                                                                                                        ((TResultColumn) resultColumn.getColumnObject()).getExpr()
9904                                                                                                                        .getConstantOperand(),
9905                                                                                                        i);
9906                                                                                }
9907
9908                                                                                if (DlineageUtil.isTempTable(tableModel, option.getVendor()) && sqlenv != null
9909                                                                                                && tableModel.getDatabase() != null && tableModel.getSchema() != null) {
9910                                                                                        TSQLSchema schema = sqlenv.getSQLSchema(
9911                                                                                                        tableModel.getDatabase() + "." + tableModel.getSchema(), true);
9912                                                                                        if (schema != null) {
9913                                                                                                TSQLTable tempTable = schema.createTable(
9914                                                                                                                DlineageUtil.getSimpleTableName(tableModel.getName()));
9915                                                                                                tempTable.addColumn(tableColumn.getName());
9916                                                                                        }
9917                                                                                }
9918                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9919                                                                                relation.setEffectType(effectType);
9920                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9921                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9922                                                                                Process process = modelFactory.createProcess(stmt);
9923                                                                                relation.setProcess(process);
9924                                                                        } else {
9925                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9926                                                                                relation.setEffectType(effectType);
9927                                                                                relation.setTarget(
9928                                                                                                new TableColumnRelationshipElement(tableModel.getColumns().get(i)));
9929                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9930                                                                                Process process = modelFactory.createProcess(stmt);
9931                                                                                relation.setProcess(process);
9932                                                                        }
9933                                                                } else {
9934                                                                        if (!initColumn) {
9935                                                                                TableColumn tableColumn;
9936                                                                                if (tableModel.isCreateTable() && !containStarColumn(tableModel.getColumns())) {
9937                                                                                        if (tableModel.getColumns().size() <= i) {
9938                                                                                                continue;
9939                                                                                        }
9940                                                                                        tableColumn = tableModel.getColumns().get(i);
9941                                                                                } else {
9942                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
9943                                                                                                        ((TResultColumn) resultColumn.getColumnObject()).getExpr(), i);
9944                                                                                }
9945                                                                                if (DlineageUtil.isTempTable(tableModel, option.getVendor()) && sqlenv != null
9946                                                                                                && tableModel.getDatabase() != null && tableModel.getSchema() != null) {
9947                                                                                        TSQLSchema schema = sqlenv.getSQLSchema(
9948                                                                                                        tableModel.getDatabase() + "." + tableModel.getSchema(), true);
9949                                                                                        if (schema != null) {
9950                                                                                                TSQLTable tempTable = schema.createTable(
9951                                                                                                                DlineageUtil.getSimpleTableName(tableModel.getName()));
9952                                                                                                tempTable.addColumn(tableColumn.getName());
9953                                                                                        }
9954                                                                                }
9955                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9956                                                                                relation.setEffectType(effectType);
9957                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
9958                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9959                                                                                Process process = modelFactory.createProcess(stmt);
9960                                                                                relation.setProcess(process);
9961                                                                        } else {
9962                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9963                                                                                relation.setEffectType(effectType);
9964                                                                                relation.setTarget(
9965                                                                                                new TableColumnRelationshipElement(tableModel.getColumns().get(i)));
9966                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
9967                                                                                Process process = modelFactory.createProcess(stmt);
9968                                                                                relation.setProcess(process);
9969                                                                        }
9970                                                                }
9971                                                        }
9972                                                }
9973                                        } else if (stmt.getSubQuery() != null) {
9974                                                ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt.getSubQuery());
9975                                                if (resultSetModel != null) {
9976
9977                                                        if (!resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
9978                                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
9979                                                                impactRelation.setEffectType(effectType);
9980                                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
9981                                                                                resultSetModel.getRelationRows()));
9982                                                                impactRelation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(
9983                                                                                tableModel.getRelationRows()));
9984                                                        }
9985
9986                                                        if(stmt.getColumnList()!=null && stmt.getColumnList().size()>0) {
9987                                                                for(int i=0;i<stmt.getColumnList().size();i++) {
9988                                                                        TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
9989                                                                                        stmt.getColumnList().getObjectName(i));
9990                                                                }
9991                                                                
9992                                                                List<TableColumn> columns = tableModel.getColumns();
9993                                                                int resultSetSize = resultSetModel.getColumns().size();
9994                                                                int starIndex = 0; 
9995                                                                int j = 0;
9996                                                                boolean fromStruct = false;
9997                                                                for (int i = 0; i < columns.size() && j < resultSetSize; i++) {
9998                                                                        String column = columns.get(i).getName();
9999                                                                        ResultColumn resultColumn = resultSetModel.getColumns().get(j);
10000                                                                        if (!resultColumn.getName().contains("*")) {
10001                                                                                if (resultColumn.getName().equals(resultColumn.getRefColumnName())
10002                                                                                                && resultColumn.getColumnObject().toString().endsWith("*")
10003                                                                                                && resultSetSize == 1) {
10004                                                                                        starIndex++;
10005                                                                                        if (resultSetSize - j == columns.size() - i) {
10006                                                                                                j++;
10007
10008                                                                                        }
10009                                                                                }
10010                                                                                else {
10011                                                                                        j++;
10012                                                                                }
10013                                                                        } else {
10014                                                                                starIndex++;
10015                                                                                if (resultSetSize - j == columns.size() - i) {
10016                                                                                        j++;
10017
10018                                                                                }
10019                                                                        }
10020                                                                        if (column != null) {
10021                                                                                TableColumn tableColumn;
10022                                                                                // if (!initColumn) {
10023                                                                                tableColumn = matchColumn(tableModel.getColumns(), columns.get(i));
10024                                                                                if (tableColumn == null) {
10025                                                                                        if (tableModel.isCreateTable()
10026                                                                                                        && !containStarColumn(tableModel.getColumns())) {
10027                                                                                                if (tableModel.getColumns().size() <= i) {
10028                                                                                                        continue;
10029                                                                                                }
10030                                                                                                tableColumn = tableModel.getColumns().get(i);
10031                                                                                        } else {
10032                                                                                                TObjectName columnName = new TObjectName();
10033                                                                                                columnName.setString(column);
10034                                                                                                tableColumn = modelFactory.createTableColumn(tableModel, columnName,
10035                                                                                                                false);
10036                                                                                        }
10037                                                                                }
10038                                                                                else if (!resultColumn.isStruct() && tableColumn.isStruct() && columns.size() != resultSetSize) {
10039                                                                                        j--;
10040                                                                                        fromStruct = true;
10041                                                                                }
10042                                                                                if(fromStruct && !tableColumn.isStruct()) {
10043                                                                                        fromStruct = false;
10044                                                                                        resultColumn = resultSetModel.getColumns().get(j);
10045                                                                                        j++;
10046                                                                                }
10047                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10048                                                                                relation.setEffectType(effectType);
10049                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
10050                                                                                if (resultColumn.hasStarLinkColumn()
10051                                                                                                && resultColumn.getStarLinkColumnNames().size() > starIndex - 1) {
10052                                                                                        ResultColumn expandStarColumn = modelFactory.createResultColumn(
10053                                                                                                        resultSetModel, resultColumn.getStarLinkColumnName(starIndex - 1),
10054                                                                                                        false);
10055                                                                                        relation.addSource(new ResultColumnRelationshipElement(expandStarColumn));
10056                                                                                } else {
10057                                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn, starIndex - 1));
10058                                                                                }
10059                                                                                Process process = modelFactory.createProcess(stmt);
10060                                                                                relation.setProcess(process);
10061                                                                        }
10062                                                                }
10063                                                        }
10064                                                        else {
10065                                                                for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
10066                                                                        ResultColumn resultColumn = resultSetModel.getColumns().get(i);
10067                                                                        TAliasClause alias = null;
10068                                                                        if(resultColumn.getColumnObject() instanceof TResultColumn) {
10069                                                                                alias = ((TResultColumn) resultColumn.getColumnObject()).getAliasClause();
10070                                                                        }
10071                                                                        if (stmt.getColumnList() != null) {
10072                                                                                if (i < stmt.getColumnList().size()) {
10073                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10074                                                                                        relation.setEffectType(effectType);
10075                                                                                        TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
10076                                                                                                        stmt.getColumnList().getObjectName(i));
10077                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
10078                                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
10079                                                                                        Process process = modelFactory.createProcess(stmt);
10080                                                                                        relation.setProcess(process);
10081                                                                                }
10082                                                                        } else {
10083                                                                                if (alias != null && alias.getAliasName() != null) {
10084                                                                                        TableColumn tableColumn;
10085                                                                                        if (!initColumn) {
10086                                                                                                if (tableModel.isCreateTable()
10087                                                                                                                && !containStarColumn(tableModel.getColumns())) {
10088                                                                                                        if (tableModel.getColumns().size() <= i) {
10089                                                                                                                continue;
10090                                                                                                        }
10091                                                                                                        tableColumn = tableModel.getColumns().get(i);
10092                                                                                                } else {
10093                                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
10094                                                                                                                        alias.getAliasName());
10095                                                                                                }
10096                                                                                        } else {
10097                                                                                                tableColumn = matchColumn(tableColumns, alias.getAliasName());
10098                                                                                                if (tableColumn == null) {
10099                                                                                                        continue;
10100                                                                                                }
10101                                                                                        }
10102        
10103                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10104                                                                                        relation.setEffectType(effectType);
10105                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
10106                                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
10107                                                                                        Process process = modelFactory.createProcess(stmt);
10108                                                                                        relation.setProcess(process);
10109                                                                                } else if (resultColumn.getColumnObject() instanceof TObjectName 
10110                                                                                                || ( resultColumn.getColumnObject() instanceof TResultColumn && ((TResultColumn) resultColumn.getColumnObject())
10111                                                                                                .getFieldAttr() != null)) {
10112                                                                                        TObjectName fieldAttr = null;
10113                                                                                        if (resultColumn.getColumnObject() instanceof TObjectName) {
10114                                                                                                fieldAttr = (TObjectName)resultColumn.getColumnObject();
10115                                                                                        }
10116                                                                                        else if (resultColumn.getColumnObject() instanceof TResultColumn) {
10117                                                                                                fieldAttr = ((TResultColumn) resultColumn.getColumnObject())
10118                                                                                                                .getFieldAttr();
10119                                                                                        }
10120                                
10121                                                                                        TableColumn tableColumn;
10122                                                                                        if (!initColumn) {
10123                                                                                                if (tableModel.isCreateTable()
10124                                                                                                                && !containStarColumn(tableModel.getColumns())) {
10125                                                                                                        if (tableModel.getColumns().size() <= i) {
10126                                                                                                                continue;
10127                                                                                                        }
10128                                                                                                        tableColumn = tableModel.getColumns().get(i);
10129                                                                                                } else {
10130                                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
10131                                                                                                                        fieldAttr);
10132                                                                                                }
10133                                                                                        } else {
10134                                                                                                tableColumn = matchColumn(tableColumns, fieldAttr);
10135                                                                                                if (tableColumn == null) {
10136                                                                                                        continue;
10137                                                                                                }
10138                                                                                        }
10139        
10140                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10141                                                                                        relation.setEffectType(effectType);
10142                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
10143                                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
10144                                                                                        Process process = modelFactory.createProcess(stmt);
10145                                                                                        relation.setProcess(process);
10146                                                                                } else if (((TResultColumn) resultColumn.getColumnObject()).getExpr()
10147                                                                                                .getExpressionType() == EExpressionType.simple_constant_t) {
10148                                                                                        if (!initColumn) {
10149                                                                                                TableColumn tableColumn;
10150                                                                                                if (tableModel.isCreateTable()
10151                                                                                                                && !containStarColumn(tableModel.getColumns())) {
10152                                                                                                        if (tableModel.getColumns().size() <= i) {
10153                                                                                                                continue;
10154                                                                                                        }
10155                                                                                                        tableColumn = tableModel.getColumns().get(i);
10156                                                                                                } else {
10157                                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
10158                                                                                                                        ((TResultColumn) resultColumn.getColumnObject()).getExpr()
10159                                                                                                                                        .getConstantOperand(),
10160                                                                                                                        i);
10161                                                                                                }
10162                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10163                                                                                                relation.setEffectType(effectType);
10164                                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
10165                                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
10166                                                                                                Process process = modelFactory.createProcess(stmt);
10167                                                                                                relation.setProcess(process);
10168                                                                                        }
10169                                                                                } else {
10170                                                                                        if (!initColumn) {
10171                                                                                                TableColumn tableColumn;
10172                                                                                                if (tableModel.isCreateTable()
10173                                                                                                                && !containStarColumn(tableModel.getColumns())) {
10174                                                                                                        if (tableModel.getColumns().size() <= i) {
10175                                                                                                                continue;
10176                                                                                                        }
10177                                                                                                        tableColumn = tableModel.getColumns().get(i);
10178                                                                                                } else {
10179                                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
10180                                                                                                                        ((TResultColumn) resultColumn.getColumnObject()).getExpr(), i);
10181                                                                                                }
10182                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10183                                                                                                relation.setEffectType(effectType);
10184                                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
10185                                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
10186                                                                                                Process process = modelFactory.createProcess(stmt);
10187                                                                                                relation.setProcess(process);
10188                                                                                        }
10189                                                                                }
10190                                                                        }
10191                                                                }
10192                                                        }
10193                                                }
10194                                        }
10195                                }
10196                        } else if (stmt.getColumnList() != null && stmt.getColumnList().size() > 0) {
10197                                TObjectNameList items = stmt.getColumnList();
10198                                TMultiTargetList values = stmt.getValues();
10199                                if (values != null) {
10200                                        for (int k = 0; values != null && k < values.size(); k++) {
10201                                                int j = 0;
10202                                                for (int i = 0; i < items.size(); i++) {
10203                                                        TObjectName column = items.getObjectName(i);
10204                                                        TableColumn tableColumn;
10205                                                        if (!initColumn) {
10206                                                                if (tableModel.isCreateTable() && !containStarColumn(tableModel.getColumns())) {
10207                                                                        if (tableModel.getColumns().size() <= i) {
10208                                                                                continue;
10209                                                                        }
10210                                                                        tableColumn = tableModel.getColumns().get(i);
10211                                                                } else {
10212                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel, column);
10213                                                                }
10214                                                        } else {
10215                                                                tableColumn = matchColumn(tableColumns, column);
10216                                                                if (tableColumn == null) {
10217                                                                        continue;
10218                                                                }
10219                                                        }
10220                                                        TResultColumn columnObject = values.getMultiTarget(k).getColumnList().getResultColumn(j);
10221                                                        if (columnObject == null) {
10222                                                                continue;
10223                                                        }
10224                                                        TExpression valueExpr = columnObject.getExpr();
10225                                                        columnsInExpr visitor = new columnsInExpr();
10226                                                        valueExpr.inOrderTraverse(visitor);
10227                                                        List<TObjectName> objectNames = visitor.getObjectNames();
10228                                                        List<TParseTreeNode> constants = visitor.getConstants();
10229                                                        List<TParseTreeNode> functions = visitor.getFunctions();
10230                                                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
10231
10232                                                        Process process = modelFactory.createProcess(stmt);
10233
10234
10235
10236                                                        if (functions != null && !functions.isEmpty()) {
10237                                                                analyzeFunctionDataFlowRelation(tableColumn, functions, effectType, process);
10238                                                        }
10239
10240                                                        if (subquerys != null && !subquerys.isEmpty()) {
10241                                                                analyzeSubqueryDataFlowRelation(tableColumn, subquerys, effectType, process);
10242                                                        }
10243                                                        if (objectNames != null && !objectNames.isEmpty()) {
10244                                                                analyzeDataFlowRelation(tableColumn, objectNames, null, effectType, functions,
10245                                                                                process, i);
10246                                                        }
10247                                                        //insert into values generate too many constant relations, ignore constant relations.
10248                                                        if (constants != null && !constants.isEmpty()) {
10249                                                                if (!option.isIgnoreInsertIntoValues() || stmt.getParentStmt() != null) {
10250                                                                        analyzeConstantDataFlowRelation(tableColumn, constants, effectType,
10251                                                                                        functions, process);
10252                                                                }
10253                                                        }
10254                                                        j++;
10255                                                }
10256                                        }
10257                                } else if (stmt.getExecuteStmt() != null && stmt.getExecuteStmt().getModuleName() != null) {
10258                                        analyzeCustomSqlStmt(stmt.getExecuteStmt());
10259                                        Procedure procedure = modelManager.getProcedureByName(DlineageUtil
10260                                                        .getIdentifierNormalTableName(stmt.getExecuteStmt().getModuleName().toString()));
10261                                        if (procedure!=null && procedure.getProcedureObject() instanceof TStoredProcedureSqlStatement) {
10262                                                TStoredProcedureSqlStatement procedureStmt = (TStoredProcedureSqlStatement) procedure
10263                                                                .getProcedureObject();
10264                                                List<TSelectSqlStatement> stmtItems = getLastSelectStmt(procedureStmt);
10265                                                if (stmtItems != null) {
10266                                                        for(TSelectSqlStatement stmtItem: stmtItems) {
10267                                                                ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmtItem);
10268                                                                if (resultSetModel != null) {
10269                                                                        for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
10270                                                                                ResultColumn resultColumn = resultSetModel.getColumns().get(i);
10271                                                                                Transform transform = new Transform();
10272                                                                                transform.setType(Transform.FUNCTION);
10273                                                                                transform.setCode(stmt.getExecuteStmt().getModuleName());
10274                                                                                resultColumn.setTransform(transform);
10275                                                                                
10276                                                                                if (stmt.getColumnList() != null) {
10277                                                                                        if (i < stmt.getColumnList().size()) {
10278                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10279                                                                                                relation.setEffectType(effectType);
10280                                                                                                TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
10281                                                                                                                stmt.getColumnList().getObjectName(i));                                                                                         
10282                                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
10283                                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
10284                                                                                                Process process = modelFactory.createProcess(stmt);
10285                                                                                                relation.setProcess(process);                                   
10286                                                                                        }
10287                                                                                } else {
10288                                                                                        if (resultColumn.getColumnObject() instanceof TObjectName) {
10289                                                                                                TObjectName fieldAttr = ((TObjectName) resultColumn.getColumnObject());
10290                                                                                                TableColumn tableColumn;
10291                                                                                                if (!initColumn) {
10292                                                                                                        if (tableModel.isCreateTable()
10293                                                                                                                        && !containStarColumn(tableModel.getColumns())) {
10294                                                                                                                if (tableModel.getColumns().size() <= i) {
10295                                                                                                                        continue;
10296                                                                                                                }
10297                                                                                                                tableColumn = tableModel.getColumns().get(i);
10298                                                                                                        } else {
10299                                                                                                                tableColumn = modelFactory.createInsertTableColumn(tableModel,
10300                                                                                                                                fieldAttr);
10301                                                                                                        }
10302                                                                                                } else {
10303                                                                                                        tableColumn = matchColumn(tableColumns, fieldAttr);
10304                                                                                                        if (tableColumn == null) {
10305                                                                                                                continue;
10306                                                                                                        }
10307                                                                                                }
10308        
10309                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10310                                                                                                relation.setEffectType(effectType);
10311                                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
10312                                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
10313                                                                                                Process process = modelFactory.createProcess(stmt);
10314                                                                                                relation.setProcess(process);
10315
10316                                                                                                
10317                                                                                        } else {
10318                                                                                                TAliasClause alias = ((TResultColumn) resultColumn.getColumnObject())
10319                                                                                                                .getAliasClause();
10320                                                                                                if (alias != null && alias.getAliasName() != null) {
10321                                                                                                        TableColumn tableColumn;
10322                                                                                                        if (!initColumn) {
10323                                                                                                                if (tableModel.isCreateTable()
10324                                                                                                                                && !containStarColumn(tableModel.getColumns())) {
10325                                                                                                                        if (tableModel.getColumns().size() <= i) {
10326                                                                                                                                continue;
10327                                                                                                                        }
10328                                                                                                                        tableColumn = tableModel.getColumns().get(i);
10329                                                                                                                } else {
10330                                                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
10331                                                                                                                                        alias.getAliasName());
10332                                                                                                                }
10333                                                                                                        } else {
10334                                                                                                                tableColumn = matchColumn(tableColumns, alias.getAliasName());
10335                                                                                                                if (tableColumn == null) {
10336                                                                                                                        continue;
10337                                                                                                                }
10338                                                                                                        }
10339                                                                                                        
10340                                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10341                                                                                                        relation.setEffectType(effectType);
10342                                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
10343                                                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
10344                                                                                                        Process process = modelFactory.createProcess(stmt);
10345                                                                                                        relation.setProcess(process);
10346                                                                                                } else if (((TResultColumn) resultColumn.getColumnObject())
10347                                                                                                                .getFieldAttr() != null) {
10348                                                                                                        TObjectName fieldAttr = ((TResultColumn) resultColumn.getColumnObject())
10349                                                                                                                        .getFieldAttr();
10350                                                                                                        TableColumn tableColumn;
10351                                                                                                        if (!initColumn) {
10352                                                                                                                if (tableModel.isCreateTable()
10353                                                                                                                                && !containStarColumn(tableModel.getColumns())) {
10354                                                                                                                        if (tableModel.getColumns().size() <= i) {
10355                                                                                                                                continue;
10356                                                                                                                        }
10357                                                                                                                        tableColumn = tableModel.getColumns().get(i);
10358                                                                                                                } else {
10359                                                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
10360                                                                                                                                        fieldAttr);
10361                                                                                                                }
10362                                                                                                        } else {
10363                                                                                                                tableColumn = matchColumn(tableColumns, fieldAttr);
10364                                                                                                                if (tableColumn == null) {
10365                                                                                                                        continue;
10366                                                                                                                }
10367                                                                                                        }
10368                                                                                                        
10369                                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10370                                                                                                        relation.setEffectType(effectType);
10371                                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
10372                                                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
10373                                                                                                        Process process = modelFactory.createProcess(stmt);
10374                                                                                                        relation.setProcess(process);                                                                                                   
10375                                                                                                } else if (((TResultColumn) resultColumn.getColumnObject()).getExpr()
10376                                                                                                                .getExpressionType() == EExpressionType.simple_constant_t) {
10377                                                                                                        if (!initColumn) {
10378                                                                                                                TableColumn tableColumn;
10379                                                                                                                if (tableModel.isCreateTable()
10380                                                                                                                                && !containStarColumn(tableModel.getColumns())) {
10381                                                                                                                        if (tableModel.getColumns().size() <= i) {
10382                                                                                                                                continue;
10383                                                                                                                        }
10384                                                                                                                        tableColumn = tableModel.getColumns().get(i);
10385                                                                                                                } else {
10386                                                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
10387                                                                                                                                        ((TResultColumn) resultColumn.getColumnObject()).getExpr()
10388                                                                                                                                                        .getConstantOperand(),
10389                                                                                                                                        i);
10390                                                                                                                }
10391                                                                                                                
10392                                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10393                                                                                                                relation.setEffectType(effectType);
10394                                                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
10395                                                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
10396                                                                                                                Process process = modelFactory.createProcess(stmt);
10397                                                                                                                relation.setProcess(process);
10398                                                                                                        }
10399                                                                                                } else {
10400                                                                                                        if (!initColumn) {
10401                                                                                                                TableColumn tableColumn;
10402                                                                                                                if (tableModel.isCreateTable()
10403                                                                                                                                && !containStarColumn(tableModel.getColumns())) {
10404                                                                                                                        if (tableModel.getColumns().size() <= i) {
10405                                                                                                                                continue;
10406                                                                                                                        }
10407                                                                                                                        tableColumn = tableModel.getColumns().get(i);
10408                                                                                                                } else {
10409                                                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
10410                                                                                                                                        ((TResultColumn) resultColumn.getColumnObject()).getExpr(),
10411                                                                                                                                        i);
10412                                                                                                                }
10413                                                                                                                
10414                                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10415                                                                                                                relation.setEffectType(effectType);
10416                                                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
10417                                                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
10418                                                                                                                Process process = modelFactory.createProcess(stmt);
10419                                                                                                                relation.setProcess(process);
10420                                                                                                        }
10421                                                                                                }
10422                                                                                        }
10423                                                                                }
10424                                                                        }
10425                                                                }
10426                                                        }
10427                                                }
10428                                        }
10429                                        else if (procedure!=null && procedure.getProcedureObject() instanceof TObjectName) {
10430                                                TObjectName functionName = new TObjectName();
10431                                                functionName.setString(procedure.getName());
10432                                                Function function = (Function)createFunction(functionName);
10433                                                if (stmt.getColumnList() != null) {
10434                                                        for (int i = 0; i < stmt.getColumnList().size(); i++) {
10435                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10436                                                                relation.setEffectType(effectType);
10437                                                                TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
10438                                                                                stmt.getColumnList().getObjectName(i));
10439                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
10440                                                                relation.addSource(new ResultColumnRelationshipElement(function.getColumns().get(0)));
10441                                                                Process process = modelFactory.createProcess(stmt);
10442                                                                relation.setProcess(process);
10443                                                        }
10444                                                }
10445                                        }
10446                                }
10447                        } else if (stmt.getValues() != null && stmt.getValues().size() > 0 && tableModel.isCreateTable() && !tableModel.getColumns().isEmpty()) {
10448                                for (int k = 0; stmt.getValues() != null && k < stmt.getValues().size(); k++) {
10449                                        TResultColumnList columns = stmt.getValues().getMultiTarget(k).getColumnList();
10450                                        boolean allConstant = true;
10451                                        Process process = modelFactory.createProcess(stmt);
10452                                        for (int x = 0; x < columns.size(); x++) {
10453                                                TableColumn tableColumn = tableModel.getColumns().get(x);
10454                                                TResultColumn columnObject = columns.getResultColumn(x);
10455                                                if (columnObject == null) {
10456                                                        continue;
10457                                                }
10458                                                TExpression valueExpr = columnObject.getExpr();
10459                                                columnsInExpr visitor = new columnsInExpr();
10460                                                valueExpr.inOrderTraverse(visitor);
10461                                                List<TObjectName> objectNames = visitor.getObjectNames();
10462                                                List<TParseTreeNode> constants = visitor.getConstants();
10463                                                List<TParseTreeNode> functions = visitor.getFunctions();
10464                                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
10465
10466                                                if (functions != null && !functions.isEmpty()) {
10467                                                        analyzeFunctionDataFlowRelation(tableColumn, functions, effectType, process);
10468                                                        allConstant = false;
10469                                                }
10470
10471                                                if (subquerys != null && !subquerys.isEmpty()) {
10472                                                        analyzeSubqueryDataFlowRelation(tableColumn, subquerys, effectType, process);
10473                                                        allConstant = false;
10474                                                }
10475                                                if (objectNames != null && !objectNames.isEmpty()) {
10476                                                        analyzeDataFlowRelation(tableColumn, objectNames, null, effectType, functions,
10477                                                                        process);
10478                                                        allConstant = false;
10479                                                }
10480                                                //insert into values generate too many constant relations, ignore constant relations.
10481                                                if (constants != null && !constants.isEmpty() && stmt.getParentStmt() != null) {
10482                                                        analyzeConstantDataFlowRelation(tableColumn, constants, effectType, functions,
10483                                                                        process);
10484                                                        allConstant = false;
10485                                                }
10486                                        }
10487                                        
10488                                        if(allConstant) {
10489                                                modelManager.unbindProcessModel(stmt);
10490                                                tableModel.removeProcess(process);
10491                                        }
10492                                }
10493                        } else if (stmt.getRecordName() != null) {
10494                                String procedureName = DlineageUtil.getProcedureParentName(stmt);
10495                                String variableString = stmt.getRecordName().toString();
10496                                if (variableString.startsWith(":")) {
10497                                        variableString = variableString.substring(variableString.indexOf(":") + 1);
10498                                }
10499                                if (!SQLUtil.isEmpty(procedureName)) {
10500                                        variableString = procedureName + "." + SQLUtil.getIdentifierNormalTableName(variableString);
10501                                }
10502                                
10503                                Table recordTable = modelManager
10504                                                .getTableByName(DlineageUtil.getTableFullName(variableString));
10505                                if (recordTable != null) {
10506                                        for (int i = 0; i < recordTable.getColumns().size(); i++) {
10507                                                TableColumn sourceTableColumn = recordTable.getColumns().get(i);
10508                                                TableColumn targetTableColumn = modelFactory.createTableColumn(tableModel,
10509                                                                sourceTableColumn.getColumnObject(), false);
10510                                                if (targetTableColumn != null) {
10511                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10512                                                        relation.setEffectType(effectType);
10513                                                        relation.setTarget(new TableColumnRelationshipElement(targetTableColumn));
10514                                                        relation.addSource(new TableColumnRelationshipElement(sourceTableColumn));
10515                                                        Process process = modelFactory.createProcess(stmt);
10516                                                        relation.setProcess(process);
10517                                                } else if (sourceTableColumn.getName().endsWith("*") && tableModel.isCreateTable()) {
10518                                                        for (TableColumn column : tableModel.getColumns()) {
10519                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10520                                                                relation.setEffectType(effectType);
10521                                                                relation.setTarget(new TableColumnRelationshipElement(column));
10522                                                                relation.addSource(new TableColumnRelationshipElement(sourceTableColumn));
10523                                                                Process process = modelFactory.createProcess(stmt);
10524                                                                relation.setProcess(process);
10525                                                        }
10526                                                }
10527                                        }
10528                                }
10529                        } else if (stmt.getInsertSource() == EInsertSource.values_function && stmt.getFunctionCall() != null) {
10530                                Table cursor = modelManager.getTableByName(
10531                                                DlineageUtil.getTableFullName(stmt.getFunctionCall().getFunctionName().toString()));
10532                                if (cursor != null) {
10533                                        TObjectName starColumn = new TObjectName();
10534                                        starColumn.setString("*");
10535                                        TableColumn insertColumn = modelFactory.createTableColumn(tableModel, starColumn, true);
10536                                        insertColumn.setShowStar(false);
10537                                        insertColumn.setExpandStar(true);
10538                                        for (int j = 0; j < cursor.getColumns().size(); j++) {
10539                                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
10540                                                dataflowRelation.setEffectType(effectType);
10541                                                dataflowRelation.addSource(new TableColumnRelationshipElement(cursor.getColumns().get(j)));
10542                                                dataflowRelation.setTarget(new TableColumnRelationshipElement(insertColumn));
10543                                                Process process = modelFactory.createProcess(stmt);
10544                                                dataflowRelation.setProcess(process);
10545                                        }
10546                                }
10547
10548                        } else if (stmt.getInsertSource() == EInsertSource.values && stmt.getValues() != null) {
10549                                TObjectName starColumn = new TObjectName();
10550                                starColumn.setString("*");
10551                                TableColumn insertColumn = modelFactory.createTableColumn(tableModel, starColumn, true);
10552                                insertColumn.setShowStar(false);
10553                                insertColumn.setExpandStar(true);
10554                                for (int k = 0; stmt.getValues() != null && k < stmt.getValues().size(); k++) {
10555                                        TResultColumnList columns = stmt.getValues().getMultiTarget(k).getColumnList();
10556                                        for (int x = 0; x < columns.size(); x++) {
10557                                                TResultColumn columnObject = columns.getResultColumn(x);
10558                                                if (columnObject == null) {
10559                                                        continue;
10560                                                }
10561                                                TExpression valueExpr = columnObject.getExpr();
10562                                                columnsInExpr visitor = new columnsInExpr();
10563                                                valueExpr.inOrderTraverse(visitor);
10564                                                List<TObjectName> objectNames = visitor.getObjectNames();
10565                                                List<TParseTreeNode> constants = visitor.getConstants();
10566                                                List<TParseTreeNode> functions = visitor.getFunctions();
10567                                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
10568
10569                                                Process process = modelFactory.createProcess(stmt);
10570                                                if (functions != null && !functions.isEmpty()) {
10571                                                        analyzeFunctionDataFlowRelation(insertColumn, functions, effectType, process);
10572                                                }
10573
10574                                                if (subquerys != null && !subquerys.isEmpty()) {
10575                                                        analyzeSubqueryDataFlowRelation(insertColumn, subquerys, effectType, process);
10576                                                }
10577                                                if (objectNames != null && !objectNames.isEmpty()) {
10578                                                        analyzeDataFlowRelation(insertColumn, objectNames, null, effectType, functions,
10579                                                                        process);
10580                                                }
10581                                                //insert into values generate too many constant relations, ignore constant relations.
10582                                                if (constants != null && !constants.isEmpty() && stmt.getParentStmt() != null) {
10583                                                        analyzeConstantDataFlowRelation(insertColumn, constants, effectType, functions,
10584                                                                        process);
10585                                                }
10586                                        }
10587                                }
10588                        }else if (stmt.getExecuteStmt() != null && stmt.getExecuteStmt().getModuleName() != null) {
10589                                analyzeCustomSqlStmt(stmt.getExecuteStmt());
10590                                Procedure procedure = modelManager.getProcedureByName(DlineageUtil
10591                                                .getIdentifierNormalTableName(stmt.getExecuteStmt().getModuleName().toString()));
10592                                if (procedure!=null && procedure.getProcedureObject() instanceof TStoredProcedureSqlStatement) {
10593                                        TStoredProcedureSqlStatement procedureStmt = (TStoredProcedureSqlStatement) procedure
10594                                                        .getProcedureObject();
10595                                        List<TSelectSqlStatement> stmtItems = getLastSelectStmt(procedureStmt);
10596                                        if (stmtItems != null) {
10597                                                for(TSelectSqlStatement stmtItem: stmtItems) {
10598                                                        ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmtItem);
10599                                                        if (resultSetModel != null) {
10600                                                                for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
10601                                                                        ResultColumn resultColumn = resultSetModel.getColumns().get(i);
10602                                                                        
10603                                                                        Transform transform = new Transform();
10604                                                                        transform.setType(Transform.FUNCTION);
10605                                                                        transform.setCode(stmt.getExecuteStmt().getModuleName());
10606                                                                        resultColumn.setTransform(transform);
10607                                                                        
10608                                                                        TAliasClause alias = null;
10609
10610                                                                        if (resultColumn.getColumnObject() instanceof TResultColumn) {
10611                                                                                alias = ((TResultColumn) resultColumn.getColumnObject())
10612                                                                                                .getAliasClause();
10613                                                                        }
10614                                                                        
10615                                                                        if (alias != null && alias.getAliasName() != null) {
10616                                                                                TableColumn tableColumn;
10617                                                                                if (!initColumn) {
10618                                                                                        if (tableModel.isCreateTable()
10619                                                                                                        && !containStarColumn(tableModel.getColumns())) {
10620                                                                                                if(resultColumn.getName().endsWith("*")) {
10621                                                                                                        for(TableColumn tableColumnItem: tableModel.getColumns()) {
10622                                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10623                                                                                                                relation.setEffectType(effectType);
10624                                                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumnItem));
10625                                                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
10626                                                                                                                Process process = modelFactory.createProcess(stmt);
10627                                                                                                                relation.setProcess(process);
10628                                                                                                        }
10629                                                                                                        continue;
10630                                                                                                }
10631                                                                                                else {
10632                                                                                                        if (tableModel.getColumns().size() <= i) {
10633                                                                                                                continue;
10634                                                                                                        }
10635                                                                                                        tableColumn = tableModel.getColumns().get(i);
10636                                                                                                }
10637                                                                                        } else {
10638                                                                                                tableColumn = modelFactory.createInsertTableColumn(tableModel,
10639                                                                                                                alias.getAliasName());
10640                                                                                        }
10641                                                                                } else {
10642                                                                                        tableColumn = matchColumn(tableColumns, alias.getAliasName());
10643                                                                                        if (tableColumn == null) {
10644                                                                                                continue;
10645                                                                                        }
10646                                                                                }
10647                                        
10648                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10649                                                                                relation.setEffectType(effectType);
10650                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
10651                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
10652                                                                                Process process = modelFactory.createProcess(stmt);
10653                                                                                relation.setProcess(process);
10654                                                                        } else if (resultColumn.getColumnObject() instanceof TObjectName
10655                                                                                        || (resultColumn.getColumnObject() instanceof TResultColumn
10656                                                                                                        && ((TResultColumn) resultColumn.getColumnObject())
10657                                                                                                                        .getFieldAttr() != null)) {
10658                                                                                TObjectName fieldAttr = null;
10659                                                                                if (resultColumn.getColumnObject() instanceof TObjectName) {
10660                                                                                        fieldAttr = (TObjectName) resultColumn.getColumnObject();
10661                                                                                } else {
10662                                                                                        fieldAttr = ((TResultColumn) resultColumn.getColumnObject()).getFieldAttr();
10663                                                                                }
10664                                                                                TableColumn tableColumn;
10665                                                                                if (!initColumn) {
10666                                                                                        if (tableModel.isCreateTable()
10667                                                                                                        && !containStarColumn(tableModel.getColumns())) {
10668                                                                                                if(resultColumn.getName().endsWith("*")) {
10669                                                                                                        for(TableColumn tableColumnItem: tableModel.getColumns()) {
10670                                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10671                                                                                                                relation.setEffectType(effectType);
10672                                                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumnItem));
10673                                                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
10674                                                                                                                Process process = modelFactory.createProcess(stmt);
10675                                                                                                                relation.setProcess(process);
10676                                                                                                        }
10677                                                                                                        continue;
10678                                                                                                }
10679                                                                                                else {
10680                                                                                                        if (tableModel.getColumns().size() <= i) {
10681                                                                                                                continue;
10682                                                                                                        }
10683                                                                                                        tableColumn = tableModel.getColumns().get(i);
10684                                                                                                }
10685                                                                                        } else {
10686                                                                                                tableColumn = modelFactory.createInsertTableColumn(tableModel,
10687                                                                                                                fieldAttr);
10688                                                                                        }
10689                                                                                } else {
10690                                                                                        tableColumn = matchColumn(tableColumns, fieldAttr);
10691                                                                                        if (tableColumn == null) {
10692                                                                                                continue;
10693                                                                                        }
10694                                                                                }
10695                                                                                
10696                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10697                                                                                relation.setEffectType(effectType);
10698                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
10699                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
10700                                                                                Process process = modelFactory.createProcess(stmt);
10701                                                                                relation.setProcess(process);
10702                                                                        } else if (((TResultColumn) resultColumn.getColumnObject()).getExpr()
10703                                                                                        .getExpressionType() == EExpressionType.simple_constant_t) {
10704                                                                                if (!initColumn) {
10705                                                                                        TableColumn tableColumn;
10706                                                                                        if (tableModel.isCreateTable()
10707                                                                                                        && !containStarColumn(tableModel.getColumns())) {
10708                                                                                                if(resultColumn.getName().endsWith("*")) {
10709                                                                                                        for(TableColumn tableColumnItem: tableModel.getColumns()) {
10710                                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10711                                                                                                                relation.setEffectType(effectType);
10712                                                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumnItem));
10713                                                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
10714                                                                                                                Process process = modelFactory.createProcess(stmt);
10715                                                                                                                relation.setProcess(process);
10716                                                                                                        }
10717                                                                                                        continue;
10718                                                                                                }
10719                                                                                                else {
10720                                                                                                        if (tableModel.getColumns().size() <= i) {
10721                                                                                                                continue;
10722                                                                                                        }
10723                                                                                                        tableColumn = tableModel.getColumns().get(i);
10724                                                                                                }
10725                                                                                        } else {
10726                                                                                                tableColumn = modelFactory.createInsertTableColumn(tableModel,
10727                                                                                                                ((TResultColumn) resultColumn.getColumnObject()).getExpr()
10728                                                                                                                                .getConstantOperand(),
10729                                                                                                                i);
10730                                                                                        }
10731                                                                                        
10732                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10733                                                                                        relation.setEffectType(effectType);
10734                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
10735                                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
10736                                                                                        Process process = modelFactory.createProcess(stmt);
10737                                                                                        relation.setProcess(process);
10738                                                                                }
10739                                                                        } else {
10740                                                                                if (!initColumn) {
10741                                                                                        TableColumn tableColumn;
10742                                                                                        if (tableModel.isCreateTable()
10743                                                                                                        && !containStarColumn(tableModel.getColumns())) {
10744                                                                                                if(resultColumn.getName().endsWith("*")) {
10745                                                                                                        for(TableColumn tableColumnItem: tableModel.getColumns()) {
10746                                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10747                                                                                                                relation.setEffectType(effectType);
10748                                                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumnItem));
10749                                                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
10750                                                                                                                Process process = modelFactory.createProcess(stmt);
10751                                                                                                                relation.setProcess(process);
10752                                                                                                        }
10753                                                                                                        continue;
10754                                                                                                }
10755                                                                                                else {
10756                                                                                                        if (tableModel.getColumns().size() <= i) {
10757                                                                                                                continue;
10758                                                                                                        }
10759                                                                                                        tableColumn = tableModel.getColumns().get(i);
10760                                                                                                }
10761                                                                                        } else {
10762                                                                                                tableColumn = modelFactory.createInsertTableColumn(tableModel,
10763                                                                                                                ((TResultColumn) resultColumn.getColumnObject()).getExpr(),
10764                                                                                                                i);
10765                                                                                        }
10766                                                                                        
10767                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10768                                                                                        relation.setEffectType(effectType);
10769                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
10770                                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
10771                                                                                        Process process = modelFactory.createProcess(stmt);
10772                                                                                        relation.setProcess(process);
10773                                                                                }
10774                                                                        }
10775                                                                }
10776                                                        }
10777                                                }
10778                                        }
10779                                }
10780                                else if (procedure!=null && procedure.getProcedureObject() instanceof TObjectName) {
10781                                        TObjectName functionName = new TObjectName();
10782                                        functionName.setString(procedure.getName());
10783                                        Function function = (Function)createFunction(functionName);
10784                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
10785                                        relation.setEffectType(effectType);
10786                                        TObjectName starColumn = new TObjectName();
10787                                        starColumn.setString("*");
10788                                        TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
10789                                                        starColumn);
10790                                        tableColumn.setExpandStar(false);
10791                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
10792                                        relation.addSource(new ResultColumnRelationshipElement(function.getColumns().get(0)));
10793                                        Process process = modelFactory.createProcess(stmt);
10794                                        relation.setProcess(process);
10795                                }
10796                        }
10797                }
10798                
10799                if(stmt.getOnDuplicateKeyUpdate()!=null) {
10800                        TTable table = stmt.getTargetTable();
10801                        Table tableModel = modelFactory.createTable(table);
10802                        for(TResultColumn column: stmt.getOnDuplicateKeyUpdate()) {
10803                                if(column.getExpr()==null || column.getExpr().getExpressionType() != EExpressionType.assignment_t) {
10804                                        continue;
10805                                }
10806                                TExpression left = column.getExpr().getLeftOperand();
10807                                TExpression right = column.getExpr().getRightOperand();
10808                                TObjectName columnObject = left.getObjectOperand();
10809                                if (columnObject != null) {
10810                                        TableColumn tableColumn = modelFactory.createTableColumn(tableModel, columnObject, false);
10811                                        if (tableColumn != null) {
10812                                                columnsInExpr visitor = new columnsInExpr();
10813                                                right.inOrderTraverse(visitor);
10814                                                List<TObjectName> objectNames = visitor.getObjectNames();
10815                                                List<TParseTreeNode> functions = visitor.getFunctions();
10816                                                List<TParseTreeNode> constants = visitor.getConstants();
10817                                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
10818
10819                                                if (functions != null && !functions.isEmpty()) {
10820                                                        analyzeFunctionDataFlowRelation(tableColumn, functions, EffectType.update);
10821                                                }
10822                                                if (subquerys != null && !subquerys.isEmpty()) {
10823                                                        analyzeSubqueryDataFlowRelation(tableColumn, subquerys, EffectType.update);
10824                                                }
10825                                                if (objectNames != null && !objectNames.isEmpty()) {
10826                                                        analyzeDataFlowRelation(tableColumn, objectNames, EffectType.update, functions);
10827                                                }
10828                                                if (constants != null && !constants.isEmpty()) {
10829                                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.update, functions);
10830                                                }
10831                                        }
10832                                }
10833                        }
10834                }
10835
10836                if (!expressions.isEmpty() && stmt.getSubQuery() != null) {
10837                        analyzeInsertImpactRelation(stmt.getSubQuery(), tableColumnMap, expressions, effectType);
10838                }
10839        }
10840
10841        private List<TSelectSqlStatement> getLastSelectStmt(TStoredProcedureSqlStatement procedureStmt) {
10842                List<TSelectSqlStatement> stmts = new ArrayList<TSelectSqlStatement>();
10843                if (procedureStmt.getBodyStatements().size() > 0) {
10844                        for (int j = procedureStmt.getBodyStatements().size() - 1; j >= 0; j--) {
10845                                TCustomSqlStatement stmtItem = procedureStmt.getBodyStatements().get(j);
10846                                if (stmtItem instanceof TReturnStmt || stmtItem instanceof TMssqlReturn) {
10847                                        if (stmtItem.getStatements() != null) {
10848                                                List<TSelectSqlStatement> item = getLastSelectStmt(stmtItem);
10849                                                if (item != null && !item.isEmpty()) {
10850                                                        stmts.addAll(item);
10851                                                        if(option.getVendor()!=EDbVendor.dbvmssql && option.getVendor()!=EDbVendor.dbvazuresql) {
10852                                                                break;
10853                                                        }
10854                                                }
10855                                        }
10856                                        break;
10857                                }
10858                                if (stmtItem instanceof TSelectSqlStatement) {
10859                                        stmts.add((TSelectSqlStatement) stmtItem);
10860                                        if(option.getVendor()!=EDbVendor.dbvmssql && option.getVendor()!=EDbVendor.dbvazuresql) {
10861                                                break;
10862                                        }
10863                                } else if (stmtItem.getStatements() != null) {
10864                                        List<TSelectSqlStatement> item = getLastSelectStmt(stmtItem);
10865                                        if (item != null && !item.isEmpty()) {
10866                                                stmts.addAll(item);
10867                                                if(option.getVendor()!=EDbVendor.dbvmssql && option.getVendor()!=EDbVendor.dbvazuresql) {
10868                                                        break;
10869                                                }
10870                                        }
10871                                }
10872                        }
10873                }
10874                return stmts;
10875        }
10876
10877        private List<TSelectSqlStatement> getLastSelectStmt(TCustomSqlStatement stmt) {
10878                List<TSelectSqlStatement> stmts = new ArrayList<TSelectSqlStatement>();
10879                for (int j = stmt.getStatements().size() - 1; j >= 0; j--) {
10880                        TCustomSqlStatement stmtItem = stmt.getStatements().get(j);
10881                        if (stmtItem instanceof TReturnStmt || stmtItem instanceof TMssqlReturn) {
10882                                if (stmtItem.getStatements() != null) {
10883                                        List<TSelectSqlStatement> item = getLastSelectStmt(stmtItem);
10884                                        if (item != null && !item.isEmpty()) {
10885                                                stmts.addAll(item);
10886                                                if(option.getVendor()!=EDbVendor.dbvmssql && option.getVendor()!=EDbVendor.dbvazuresql) {
10887                                                        break;
10888                                                }
10889                                        }
10890                                }
10891                                break;
10892                        }
10893                        if (stmtItem instanceof TSelectSqlStatement) {
10894                                stmts.add((TSelectSqlStatement) stmtItem);
10895                                if(option.getVendor()!=EDbVendor.dbvmssql && option.getVendor()!=EDbVendor.dbvazuresql) {
10896                                        break;
10897                                }
10898                        } else if (stmtItem.getStatements() != null) {
10899                                List<TSelectSqlStatement> item = getLastSelectStmt(stmtItem);
10900                                if (item != null && !item.isEmpty()) {
10901                                        stmts.addAll(item);
10902                                        if(option.getVendor()!=EDbVendor.dbvmssql && option.getVendor()!=EDbVendor.dbvazuresql) {
10903                                                break;
10904                                        }
10905                                }
10906                        }
10907                }
10908                return stmts;
10909        }
10910
10911        private TableColumn getStarColumn(List<TableColumn> columns) {
10912                for (TableColumn column : columns) {
10913                        if (column.getName().endsWith("*")) {
10914                                return column;
10915                        }
10916                }
10917                return null;
10918        }
10919
10920        private boolean containStarColumn(List<TableColumn> columns) {
10921                if (columns == null)
10922                        return false;
10923                for (TableColumn column : columns) {
10924                        if (column.getName().endsWith("*")) {
10925                                return true;
10926                        }
10927                }
10928                return false;
10929        }
10930
10931        private boolean containStarColumn(ResultSet resultSet) {
10932                if (resultSet == null || resultSet.getColumns() == null)
10933                        return false;
10934                for (ResultColumn column : resultSet.getColumns()) {
10935                        if (column.getName().endsWith("*")) {
10936                                return true;
10937                        }
10938                }
10939                return false;
10940        }
10941
10942        private int indexOfColumn(List<TResultColumn> columns, TObjectName objectName) {
10943                for (int i = 0; i < columns.size(); i++) {
10944                        if (columns.get(i).toString().trim().equalsIgnoreCase(objectName.toString().trim())) {
10945                                return i;
10946                        }
10947                }
10948                return -1;
10949        }
10950
10951        private boolean isEmptyCollection(Collection<?> keyMap) {
10952                return keyMap == null || keyMap.isEmpty();
10953        }
10954
10955        private void analyzeInsertImpactRelation(TSelectSqlStatement stmt, Map<String, List<TableColumn>> insertMap,
10956                        List<TExpression> expressions, EffectType effectType) {
10957                List<TObjectName> objectNames = new ArrayList<TObjectName>();
10958                for (int i = 0; i < expressions.size(); i++) {
10959                        TExpression condition = expressions.get(i);
10960                        columnsInExpr visitor = new columnsInExpr();
10961                        condition.inOrderTraverse(visitor);
10962                        objectNames.addAll(visitor.getObjectNames());
10963                }
10964
10965                Iterator<String> iter = insertMap.keySet().iterator();
10966                while (iter.hasNext()) {
10967                        String table = iter.next();
10968                        List<TableColumn> tableColumns = insertMap.get(table);
10969                        for (int i = 0; i < tableColumns.size(); i++) {
10970
10971                                TableColumn column = tableColumns.get(i);
10972                                ImpactRelationship relation = modelFactory.createImpactRelation();
10973                                relation.setEffectType(effectType);
10974                                relation.setTarget(new TableColumnRelationshipElement(column));
10975
10976                                for (int j = 0; j < objectNames.size(); j++) {
10977                                        TObjectName columnName = objectNames.get(j);
10978                                        Object model = modelManager.getModel(stmt);
10979                                        if (model instanceof SelectResultSet) {
10980                                                SelectResultSet queryTable = (SelectResultSet) model;
10981                                                List<ResultColumn> columns = queryTable.getColumns();
10982                                                for (int k = 0; k < columns.size(); k++) {
10983                                                        ResultColumn resultColumn = columns.get(k);
10984                                                        if (resultColumn.getAlias() != null
10985                                                                        && columnName.toString().equalsIgnoreCase(resultColumn.getAlias())) {
10986                                                                relation.addSource(
10987                                                                                new ResultColumnRelationshipElement(resultColumn, columnName.getLocation()));
10988                                                        } else if (resultColumn.getName() != null
10989                                                                        && columnName.toString().equalsIgnoreCase(resultColumn.getName())) {
10990                                                                relation.addSource(
10991                                                                                new ResultColumnRelationshipElement(resultColumn, columnName.getLocation()));
10992                                                        }
10993                                                }
10994                                        }
10995                                }
10996                        }
10997                }
10998        }
10999
11000        private void analyzeUpdateStmt(TUpdateSqlStatement stmt) {
11001                if (stmt.getResultColumnList() == null)
11002                        return;
11003
11004                TTable table = stmt.getTargetTable();
11005                while (table.getCTE() != null || table.getSubquery() != null || (table.getLinkTable() != null && table.getLinkTable().getSubquery() != null)) {
11006                        if (table.getCTE() != null) {
11007                                table = table.getCTE().getSubquery().getTables().getTable(0);
11008                        } else if (table.getLinkTable() != null && table.getLinkTable().getSubquery() != null) {
11009                                table = table.getLinkTable().getSubquery().getTables().getTable(0);
11010                        } else if (table.getSubquery() != null) {
11011                                table = table.getSubquery().getTables().getTable(0);
11012                        }
11013                }
11014                Table tableModel = modelFactory.createTable(table);
11015                Process process = modelFactory.createProcess(stmt);
11016                tableModel.addProcess(process);
11017
11018                for (int i = 0; i < stmt.tables.size(); i++) {
11019                        TTable tableElement = stmt.tables.getTable(i);
11020                        if (tableElement.getSubquery() != null) {
11021                                QueryTable queryTable = modelFactory.createQueryTable(tableElement);
11022                                TSelectSqlStatement subquery = tableElement.getSubquery();
11023                                analyzeSelectStmt(subquery);
11024
11025                                if (subquery.getSetOperatorType() != ESetOperatorType.none) {
11026                                        SelectSetResultSet selectSetResultSetModel = (SelectSetResultSet) modelManager.getModel(subquery);
11027                                        for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) {
11028                                                ResultColumn sourceColumn = selectSetResultSetModel.getColumns().get(j);
11029                                                ResultColumn targetColumn = modelFactory.createSelectSetResultColumn(queryTable, sourceColumn);
11030                                                DataFlowRelationship selectSetRalation = modelFactory.createDataFlowRelation();
11031                                                selectSetRalation.setEffectType(EffectType.select);
11032                                                selectSetRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
11033                                                selectSetRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
11034                                                selectSetRalation.setProcess(process);
11035                                        }
11036                                }
11037
11038                                ResultSet resultSetModel = (ResultSet) modelManager.getModel(tableElement.getSubquery());
11039                                if (resultSetModel != null && resultSetModel != queryTable
11040                                                && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
11041                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
11042                                        impactRelation.setEffectType(EffectType.update);
11043                                        impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
11044                                                        resultSetModel.getRelationRows()));
11045                                        impactRelation.setTarget(
11046                                                        new RelationRowsRelationshipElement<ResultSetRelationRows>(queryTable.getRelationRows()));
11047                                }
11048
11049                        } else if (tableElement.getCTE() != null) {
11050                                QueryTable queryTable = modelFactory.createQueryTable(tableElement);
11051
11052                                TObjectNameList cteColumns = tableElement.getCTE().getColumnList();
11053                                if (cteColumns != null) {
11054                                        for (int j = 0; j < cteColumns.size(); j++) {
11055                                                modelFactory.createResultColumn(queryTable, cteColumns.getObjectName(j));
11056                                        }
11057                                }
11058                                TSelectSqlStatement subquery = tableElement.getCTE().getSubquery();
11059                                if (subquery != null && !stmtStack.contains(subquery)) {
11060                                        analyzeSelectStmt(subquery);
11061
11062                                        ResultSet resultSetModel = (ResultSet) modelManager.getModel(subquery);
11063                                        if (resultSetModel != null && resultSetModel != queryTable
11064                                                        && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
11065                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
11066                                                impactRelation.setEffectType(EffectType.select);
11067                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
11068                                                                resultSetModel.getRelationRows()));
11069                                                impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
11070                                                                queryTable.getRelationRows()));
11071                                        }
11072
11073                                        if (subquery.getSetOperatorType() != ESetOperatorType.none) {
11074                                                SelectSetResultSet selectSetResultSetModel = (SelectSetResultSet) modelManager
11075                                                                .getModel(subquery);
11076                                                for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) {
11077                                                        ResultColumn sourceColumn = selectSetResultSetModel.getColumns().get(j);
11078                                                        ResultColumn targetColumn = null;
11079                                                        if (cteColumns != null) {
11080                                                                targetColumn = queryTable.getColumns().get(j);
11081                                                        } else {
11082                                                                targetColumn = modelFactory.createSelectSetResultColumn(queryTable, sourceColumn);
11083                                                        }
11084                                                        for (Set<TObjectName> starLinkColumns : sourceColumn.getStarLinkColumns().values()) {
11085                                                                for (TObjectName starLinkColumn : starLinkColumns) {
11086                                                                        targetColumn.bindStarLinkColumn(starLinkColumn);
11087                                                                }
11088                                                        }
11089                                                        DataFlowRelationship selectSetRalation = modelFactory.createDataFlowRelation();
11090                                                        selectSetRalation.setEffectType(EffectType.select);
11091                                                        selectSetRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
11092                                                        selectSetRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
11093                                                        selectSetRalation.setProcess(process);
11094                                                }
11095                                        } else {
11096                                                for (int j = 0; j < resultSetModel.getColumns().size(); j++) {
11097                                                        ResultColumn sourceColumn = resultSetModel.getColumns().get(j);
11098                                                        ResultColumn targetColumn = null;
11099                                                        if (cteColumns != null) {
11100                                                                targetColumn = queryTable.getColumns().get(j);
11101                                                        } else {
11102                                                                targetColumn = modelFactory.createSelectSetResultColumn(queryTable, sourceColumn);
11103                                                        }
11104                                                        for (TObjectName starLinkColumn : sourceColumn.getStarLinkColumnList()) {
11105                                                                targetColumn.bindStarLinkColumn(starLinkColumn);
11106                                                        }
11107                                                        DataFlowRelationship selectSetRalation = modelFactory.createDataFlowRelation();
11108                                                        selectSetRalation.setEffectType(EffectType.select);
11109                                                        selectSetRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
11110                                                        selectSetRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
11111                                                        selectSetRalation.setProcess(process);
11112                                                }
11113                                        }
11114                                } else if (tableElement.getCTE().getUpdateStmt() != null) {
11115                                        analyzeCustomSqlStmt(tableElement.getCTE().getUpdateStmt());
11116                                } else if (tableElement.getCTE().getInsertStmt() != null) {
11117                                        analyzeCustomSqlStmt(tableElement.getCTE().getInsertStmt());
11118                                } else if (tableElement.getCTE().getDeleteStmt() != null) {
11119                                        analyzeCustomSqlStmt(tableElement.getCTE().getDeleteStmt());
11120                                }
11121                        } else {
11122                                modelFactory.createTable(stmt.tables.getTable(i));
11123                        }
11124                }
11125
11126                for (int i = 0; i < stmt.getResultColumnList().size(); i++) {
11127                        TResultColumn field = stmt.getResultColumnList().getResultColumn(i);
11128
11129                        if (field.getExpr().getExpressionType() == EExpressionType.function_t) {
11130                                // Handle SQL Server XML modify() method for data lineage
11131                                TFunctionCall funcCall = field.getExpr().getFunctionCall();
11132                                if (funcCall != null && funcCall.getFunctionType() == EFunctionType.xmlmodify_t) {
11133                                        analyzeXmlModifyFunction(stmt, tableModel, process, funcCall);
11134                                }
11135                                continue;
11136                        }
11137
11138                        TExpression expression = field.getExpr().getLeftOperand();
11139                        if (expression == null) {
11140                                ErrorInfo errorInfo = new ErrorInfo();
11141                                errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
11142                                errorInfo.setErrorMessage(
11143                                                "Can't get result column expression. Expression is " + field.getExpr().toString());
11144                                errorInfo.setStartPosition(new Pair3<Long, Long, String>(field.getExpr().getStartToken().lineNo,
11145                                                field.getExpr().getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
11146                                errorInfo.setEndPosition(new Pair3<Long, Long, String>(field.getExpr().getEndToken().lineNo,
11147                                                field.getExpr().getEndToken().columnNo + field.getExpr().getEndToken().getAstext().length(),
11148                                                ModelBindingManager.getGlobalHash()));
11149                                errorInfo.fillInfo(this);
11150                                errorInfos.add(errorInfo);
11151                                continue;
11152                        }
11153                        if (expression.getExpressionType() == EExpressionType.list_t) {
11154                                TExpression setExpression = field.getExpr().getRightOperand();
11155                                if (setExpression != null && setExpression.getSubQuery() != null) {
11156                                        TSelectSqlStatement query = setExpression.getSubQuery();
11157                                        analyzeSelectStmt(query);
11158
11159                                        SelectResultSet resultSetModel = (SelectResultSet) modelManager
11160                                                        .getModel(query.getResultColumnList());
11161
11162                                        if (!resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
11163                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
11164                                                impactRelation.setEffectType(EffectType.update);
11165                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
11166                                                                resultSetModel.getRelationRows()));
11167                                                impactRelation.setTarget(
11168                                                                new RelationRowsRelationshipElement<TableRelationRows>(tableModel.getRelationRows()));
11169                                        }
11170
11171                                        TExpressionList columnList = expression.getExprList();
11172                                        for (int j = 0; j < columnList.size(); j++) {
11173                                                TObjectName column = columnList.getExpression(j).getObjectOperand();
11174
11175                                                if (column.getDbObjectType() == EDbObjectType.variable) {
11176                                                        continue;
11177                                                }
11178
11179                                                if (column.getColumnNameOnly().startsWith("@") && (option.getVendor() == EDbVendor.dbvmssql
11180                                                                || option.getVendor() == EDbVendor.dbvazuresql)) {
11181                                                        continue;
11182                                                }
11183
11184                                                if (column.getColumnNameOnly().startsWith(":") && (option.getVendor() == EDbVendor.dbvhana
11185                                                                || option.getVendor() == EDbVendor.dbvteradata)) {
11186                                                        continue;
11187                                                }
11188
11189                                                ResultColumn resultColumn = resultSetModel.getColumns().get(j);
11190                                                TableColumn tableColumn = modelFactory.createTableColumn(tableModel, column, false);
11191                                                if (tableColumn != null) {
11192                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11193                                                        relation.setEffectType(EffectType.update);
11194                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
11195                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11196                                                        relation.setProcess(process);
11197                                                }
11198
11199                                        }
11200                                }
11201                        } else if (expression.getExpressionType() == EExpressionType.simple_object_name_t) {
11202                                TExpression setExpression = field.getExpr().getRightOperand();
11203                                if (setExpression != null && setExpression.getSubQuery() != null) {
11204                                        TSelectSqlStatement query = setExpression.getSubQuery();
11205                                        analyzeSelectStmt(query);
11206
11207                                        SelectResultSet resultSetModel = (SelectResultSet) modelManager
11208                                                        .getModel(query.getResultColumnList());
11209
11210                                        if (!resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
11211                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
11212                                                impactRelation.setEffectType(EffectType.update);
11213                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
11214                                                                resultSetModel.getRelationRows()));
11215                                                impactRelation.setTarget(
11216                                                                new RelationRowsRelationshipElement<TableRelationRows>(tableModel.getRelationRows()));
11217                                        }
11218
11219                                        TObjectName column = expression.getObjectOperand();
11220                                        ResultColumn resultColumn = resultSetModel.getColumns().get(0);
11221                                        TableColumn tableColumn = modelFactory.createTableColumn(tableModel, column, false);
11222                                        if (tableColumn != null) {
11223                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11224                                                relation.setEffectType(EffectType.update);
11225                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
11226                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11227                                                relation.setProcess(process);
11228                                        }
11229                                } else if (setExpression != null) {
11230                                        // ResultSet resultSet = modelFactory.createResultSet(stmt,
11231                                        // true);
11232
11233                                        ResultSet resultSet = modelFactory.createResultSet(stmt, false);
11234
11235                                        createPseudoImpactRelation(stmt, resultSet, EffectType.update);
11236
11237                                        TObjectName columnObject = expression.getObjectOperand();
11238
11239                                        ResultColumn updateColumn = modelFactory.createUpdateResultColumn(resultSet, columnObject);
11240
11241                                        columnsInExpr visitor = new columnsInExpr();
11242                                        field.getExpr().getRightOperand().inOrderTraverse(visitor);
11243
11244                                        List<TObjectName> objectNames = visitor.getObjectNames();
11245                                        List<TParseTreeNode> functions = visitor.getFunctions();
11246                                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
11247
11248                                        if (functions != null && !functions.isEmpty()) {
11249                                                analyzeFunctionDataFlowRelation(updateColumn, functions, EffectType.update);
11250                                        }
11251
11252                                        if (subquerys != null && !subquerys.isEmpty()) {
11253                                                analyzeSubqueryDataFlowRelation(updateColumn, subquerys, EffectType.update);
11254                                        }
11255
11256                                        Transform transform = new Transform();
11257                                        transform.setType(Transform.EXPRESSION);
11258                                        transform.setCode(setExpression);
11259                                        updateColumn.setTransform(transform);
11260                                        analyzeDataFlowRelation(updateColumn, objectNames, EffectType.update, functions);
11261
11262                                        List<TParseTreeNode> constants = visitor.getConstants();
11263                                        analyzeConstantDataFlowRelation(updateColumn, constants, EffectType.update, functions);
11264
11265                                        TableColumn tableColumn = modelFactory.createTableColumn(tableModel, columnObject, false);
11266                                        if(tableColumn!=null) {
11267                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11268                                                relation.setEffectType(EffectType.update);
11269                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
11270                                                relation.addSource(new ResultColumnRelationshipElement(updateColumn));
11271                                                relation.setProcess(process);
11272                                        }
11273                                        
11274                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
11275                                        impactRelation.setEffectType(EffectType.update);
11276                                        impactRelation.addSource(
11277                                                        new RelationRowsRelationshipElement<ResultSetRelationRows>(resultSet.getRelationRows()));
11278                                        impactRelation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(
11279                                                        tableModel.getRelationRows()));
11280                                }
11281                        }
11282                }
11283
11284                if (stmt.getJoins() != null && stmt.getJoins().size() > 0) {
11285                        for (int i = 0; i < stmt.getJoins().size(); i++) {
11286                                TJoin join = stmt.getJoins().getJoin(i);
11287                                if (join.getJoinItems() != null) {
11288                                        for (int j = 0; j < join.getJoinItems().size(); j++) {
11289                                                TJoinItem joinItem = join.getJoinItems().getJoinItem(j);
11290                                                TExpression expr = joinItem.getOnCondition();
11291                                                analyzeFilterCondition(null, expr, joinItem.getJoinType(), JoinClauseType.on,
11292                                                                EffectType.update);
11293                                        }
11294                                }
11295                        }
11296                }
11297
11298                if (stmt.getWhereClause() != null && stmt.getWhereClause().getCondition() != null) {
11299                        analyzeFilterCondition(null, stmt.getWhereClause().getCondition(), null, JoinClauseType.where,
11300                                        EffectType.update);
11301                }
11302
11303                if (stmt.getOutputClause() != null) {
11304                        TOutputClause outputClause = stmt.getOutputClause();
11305                        if (outputClause.getSelectItemList() != null) {
11306                                ResultSet resultSet = modelFactory.createResultSet(outputClause, false);
11307                                for (int j = 0; j < outputClause.getSelectItemList().size(); j++) {
11308                                        TResultColumn sourceColumn = outputClause.getSelectItemList().getResultColumn(j);
11309                                        ResultColumn sourceColumnModel = modelFactory.createResultColumn(resultSet, sourceColumn);
11310                                        analyzeResultColumn(sourceColumn, EffectType.select);
11311
11312                                        if (outputClause.getIntoTable() != null) {
11313                                                Table intoTableModel = modelFactory.createTableByName(outputClause.getIntoTable());
11314                                                intoTableModel.addProcess(process);
11315                                                if (outputClause.getIntoColumnList() != null) {
11316                                                        TableColumn intoTableColumn = modelFactory.createInsertTableColumn(intoTableModel,
11317                                                                        outputClause.getIntoColumnList().getObjectName(j));
11318
11319                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11320                                                        relation.setEffectType(EffectType.insert);
11321                                                        relation.setTarget(new TableColumnRelationshipElement(intoTableColumn));
11322                                                        relation.addSource(new ResultColumnRelationshipElement(sourceColumnModel));
11323                                                } else if (sourceColumn.getAliasClause() != null
11324                                                                || sourceColumn.getExpr().getObjectOperand() != null) {
11325                                                        TObjectName tableColumnObject = null;
11326                                                        if (sourceColumn.getAliasClause() != null) {
11327                                                                tableColumnObject = sourceColumn.getAliasClause().getAliasName();
11328                                                        } else {
11329                                                                tableColumnObject = sourceColumn.getExpr().getObjectOperand();
11330                                                        }
11331
11332                                                        TableColumn intoTableColumn = modelFactory.createInsertTableColumn(intoTableModel,
11333                                                                        tableColumnObject);
11334                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11335                                                        relation.setEffectType(EffectType.insert);
11336                                                        relation.setTarget(new TableColumnRelationshipElement(intoTableColumn));
11337                                                        relation.addSource(new ResultColumnRelationshipElement(sourceColumnModel));
11338                                                }
11339                                        }
11340                                }
11341                        }
11342                }
11343        }
11344
11345        /**
11346         * Analyzes SQL Server XML modify() function to extract data lineage from sql:column() references
11347         * in XQuery expressions.
11348         *
11349         * Example SQL:
11350         * SET [Demographics].modify('... sql:column("deleted.LineTotal") ...')
11351         *
11352         * This extracts:
11353         * - Target column: Demographics (from the XML column being modified)
11354         * - Source column: deleted.LineTotal (from sql:column() reference in XQuery)
11355         */
11356        private void analyzeXmlModifyFunction(TUpdateSqlStatement stmt, Table tableModel, Process process, TFunctionCall funcCall) {
11357                TObjectName funcName = funcCall.getFunctionName();
11358                if (funcName == null || funcName.getPartToken() == null) {
11359                        return;
11360                }
11361
11362                // Get the XML column being modified (e.g., [Demographics] from "[Demographics].modify")
11363                String xmlColumnName = funcName.getPartToken().astext;
11364
11365                // Get the XQuery argument
11366                if (funcCall.getArgs() == null || funcCall.getArgs().size() == 0) {
11367                        return;
11368                }
11369
11370                String xqueryString = funcCall.getArgs().getExpression(0).toString();
11371
11372                // Extract sql:column() references from the XQuery string
11373                List<String> sqlColumnRefs = extractSqlColumnReferences(xqueryString);
11374                if (sqlColumnRefs.isEmpty()) {
11375                        return;
11376                }
11377
11378                // Extract XPath target from XQuery (e.g., /IndividualSurvey/TotalPurchaseYTD)
11379                String xpathTarget = extractXPathTarget(xqueryString);
11380
11381                // Build full target column name including XML path
11382                String fullTargetColumnName = xmlColumnName;
11383                if (xpathTarget != null && !xpathTarget.isEmpty()) {
11384                        fullTargetColumnName = xmlColumnName + "." + xpathTarget;
11385                }
11386
11387                // Create the target table column for the XML column being modified
11388                TableColumn targetTableColumn = modelFactory.createInsertTableColumn(tableModel, fullTargetColumnName);
11389
11390                // Create source-to-target relationships for each sql:column reference
11391                for (String sqlColRef : sqlColumnRefs) {
11392                        // Parse the column reference (e.g., "deleted.LineTotal" -> table="deleted", column="LineTotal")
11393                        String[] parts = sqlColRef.split("\\.", 2);
11394                        String sourceTableName = parts.length > 1 ? parts[0] : null;
11395                        String sourceColumnName = parts.length > 1 ? parts[1] : parts[0];
11396
11397                        // Find the source table in the statement's tables
11398                        TTable sourceTable = null;
11399                        if (sourceTableName != null && stmt.tables != null) {
11400                                for (int j = 0; j < stmt.tables.size(); j++) {
11401                                        TTable t = stmt.tables.getTable(j);
11402                                        String tableName = t.getTableName().toString();
11403                                        String alias = t.getAliasName();
11404                                        if (tableName.equalsIgnoreCase(sourceTableName) ||
11405                                                (alias != null && alias.equalsIgnoreCase(sourceTableName))) {
11406                                                sourceTable = t;
11407                                                break;
11408                                        }
11409                                }
11410                        }
11411
11412                        if (sourceTable != null && targetTableColumn != null) {
11413                                // Create a table model for the source if needed
11414                                Table sourceTableModel = modelFactory.createTable(sourceTable);
11415                                TableColumn sourceColumn = modelFactory.createInsertTableColumn(sourceTableModel, sourceColumnName);
11416
11417                                if (sourceColumn != null) {
11418                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11419                                        relation.setEffectType(EffectType.update);
11420                                        relation.setTarget(new TableColumnRelationshipElement(targetTableColumn));
11421                                        relation.addSource(new TableColumnRelationshipElement(sourceColumn));
11422                                        relation.setProcess(process);
11423                                        relation.setFunction("modify");
11424                                }
11425                        }
11426                }
11427        }
11428
11429        /**
11430         * Extracts sql:column() references from an XQuery string.
11431         * Example: 'sql:column("deleted.LineTotal")' -> ["deleted.LineTotal"]
11432         */
11433        private List<String> extractSqlColumnReferences(String xquery) {
11434                List<String> refs = new ArrayList<>();
11435                if (xquery == null) {
11436                        return refs;
11437                }
11438
11439                int startIdx = 0;
11440                while ((startIdx = xquery.indexOf("sql:column(", startIdx)) >= 0) {
11441                        int parenStart = startIdx + "sql:column(".length();
11442                        int parenEnd = xquery.indexOf(")", parenStart);
11443                        if (parenEnd < 0) {
11444                                break;
11445                        }
11446
11447                        String arg = xquery.substring(parenStart, parenEnd).trim();
11448                        // Remove quotes (single or double)
11449                        if ((arg.startsWith("\"") && arg.endsWith("\"")) ||
11450                                (arg.startsWith("'") && arg.endsWith("'"))) {
11451                                arg = arg.substring(1, arg.length() - 1);
11452                        }
11453
11454                        if (!arg.isEmpty()) {
11455                                refs.add(arg);
11456                        }
11457
11458                        startIdx = parenEnd + 1;
11459                }
11460
11461                return refs;
11462        }
11463
11464        /**
11465         * Extracts the XPath target from an XQuery modify expression.
11466         * Example: 'replace value of (/IndividualSurvey/TotalPurchaseYTD)[1]' -> "IndividualSurvey.TotalPurchaseYTD"
11467         */
11468        private String extractXPathTarget(String xquery) {
11469                if (xquery == null) {
11470                        return null;
11471                }
11472
11473                // Look for patterns like "(/path/to/element)" or "(/path/to/element)[1]"
11474                int replaceIdx = xquery.indexOf("replace value of");
11475                if (replaceIdx < 0) {
11476                        return null;
11477                }
11478
11479                int parenStart = xquery.indexOf("(/", replaceIdx);
11480                if (parenStart < 0) {
11481                        return null;
11482                }
11483
11484                int parenEnd = xquery.indexOf(")", parenStart);
11485                if (parenEnd < 0) {
11486                        return null;
11487                }
11488
11489                String xpath = xquery.substring(parenStart + 1, parenEnd);
11490                // Remove any predicates like [1]
11491                int bracketIdx = xpath.indexOf("[");
11492                if (bracketIdx > 0) {
11493                        xpath = xpath.substring(0, bracketIdx);
11494                }
11495
11496                // Convert XPath to dot notation (e.g., /IndividualSurvey/TotalPurchaseYTD -> IndividualSurvey.TotalPurchaseYTD)
11497                if (xpath.startsWith("/")) {
11498                        xpath = xpath.substring(1);
11499                }
11500                xpath = xpath.replace("/", ".");
11501
11502                return xpath;
11503        }
11504
11505        private void analyzeConstantDataFlowRelation(Object modelObject, List<TParseTreeNode> constants,
11506                        EffectType effectType, List<TParseTreeNode> functions) {
11507                analyzeConstantDataFlowRelation(modelObject, constants, effectType, functions, null);
11508        }
11509
11510        private void analyzeConstantDataFlowRelation(Object modelObject, List<TParseTreeNode> constants,
11511                        EffectType effectType, List<TParseTreeNode> functions, Process process) {
11512                if (constants == null || constants.size() == 0)
11513                        return;
11514
11515                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11516                relation.setEffectType(effectType);
11517                relation.setProcess(process);
11518
11519                if (functions != null && !functions.isEmpty()) {
11520                        relation.setFunction(getFunctionName(functions.get(0)));
11521                }
11522
11523                if (modelObject instanceof ResultColumn) {
11524                        relation.setTarget(new ResultColumnRelationshipElement((ResultColumn) modelObject));
11525
11526                } else if (modelObject instanceof TableColumn) {
11527                        relation.setTarget(new TableColumnRelationshipElement((TableColumn) modelObject));
11528
11529                } else {
11530                        throw new UnsupportedOperationException();
11531                }
11532
11533                if (option.isShowConstantTable()) {
11534                        Table constantTable = null;
11535                        if(modelObject instanceof FunctionResultColumn && ((FunctionResultColumn)modelObject).getFunction() instanceof TFunctionCall) {
11536                                TFunctionCall function = (TFunctionCall)((FunctionResultColumn)modelObject).getFunction();
11537                                if(function.getFunctionType() == EFunctionType.struct_t) {
11538                                        constantTable = modelFactory.createConstantsTable(String.valueOf(function.toString().hashCode()));
11539                                }
11540                        }
11541                        if(constantTable == null) {
11542                                constantTable = modelFactory.createConstantsTable(stmtStack.peek());
11543                        }
11544                        for (int i = 0; i < constants.size(); i++) {
11545                                TParseTreeNode constant = constants.get(i);
11546                                if (constant instanceof TConstant) {
11547                                        TableColumn constantColumn = modelFactory.createTableColumn(constantTable, (TConstant) constant);
11548                                        relation.addSource(new ConstantRelationshipElement(constantColumn));
11549                                } else if (constant instanceof TObjectName) {
11550                                        TableColumn constantColumn = modelFactory.createTableColumn(constantTable, (TObjectName) constant,
11551                                                        false);
11552                                        if(constantColumn == null) {
11553                                                continue;
11554                                        }
11555                                        relation.addSource(new ConstantRelationshipElement(constantColumn));
11556                                }
11557                        }
11558                }
11559
11560        }
11561
11562        private String getFunctionName(TParseTreeNode parseTreeNode) {
11563                if (parseTreeNode instanceof TFunctionCall) {
11564                        return ((TFunctionCall) parseTreeNode).getFunctionName().toString();
11565                }
11566                if (parseTreeNode instanceof TCaseExpression) {
11567                        return "case-when";
11568                }
11569                return null;
11570        }
11571
11572        private void analyzeCreateViewStmt(TCustomSqlStatement stmt, TSelectSqlStatement subquery,
11573                        TViewAliasClause viewAlias, TObjectName viewName) {
11574
11575                if (subquery != null) {
11576                        TTableList tables = subquery.getTables();
11577                        if (tables != null) {
11578                                for (int i = 0; i < tables.size(); i++) {
11579                                        TTable table = tables.getTable(i);
11580                                        TCustomSqlStatement createView = viewDDLMap
11581                                                        .get(DlineageUtil.getTableFullName(table.getTableName().toString()));
11582                                        if (createView != null) {
11583                                                analyzeCustomSqlStmt(createView);
11584                                        }
11585                                }
11586                        }
11587                        analyzeSelectStmt(subquery);
11588                }
11589
11590        
11591                if (viewAlias != null && viewAlias.getViewAliasItemList() != null) {
11592                        TViewAliasItemList viewItems = viewAlias.getViewAliasItemList();
11593                        Table viewModel = modelFactory.createView(stmt, viewName, true);
11594                        viewModel.setFromDDL(true);
11595                        viewModel.setDetermined(true);
11596                        Process process = modelFactory.createProcess(stmt);
11597                        viewModel.addProcess(process);
11598                        ResultSet resultSetModel = (ResultSet) modelManager.getModel(subquery);
11599                        if (resultSetModel != null) {
11600                                int resultSetSize = resultSetModel.getColumns().size();
11601                                int viewItemSize = viewItems.size();
11602                                int j = 0;
11603                                int viewColumnSize = viewItemSize;
11604                                if (resultSetModel.isDetermined() && resultSetSize > viewColumnSize) {
11605                                        viewColumnSize = resultSetSize;
11606                                }
11607                                for (int i = 0; i < viewColumnSize && j < resultSetSize; i++) {
11608                                        ResultColumn resultColumn = resultSetModel.getColumns().get(j);                                 
11609                                        if (i < viewItemSize) {
11610                                                TObjectName alias = viewItems.getViewAliasItem(i).getAlias();
11611
11612                                                if (!resultSetModel.getColumns().get(j).getName().contains("*")) {
11613                                                        j++;
11614                                                } else {
11615                                                        if (resultSetSize - j == viewItems.size() - i) {
11616                                                                j++;
11617                                                        }
11618                                                }
11619
11620                                                if (alias != null) {
11621                                                        TableColumn viewColumn = modelFactory.createViewColumn(viewModel, alias, i, true);
11622                                                        appendTableColumnToSQLEnv(viewModel, viewColumn);
11623                                                        if (resultColumn != null) {
11624                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11625                                                                relation.setEffectType(EffectType.create_view);
11626                                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
11627                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11628                                                                relation.setProcess(process);
11629                                                        }
11630                                                } else if (resultColumn.getColumnObject() instanceof TObjectName) {
11631                                                        TableColumn viewColumn = modelFactory.createViewColumn(viewModel,
11632                                                                        (TObjectName) resultColumn.getColumnObject(), i, true);
11633                                                        appendTableColumnToSQLEnv(viewModel, viewColumn);
11634                                                        if (resultColumn != null) {
11635                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11636                                                                relation.setEffectType(EffectType.create_view);
11637                                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
11638                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11639                                                                relation.setProcess(process);
11640                                                        }
11641                                                } else if (resultColumn.getColumnObject() instanceof TResultColumn) {
11642                                                        TableColumn viewColumn = modelFactory.createViewColumn(viewModel,
11643                                                                        ((TResultColumn) resultColumn.getColumnObject()).getFieldAttr(), i, true);
11644                                                        appendTableColumnToSQLEnv(viewModel, viewColumn);
11645                                                        ResultColumn column = (ResultColumn) modelManager.getModel(resultColumn.getColumnObject());
11646                                                        if (column != null && !column.getStarLinkColumns().isEmpty()) {
11647                                                                viewColumn.bindStarLinkColumns(column.getStarLinkColumns());
11648                                                        }
11649                                                        if (resultColumn != null) {
11650                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11651                                                                relation.setEffectType(EffectType.create_view);
11652                                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
11653                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11654                                                                relation.setProcess(process);
11655                                                        }
11656                                                }
11657                                        }
11658                                        else if(resultSetModel.isDetermined()){
11659                                                TObjectName viewColumnName = new TObjectName();
11660                                                viewColumnName.setString(resultColumn.getName());
11661                                                TableColumn viewColumn = modelFactory.createViewColumn(viewModel, viewColumnName, viewModel.getColumns().size(), true);
11662                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11663                                                relation.setEffectType(EffectType.create_view);
11664                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
11665                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11666                                                relation.setProcess(process);
11667                                                j++;
11668                                        }
11669                                }
11670                                if (resultSetModel != null && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
11671                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
11672                                        impactRelation.setEffectType(EffectType.create_view);
11673                                        impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
11674                                                        resultSetModel.getRelationRows()));
11675                                        impactRelation.setTarget(
11676                                                        new RelationRowsRelationshipElement<TableRelationRows>(viewModel.getRelationRows()));
11677                                }
11678                        }
11679
11680                        if (subquery.getResultColumnList() == null && subquery.getValueClause() != null
11681                                        && subquery.getValueClause().getValueRows().size() == viewItems.size()) {
11682                                for (int i = 0; i < viewItems.size(); i++) {
11683                                        TObjectName alias = viewItems.getViewAliasItem(i).getAlias();
11684
11685                                        if (alias != null) {
11686                                                TableColumn viewColumn = modelFactory.createViewColumn(viewModel, alias, i, true);
11687                                                appendTableColumnToSQLEnv(viewModel, viewColumn);
11688                                                TExpression expression = subquery.getValueClause().getValueRows().getValueRowItem(i).getExpr();
11689
11690                                                columnsInExpr visitor = new columnsInExpr();
11691                                                expression.inOrderTraverse(visitor);
11692                                                List<TObjectName> objectNames = visitor.getObjectNames();
11693                                                List<TParseTreeNode> functions = visitor.getFunctions();
11694
11695                                                if (functions != null && !functions.isEmpty()) {
11696                                                        analyzeFunctionDataFlowRelation(viewColumn, functions, EffectType.select);
11697
11698                                                }
11699
11700                                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
11701                                                if (subquerys != null && !subquerys.isEmpty()) {
11702                                                        analyzeSubqueryDataFlowRelation(viewColumn, subquerys, EffectType.select);
11703                                                }
11704
11705                                                analyzeDataFlowRelation(viewColumn, objectNames, EffectType.select, functions);
11706                                                List<TParseTreeNode> constants = visitor.getConstants();
11707                                                analyzeConstantDataFlowRelation(viewColumn, constants, EffectType.select, functions);
11708                                        }
11709                                }
11710                        }
11711
11712                } else {
11713                        if (viewName == null) {
11714                                ErrorInfo errorInfo = new ErrorInfo();
11715                                errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
11716                                errorInfo.setErrorMessage("Can't get view name. CreateView is " + stmt.toString());
11717                                errorInfo.setStartPosition(new Pair3<Long, Long, String>(stmt.getStartToken().lineNo,
11718                                                stmt.getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
11719                                errorInfo.setEndPosition(new Pair3<Long, Long, String>(stmt.getEndToken().lineNo,
11720                                                stmt.getEndToken().columnNo + stmt.getEndToken().getAstext().length(),
11721                                                ModelBindingManager.getGlobalHash()));
11722                                errorInfo.fillInfo(this);
11723                                errorInfos.add(errorInfo);
11724                                return;
11725                        }
11726                        Table viewModel = modelFactory.createView(stmt, viewName);
11727                        Process process = modelFactory.createProcess(stmt);
11728                        viewModel.addProcess(process);
11729                        if (subquery != null && !subquery.isCombinedQuery()) {
11730                                SelectResultSet resultSetModel = (SelectResultSet) modelManager
11731                                                .getModel(subquery.getResultColumnList());
11732
11733                                boolean determined = false;
11734                                if (!containStarColumn(resultSetModel)) {
11735                                        viewModel.setCreateTable(true);
11736                                        determined = true;
11737                                        viewModel.setFromDDL(true);
11738                                }
11739                                for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
11740                                        ResultColumn resultColumn = resultSetModel.getColumns().get(i);
11741                                        if (resultColumn.getColumnObject() instanceof TResultColumn) {
11742                                                TResultColumn columnObject = ((TResultColumn) resultColumn.getColumnObject());
11743
11744                                                TAliasClause alias = ((TResultColumn) resultColumn.getColumnObject()).getAliasClause();
11745                                                if (alias != null && alias.getAliasName() != null) {
11746                                                        TableColumn viewColumn = modelFactory.createViewColumn(viewModel, alias.getAliasName(), i,
11747                                                                        determined);
11748                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11749                                                        relation.setEffectType(EffectType.create_view);
11750                                                        relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
11751                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11752                                                        relation.setProcess(process);
11753                                                        if (determined) {
11754                                                                appendTableColumnToSQLEnv(viewModel, viewColumn);
11755                                                        }
11756                                                } else if (columnObject.getFieldAttr() != null) {
11757                                                        TObjectName viewColumnObject = columnObject.getFieldAttr();
11758                                                        TableColumn viewColumn;
11759                                                        Object model = modelManager.getModel(resultColumn.getColumnObject());
11760                                                        if ("*".equals(viewColumnObject.getColumnNameOnly())) {
11761                                                                if (model instanceof LinkedHashMap) {
11762                                                                        String columnName = getColumnNameOnly(resultColumn.getName());
11763                                                                        LinkedHashMap<String, ResultColumn> resultColumns = (LinkedHashMap<String, ResultColumn>) model;
11764                                                                        if ("*".equals(columnName)) {
11765                                                                                for (String key : resultColumns.keySet()) {
11766                                                                                        ResultColumn column = resultColumns.get(key);
11767                                                                                        viewColumn = modelFactory.createInsertTableColumn(viewModel, column.getName());
11768                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11769                                                                                        relation.setEffectType(EffectType.create_view);
11770                                                                                        relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
11771                                                                                        relation.addSource(new ResultColumnRelationshipElement(column));
11772                                                                                        relation.setProcess(process);
11773                                                                                }
11774                                                                                continue;
11775                                                                        } else if (resultColumns.containsKey(columnName)) {
11776                                                                                ResultColumn column = resultColumns.get(columnName);
11777                                                                                viewColumn = modelFactory.createInsertTableColumn(viewModel, column.getName());
11778                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11779                                                                                relation.setEffectType(EffectType.create_view);
11780                                                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
11781                                                                                relation.addSource(new ResultColumnRelationshipElement(column));
11782                                                                                relation.setProcess(process);
11783                                                                                continue;
11784                                                                        }
11785                                                                }
11786                                                        }
11787                                                        
11788                                                        if (!SQLUtil.isEmpty(viewColumnObject.getColumnNameOnly())
11789                                                                        && viewColumnObject.getPropertyToken() != null) {
11790                                                                TObjectName object = new TObjectName();
11791                                                                // object.setString(viewColumnObject.getPropertyToken().astext);
11792                                                                object.setPartToken(viewColumnObject.getPropertyToken());
11793                                                                object.setStartToken(viewColumnObject.getPropertyToken());
11794                                                                object.setEndToken(viewColumnObject.getPropertyToken());
11795                                                                viewColumn = modelFactory.createViewColumn(viewModel, object, i, determined);
11796                                                        } else {
11797                                                                viewColumn = modelFactory.createViewColumn(viewModel, columnObject.getFieldAttr(),
11798                                                                                i, determined);
11799                                                        }
11800                                                        
11801                                                        if(determined) {
11802                                                                appendTableColumnToSQLEnv(viewModel, viewColumn);
11803                                                        }
11804                                                        
11805                                                        if (model instanceof ResultColumn) {
11806                                                                ResultColumn column = (ResultColumn) modelManager
11807                                                                                .getModel(resultColumn.getColumnObject());
11808                                                                if (column != null && !column.getStarLinkColumns().isEmpty()) {
11809                                                                        viewColumn.bindStarLinkColumns(column.getStarLinkColumns());
11810                                                                }
11811                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11812                                                                relation.setEffectType(EffectType.create_view);
11813                                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
11814                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11815                                                                if (sqlenv == null) {
11816                                                                        if (resultColumn.isShowStar()) {
11817                                                                                relation.setShowStarRelation(true);
11818                                                                                viewColumn.setShowStar(true);
11819                                                                                resultColumn.setShowStar(true);
11820                                                                                setSourceShowStar(resultColumn);
11821                                                                        }
11822                                                                }
11823                                                                if (viewColumn.getName().endsWith("*") && resultColumn.getName().endsWith("*")) {
11824                                                                        viewModel.setStarStmt("create_view");
11825                                                                }
11826                                                                relation.setProcess(process);
11827                                                        }
11828                                                } else if (resultColumn.getAlias() != null && columnObject.getExpr()
11829                                                                .getExpressionType() == EExpressionType.sqlserver_proprietary_column_alias_t) {
11830                                                        TableColumn viewColumn = modelFactory.createViewColumn(viewModel,
11831                                                                        columnObject.getExpr().getLeftOperand().getObjectOperand(), i, determined);
11832                                                        if(determined) {
11833                                                                appendTableColumnToSQLEnv(viewModel, viewColumn);
11834                                                        }
11835                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11836                                                        relation.setEffectType(EffectType.create_view);
11837                                                        relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
11838                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11839                                                        relation.setProcess(process);
11840                                                } else {
11841                                                        TGSqlParser parser = columnObject.getGsqlparser();
11842                                                        TObjectName viewColumnName = parser
11843                                                                        .parseObjectName(generateQuotedName(parser, columnObject.toString()));
11844                                                        if (viewColumnName == null) {
11845                                                                ErrorInfo errorInfo = new ErrorInfo();
11846                                                                errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
11847                                                                errorInfo.setErrorMessage(
11848                                                                                "Can't parse view column. Column is " + columnObject.toString());
11849                                                                errorInfo.setStartPosition(new Pair3<Long, Long, String>(
11850                                                                                columnObject.getStartToken().lineNo, columnObject.getStartToken().columnNo,
11851                                                                                ModelBindingManager.getGlobalHash()));
11852                                                                errorInfo
11853                                                                                .setEndPosition(new Pair3<Long, Long, String>(columnObject.getEndToken().lineNo,
11854                                                                                                columnObject.getEndToken().columnNo
11855                                                                                                                + columnObject.getEndToken().getAstext().length(),
11856                                                                                                ModelBindingManager.getGlobalHash()));
11857                                                                errorInfo.fillInfo(this);
11858                                                                errorInfos.add(errorInfo);
11859                                                        } else {
11860                                                                TableColumn viewColumn = modelFactory.createViewColumn(viewModel, viewColumnName, i,
11861                                                                                determined);
11862                                                                if(determined) {
11863                                                                        appendTableColumnToSQLEnv(viewModel, viewColumn);
11864                                                                }
11865                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11866                                                                relation.setEffectType(EffectType.create_view);
11867                                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
11868                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11869                                                                relation.setProcess(process);
11870                                                        }
11871                                                }
11872                                        } else if (resultColumn.getColumnObject() instanceof TObjectName) {
11873                                                TableColumn viewColumn = modelFactory.createViewColumn(viewModel,
11874                                                                (TObjectName) resultColumn.getColumnObject(), i, determined);
11875                                                if(determined) {
11876                                                        appendTableColumnToSQLEnv(viewModel, viewColumn);
11877                                                }
11878                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11879                                                relation.setEffectType(EffectType.create_view);
11880                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
11881                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11882                                                relation.setProcess(process);
11883                                        }
11884                                }
11885                                if (!resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
11886                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
11887                                        impactRelation.setEffectType(EffectType.create_view);
11888                                        impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
11889                                                        resultSetModel.getRelationRows()));
11890                                        impactRelation.setTarget(
11891                                                        new RelationRowsRelationshipElement<TableRelationRows>(viewModel.getRelationRows()));
11892                                }
11893                        } else if (subquery != null && subquery.isCombinedQuery()) {
11894                                SelectSetResultSet resultSetModel = (SelectSetResultSet) modelManager.getModel(subquery);
11895
11896                                boolean determined = false;
11897                                if (!containStarColumn(resultSetModel)) {
11898                                        viewModel.setCreateTable(true);
11899                                        viewModel.setFromDDL(true);
11900                                        determined = true;
11901                                }
11902
11903                                for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
11904                                        ResultColumn resultColumn = resultSetModel.getColumns().get(i);
11905
11906                                        if (resultColumn.getColumnObject() instanceof TResultColumn) {
11907                                                TResultColumn columnObject = ((TResultColumn) resultColumn.getColumnObject());
11908
11909                                                TAliasClause alias = columnObject.getAliasClause();
11910                                                if (alias != null && alias.getAliasName() != null) {
11911                                                        TableColumn viewColumn = modelFactory.createViewColumn(viewModel, alias.getAliasName(), i,
11912                                                                        determined);
11913                                                        if(determined) {
11914                                                                appendTableColumnToSQLEnv(viewModel, viewColumn);
11915                                                        }
11916                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11917                                                        relation.setEffectType(EffectType.create_view);
11918                                                        relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
11919                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11920                                                        relation.setProcess(process);
11921                                                } else if (columnObject.getFieldAttr() != null) {
11922                                                        TableColumn viewColumn = modelFactory.createViewColumn(viewModel,
11923                                                                        columnObject.getFieldAttr(), i, determined);
11924                                                        if(determined) {
11925                                                                appendTableColumnToSQLEnv(viewModel, viewColumn);
11926                                                        }
11927                                                        ResultColumn column = (ResultColumn) modelManager.getModel(resultColumn.getColumnObject());
11928                                                        if (column != null && !column.getStarLinkColumns().isEmpty()) {
11929                                                                viewColumn.bindStarLinkColumns(column.getStarLinkColumns());
11930                                                        }
11931                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11932                                                        relation.setEffectType(EffectType.create_view);
11933                                                        relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
11934                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11935                                                        relation.setProcess(process);
11936                                                } else if (resultColumn.getAlias() != null && columnObject.getExpr()
11937                                                                .getExpressionType() == EExpressionType.sqlserver_proprietary_column_alias_t) {
11938                                                        TableColumn viewColumn = modelFactory.createViewColumn(viewModel,
11939                                                                        columnObject.getExpr().getLeftOperand().getObjectOperand(), i, determined);
11940                                                        if(determined) {
11941                                                                appendTableColumnToSQLEnv(viewModel, viewColumn);
11942                                                        }
11943                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11944                                                        relation.setEffectType(EffectType.create_view);
11945                                                        relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
11946                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11947                                                        relation.setProcess(process);
11948                                                } else {
11949                                                        TGSqlParser parser = columnObject.getGsqlparser();
11950                                                        TObjectName viewColumnName = parser
11951                                                                        .parseObjectName(generateQuotedName(parser, columnObject.toString()));
11952                                                        TableColumn viewColumn = modelFactory.createViewColumn(viewModel, viewColumnName, i,
11953                                                                        determined);
11954                                                        if(determined) {
11955                                                                appendTableColumnToSQLEnv(viewModel, viewColumn);
11956                                                        }
11957                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11958                                                        relation.setEffectType(EffectType.create_view);
11959                                                        relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
11960                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11961                                                        relation.setProcess(process);
11962                                                }
11963                                        } else if (resultColumn.getColumnObject() instanceof TObjectName) {
11964                                                TableColumn viewColumn = modelFactory.createViewColumn(viewModel,
11965                                                                (TObjectName) resultColumn.getColumnObject(), i, determined);
11966                                                if(viewColumn!=null) {
11967                                                        if(determined) {
11968                                                                appendTableColumnToSQLEnv(viewModel, viewColumn);
11969                                                        }
11970                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11971                                                        relation.setEffectType(EffectType.create_view);
11972                                                        relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
11973                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11974                                                        relation.setProcess(process);
11975                                                }
11976                                        }
11977                                }
11978                                if (!resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
11979                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
11980                                        impactRelation.setEffectType(EffectType.create_view);
11981                                        impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
11982                                                        resultSetModel.getRelationRows()));
11983                                        impactRelation.setTarget(
11984                                                        new RelationRowsRelationshipElement<TableRelationRows>(viewModel.getRelationRows()));
11985                                }
11986                        }
11987                }
11988        }
11989
11990        private void setSourceShowStar(Object resultColumn) {
11991                for (Relationship relation : modelManager.getRelations()) {
11992                        Collection<RelationshipElement<?>> sources = (Collection<RelationshipElement<?>>)relation.getSources();
11993                        if (relation.getTarget().getElement() == resultColumn && sources != null) {
11994
11995                                ((AbstractRelationship) relation).setShowStarRelation(true);
11996                                for (RelationshipElement<?> source : sources) {
11997                                        Object column = source.getElement();
11998                                        if (column instanceof TableColumn) {
11999                                                if (((TableColumn) column).isShowStar())
12000                                                        continue;
12001                                                ((TableColumn) column).setShowStar(true);
12002                                        }
12003                                        if (column instanceof ResultColumn) {
12004                                                if (((ResultColumn) column).isShowStar())
12005                                                        continue;
12006                                                ((ResultColumn) column).setShowStar(true);
12007                                        }
12008                                        setSourceShowStar(column);
12009                                }
12010                        }
12011                }
12012        }
12013
12014        private String generateQuotedName(TGSqlParser parser, String name) {
12015                return "\"" + name + "\"";
12016        }
12017
12018        private void appendRelations(dataflow dataflow) {
12019                Relationship[] relations = modelManager.getRelations();
12020
12021                appendRelation(dataflow, relations, DataFlowRelationship.class);
12022                appendRelation(dataflow, relations, IndirectImpactRelationship.class);
12023                appendRecordSetRelation(dataflow, relations);
12024                appendCallRelation(dataflow, relations);
12025                appendRelation(dataflow, relations, ImpactRelationship.class);
12026                appendRelation(dataflow, relations, JoinRelationship.class);
12027                appendRelation(dataflow, relations, ERRelationship.class);
12028                appendRelation(dataflow, relations, CrudRelationship.class);
12029        }
12030        
12031        
12032        private relationship cloneRelationshipWithSingleSource(relationship orig, sourceColumn src) {
12033                relationship rel = new relationship();
12034                rel.setType(orig.getType());
12035                rel.setEffectType(orig.getEffectType());
12036                rel.setId(orig.getId() + "_" + src.getColumn());
12037                rel.setSqlHash(orig.getSqlHash());
12038                rel.setSqlComment(orig.getSqlComment());
12039                rel.setProcessId(orig.getProcessId());
12040                rel.setProcessType(orig.getProcessType());
12041                rel.setFunction(orig.getFunction());
12042                rel.setProcedureId(orig.getProcedureId());
12043
12044                targetColumn t = new targetColumn();
12045                t.setId(orig.getTarget().getId());
12046                t.setColumn(orig.getTarget().getColumn());
12047                t.setParent_id(orig.getTarget().getParent_id());
12048                t.setParent_name(orig.getTarget().getParent_name());
12049                t.setParent_alias(orig.getTarget().getParent_alias());
12050                t.setCoordinate(orig.getTarget().getCoordinate());
12051                rel.setTarget(t);
12052
12053                sourceColumn s = new sourceColumn();
12054                s.setId(src.getId());
12055                s.setColumn(src.getColumn());
12056                s.setParent_id(src.getParent_id());
12057                s.setParent_name(src.getParent_name());
12058                s.setCoordinate(src.getCoordinate());
12059                rel.addSource(s);
12060                return rel;
12061        }
12062
12063        private Set<String> appendStarColumns = new HashSet<String>();
12064        private void appendRelation(dataflow dataflow, Relationship[] relations, Class<? extends Relationship> clazz) {
12065                for (int i = 0; i < relations.length; i++) {
12066                        AbstractRelationship relation = (AbstractRelationship) relations[i];
12067                        if (relation.getClass() == clazz) {
12068                                if (relation.getSources() == null || relation.getTarget() == null) {
12069                                        continue;
12070                                }
12071                                Object targetElement = relation.getTarget().getElement();
12072                                TObjectName targetColumnName = null;
12073                                if(relation.getTarget() instanceof ResultColumnRelationshipElement) {
12074                                        targetColumnName = ((ResultColumnRelationshipElement) relation.getTarget()).getColumnName();
12075                                }
12076                                if (targetElement instanceof ResultColumn) {
12077                                        ResultColumn targetColumn = (ResultColumn) targetElement;
12078                                        if (!targetColumn.isPseduo()) 
12079                                        {
12080                                                if (targetColumnName == null) 
12081                                                {
12082                                                        if ("*".equals(targetColumn.getName())) {
12083                                                                updateResultColumnStarLinks(dataflow, relation, -1);
12084                                                        }
12085
12086                                                        if (targetColumn.hasStarLinkColumn()) {
12087                                                                for (int j = 0; j < targetColumn.getStarLinkColumnNames().size(); j++) {
12088                                                                        appendStarRelation(dataflow, relation, j);
12089                                                                }
12090
12091                                                                if (!containsStar(relation.getSources())) {
12092                                                                        continue;
12093                                                                }
12094                                                        }
12095                                                }
12096                                                else {
12097                                                        String columnName = DlineageUtil.getColumnName(targetColumnName);
12098                                                        int index = targetColumn.getStarLinkColumnNames().indexOf(columnName);
12099                                                        if (index != -1) {
12100                                                                int size = targetColumn.getStarLinkColumnNames().size();
12101                                                                if ("*".equals(targetColumn.getName())) {
12102                                                                        if (appendStarColumns.contains(columnName)) {
12103                                                                                updateResultColumnStarLinks(dataflow, relation, index);
12104                                                                        }
12105                                                                        else {
12106                                                                                updateResultColumnStarLinks(dataflow, relation, -1);
12107                                                                                appendStarColumns.add(columnName);
12108                                                                        }
12109                                                                }
12110                                                                appendStarRelation(dataflow, relation, index);
12111                                                                for(int j= size; j < targetColumn.getStarLinkColumnNames().size(); j++) {
12112                                                                        appendStarRelation(dataflow, relation, j);
12113                                                                }
12114                                                                if (!containsStar(relation.getSources())) {
12115                                                                        continue;
12116                                                                }
12117                                                        }
12118                                                }
12119                                        }
12120                                } else if (targetElement instanceof TableColumn) {
12121                                        TableColumn targetColumn = (TableColumn) targetElement;
12122                                        if (!targetColumn.isPseduo()) {
12123                                                if ("*".equals(targetColumn.getName())) {
12124                                                        updateTableColumnStarLinks(dataflow, relation);
12125                                                }
12126
12127                                                if (targetColumn.hasStarLinkColumn() && !targetColumn.isVariant()
12128                                                                && targetColumn.isExpandStar()) {
12129                                                        for (int j = 0; j < targetColumn.getStarLinkColumnNames().size(); j++) {
12130                                                                appendStarRelation(dataflow, relation, j);
12131                                                        }
12132                                                        if (!containsStar(relation.getSources())) {
12133                                                                continue;
12134                                                        }
12135                                                }
12136                                        }
12137                                }
12138
12139                                relationship relationElement = new relationship();
12140                                relationElement.setType(relation.getRelationshipType().name());
12141                                if (relation.getEffectType() != null) {
12142                                        relationElement.setEffectType(relation.getEffectType().name());
12143                                }
12144                                if (relation.getFunction() != null) {
12145                                        relationElement.setFunction(relation.getFunction());
12146                                }
12147                                relationElement.setSqlHash(relation.getSqlHash());
12148                                relationElement.setSqlComment(relation.getSqlComment());
12149
12150                                if (relation.getProcedureId() != null) {
12151                                        relationElement.setProcedureId(String.valueOf(relation.getProcedureId()));
12152                                }
12153                                relationElement.setId(String.valueOf(relation.getId()));
12154                                if (relation.getProcess() != null) {
12155                                        relationElement.setProcessId(String.valueOf(relation.getProcess().getId()));
12156                                        if (relation.getProcess().getGspObject() != null) {
12157                                                relationElement.setProcessType(relation.getProcess().getGspObject().sqlstatementtype.name());
12158                                        }
12159                                }
12160
12161                                if (relation.getPartition() != null) {
12162                                        relationElement.setPartition(relation.getPartition());
12163                                }
12164                                relationElement.setSqlHash(relation.getSqlHash());
12165                                relationElement.setSqlComment(relation.getSqlComment());
12166                                
12167                                if (relation.getProcedureId() != null) {
12168                                        relationElement.setProcedureId(String.valueOf(relation.getProcedureId()));
12169                                }
12170                                
12171                                if (relation instanceof JoinRelationship) {
12172                                        relationElement.setCondition(((JoinRelationship) relation).getJoinCondition());
12173                                        relationElement.setJoinType(((JoinRelationship) relation).getJoinType().name());
12174                                        relationElement.setClause(((JoinRelationship) relation).getJoinClauseType().name());
12175                                }
12176
12177                                if(relation instanceof ImpactRelationship){
12178                                        ImpactRelationship impactRelationship = (ImpactRelationship)relation;
12179                                        if(impactRelationship.getJoinClauseType()!=null){
12180                                                relationElement.setClause(impactRelationship.getJoinClauseType().name());
12181                                        }
12182                                }
12183
12184                                String targetName = null;
12185                                Object columnObject = null;
12186                                List<TObjectName> targetObjectNames = null;
12187
12188                                if (targetElement instanceof ResultSetRelationRows) {
12189                                        ResultSetRelationRows targetColumn = (ResultSetRelationRows) targetElement;
12190                                        targetColumn target = new targetColumn();
12191                                        target.setId(String.valueOf(targetColumn.getId()));
12192                                        target.setColumn(targetColumn.getName());
12193                                        target.setParent_id(String.valueOf(targetColumn.getHolder().getId()));
12194                                        target.setParent_name(getResultSetName(targetColumn.getHolder()));
12195                                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
12196                                                target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
12197                                                                + convertCoordinate(targetColumn.getEndPosition()));
12198                                        }
12199                                        if (relation instanceof RecordSetRelationship) {
12200                                                target.setFunction(((RecordSetRelationship) relation).getAggregateFunction());
12201                                        }
12202                                        target.setSource("system");
12203                                        targetName = targetColumn.getName();
12204                                        relationElement.setTarget(target);
12205                                } else if (targetElement instanceof TableRelationRows) {
12206                                        TableRelationRows targetColumn = (TableRelationRows) targetElement;
12207                                        targetColumn target = new targetColumn();
12208                                        target.setId(String.valueOf(targetColumn.getId()));
12209                                        target.setColumn(targetColumn.getName());
12210                                        target.setParent_id(String.valueOf(targetColumn.getHolder().getId()));
12211                                        target.setParent_name(getTableName(targetColumn.getHolder()));
12212                                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
12213                                                target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
12214                                                                + convertCoordinate(targetColumn.getEndPosition()));
12215                                        }
12216                                        if (relation instanceof RecordSetRelationship) {
12217                                                target.setFunction(((RecordSetRelationship) relation).getAggregateFunction());
12218                                        }
12219                                        target.setSource("system");
12220                                        targetName = targetColumn.getName();
12221                                        relationElement.setTarget(target);
12222                                } else if (targetElement instanceof ResultColumn) {
12223                                        ResultColumn targetColumn = (ResultColumn) targetElement;
12224                                        targetColumn target = new targetColumn();
12225                                        target.setId(String.valueOf(targetColumn.getId()));
12226                                        target.setColumn(targetColumn.getName());
12227                                        target.setStruct(targetColumn.isStruct());
12228                                        target.setParent_id(String.valueOf(targetColumn.getResultSet().getId()));
12229                                        target.setParent_name(getResultSetName(targetColumn.getResultSet()));
12230                                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
12231                                                target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
12232                                                                + convertCoordinate(targetColumn.getEndPosition()));
12233                                        }
12234                                        if (relation instanceof RecordSetRelationship) {
12235                                                target.setFunction(((RecordSetRelationship) relation).getAggregateFunction());
12236                                        }
12237                                        targetName = targetColumn.getName();
12238                                        if (((ResultColumn) targetColumn).getColumnObject() instanceof TResultColumn) {
12239                                                columnsInExpr visitor = new columnsInExpr();
12240                                                ((TResultColumn) ((ResultColumn) targetColumn).getColumnObject()).getExpr()
12241                                                                .inOrderTraverse(visitor);
12242                                                targetObjectNames = visitor.getObjectNames();
12243                                        }
12244
12245                                        if (targetElement instanceof FunctionResultColumn) {
12246                                                columnObject = ((FunctionResultColumn) targetElement).getColumnObject();
12247                                        }
12248                                        if(targetColumn.isPseduo()) {
12249                                                target.setSource("system");
12250                                        }
12251                                        relationElement.setTarget(target);
12252                                } else if (targetElement instanceof TableColumn) {
12253                                        TableColumn targetColumn = (TableColumn) targetElement;
12254                                        targetColumn target = new targetColumn();
12255                                        target.setId(String.valueOf(targetColumn.getId()));
12256                                        target.setColumn(targetColumn.getName());
12257                                        target.setStruct(targetColumn.isStruct());
12258                                        target.setParent_id(String.valueOf(targetColumn.getTable().getId()));
12259                                        target.setParent_name(getTableName(targetColumn.getTable()));
12260                                        if (relation.getTarget() instanceof TableColumnRelationshipElement) {
12261                                                target.setParent_alias(((TableColumnRelationshipElement) relation.getTarget()).getTableAlias());
12262                                        }
12263                                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
12264                                                target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
12265                                                                + convertCoordinate(targetColumn.getEndPosition()));
12266                                        }
12267                                        if (relation instanceof RecordSetRelationship) {
12268                                                target.setFunction(((RecordSetRelationship) relation).getAggregateFunction());
12269                                        }
12270                                        if(targetColumn.isPseduo()) {
12271                                                target.setSource("system");
12272                                        }
12273                                        targetName = targetColumn.getName();
12274                                        relationElement.setTarget(target);
12275                                } else if (targetElement instanceof Argument) {
12276                                        Argument targetColumn = (Argument) targetElement;
12277                                        targetColumn target = new targetColumn();
12278                                        target.setId(String.valueOf(targetColumn.getId()));
12279                                        target.setColumn(targetColumn.getName());
12280                                        target.setParent_id(String.valueOf(targetColumn.getProcedure().getId()));
12281                                        target.setParent_name(targetColumn.getProcedure().getName());
12282                                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
12283                                                target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
12284                                                                + convertCoordinate(targetColumn.getEndPosition()));
12285                                        }
12286                                        if (relation instanceof RecordSetRelationship) {
12287                                                target.setFunction(((RecordSetRelationship) relation).getAggregateFunction());
12288                                        }
12289                                        targetName = targetColumn.getName();
12290                                        relationElement.setTarget(target);
12291                                } else if (targetElement instanceof Table) {
12292                                        Table table = (Table) targetElement;
12293                                        targetColumn target = new targetColumn();
12294                                        target.setTarget_id(String.valueOf(table.getId()));
12295                                        target.setTarget_name(getTableName(table));
12296                                        if (table.getStartPosition() != null && table.getEndPosition() != null) {
12297                                                target.setCoordinate(convertCoordinate(table.getStartPosition()) + ","
12298                                                                + convertCoordinate(table.getEndPosition()));
12299                                        }
12300                                        relationElement.setTarget(target);
12301                                } else {
12302                                        continue;
12303                                }
12304
12305                                Collection<RelationshipElement<?>> sourceElements = relation.getSources();
12306                                if (sourceElements.size() == 0) {
12307                                        if(clazz == CrudRelationship.class && option.getAnalyzeMode() == AnalyzeMode.crud) {
12308                                                dataflow.getRelationships().add(relationElement);
12309                                        }
12310                                        continue;
12311                                }
12312
12313                                boolean append = false;
12314                                for (RelationshipElement<?> sourceItem: relation.getSources()) {
12315                                        Object sourceElement = sourceItem.getElement();
12316                                        TObjectName sourceColumnName = null;
12317                                        if (sourceItem instanceof ResultColumnRelationshipElement) {
12318                                                sourceColumnName = ((ResultColumnRelationshipElement) sourceItem).getColumnName();
12319                                        }
12320//                                      if (sourceItem instanceof TableColumnRelationElement
12321//                                                      && (((TableColumnRelationElement) sourceItem).getColumnIndex() != null)) {
12322//                                              TableColumnRelationElement tableColumnRelationElement = (TableColumnRelationElement) sourceItem;
12323//                                              sourceElement = tableColumnRelationElement.getElement().getTable().getColumns()
12324//                                                              .get(tableColumnRelationElement.getColumnIndex() + 1);
12325//                                      }
12326                                        if (sourceElement instanceof ResultColumn) {
12327                                                ResultColumn sourceColumn = (ResultColumn) sourceElement;
12328                                                if (sourceColumn.hasStarLinkColumn() && !relation.isShowStarRelation() && !sourceColumn.isPseduo()) {
12329                                                        List<String> sourceStarColumnNames = sourceColumn.getStarLinkColumnNames();
12330                                                        sourceColumn source = new sourceColumn();
12331                                                        if (targetObjectNames != null && !targetObjectNames.isEmpty()) {
12332
12333                                                                for (int k = 0; k < targetObjectNames.size(); k++) {
12334                                                                        String targetObjectName = getColumnName(targetObjectNames.get(k));
12335                                                                        if(sourceColumn.getResultSet()!=null && sourceColumn.getResultSet().getColumns()!=null) {
12336                                                                                boolean find = false;
12337                                                                                for(ResultColumn column: sourceColumn.getResultSet().getColumns()) {
12338                                                                                        if(column.getName().equalsIgnoreCase(targetObjectName)) {
12339                                                                                                source = new sourceColumn();
12340                                                                                                source.setId(String.valueOf(column.getId()));
12341                                                                                                source.setColumn(targetObjectName);
12342                                                                                                source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
12343                                                                                                source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
12344                                                                                                if (sourceColumn.getStartPosition() != null
12345                                                                                                                && sourceColumn.getEndPosition() != null) {
12346                                                                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition())
12347                                                                                                                        + "," + convertCoordinate(sourceColumn.getEndPosition()));
12348                                                                                                }
12349                                                                                                append = true;
12350                                                                                                relationElement.addSource(source);
12351                                                                                                find = true;
12352                                                                                                break;
12353                                                                                        }
12354                                                                                }
12355                                                                                if(find) {
12356                                                                                        continue;
12357                                                                                }
12358                                                                        }
12359                                                                        if (sourceColumn.getStarLinkColumns().containsKey(targetObjectName)) {
12360                                                                                source = new sourceColumn();
12361                                                                                source.setId(String.valueOf(sourceColumn.getId()) + "_"
12362                                                                                                + sourceStarColumnNames.indexOf(targetObjectName));
12363                                                                                source.setColumn(targetObjectName);
12364                                                                                source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
12365                                                                                source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
12366                                                                                if (sourceColumn.getStartPosition() != null
12367                                                                                                && sourceColumn.getEndPosition() != null) {
12368                                                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition())
12369                                                                                                        + "," + convertCoordinate(sourceColumn.getEndPosition()));
12370                                                                                }
12371                                                                                append = true;
12372                                                                                relationElement.addSource(source);
12373                                                                        } else {
12374                                                                                source = new sourceColumn();
12375                                                                                source.setId(String.valueOf(sourceColumn.getId()));
12376                                                                                source.setColumn(relationElement.getTarget().getColumn());
12377                                                                                source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
12378                                                                                source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
12379                                                                                if (sourceColumn.getStartPosition() != null
12380                                                                                                && sourceColumn.getEndPosition() != null) {
12381                                                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition())
12382                                                                                                        + "," + convertCoordinate(sourceColumn.getEndPosition()));
12383                                                                                }
12384                                                                                append = true;
12385                                                                                relationElement.addSource(source);
12386                                                                        }
12387                                                                }
12388                                                        } else {
12389                                                                if (columnObject instanceof TWhenClauseItemList) {
12390                                                                        TCaseExpression expr = (TCaseExpression)((FunctionResultColumn) targetElement).getResultSet().getGspObject();
12391                                                                        List<TExpression> directExpressions = new ArrayList<TExpression>();
12392                                                                        
12393//                                                                      TExpression inputExpr = expr.getInput_expr();
12394//                                                                      if (inputExpr != null) {
12395//                                                                              directExpressions.add(inputExpr);
12396//                                                                      }
12397                                                                        TExpression defaultExpr = expr.getElse_expr();
12398                                                                        if (defaultExpr != null) {
12399                                                                                directExpressions.add(defaultExpr);
12400                                                                        }
12401                                                                        TWhenClauseItemList list = expr.getWhenClauseItemList();
12402                                                                        for (int k = 0; k < list.size(); k++) {
12403                                                                                TWhenClauseItem element = list.getWhenClauseItem(k);
12404                                                                                directExpressions.add(element.getReturn_expr());
12405                                                                        }
12406                                                                        
12407                                                                        for (int k = 0; k < directExpressions.size(); k++) {
12408                                                                                columnsInExpr visitor = new columnsInExpr();
12409                                                                                directExpressions.get(k).inOrderTraverse(visitor);
12410                                                                                List<TObjectName> objectNames = visitor.getObjectNames();
12411                                                                                if (objectNames == null) {
12412                                                                                        continue;
12413                                                                                }
12414                                                                                for (int x = 0; x < objectNames.size(); x++) {
12415                                                                                        String objectName = getColumnName(objectNames.get(x));
12416                                                                                        if (sourceColumn.getStarLinkColumns().containsKey(objectName)) {
12417
12418                                                                                                if(sourceColumn.getResultSet()!=null && sourceColumn.getResultSet().getColumns()!=null) {
12419                                                                                                        boolean find = false;
12420                                                                                                        for(ResultColumn column: sourceColumn.getResultSet().getColumns()) {
12421                                                                                                                if(column.getName().equalsIgnoreCase(objectName)) {
12422                                                                                                                        source = new sourceColumn();
12423                                                                                                                        source.setId(String.valueOf(column.getId()));
12424                                                                                                                        source.setColumn(objectName);
12425                                                                                                                        source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
12426                                                                                                                        source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
12427                                                                                                                        if (sourceColumn.getStartPosition() != null
12428                                                                                                                                        && sourceColumn.getEndPosition() != null) {
12429                                                                                                                                source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition())
12430                                                                                                                                                + "," + convertCoordinate(sourceColumn.getEndPosition()));
12431                                                                                                                        }
12432                                                                                                                        append = true;
12433                                                                                                                        relationElement.addSource(source);
12434                                                                                                                        find = true;
12435                                                                                                                        break;
12436                                                                                                                }
12437                                                                                                        }
12438                                                                                                        if(find) {
12439                                                                                                                continue;
12440                                                                                                        }
12441                                                                                                }
12442                                                                                                
12443                                                                                                source.setId(String.valueOf(sourceColumn.getId()) + "_"
12444                                                                                                                + sourceStarColumnNames.indexOf(objectName));
12445                                                                                                source.setColumn(objectName);
12446                                                                                        } else {
12447                                                                                                source.setId(String.valueOf(sourceColumn.getId()));
12448                                                                                                source.setColumn(sourceColumn.getName());
12449                                                                                        }
12450                                                                                        source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
12451                                                                                        source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
12452                                                                                        if (sourceColumn.getStartPosition() != null
12453                                                                                                        && sourceColumn.getEndPosition() != null) {
12454                                                                                                source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition())
12455                                                                                                                + "," + convertCoordinate(sourceColumn.getEndPosition()));
12456                                                                                        }
12457                                                                                        append = true;
12458                                                                                        relationElement.addSource(source);
12459                                                                                }
12460                                                                        }
12461                                                                } else {
12462                                                                        String objectName = getColumnName(targetName);
12463                                                                        boolean find = false;
12464                                                                        
12465                                                                        if (!find && sourceColumn.getStarLinkColumns().containsKey(objectName)) {
12466                                                                                source.setId(String.valueOf(sourceColumn.getId()) + "_"
12467                                                                                                + sourceStarColumnNames.indexOf(objectName));
12468                                                                                source.setColumn(objectName);
12469                                                                                find = true;
12470                                                                        }
12471
12472                                                                        if (!find && sourceColumnName != null) {
12473                                                                                objectName = getColumnName(sourceColumnName);
12474                                                                                
12475                                                                                if(sourceColumn.getResultSet()!=null && sourceColumn.getResultSet().getColumns()!=null) {
12476                                                                                        for(ResultColumn column: sourceColumn.getResultSet().getColumns()) {
12477                                                                                                if(column.getName().equalsIgnoreCase(objectName)) {
12478                                                                                                        source = new sourceColumn();
12479                                                                                                        source.setId(String.valueOf(column.getId()));
12480                                                                                                        source.setColumn(objectName);
12481                                                                                                        source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
12482                                                                                                        source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
12483                                                                                                        if (sourceColumn.getStartPosition() != null
12484                                                                                                                        && sourceColumn.getEndPosition() != null) {
12485                                                                                                                source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition())
12486                                                                                                                                + "," + convertCoordinate(sourceColumn.getEndPosition()));
12487                                                                                                        }
12488                                                                                                        append = true;
12489                                                                                                        relationElement.addSource(source);
12490                                                                                                        find = true;
12491                                                                                                        break;
12492                                                                                                }
12493                                                                                        }
12494                                                                                        if(find) {
12495                                                                                                continue;
12496                                                                                        }
12497                                                                                }
12498                                                                                
12499                                                                                if (sourceColumn.getStarLinkColumns().containsKey(objectName)) {
12500                                                                                        source.setId(String.valueOf(sourceColumn.getId()) + "_"
12501                                                                                                        + sourceStarColumnNames.indexOf(objectName));
12502                                                                                        source.setColumn(objectName);
12503                                                                                        find = true;
12504                                                                                }
12505                                                                        }
12506                                                                        
12507                                                                        if (!find && sourceItem instanceof ResultColumnRelationshipElement) {
12508                                                                                int starIndex = ((ResultColumnRelationshipElement) sourceItem)
12509                                                                                                .getStarIndex();
12510                                                                                if (starIndex > -1 && sourceColumn.getStarLinkColumnList().size() > starIndex) {
12511                                                                                        objectName = getColumnName(sourceColumn.getStarLinkColumnList().get(starIndex));
12512                                                                                        source.setId(String.valueOf(sourceColumn.getId()) + "_"
12513                                                                                                        + starIndex);
12514                                                                                        source.setColumn(objectName);
12515                                                                                        find = true;
12516                                                                                }
12517                                                                        }
12518
12519                                                                        if (!find) {
12520                                                                                source.setId(String.valueOf(sourceColumn.getId()));
12521                                                                                source.setColumn(sourceColumn.getName());
12522                                                                        }
12523
12524                                                                        source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
12525                                                                        source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
12526                                                                        if (sourceColumn.getStartPosition() != null
12527                                                                                        && sourceColumn.getEndPosition() != null) {
12528                                                                                source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
12529                                                                                                + convertCoordinate(sourceColumn.getEndPosition()));
12530                                                                        }
12531                                                                        append = true;
12532                                                                        relationElement.addSource(source);
12533                                                                }
12534                                                        }
12535
12536                                                } else {
12537                                                        sourceColumn source = new sourceColumn();
12538                                                        source.setId(String.valueOf(sourceColumn.getId()));
12539                                                        source.setColumn(sourceColumn.getName());
12540                                                        source.setStruct(sourceColumn.isStruct());
12541                                                        source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
12542                                                        source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
12543                                                        if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
12544                                                                source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
12545                                                                                + convertCoordinate(sourceColumn.getEndPosition()));
12546                                                        }
12547                                                        append = true;
12548                                                        if (sourceItem.getTransforms() != null) {
12549                                                                for (Transform transform : sourceItem.getTransforms()) {
12550                                                                        source.addTransform(transform);
12551                                                                }
12552                                                        }
12553                                                        if(sourceColumn.isPseduo()) {
12554                                                                source.setSource("system");
12555                                                        }
12556                                                        relationElement.addSource(source);
12557                                                }
12558                                        } else if (sourceElement instanceof TableColumn) {
12559                                                TableColumn sourceColumn = (TableColumn) sourceElement;
12560                                                if (!sourceColumn.isPseduo() && sourceColumn.hasStarLinkColumn()
12561                                                                && sourceItem instanceof TableColumnRelationshipElement) {
12562                                                        sourceColumn source = new sourceColumn();
12563                                                        boolean find = false;
12564                                                        if (((TableColumnRelationshipElement) sourceItem).getColumnIndex() != null) {
12565                                                                int columnIndex = ((TableColumnRelationshipElement) sourceItem).getColumnIndex();
12566                                                                if (sourceColumn.getStarLinkColumns().size() > columnIndex) {
12567                                                                        source = new sourceColumn();
12568                                                                        source.setId(String.valueOf(sourceColumn.getId()) + "_" + columnIndex);
12569                                                                        String targetObjectName = getColumnName(
12570                                                                                        sourceColumn.getStarLinkColumnList().get(columnIndex));
12571                                                                        source.setColumn(targetObjectName);
12572                                                                        source.setParent_id(String.valueOf(sourceColumn.getTable().getId()));
12573                                                                        source.setParent_name(sourceColumn.getTable().getName());
12574                                                                        if (sourceItem instanceof TableColumnRelationshipElement) {
12575                                                                                source.setParent_alias(
12576                                                                                                ((TableColumnRelationshipElement) sourceItem).getTableAlias());
12577                                                                        }
12578                                                                        if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
12579                                                                                source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
12580                                                                                                + convertCoordinate(sourceColumn.getEndPosition()));
12581                                                                        }
12582                                                                        append = true;
12583                                                                        relationElement.addSource(source);
12584                                                                        find = true;
12585                                                                }
12586                                                        }
12587
12588                                                        if (!find) {
12589                                                                String objectName = getColumnName(targetName);
12590
12591                                                                table tableElement = null;
12592                                                                if (dataflow.getTables() != null) {
12593                                                                        for (table t : dataflow.getTables()) {
12594                                                                                if (t.getId().equals(String.valueOf(sourceColumn.getTable().getId()))) {
12595                                                                                        tableElement = t;
12596                                                                                        break;
12597                                                                                }
12598                                                                        }
12599                                                                }
12600                                                                if (tableElement == null && dataflow.getViews() != null) {
12601                                                                        for (table t : dataflow.getViews()) {
12602                                                                                if (t.getId().equals(String.valueOf(sourceColumn.getTable().getId()))) {
12603                                                                                        tableElement = t;
12604                                                                                        break;
12605                                                                                }
12606                                                                        }
12607                                                                }
12608                                                                if (tableElement == null && dataflow.getVariables() != null) {
12609                                                                        for (table t : dataflow.getVariables()) {
12610                                                                                if (t.getId().equals(String.valueOf(sourceColumn.getTable().getId()))) {
12611                                                                                        tableElement = t;
12612                                                                                        break;
12613                                                                                }
12614                                                                        }
12615                                                                }
12616
12617                                                                if(tableElement!=null) {
12618                                                                        for (column column : tableElement.getColumns()) {
12619                                                                                if (column.getName() != null && column.getName().equalsIgnoreCase(objectName)) {
12620                                                                                        source = new sourceColumn();
12621                                                                                        source.setId(String.valueOf(column.getId()));
12622                                                                                        source.setColumn(objectName);
12623                                                                                        source.setParent_id(String.valueOf(sourceColumn.getTable().getId()));
12624                                                                                        source.setParent_name(sourceColumn.getTable().getName());
12625                                                                                        if (sourceItem instanceof TableColumnRelationshipElement) {
12626                                                                                                source.setParent_alias(
12627                                                                                                                ((TableColumnRelationshipElement) sourceItem).getTableAlias());
12628                                                                                        }
12629                                                                                        if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
12630                                                                                                source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
12631                                                                                                                + convertCoordinate(sourceColumn.getEndPosition()));
12632                                                                                        }
12633                                                                                        if (sourceItem.getTransforms() != null) {
12634                                                                                                for (Transform transform : sourceItem.getTransforms()) {
12635                                                                                                        source.addTransform(transform);
12636                                                                                                }
12637                                                                                        }
12638                                                                                        if (sourceColumn.getCandidateParents() != null) {
12639                                                                                                for(Object item: sourceColumn.getCandidateParents()) {
12640                                                                                                        candidateTable candidateParent = new candidateTable();
12641                                                                                                        if(item instanceof Table) {
12642                                                                                                                candidateParent.setId(String.valueOf(((Table)item).getId()));
12643                                                                                                                candidateParent.setName(getTableName((Table)item));
12644                                                                                                                source.addCandidateParent(candidateParent);
12645                                                                                                        }
12646                                                                                                        else if(item instanceof ResultSet) {
12647                                                                                                                candidateParent.setId(String.valueOf(((ResultSet)item).getId()));
12648                                                                                                                candidateParent.setName(getResultSetName((ResultSet)item));
12649                                                                                                                source.addCandidateParent(candidateParent);
12650                                                                                                        }
12651                                                                                                }
12652                                                                                        }
12653                                                                                        append = true;
12654                                                                                        relationElement.addSource(source);
12655                                                                                        find = true;
12656                                                                                        break;
12657                                                                                }
12658                                                                        }
12659                                                                }
12660                                                        }
12661
12662                                                        if(!find) {
12663                                                                source = new sourceColumn();
12664                                                                source.setId(String.valueOf(sourceColumn.getId()));
12665                                                                source.setColumn(sourceColumn.getName());
12666                                                                source.setParent_id(String.valueOf(sourceColumn.getTable().getId()));
12667                                                                source.setParent_name(sourceColumn.getTable().getName());
12668                                                                if (sourceItem instanceof TableColumnRelationshipElement) {
12669                                                                        source.setParent_alias(
12670                                                                                        ((TableColumnRelationshipElement) sourceItem).getTableAlias());
12671                                                                }
12672                                                                if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
12673                                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
12674                                                                                        + convertCoordinate(sourceColumn.getEndPosition()));
12675                                                                }
12676                                                                if (sourceItem.getTransforms() != null) {
12677                                                                        for (Transform transform : sourceItem.getTransforms()) {
12678                                                                                source.addTransform(transform);
12679                                                                        }
12680                                                                }
12681                                                                if (sourceColumn.getCandidateParents() != null) {
12682                                                                        for(Object item: sourceColumn.getCandidateParents()) {
12683                                                                                candidateTable candidateParent = new candidateTable();
12684                                                                                if(item instanceof Table) {
12685                                                                                        candidateParent.setId(String.valueOf(((Table)item).getId()));
12686                                                                                        candidateParent.setName(getTableName((Table)item));
12687                                                                                        source.addCandidateParent(candidateParent);
12688                                                                                }
12689                                                                                else if(item instanceof ResultSet) {
12690                                                                                        candidateParent.setId(String.valueOf(((ResultSet)item).getId()));
12691                                                                                        candidateParent.setName(getResultSetName((ResultSet)item));
12692                                                                                        source.addCandidateParent(candidateParent);
12693                                                                                }
12694                                                                        }
12695                                                                }
12696                                                                append = true;
12697                                                                relationElement.addSource(source);
12698                                                        }
12699
12700                                                } else {
12701                                                        sourceColumn source = new sourceColumn();
12702                                                        source.setId(String.valueOf(sourceColumn.getId()));
12703                                                        source.setColumn(sourceColumn.getName());
12704                                                        source.setStruct(sourceColumn.isStruct());
12705                                                        source.setParent_id(String.valueOf(sourceColumn.getTable().getId()));
12706                                                        source.setParent_name(getTableName(sourceColumn.getTable()));
12707                                                        if (sourceItem instanceof TableColumnRelationshipElement) {
12708                                                                source.setParent_alias(
12709                                                                                ((TableColumnRelationshipElement) sourceItem).getTableAlias());
12710                                                        }
12711                                                        if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
12712                                                                source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
12713                                                                                + convertCoordinate(sourceColumn.getEndPosition()));
12714                                                        }
12715                                                        if (sourceItem.getTransforms() != null) {
12716                                                                for (Transform transform : sourceItem.getTransforms()) {
12717                                                                        source.addTransform(transform);
12718                                                                }
12719                                                        }
12720                                                        if(sourceColumn.isPseduo()) {
12721                                                                source.setSource("system");
12722                                                        }
12723                                                        if (sourceColumn.getCandidateParents() != null) {
12724                                                                for(Object item: sourceColumn.getCandidateParents()) {
12725                                                                        candidateTable candidateParent = new candidateTable();
12726                                                                        if(item instanceof Table) {
12727                                                                                candidateParent.setId(String.valueOf(((Table)item).getId()));
12728                                                                                candidateParent.setName(getTableName((Table)item));
12729                                                                                source.addCandidateParent(candidateParent);
12730                                                                        }
12731                                                                        else if(item instanceof ResultSet) {
12732                                                                                candidateParent.setId(String.valueOf(((ResultSet)item).getId()));
12733                                                                                candidateParent.setName(getResultSetName((ResultSet)item));
12734                                                                                source.addCandidateParent(candidateParent);
12735                                                                        }
12736                                                                }
12737                                                        }
12738                                                        append = true;
12739                                                        relationElement.addSource(source);
12740                                                }
12741                                        } else if (sourceElement instanceof Argument) {
12742                                                Argument sourceColumn = (Argument) sourceElement;
12743                                                sourceColumn source = new sourceColumn();
12744                                                source.setId(String.valueOf(sourceColumn.getId()));
12745                                                source.setColumn(sourceColumn.getName());
12746                                                source.setParent_id(String.valueOf(sourceColumn.getProcedure().getId()));
12747                                                source.setParent_name(sourceColumn.getProcedure().getName());
12748                                                if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
12749                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
12750                                                                        + convertCoordinate(sourceColumn.getEndPosition()));
12751                                                }
12752                                                append = true;
12753                                                relationElement.addSource(source);
12754                                        } else if (sourceElement instanceof TableRelationRows) {
12755                                                TableRelationRows sourceColumn = (TableRelationRows) sourceElement;
12756                                                sourceColumn source = new sourceColumn();
12757                                                source.setId(String.valueOf(sourceColumn.getId()));
12758                                                source.setColumn(sourceColumn.getName());
12759                                                source.setParent_id(String.valueOf(sourceColumn.getHolder().getId()));
12760                                                source.setParent_name(getTableName(sourceColumn.getHolder()));
12761                                                if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
12762                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
12763                                                                        + convertCoordinate(sourceColumn.getEndPosition()));
12764                                                }
12765                                                source.setSource("system");
12766                                                append = true;
12767                                                relationElement.addSource(source);
12768                                        } else if (sourceElement instanceof ResultSetRelationRows) {
12769                                                ResultSetRelationRows sourceColumn = (ResultSetRelationRows) sourceElement;
12770                                                sourceColumn source = new sourceColumn();
12771                                                source.setId(String.valueOf(sourceColumn.getId()));
12772                                                source.setColumn(sourceColumn.getName());
12773                                                source.setParent_id(String.valueOf(sourceColumn.getHolder().getId()));
12774                                                source.setParent_name(getResultSetName(sourceColumn.getHolder()));
12775                                                if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
12776                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
12777                                                                        + convertCoordinate(sourceColumn.getEndPosition()));
12778                                                }
12779                                                source.setSource("system");
12780                                                append = true;
12781                                                relationElement.addSource(source);
12782                                        } else if (sourceElement instanceof Constant) {
12783                                                Constant sourceColumn = (Constant) sourceElement;
12784                                                sourceColumn source = new sourceColumn();
12785                                                source.setId(String.valueOf(sourceColumn.getId()));
12786                                                source.setColumn(sourceColumn.getName());
12787                                                source.setColumn_type("constant");
12788                                                if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
12789                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
12790                                                                        + convertCoordinate(sourceColumn.getEndPosition()));
12791                                                }
12792                                                append = true;
12793                                                relationElement.addSource(source);
12794                                        } else if (sourceElement instanceof Table) {
12795                                                Table table = (Table) sourceElement;
12796                                                sourceColumn source = new sourceColumn();
12797                                                source.setSource_id(String.valueOf(table.getId()));
12798                                                source.setSource_name(getTableName(table));
12799                                                if (table.getStartPosition() != null && table.getEndPosition() != null) {
12800                                                        source.setCoordinate(convertCoordinate(table.getStartPosition()) + ","
12801                                                                        + convertCoordinate(table.getEndPosition()));
12802                                                }
12803                                                append = true;
12804                                                relationElement.addSource(source);
12805                                        }
12806
12807                                        if (relation instanceof ImpactRelationship) {
12808                                                ESqlClause clause = getSqlClause(sourceItem);
12809                                                if (clause != null
12810                                                                && (relationElement.getSources() != null && !relationElement.getSources().isEmpty())) {
12811                                                        relationElement.getSources().get(relationElement.getSources().size() - 1)
12812                                                                        .setClauseType(clause.name());
12813                                                }
12814                                        }
12815                                }
12816                                if (append)
12817                                        dataflow.getRelationships().add(relationElement);
12818                        }
12819                }
12820        }
12821
12822        private boolean containsStar(Collection<RelationshipElement<?>> elements) {
12823                if (elements == null || elements.size() == 0)
12824                        return false;
12825
12826                for (RelationshipElement<?> element : elements) {
12827                        if (element.getElement() instanceof ResultColumn) {
12828                                ResultColumn object = (ResultColumn) element.getElement();
12829                                if (object.getName().endsWith("*") && object.isShowStar())
12830                                        return true;
12831                        } else if (element.getElement() instanceof TableColumn) {
12832                                TableColumn object = (TableColumn) element.getElement();
12833                                if (object.getName().endsWith("*") && object.isShowStar())
12834                                        return true;
12835                        }
12836                }
12837                return false;
12838        }
12839
12840        private Map<String, Set<String>> appendTableStarColumns = new HashMap<String, Set<String>>();
12841
12842        // Tracks column names that have explicit (non-star) sources per target star column.
12843        // Used across relationships to prevent phantom star expansions.
12844        // Key: target star column ID, Value: set of column names with explicit sources.
12845        private Map<Long, Set<String>> explicitStarTargetColumns = new HashMap<Long, Set<String>>();
12846        private void updateResultColumnStarLinks(dataflow dataflow, AbstractRelationship relation, int index) {
12847        try {
12848            if (option.getAnalyzeMode() == AnalyzeMode.crud) {
12849                return;
12850            }
12851
12852            ResultColumn targetColumn = (ResultColumn) relation.getTarget().getElement();
12853
12854            Collection<RelationshipElement<?>> sourceElements = (Collection<RelationshipElement<?>>) relation.getSources();
12855            if (sourceElements == null || sourceElements.size() == 0)
12856                return;
12857
12858            for (RelationshipElement<?> sourceItem : sourceElements) {
12859                Object sourceElement = sourceItem.getElement();
12860                if (sourceElement instanceof ResultColumn) {
12861                    ResultColumn source = (ResultColumn) sourceElement;
12862                    if (source.hasStarLinkColumn()) {
12863                        for (Map.Entry<String, Set<TObjectName>> item : source.getStarLinkColumns().entrySet()) {
12864                            if (!targetColumn.getStarLinkColumns().containsKey(item.getKey())) {
12865                                targetColumn.getStarLinkColumns().put(item.getKey(), new LinkedHashSet<TObjectName>());
12866                            }
12867                            targetColumn.getStarLinkColumns().get(item.getKey()).addAll(item.getValue());
12868                        }
12869                        if (!source.isShowStar()) {
12870                            targetColumn.setShowStar(false);
12871                            relation.setShowStarRelation(false);
12872                        }
12873                    } else if (!"*".equals(source.getName())) {
12874                        if (source.getColumnObject() instanceof TObjectName) {
12875                            if (source instanceof FunctionResultColumn) {
12876
12877                            } else {
12878                                targetColumn.bindStarLinkColumn((TObjectName) source.getColumnObject());
12879                            }
12880                        } else if (source.getColumnObject() instanceof TResultColumn) {
12881                            TResultColumn sourceColumn = (TResultColumn) source.getColumnObject();
12882                            if (sourceColumn.getAliasClause() != null) {
12883                                targetColumn.bindStarLinkColumn(sourceColumn.getAliasClause().getAliasName());
12884                            } else if (sourceColumn.getFieldAttr() != null) {
12885                                targetColumn.bindStarLinkColumn(sourceColumn.getFieldAttr());
12886                            } else {
12887                                TObjectName column = new TObjectName();
12888                                if (sourceColumn.getExpr().getExpressionType() == EExpressionType.typecast_t) {
12889                                    column.setString(sourceColumn.getExpr().getLeftOperand().toString());
12890                                } else {
12891                                    column.setString(sourceColumn.toString());
12892                                }
12893                                targetColumn.bindStarLinkColumn(column);
12894                            }
12895                        }
12896                    }
12897                } else if (sourceElement instanceof TableColumn && !targetColumn.isStruct()) {
12898                    TableColumn source = (TableColumn) sourceElement;
12899                    if (!source.isPseduo() && source.hasStarLinkColumn()) {
12900                                                for (Map.Entry<String, Set<TObjectName>> item : source.getStarLinkColumns().entrySet()) {
12901                                                        if (!targetColumn.getStarLinkColumns().containsKey(item.getKey())) {
12902                                                                targetColumn.getStarLinkColumns().put(item.getKey(),
12903                                                                                new LinkedHashSet<TObjectName>());
12904                                                        }
12905                                                        targetColumn.getStarLinkColumns().get(item.getKey()).addAll(item.getValue());
12906                                                }
12907                                                if ((source.getTable().isCreateTable() || source.getTable().hasSQLEnv())
12908                                                                && !source.isShowStar()) {
12909                                                        targetColumn.setShowStar(false);
12910                                                        relation.setShowStarRelation(false);
12911                                                }
12912                    } else if (!"*".equals(source.getName())) {
12913                        targetColumn.bindStarLinkColumn(source.getColumnObject());
12914                    }
12915                }
12916            }
12917
12918            if (targetColumn.hasStarLinkColumn()) {
12919                table resultSetElement = null;
12920                for (table t : dataflow.getResultsets()) {
12921                    if (t.getId().equals(String.valueOf(targetColumn.getResultSet().getId()))) {
12922                        resultSetElement = t;
12923                        break;
12924                    }
12925                }
12926
12927                int starColumnCount = 0;
12928                for (column item : resultSetElement.getColumns()) {
12929                    if (item.getName() != null && item.getName().endsWith("*")) {
12930                        starColumnCount += 1;
12931                    }
12932                }
12933
12934                if (resultSetElement != null && starColumnCount <= 1) {
12935                    List<String> columns = targetColumn.getStarLinkColumnNames();
12936                    if (index == -1) {
12937                        for (int k = 0; k < columns.size(); k++) {
12938                            String columnName = columns.get(k);
12939                            String id = String.valueOf(targetColumn.getId()) + "_" + k;
12940                            if (appendTableStarColumns.containsKey(resultSetElement.getId()) && appendTableStarColumns.get(resultSetElement.getId()).contains(id)) {
12941                                continue;
12942                            }
12943                            column columnElement = new column();
12944                            columnElement.setId(id);
12945                            columnElement.setName(columnName);
12946                            if (targetColumn.isFunction()) {
12947                                columnElement.setIsFunction(String.valueOf(targetColumn.isFunction()));
12948                            }
12949                            if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
12950                                columnElement.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
12951                                        + convertCoordinate(targetColumn.getEndPosition()));
12952                            }
12953
12954                            if (targetColumn.getResultSet() != null && targetColumn.getResultSet().getColumns() != null) {
12955                                boolean find = false;
12956                                for (int i = 0; i < targetColumn.getResultSet().getColumns().size(); i++) {
12957                                    ResultColumn columnModel = targetColumn.getResultSet().getColumns().get(i);
12958                                    if (DlineageUtil.getIdentifierNormalColumnName(columnModel.getName())
12959                                            .equals(columnName)) {
12960                                        find = true;
12961                                        break;
12962                                    }
12963                                }
12964                                if (find) {
12965                                    continue;
12966                                }
12967                            }
12968
12969                            if (!resultSetElement.getColumns().contains(columnElement)) {
12970                                resultSetElement.getColumns().add(columnElement);
12971                                appendTableStarColumns.putIfAbsent(resultSetElement.getId(), new HashSet<String>());
12972                                appendTableStarColumns.get(resultSetElement.getId()).add(id);
12973                            }
12974                        }
12975                    } else {
12976                        int k = index;
12977                        String columnName = columns.get(k);
12978                        column columnElement = new column();
12979                        columnElement.setId(String.valueOf(targetColumn.getId()) + "_" + k);
12980                        columnElement.setName(columnName);
12981                        if (targetColumn.isFunction()) {
12982                            columnElement.setIsFunction(String.valueOf(targetColumn.isFunction()));
12983                        }
12984                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
12985                            columnElement.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
12986                                    + convertCoordinate(targetColumn.getEndPosition()));
12987                        }
12988
12989                        if (targetColumn.getResultSet() != null && targetColumn.getResultSet().getColumns() != null) {
12990                            boolean find = false;
12991                            for (int i = 0; i < targetColumn.getResultSet().getColumns().size(); i++) {
12992                                ResultColumn columnModel = targetColumn.getResultSet().getColumns().get(i);
12993                                if (DlineageUtil.getIdentifierNormalColumnName(columnModel.getName()).equals(columnName)) {
12994                                    find = true;
12995                                    break;
12996                                }
12997                            }
12998                            if (find) {
12999                                return;
13000                            }
13001                        }
13002
13003                        if (!resultSetElement.getColumns().contains(columnElement)) {
13004                            resultSetElement.getColumns().add(columnElement);
13005                        }
13006                    }
13007                }
13008            }
13009        } catch (Exception e) {
13010            logger.error("updateResultColumnStarLinks occurs unknown exceptions.", e);
13011        }
13012        }
13013
13014        private void updateTableColumnStarLinks(dataflow dataflow, AbstractRelationship relation) {
13015                TableColumn targetColumn = (TableColumn) relation.getTarget().getElement();             
13016                Collection<RelationshipElement<?>> sourceElements = (Collection<RelationshipElement<?>>) relation.getSources();
13017                if (sourceElements == null || sourceElements.size() == 0)
13018                        return;
13019
13020                TableColumn sourceStarColumn = null;
13021
13022                for (RelationshipElement<?> sourceItem: sourceElements) {
13023                        Object sourceElement = sourceItem.getElement();
13024                        if (sourceElement instanceof ResultColumn) {
13025                                ResultColumn source = (ResultColumn) sourceElement;
13026                                if(source.getResultSet() instanceof Function) {
13027                                        continue;
13028                                }
13029                                if (source.hasStarLinkColumn()) {
13030                                        for (Map.Entry<String, Set<TObjectName>> item : source.getStarLinkColumns().entrySet()) {
13031                                                if (!targetColumn.getStarLinkColumns().containsKey(item.getKey())) {
13032                                                        targetColumn.getStarLinkColumns().put(item.getKey(), new LinkedHashSet<TObjectName>());
13033                                                }
13034                                                targetColumn.getStarLinkColumns().get(item.getKey()).addAll(item.getValue());
13035                                        }
13036                                        if (!source.isShowStar()) {
13037                                                targetColumn.setShowStar(false);
13038                                                relation.setShowStarRelation(false);
13039                                        }
13040                                } else if (!"*".equals(source.getName())) {
13041                                        if (source.getColumnObject() instanceof TObjectName) {
13042                                                targetColumn.bindStarLinkColumn((TObjectName) source.getColumnObject());
13043                                        } else if (source.getColumnObject() instanceof TResultColumn) {
13044                                                if (((TResultColumn) source.getColumnObject()).getAliasClause() != null) {
13045                                                        TObjectName field = ((TResultColumn) source.getColumnObject()).getAliasClause()
13046                                                                        .getAliasName();
13047                                                        if (field != null) {
13048                                                                targetColumn.bindStarLinkColumn(field);
13049                                                        }
13050                                                } else {
13051                                                        TObjectName field = ((TResultColumn) source.getColumnObject()).getFieldAttr();
13052                                                        if (field != null) {
13053                                                                targetColumn.bindStarLinkColumn(field);
13054                                                        } else {
13055                                                                TObjectName column = new TObjectName();
13056                                                                if (((TResultColumn) source.getColumnObject()).getExpr()
13057                                                                                .getExpressionType() == EExpressionType.typecast_t) {
13058                                                                        column.setString(((TResultColumn) source.getColumnObject()).getExpr()
13059                                                                                        .getLeftOperand().toString());
13060                                                                } else {
13061                                                                        column.setString(((TResultColumn) source.getColumnObject()).toString());
13062                                                                }
13063                                                                targetColumn.bindStarLinkColumn(column);
13064                                                        }
13065                                                }
13066                                        }
13067                                }
13068                        } else if (sourceElement instanceof TableColumn) {
13069                                TableColumn source = (TableColumn) sourceElement;
13070                                if (source.hasStarLinkColumn()) {
13071                                        for (Map.Entry<String, Set<TObjectName>> item : source.getStarLinkColumns().entrySet()) {
13072                                                if (!targetColumn.getStarLinkColumns().containsKey(item.getKey())) {
13073                                                        targetColumn.getStarLinkColumns().put(item.getKey(), new LinkedHashSet<TObjectName>());
13074                                                }
13075                                                targetColumn.getStarLinkColumns().get(item.getKey()).addAll(item.getValue());
13076                                        }
13077                                        for (Map.Entry<String, Set<TObjectName>> item : targetColumn.getStarLinkColumns().entrySet()) {
13078                                                if (!source.getStarLinkColumns().containsKey(item.getKey())) {
13079                                                        source.getStarLinkColumns().put(item.getKey(), new LinkedHashSet<TObjectName>());
13080                                                }
13081                                                source.getStarLinkColumns().get(item.getKey()).addAll(item.getValue());
13082                                        }
13083
13084                                        sourceStarColumn = source;
13085
13086                                        if (source.getTable().isCreateTable() && !source.isShowStar()) {
13087                                                targetColumn.setShowStar(false);
13088                                                relation.setShowStarRelation(false);
13089                                        }
13090                                } else if (!"*".equals(source.getName())) {
13091                                        if (source.isStruct()) {
13092                                                targetColumn.bindStarLinkColumn(source.getColumnObject());
13093                                        }
13094                                        else {
13095                                                TObjectName objectName = new TObjectName();
13096                                                objectName.setString(DlineageUtil.getColumnNameOnly(source.getName()));
13097                                                targetColumn.bindStarLinkColumn(objectName);
13098                                        }
13099                                }
13100                        }
13101                }
13102
13103                if (targetColumn.hasStarLinkColumn()) {
13104                        table tableElement = null;
13105                        if (dataflow.getTables() != null) {
13106                                for (table t : dataflow.getTables()) {
13107                                        if (t.getId().equals(String.valueOf(targetColumn.getTable().getId()))) {
13108                                                tableElement = t;
13109                                                break;
13110                                        }
13111                                }
13112                        }
13113                        if (tableElement == null && dataflow.getViews() != null) {
13114                                for (table t : dataflow.getViews()) {
13115                                        if (t.getId().equals(String.valueOf(targetColumn.getTable().getId()))) {
13116                                                tableElement = t;
13117                                                break;
13118                                        }
13119                                }
13120                        }
13121                        if (tableElement == null && dataflow.getVariables() != null) {
13122                                for (table t : dataflow.getVariables()) {
13123                                        if (t.getId().equals(String.valueOf(targetColumn.getTable().getId()))) {
13124                                                tableElement = t;
13125                                                break;
13126                                        }
13127                                }
13128                        }
13129
13130                        if (tableElement != null) {
13131                                List<String> columns = new ArrayList<String>(targetColumn.getStarLinkColumns().keySet());
13132                                for (int k = 0; k < columns.size(); k++) {
13133                                        String columnName = columns.get(k);
13134                                        if (containStarColumn(targetColumn.getTable().getColumns(), columnName)) {
13135                                                continue;
13136                                        }
13137                                        column columnElement = new column();
13138                                        columnElement.setId(String.valueOf(targetColumn.getId()) + "_" + k);
13139                                        columnElement.setName(columnName);
13140                                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
13141                                                columnElement.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
13142                                                                + convertCoordinate(targetColumn.getEndPosition()));
13143                                        }
13144                                        if (!tableElement.getColumns().contains(columnElement)) {
13145                                                tableElement.getColumns().add(columnElement);
13146                                        }
13147                                }
13148                        }
13149                }
13150
13151                if (sourceStarColumn != null) {
13152                        table tableElement = null;
13153                        if (dataflow.getTables() != null) {
13154                                for (table t : dataflow.getTables()) {
13155                                        if (t.getId().equals(String.valueOf(sourceStarColumn.getTable().getId()))) {
13156                                                tableElement = t;
13157                                                break;
13158                                        }
13159                                }
13160                        }
13161                        if (tableElement == null && dataflow.getViews() != null) {
13162                                for (table t : dataflow.getViews()) {
13163                                        if (t.getId().equals(String.valueOf(sourceStarColumn.getTable().getId()))) {
13164                                                tableElement = t;
13165                                                break;
13166                                        }
13167                                }
13168                        }
13169                        if (tableElement == null && dataflow.getVariables() != null) {
13170                                for (table t : dataflow.getVariables()) {
13171                                        if (t.getId().equals(String.valueOf(sourceStarColumn.getTable().getId()))) {
13172                                                tableElement = t;
13173                                                break;
13174                                        }
13175                                }
13176                        }
13177
13178                        if (tableElement != null) {
13179                                List<String> columns = new ArrayList<String>(sourceStarColumn.getStarLinkColumns().keySet());
13180                                for (int k = 0; k < columns.size(); k++) {
13181                                        String columnName = columns.get(k);
13182                                        if (containStarColumn(sourceStarColumn.getTable().getColumns(), columnName)) {
13183                                                continue;
13184                                        }
13185                                        column columnElement = new column();
13186                                        columnElement.setId(sourceStarColumn.getId() + "_" + k);
13187                                        columnElement.setName(columnName);
13188                                        if (sourceStarColumn.getStartPosition() != null && sourceStarColumn.getEndPosition() != null) {
13189                                                columnElement.setCoordinate(convertCoordinate(sourceStarColumn.getStartPosition()) + ","
13190                                                                + convertCoordinate(sourceStarColumn.getEndPosition()));
13191                                        }
13192                                        if (!tableElement.getColumns().contains(columnElement)) {
13193                                                tableElement.getColumns().add(columnElement);
13194                                        }
13195                                }
13196                        }
13197                }
13198        }
13199
13200        private ESqlClause getSqlClause(RelationshipElement<?> relationshipElement) {
13201                if (relationshipElement instanceof TableColumnRelationshipElement) {
13202                        return ((TableColumnRelationshipElement) relationshipElement).getRelationLocation();
13203                } else if (relationshipElement instanceof ResultColumnRelationshipElement) {
13204                        return ((ResultColumnRelationshipElement) relationshipElement).getRelationLocation();
13205                }
13206                return null;
13207        }
13208
13209        private void appendStarRelation(dataflow dataflow, AbstractRelationship relation, int index) {
13210                if(option.getAnalyzeMode() == AnalyzeMode.crud) {
13211                        return;
13212                }
13213                
13214                Object targetElement = relation.getTarget().getElement();
13215
13216                relationship relationElement = new relationship();
13217                relationElement.setType(relation.getRelationshipType().name());
13218                if (relation.getEffectType() != null) {
13219                        relationElement.setEffectType(relation.getEffectType().name());
13220                }
13221                relationElement.setSqlHash(relation.getSqlHash());
13222                relationElement.setSqlComment(relation.getSqlComment());
13223
13224                if (relation.getProcedureId() != null) {
13225                        relationElement.setProcedureId(String.valueOf(relation.getProcedureId()));
13226                }
13227                relationElement.setId(String.valueOf(relation.getId()) + "_" + index);
13228                if (relation.getProcess() != null) {
13229                        relationElement.setProcessId(String.valueOf(relation.getProcess().getId()));
13230                        if (relation.getProcess().getGspObject() != null) {
13231                                relationElement.setProcessType(relation.getProcess().getGspObject().sqlstatementtype.name());
13232                        }
13233                }
13234                if (relation instanceof DataFlowRelationship) {
13235                        relationElement.setSqlHash(((DataFlowRelationship) relation).getSqlHash());
13236                        relationElement.setSqlComment(((DataFlowRelationship) relation).getSqlComment());
13237                
13238                        if (relation.getProcedureId() != null) {
13239                                relationElement.setProcedureId(String.valueOf(relation.getProcedureId()));
13240                        }
13241                }
13242                String targetName = "";
13243
13244                if (targetElement instanceof ResultColumn) {
13245                        ResultColumn targetColumn = (ResultColumn) targetElement;
13246
13247                        targetName = targetColumn.getStarLinkColumnNames().get(index);
13248
13249                        
13250                        targetColumn target = new targetColumn();
13251                        target.setId(String.valueOf(targetColumn.getId()) + "_" + index);
13252                        if(targetColumn.getResultSet()!=null && targetColumn.getResultSet().getColumns()!=null) {
13253                                for (int i = 0; i < targetColumn.getResultSet().getColumns().size(); i++) {
13254                                        ResultColumn columnModel = targetColumn.getResultSet().getColumns().get(i);
13255                                        if (DlineageUtil.getIdentifierNormalColumnName(columnModel.getName()).equals(targetName)) {
13256                                                target.setId(String.valueOf(columnModel.getId()));
13257                                                break;
13258                                        }
13259                                }
13260                        }
13261                        target.setColumn(targetName);
13262                        target.setStruct(targetColumn.isStruct());
13263                        target.setParent_id(String.valueOf(targetColumn.getResultSet().getId()));
13264                        target.setParent_name(getResultSetName(targetColumn.getResultSet()));
13265                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
13266                                target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
13267                                                + convertCoordinate(targetColumn.getEndPosition()));
13268                        }
13269                        relationElement.setTarget(target);
13270                } else if (targetElement instanceof TableColumn) {
13271                        TableColumn targetColumn = (TableColumn) targetElement;
13272
13273                        targetName = targetColumn.getStarLinkColumnNames().get(index);
13274
13275                        TableColumn tableColumn = searchTableColumn(targetColumn.getTable().getColumns(), targetName);
13276
13277                        targetColumn target = new targetColumn();
13278                        if (tableColumn == null) {
13279                                target.setId(targetColumn.getId() + "_" + index);
13280                        } else {
13281                                target.setId(String.valueOf(tableColumn.getId()));
13282                        }
13283                        target.setStruct(targetColumn.isStruct());
13284                        target.setColumn(targetName);
13285                        target.setParent_id(String.valueOf(targetColumn.getTable().getId()));
13286                        target.setParent_name(targetColumn.getTable().getName());
13287                        if (relation.getTarget() instanceof TableColumnRelationshipElement) {
13288                                target.setParent_alias(((TableColumnRelationshipElement) relation.getTarget()).getTableAlias());
13289                        }
13290                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
13291                                target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
13292                                                + convertCoordinate(targetColumn.getEndPosition()));
13293                        }
13294                        relationElement.setTarget(target);
13295                } else {
13296                        return;
13297                }
13298
13299                Collection<RelationshipElement<?>> sourceElements = (Collection<RelationshipElement<?>>) relation.getSources();
13300                if (sourceElements.size() == 0) {
13301                        return;
13302                }
13303
13304                String targetIdentifierName = DlineageUtil.getIdentifierNormalColumnName(targetName);
13305                if (targetIdentifierName == null) {
13306                        return;
13307                }
13308
13309                // Track explicit (non-star) source columns per target star column to prevent
13310                // phantom star expansions. Skip for UNION targets where branches contribute independently.
13311                long targetColumnId = -1;
13312                boolean isUnionTarget = false;
13313                if (targetElement instanceof ResultColumn) {
13314                        ResultColumn tc = (ResultColumn) targetElement;
13315                        targetColumnId = tc.getId();
13316                        isUnionTarget = (tc.getResultSet() instanceof SelectSetResultSet);
13317                } else if (targetElement instanceof TableColumn) {
13318                        targetColumnId = ((TableColumn) targetElement).getId();
13319                }
13320
13321                if (!isUnionTarget && targetColumnId != -1) {
13322                        // Check if any source in this relationship is a non-star source matching the target
13323                        // column name, AND whose parent does NOT also contribute a star source.
13324                        // This distinguishes:
13325                        //   - "b.col_a" (definitive: b only provides explicit cols, not *) => suppress star expansion
13326                        //   - "aTab.id" (not definitive: aTab also provides *) => don't suppress
13327                        boolean hasDefinitiveExplicitSource = hasDefinitiveNonStarSource(sourceElements, targetIdentifierName);
13328                        if (hasDefinitiveExplicitSource) {
13329                                if (!explicitStarTargetColumns.containsKey(targetColumnId)) {
13330                                        explicitStarTargetColumns.put(targetColumnId, new HashSet<String>());
13331                                }
13332                                explicitStarTargetColumns.get(targetColumnId).add(targetIdentifierName);
13333                        }
13334                }
13335
13336                // Column names that have explicit (non-star) sources for this target, across all relationships
13337                Set<String> targetExplicitNames = isUnionTarget ? null : explicitStarTargetColumns.get(targetColumnId);
13338
13339                for (RelationshipElement<?> sourceItem: sourceElements) {
13340                        Object sourceElement = sourceItem.getElement();
13341                        if (sourceElement instanceof ResultColumn) {
13342                                ResultColumn sourceColumn = (ResultColumn) sourceElement;
13343                                if (sourceColumn.hasStarLinkColumn()) {
13344                                        List<String> linkColumnNames = sourceColumn.getStarLinkColumnNames();
13345                                        int linkColumnNameSize = linkColumnNames.size();
13346                                        for (int k = 0; k < linkColumnNameSize; k++) {
13347                                                String sourceName = linkColumnNames.get(k);
13348                                                if (relation.getRelationshipType() == RelationshipType.fdd) {
13349                                                        if (!targetIdentifierName.equalsIgnoreCase(sourceName) && !"*".equals(sourceName))
13350                                                                continue;
13351                                                }
13352                                                // Skip star-expanded source when an explicit (non-star) source provides
13353                                                // the same column, either in this relationship or a prior one.
13354                                                if (targetExplicitNames != null && targetExplicitNames.contains(sourceName)) {
13355                                                        continue;
13356                                                }
13357                                                sourceColumn source = new sourceColumn();                                               
13358                                                
13359                                                boolean find = false;
13360                                                if(sourceColumn.getResultSet()!=null && sourceColumn.getResultSet().getColumns()!=null) {
13361                                                        for (int i = 0; i < sourceColumn.getResultSet().getColumns().size(); i++) {
13362                                                                ResultColumn columnModel = sourceColumn.getResultSet().getColumns().get(i);
13363                                                                if (DlineageUtil.getIdentifierNormalColumnName(columnModel.getName()).equals(sourceName)) {
13364                                                                        source.setId(String.valueOf(columnModel.getId()));
13365                                                                        find = true;
13366                                                                        break;
13367                                                                }
13368                                                        }
13369                                                }
13370                                                if(!find) {
13371                                                        source.setId(String.valueOf(sourceColumn.getId()) + "_" + k);
13372                                                }
13373                                                source.setColumn(sourceName);
13374                                                source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
13375                                                source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
13376                                                if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
13377                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
13378                                                                        + convertCoordinate(sourceColumn.getEndPosition()));
13379                                                }
13380                                                if (sourceItem.getTransforms() != null) {
13381                                                        for (Transform transform : sourceItem.getTransforms()) {
13382                                                                source.addTransform(transform);
13383                                                        }
13384                                                }
13385                                                relationElement.addSource(source);
13386                                        }
13387                                        if(relationElement.getSources().isEmpty()
13388                                                && !(targetExplicitNames != null && targetExplicitNames.contains(targetIdentifierName))
13389                                                && sourceColumn.getResultSet()!=null && sourceColumn.getResultSet().getColumns()!=null) {
13390                                                for (int i = 0; i < sourceColumn.getResultSet().getColumns().size(); i++) {
13391                                                        ResultColumn columnModel = sourceColumn.getResultSet().getColumns().get(i);
13392                                                        if (DlineageUtil.getIdentifierNormalColumnName(columnModel.getName()).equalsIgnoreCase(targetIdentifierName)) {
13393                                                                sourceColumn source = new sourceColumn();
13394                                                                source.setId(String.valueOf(columnModel.getId()));
13395                                                                source.setColumn(columnModel.getName());
13396                                                                source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
13397                                                                source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
13398                                                                if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
13399                                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
13400                                                                                        + convertCoordinate(sourceColumn.getEndPosition()));
13401                                                                }
13402                                                                if (sourceItem.getTransforms() != null) {
13403                                                                        for (Transform transform : sourceItem.getTransforms()) {
13404                                                                                source.addTransform(transform);
13405                                                                        }
13406                                                                }
13407                                                                relationElement.addSource(source);
13408                                                                break;
13409                                                        }
13410                                                }
13411                                        }
13412                                        if (relationElement.getSources().isEmpty()
13413                                                && !(targetExplicitNames != null && targetExplicitNames.contains(targetIdentifierName))) {
13414                                                TObjectName sourceStarLinkColumn = new TObjectName();
13415                                                sourceStarLinkColumn.setString(targetName);
13416                                                boolean newBinding = sourceColumn.bindStarLinkColumn(sourceStarLinkColumn);
13417                                                sourceColumn source = new sourceColumn();
13418                                                String sourceName = DlineageUtil.getColumnName(sourceStarLinkColumn);
13419                                                if (!newBinding) {
13420                                                        source.setId(String.valueOf(sourceColumn.getId()) + "_"
13421                                                                        + sourceColumn.indexOfStarLinkColumn(sourceStarLinkColumn));
13422                                                } else {
13423                                                        source.setId(String.valueOf(sourceColumn.getId()) + "_" + linkColumnNameSize);
13424                                                }
13425                                                source.setColumn(sourceName);
13426                                                source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
13427                                                source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
13428                                                if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
13429                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
13430                                                                        + convertCoordinate(sourceColumn.getEndPosition()));
13431                                                }
13432                                                if (sourceItem.getTransforms() != null) {
13433                                                        for (Transform transform : sourceItem.getTransforms()) {
13434                                                                source.addTransform(transform);
13435                                                        }
13436                                                }
13437                                                relationElement.addSource(source);
13438
13439                                                if (newBinding) {
13440                                                        table resultSetElement = null;
13441                                                        for (table t : dataflow.getResultsets()) {
13442                                                                if (t.getId().equals(String.valueOf(sourceColumn.getResultSet().getId()))) {
13443                                                                        resultSetElement = t;
13444                                                                        break;
13445                                                                }
13446                                                        }
13447                                                        if (resultSetElement != null) {
13448                                                                column columnElement = new column();
13449                                                                columnElement.setId(String.valueOf(sourceColumn.getId()) + "_" + linkColumnNameSize);
13450                                                                columnElement.setName(sourceName);
13451                                                                if (sourceColumn.isFunction()) {
13452                                                                        columnElement.setIsFunction(String.valueOf(sourceColumn.isFunction()));
13453                                                                }
13454                                                                if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
13455                                                                        columnElement.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
13456                                                                                        + convertCoordinate(sourceColumn.getEndPosition()));
13457                                                                }
13458                                                                resultSetElement.getColumns().add(columnElement);
13459                                                        }
13460                                                }
13461                                        }
13462                                } else {
13463                                        sourceColumn source = new sourceColumn();
13464                                        source.setId(String.valueOf(sourceColumn.getId()));
13465                                        source.setColumn(sourceColumn.getName());
13466                                        source.setStruct(sourceColumn.isStruct());
13467                                        source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
13468                                        source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
13469                                        if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
13470                                                source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
13471                                                                + convertCoordinate(sourceColumn.getEndPosition()));
13472                                        }
13473                                        if (relation.getRelationshipType() == RelationshipType.fdd) {
13474                                                if (!targetIdentifierName
13475                                                                .equalsIgnoreCase(DlineageUtil.getIdentifierNormalColumnName(sourceColumn.getName()))) {
13476                                                        if (!"*".equals(sourceColumn.getName())) {
13477                                                                continue;
13478                                                        }
13479                                                        else {
13480                                                                boolean flag = false;
13481                                                                for(ResultColumn column: sourceColumn.getResultSet().getColumns()) {
13482                                                                        if(targetIdentifierName
13483                                                                                        .equalsIgnoreCase(DlineageUtil.getIdentifierNormalColumnName(column.getName()))) {
13484                                                                                flag = true;
13485                                                                                break;
13486                                                                        }
13487                                                                }
13488                                                                if(flag) {
13489                                                                        continue;
13490                                                                }
13491                                                        }
13492                                                }
13493                                        }
13494                                        if (sourceItem.getTransforms() != null) {
13495                                                for (Transform transform : sourceItem.getTransforms()) {
13496                                                        source.addTransform(transform);
13497                                                }
13498                                        }
13499                                        relationElement.addSource(source);
13500                                }
13501                        } else if (sourceElement instanceof TableColumn) {
13502                                TableColumn sourceColumn = (TableColumn) sourceElement;
13503                                if (!sourceColumn.isPseduo() && sourceColumn.hasStarLinkColumn()) {
13504                                        List<String> linkColumnNames = sourceColumn.getStarLinkColumnNames();
13505                                        int linkColumnNameSize = linkColumnNames.size();
13506                                        for (int k = 0; k < linkColumnNameSize; k++) {
13507                                                String sourceName = linkColumnNames.get(k);
13508                                                if (relation.getRelationshipType() == RelationshipType.fdd) {
13509                                                        if (!targetIdentifierName.equalsIgnoreCase(sourceName) && !"*".equals(sourceName))
13510                                                                continue;
13511                                                }
13512
13513                                                TableColumn tableColumn = searchTableColumn(sourceColumn.getTable().getColumns(), sourceName);
13514
13515                                                sourceColumn source = new sourceColumn();
13516                                                if (tableColumn == null) {
13517                                                        source.setId(sourceColumn.getId() + "_" + k);
13518                                                } else {
13519                                                        source.setId(String.valueOf(tableColumn.getId()));
13520                                                }
13521                                                source.setColumn(sourceName);
13522                                                source.setStruct(sourceColumn.isStruct());
13523                                                if (containStarColumn(sourceElements, sourceName)) {
13524                                                        continue;
13525                                                }
13526                                                // Cross-relationship check: skip if explicit source was found in prior relationship
13527                                                if (targetExplicitNames != null && targetExplicitNames.contains(sourceName)) {
13528                                                        continue;
13529                                                }
13530                                                if (sourceColumn.getTable().getColumns().size() > 1) {
13531                                                        for (int y = 0; y < sourceColumn.getTable().getColumns().size(); y++) {
13532                                                                if (sourceColumn.getTable().getColumns().get(y).getName()
13533                                                                                .equalsIgnoreCase(sourceName)) {
13534                                                                        source.setId(String.valueOf(sourceColumn.getTable().getColumns().get(y).getId()));
13535                                                                        break;
13536                                                                }
13537                                                        }
13538                                                }
13539                                                source.setParent_id(String.valueOf(sourceColumn.getTable().getId()));
13540                                                source.setParent_name(getTableName(sourceColumn.getTable()));
13541                                                if (sourceItem instanceof TableColumnRelationshipElement) {
13542                                                        source.setParent_alias(
13543                                                                        ((TableColumnRelationshipElement) sourceItem).getTableAlias());
13544                                                }
13545                                                if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
13546                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
13547                                                                        + convertCoordinate(sourceColumn.getEndPosition()));
13548                                                }
13549                                                if (sourceItem.getTransforms() != null) {
13550                                                        for (Transform transform : sourceItem.getTransforms()) {
13551                                                                source.addTransform(transform);
13552                                                        }
13553                                                }
13554                                                relationElement.addSource(source);
13555                                        }
13556                                } else {
13557                                        sourceColumn source = new sourceColumn();
13558                                        source.setId(String.valueOf(sourceColumn.getId()));
13559                                        source.setColumn(sourceColumn.getName());
13560                                        source.setStruct(sourceColumn.isStruct());
13561                                        source.setParent_id(String.valueOf(sourceColumn.getTable().getId()));
13562                                        source.setParent_name(getTableName(sourceColumn.getTable()));
13563                                        if (sourceItem instanceof TableColumnRelationshipElement) {
13564                                                source.setParent_alias(((TableColumnRelationshipElement) sourceItem).getTableAlias());
13565                                        }
13566                                        if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
13567                                                source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
13568                                                                + convertCoordinate(sourceColumn.getEndPosition()));
13569                                        }
13570                                        if (relation.getRelationshipType() == RelationshipType.fdd) {
13571                                                if (!targetIdentifierName
13572                                                                .equalsIgnoreCase(DlineageUtil.getIdentifierNormalColumnName(sourceColumn.getName()))
13573                                                                && !"*".equals(sourceColumn.getName()))
13574                                                        continue;
13575                                        }
13576                                        if (sourceItem.getTransforms() != null) {
13577                                                for (Transform transform : sourceItem.getTransforms()) {
13578                                                        source.addTransform(transform);
13579                                                }
13580                                        }
13581                                        relationElement.addSource(source);
13582                                }
13583                        }
13584                }
13585
13586                if (relationElement.getTarget() != null && relationElement.getSources() != null
13587                                && !relationElement.getSources().isEmpty()) {
13588                        dataflow.getRelationships().add(relationElement);
13589                }
13590        }
13591
13592        private String getColumnName(TObjectName column) {
13593                if (column == null) {
13594                        return null;
13595                }
13596                String name = column.getColumnNameOnly();
13597                if (name == null || "".equals(name.trim())) {
13598                        return DlineageUtil.getIdentifierNormalColumnName(column.toString().trim());
13599                } else
13600                        return DlineageUtil.getIdentifierNormalColumnName(name.trim());
13601        }
13602
13603        /**
13604         * For BigQuery/Redshift struct field access, returns the full struct path
13605         * (e.g., "customer.name" from ColumnSource with exposedName="customer", fieldPath=["name"]).
13606         * Checks StructFieldHint first (for 3+ part no-alias), then ColumnSource (for 2-part/alias).
13607         * Returns null if this is not a struct field access.
13608         */
13609        private String getStructFieldFullName(TObjectName column) {
13610                if (column == null) return null;
13611                if (getOption().getVendor() != EDbVendor.dbvbigquery
13612                        && getOption().getVendor() != EDbVendor.dbvredshift) return null;
13613                // Priority 1: StructFieldHint (side-channel, for 3+ part no-alias deep struct access)
13614                gudusoft.gsqlparser.resolver2.model.StructFieldHint hint = column.getStructFieldHint();
13615                if (hint != null && hint.getFieldPath() != null && !hint.getFieldPath().isEmpty()) {
13616                        return hint.toFullReference();
13617                }
13618                // Priority 2: ColumnSource (main resolution, for 2-part and alias cases)
13619                gudusoft.gsqlparser.resolver2.model.ColumnSource source = column.getColumnSource();
13620                if (source != null && source.isStructFieldAccess() && source.hasFieldPath()) {
13621                        return source.getFieldPath().toFullReference(source.getExposedName());
13622                }
13623                return null;
13624        }
13625
13626        /**
13627         * Get the base column name for a struct field access column.
13628         * Checks StructFieldHint first (3+ part no-alias), then ColumnSource (2-part/alias).
13629         * Returns null if not a struct field access.
13630         */
13631        private String getStructFieldBaseName(TObjectName column) {
13632                if (column == null) return null;
13633                gudusoft.gsqlparser.resolver2.model.StructFieldHint hint = column.getStructFieldHint();
13634                if (hint != null && hint.getBaseColumn() != null) {
13635                        return hint.getBaseColumn();
13636                }
13637                gudusoft.gsqlparser.resolver2.model.ColumnSource source = column.getColumnSource();
13638                if (source != null && source.isStructFieldAccess()) {
13639                        return source.getExposedName();
13640                }
13641                return null;
13642        }
13643
13644        private String getColumnName(String column) {
13645                if (column == null) {
13646                        return null;
13647                }
13648                String name = column.substring(column.lastIndexOf(".") + 1);
13649                if (name == null || "".equals(name.trim())) {
13650                        return DlineageUtil.getIdentifierNormalColumnName(column.toString().trim());
13651                } else
13652                        return DlineageUtil.getIdentifierNormalColumnName(name.trim());
13653        }
13654        
13655        private String getColumnNameOnly(String column) {
13656                if (column == null) {
13657                        return null;
13658                }
13659                return DlineageUtil.getColumnNameOnly(column);
13660        }
13661
13662        private void appendRecordSetRelation(dataflow dataflow, Relationship[] relations) {
13663                for (int i = 0; i < relations.length; i++) {
13664                        AbstractRelationship relation = (AbstractRelationship) relations[i];
13665                        relationship relationElement = new relationship();
13666                        relationElement.setType(relation.getRelationshipType().name());
13667                        if (relation.getFunction() != null) {
13668                                relationElement.setFunction(relation.getFunction());
13669                        }
13670                        if (relation.getEffectType() != null) {
13671                                relationElement.setEffectType(relation.getEffectType().name());
13672                        }
13673                        relationElement.setSqlHash(relation.getSqlHash());
13674                        relationElement.setSqlComment(relation.getSqlComment());
13675
13676                        if (relation.getProcedureId() != null) {
13677                                relationElement.setProcedureId(String.valueOf(relation.getProcedureId()));
13678                        }
13679                        relationElement.setId(String.valueOf(relation.getId()));
13680
13681                        if (relation instanceof RecordSetRelationship) {
13682                                RecordSetRelationship recordCountRelation = (RecordSetRelationship) relation;
13683
13684                                Object targetElement = recordCountRelation.getTarget().getElement();
13685                                if (targetElement instanceof ResultColumn) {
13686                                        ResultColumn targetColumn = (ResultColumn) targetElement;
13687                                        targetColumn target = new targetColumn();
13688                                        target.setId(String.valueOf(targetColumn.getId()));
13689                                        target.setColumn(targetColumn.getName());
13690                                        target.setStruct(targetColumn.isStruct());
13691                                        target.setFunction(recordCountRelation.getAggregateFunction());
13692                                        target.setParent_id(String.valueOf(targetColumn.getResultSet().getId()));
13693                                        target.setParent_name(getResultSetName(targetColumn.getResultSet()));
13694                                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
13695                                                target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
13696                                                                + convertCoordinate(targetColumn.getEndPosition()));
13697                                        }
13698                                        relationElement.setTarget(target);
13699                                } else if (targetElement instanceof TableColumn) {
13700                                        TableColumn targetColumn = (TableColumn) targetElement;
13701                                        targetColumn target = new targetColumn();
13702                                        target.setId(String.valueOf(targetColumn.getId()));
13703                                        target.setColumn(targetColumn.getName());
13704                                        target.setStruct(targetColumn.isStruct());
13705                                        target.setFunction(recordCountRelation.getAggregateFunction());
13706                                        target.setParent_id(String.valueOf(targetColumn.getTable().getId()));
13707                                        target.setParent_name(getTableName(targetColumn.getTable()));
13708                                        if (recordCountRelation.getTarget() instanceof TableColumnRelationshipElement) {
13709                                                target.setParent_alias(
13710                                                                ((TableColumnRelationshipElement) recordCountRelation.getTarget()).getTableAlias());
13711                                        }
13712                                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
13713                                                target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
13714                                                                + convertCoordinate(targetColumn.getEndPosition()));
13715                                        }
13716                                        relationElement.setTarget(target);
13717                                } else if (targetElement instanceof ResultSetRelationRows) {
13718                                        ResultSetRelationRows targetColumn = (ResultSetRelationRows) targetElement;
13719                                        targetColumn target = new targetColumn();
13720                                        target.setId(String.valueOf(targetColumn.getId()));
13721                                        target.setColumn(targetColumn.getName());
13722                                        target.setParent_id(String.valueOf(targetColumn.getHolder().getId()));
13723                                        target.setParent_name(getResultSetName(targetColumn.getHolder()));
13724                                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
13725                                                target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
13726                                                                + convertCoordinate(targetColumn.getEndPosition()));
13727                                        }
13728                                        target.setSource("system");
13729                                        relationElement.setTarget(target);
13730                                } else {
13731                                        continue;
13732                                }
13733
13734                                Collection<RelationshipElement<?>> sourceElements = (Collection<RelationshipElement<?>>)recordCountRelation.getSources();
13735                                if (sourceElements.size() == 0) {
13736                                        continue;
13737                                }
13738
13739                                boolean append = false;
13740                                for (RelationshipElement<?> sourceItem: sourceElements) {
13741                                        Object sourceElement = sourceItem.getElement();
13742                                        if (sourceElement instanceof Table) {
13743                                                Table table = (Table) sourceElement;
13744                                                sourceColumn source = new sourceColumn();
13745                                                source.setSource_id(String.valueOf(table.getId()));
13746                                                source.setSource_name(getTableName(table));
13747                                                if (table.getStartPosition() != null && table.getEndPosition() != null) {
13748                                                        source.setCoordinate(convertCoordinate(table.getStartPosition()) + ","
13749                                                                        + convertCoordinate(table.getEndPosition()));
13750                                                }
13751                                                append = true;
13752                                                relationElement.addSource(source);
13753                                        } else if (sourceElement instanceof QueryTable) {
13754                                                QueryTable table = (QueryTable) sourceElement;
13755                                                sourceColumn source = new sourceColumn();
13756                                                source.setSource_id(String.valueOf(table.getId()));
13757                                                source.setSource_name(getResultSetName(table));
13758                                                if (table.getStartPosition() != null && table.getEndPosition() != null) {
13759                                                        source.setCoordinate(convertCoordinate(table.getStartPosition()) + ","
13760                                                                        + convertCoordinate(table.getEndPosition()));
13761                                                }
13762                                                append = true;
13763                                                relationElement.addSource(source);
13764                                        } else if (sourceElement instanceof TableRelationRows) {
13765                                                TableRelationRows relationRows = (TableRelationRows) sourceElement;
13766                                                sourceColumn source = new sourceColumn();
13767                                                source.setId(String.valueOf(relationRows.getId()));
13768                                                source.setColumn(relationRows.getName());
13769                                                source.setParent_id(String.valueOf(relationRows.getHolder().getId()));
13770                                                source.setParent_name(getTableName(relationRows.getHolder()));
13771                                                if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
13772                                                        source.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
13773                                                                        + convertCoordinate(relationRows.getEndPosition()));
13774                                                }
13775                                                source.setSource("system");
13776                                                append = true;
13777                                                relationElement.addSource(source);
13778                                        } else if (sourceElement instanceof ResultSetRelationRows) {
13779                                                ResultSetRelationRows relationRows = (ResultSetRelationRows) sourceElement;
13780                                                sourceColumn source = new sourceColumn();
13781                                                source.setId(String.valueOf(relationRows.getId()));
13782                                                source.setColumn(relationRows.getName());
13783                                                source.setParent_id(String.valueOf(relationRows.getHolder().getId()));
13784                                                source.setParent_name(getResultSetName(relationRows.getHolder()));
13785                                                if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
13786                                                        source.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
13787                                                                        + convertCoordinate(relationRows.getEndPosition()));
13788                                                }
13789                                                source.setSource("system");
13790                                                append = true;
13791                                                relationElement.addSource(source);
13792                                        } else if (sourceElement instanceof TableColumn) {
13793                                                TableColumn sourceColumn = (TableColumn) sourceElement;
13794                                                sourceColumn source = new sourceColumn();
13795                                                source.setId(String.valueOf(sourceColumn.getId()));
13796                                                source.setColumn(sourceColumn.getName());
13797                                                source.setStruct(sourceColumn.isStruct());
13798                                                source.setParent_id(String.valueOf(sourceColumn.getTable().getId()));
13799                                                source.setParent_name(getTableName(sourceColumn.getTable()));
13800                                                if (sourceItem instanceof TableColumnRelationshipElement) {
13801                                                        source.setParent_alias(
13802                                                                        ((TableColumnRelationshipElement) sourceItem).getTableAlias());
13803                                                }
13804                                                if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
13805                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
13806                                                                        + convertCoordinate(sourceColumn.getEndPosition()));
13807                                                }
13808                                                append = true;
13809                                                relationElement.addSource(source);
13810                                        }
13811                                        if (sourceElement instanceof ResultColumn) {
13812                                                ResultColumn sourceColumn = (ResultColumn) sourceElement;
13813                                                sourceColumn source = new sourceColumn();
13814                                                source.setId(String.valueOf(sourceColumn.getId()));
13815                                                source.setColumn(sourceColumn.getName());
13816                                                source.setStruct(sourceColumn.isStruct());
13817                                                source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
13818                                                source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
13819                                                if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
13820                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
13821                                                                        + convertCoordinate(sourceColumn.getEndPosition()));
13822                                                }
13823                                                append = true;
13824                                                relationElement.addSource(source);
13825                                        }
13826                                }
13827
13828                                if (append)
13829                                        dataflow.getRelationships().add(relationElement);
13830                        }
13831                }
13832        }
13833
13834        private void appendCallRelation(dataflow dataflow, Relationship[] relations) {
13835                for (int i = 0; i < relations.length; i++) {
13836                        AbstractRelationship relation = (AbstractRelationship) relations[i];
13837                        relationship relationElement = new relationship();
13838                        relationElement.setType(relation.getRelationshipType().name());
13839                        if (relation.getFunction() != null) {
13840                                relationElement.setFunction(relation.getFunction());
13841                        }
13842                        if (relation.getEffectType() != null) {
13843                                relationElement.setEffectType(relation.getEffectType().name());
13844                        }
13845                        relationElement.setSqlHash(relation.getSqlHash());
13846                        relationElement.setSqlComment(relation.getSqlComment());
13847                        
13848                        if (relation.getProcedureId() != null) {
13849                                relationElement.setProcedureId(String.valueOf(relation.getProcedureId()));
13850                        }
13851                        relationElement.setId(String.valueOf(relation.getId()));
13852
13853                        if (relation instanceof CallRelationship) {
13854                                CallRelationship callRelation = (CallRelationship) relation;
13855                                
13856                                if (callRelation.getCallObject() != null) {
13857                                        relationElement.setCallStmt(callRelation.getCallObject().toString());
13858                                        if (callRelation.getStartPosition() != null && callRelation.getEndPosition() != null) {
13859                                                relationElement.setCallCoordinate(convertCoordinate(callRelation.getStartPosition()) + ","
13860                                                                + convertCoordinate(callRelation.getEndPosition()));
13861                                        }
13862                                }
13863                                
13864                                if (Boolean.TRUE.equals(callRelation.getBuiltIn())) {
13865                                        relationElement.setBuiltIn(true);
13866                                }
13867                                Object targetElement = callRelation.getTarget().getElement();
13868                                if (targetElement instanceof Procedure) {
13869                                        Procedure procedure = (Procedure) targetElement;
13870                                        targetColumn target = new targetColumn();
13871                                        target.setId(String.valueOf(procedure.getId()));
13872                                        target.setName(getProcedureName(procedure));
13873                                        if (procedure.getStartPosition() != null && procedure.getEndPosition() != null) {
13874                                                target.setCoordinate(convertCoordinate(procedure.getStartPosition()) + ","
13875                                                                + convertCoordinate(procedure.getEndPosition()));
13876                                        }
13877                                        String clazz = procedure.getProcedureObject().getClass().getSimpleName().toLowerCase();
13878                                        if (clazz.indexOf("function") != -1) {
13879                                                target.setType("function");
13880                                        } else if (clazz.indexOf("trigger") != -1) {
13881                                                target.setType("trigger");
13882                                        } else if (clazz.indexOf("macro") != -1) {
13883                                                target.setType("macro");
13884                                        } else {
13885                                                target.setType("procedure");
13886                                        }
13887                                        relationElement.setCaller(target);
13888                                } else {
13889                                        continue;
13890                                }
13891
13892                                Collection<RelationshipElement<?>> sourceElements = (Collection<RelationshipElement<?>>)callRelation.getSources();
13893                                if (sourceElements.size() == 0) {
13894                                        continue;
13895                                }
13896
13897                                boolean append = false;
13898                                for (RelationshipElement<?> sourceItem: sourceElements) {
13899                                        Object sourceElement = sourceItem.getElement();
13900                                        if (sourceElement instanceof Procedure) {
13901                                                Procedure procedure = (Procedure) sourceElement;
13902                                                sourceColumn source = new sourceColumn();
13903                                                source.setId(String.valueOf(procedure.getId()));
13904                                                source.setName(getProcedureName(procedure));
13905                                                if (procedure.getStartPosition() != null && procedure.getEndPosition() != null) {
13906                                                        source.setCoordinate(convertCoordinate(procedure.getStartPosition()) + ","
13907                                                                        + convertCoordinate(procedure.getEndPosition()));
13908                                                }
13909                                                String clazz = procedure.getProcedureObject().getClass().getSimpleName().toLowerCase();
13910                                                if (clazz.indexOf("function") != -1) {
13911                                                        source.setType("function");
13912                                                } else if (clazz.indexOf("trigger") != -1) {
13913                                                        source.setType("trigger");
13914                                                } else if (clazz.indexOf("macro") != -1) {
13915                                                        source.setType("macro");
13916                                                } else {
13917                                                        source.setType("procedure");
13918                                                }
13919                                                append = true;
13920                                                relationElement.getCallees().add(source);
13921                                        } else if (sourceElement instanceof Function) {
13922                                                Function function = (Function) sourceElement;
13923                                                sourceColumn source = new sourceColumn();
13924                                                source.setId(String.valueOf(function.getId()));
13925                                                source.setName(getFunctionName(function.getFunctionObject()));
13926                                                if (function.getStartPosition() != null && function.getEndPosition() != null) {
13927                                                        source.setCoordinate(convertCoordinate(function.getStartPosition()) + ","
13928                                                                        + convertCoordinate(function.getEndPosition()));
13929                                                }
13930                                                source.setType("function");
13931                                                append = true;
13932                                                relationElement.getCallees().add(source);
13933                                        }
13934                                }
13935
13936                                if (append)
13937                                        dataflow.getRelationships().add(relationElement);
13938                        }
13939                }
13940        }
13941
13942        private void appendResultSets(dataflow dataflow) {
13943                Set<ResultSet> resultSets = modelManager.getResultSets(); 
13944                for (ResultSet resultSet: resultSets) {
13945                        appendResultSet(dataflow, resultSet);
13946                }
13947        }
13948
13949        private void appendResultSet(dataflow dataflow, ResultSet resultSetModel) {
13950                if (!appendResultSets.contains(resultSetModel)) {
13951                        appendResultSets.add(resultSetModel);
13952                } else {
13953                        return;
13954                }
13955
13956                table resultSetElement = new table();
13957                resultSetElement.setId(String.valueOf(resultSetModel.getId()));
13958                resultSetElement.setServer(resultSetModel.getServer());
13959                if (!SQLUtil.isEmpty(resultSetModel.getDatabase())) {
13960                        resultSetElement.setDatabase(resultSetModel.getDatabase());
13961                }
13962                if (!SQLUtil.isEmpty(resultSetModel.getSchema())) {
13963                        resultSetElement.setSchema(resultSetModel.getSchema());
13964                }
13965                resultSetElement.setName(getResultSetName(resultSetModel));
13966                resultSetElement.setType(getResultSetType(resultSetModel));
13967                // if ((ignoreRecordSet || simpleOutput) && resultSetModel.isTarget()) {
13968                resultSetElement.setIsTarget(String.valueOf(resultSetModel.isTarget()));
13969                // }
13970                resultSetElement.setIsDetermined(String.valueOf(resultSetModel.isDetermined()));
13971                if (resultSetModel.getStartPosition() != null && resultSetModel.getEndPosition() != null) {
13972                        resultSetElement.setCoordinate(convertCoordinate(resultSetModel.getStartPosition()) + ","
13973                                        + convertCoordinate(resultSetModel.getEndPosition()));
13974                }
13975                dataflow.getResultsets().add(resultSetElement);
13976
13977                List<ResultColumn> columns = resultSetModel.getColumns();
13978
13979                Map<String, Integer> columnCounts = new HashMap<String, Integer>();
13980                for (ResultColumn column : columns) {
13981                        String columnName = DlineageUtil.getIdentifierNormalColumnName(column.getName());
13982                        if (!columnCounts.containsKey(columnName)) {
13983                                columnCounts.put(columnName, 0);
13984                        }
13985                        columnCounts.put(columnName, columnCounts.get(columnName) + 1);
13986                        // if (column.hasStarLinkColumn()) {
13987                        // List<String> starLinkColumns = column.getStarLinkColumnNames();
13988                        // for (int k = 0; k < starLinkColumns.size(); k++) {
13989                        // columnName = starLinkColumns.get(k);
13990                        // if (!columnCounts.containsKey(columnName)) {
13991                        // columnCounts.put(columnName, 0);
13992                        // }
13993                        // columnCounts.put(columnName, columnCounts.get(columnName) + 1);
13994                        // }
13995                        // }
13996                }
13997
13998                for (int j = 0; j < columns.size(); j++) {
13999                        ResultColumn columnModel = columns.get(j);
14000                        if (columnModel.hasStarLinkColumn()) {
14001                                // List<String> starLinkColumns =
14002                                // columnModel.getStarLinkColumnNames();
14003                                // for (int k = 0; k < starLinkColumns.size(); k++) {
14004                                // column columnElement = new column();
14005                                // columnElement.setId( String.valueOf(columnModel.getId()) +
14006                                // "_" + k);
14007                                // String columnName = starLinkColumns.get(k);
14008                                // columnElement.setName(columnName);
14009                                // if(columnModel.isFunction()){
14010                                // columnElement.setIsFunction(String.valueOf(columnModel.isFunction()));
14011                                // }
14012                                // if (columnModel.getStartPosition() != null &&
14013                                // columnModel.getEndPosition() != null) {
14014                                // columnElement.setCoordinate(
14015                                // columnModel.getStartPosition() + "," +
14016                                // columnModel.getEndPosition());
14017                                // }
14018                                // String identifier = columnName;
14019                                // if(columnCounts.containsKey(identifier) &&
14020                                // columnCounts.get(identifier)>1){
14021                                // TObjectName column =
14022                                // columnModel.getStarLinkColumns().get(columnName).iterator().next();
14023                                // if(!SQLUtil.isEmpty(getQualifiedTable(column))){
14024                                // columnElement.setQualifiedTable(getQualifiedTable(column));
14025                                // }
14026                                // }
14027                                // resultSetElement.getColumns().add(columnElement);
14028                                // }
14029                                if (columnModel.isShowStar()) {
14030                                        column columnElement = new column();
14031                                        columnElement.setId(String.valueOf(columnModel.getId()));
14032                                        columnElement.setName(columnModel.getName());
14033                                        if (columnModel.isFunction()) {
14034                                                columnElement.setIsFunction(String.valueOf(columnModel.isFunction()));
14035                                        }
14036                                        if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
14037                                                columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
14038                                                                + convertCoordinate(columnModel.getEndPosition()));
14039                                        }
14040
14041                                        String identifier = DlineageUtil.getIdentifierNormalColumnName(columnModel.getName());
14042                                        if (columnCounts.containsKey(identifier) && columnCounts.get(identifier) > 1) {
14043                                                String qualifiedTable = getQualifiedTable(columnModel);
14044                                                if (!SQLUtil.isEmpty(qualifiedTable)) {
14045                                                        columnElement.setQualifiedTable(qualifiedTable);
14046                                                }
14047                                        }
14048                                        resultSetElement.getColumns().add(columnElement);
14049                                }
14050                        } else {
14051                                column columnElement = new column();
14052                                columnElement.setId(String.valueOf(columnModel.getId()));
14053                                columnElement.setName(columnModel.getName());
14054                                if (columnModel.isFunction()) {
14055                                        columnElement.setIsFunction(String.valueOf(columnModel.isFunction()));
14056                                }
14057                                if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
14058                                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
14059                                                        + convertCoordinate(columnModel.getEndPosition()));
14060                                }
14061
14062                                String identifier = DlineageUtil.getIdentifierNormalColumnName(columnModel.getName());
14063                                if (columnCounts.containsKey(identifier) && columnCounts.get(identifier) > 1) {
14064                                        String qualifiedTable = getQualifiedTable(columnModel);
14065                                        if (!SQLUtil.isEmpty(qualifiedTable)) {
14066                                                columnElement.setQualifiedTable(qualifiedTable);
14067                                        }
14068                                }
14069                                resultSetElement.getColumns().add(columnElement);
14070                        }
14071                }
14072
14073                ResultSetRelationRows relationRows = resultSetModel.getRelationRows();
14074                if (relationRows.hasRelation()) {
14075                        column relationRowsElement = new column();
14076                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
14077                        relationRowsElement.setName(relationRows.getName());
14078                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
14079                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
14080                                                + convertCoordinate(relationRows.getEndPosition()));
14081                        }
14082                        relationRowsElement.setSource("system");
14083                        resultSetElement.getColumns().add(relationRowsElement);
14084                }
14085        }
14086
14087        private String getQualifiedTable(ResultColumn columnModel) {
14088                if (columnModel.getColumnObject() instanceof TObjectName) {
14089                        return getQualifiedTable((TObjectName) columnModel.getColumnObject());
14090                }
14091                if (columnModel.getColumnObject() instanceof TResultColumn) {
14092                        TObjectName field = ((TResultColumn) columnModel.getColumnObject()).getFieldAttr();
14093                        if (field != null) {
14094                                return getQualifiedTable(field);
14095                        }
14096                }
14097                return null;
14098        }
14099
14100        private String getQualifiedTable(TObjectName column) {
14101                if (column == null)
14102                        return null;
14103                String[] splits = column.toString().split("\\.");
14104                if (splits.length > 1) {
14105                        return splits[splits.length - 2];
14106                }
14107                return null;
14108        }
14109
14110        /**
14111         * Get the qualified prefix (schema.table) from a column name for 3-part names.
14112         * For example, for "sch.pk_constv2.c_cdsl", returns "sch.pk_constv2".
14113         * Returns null if the column doesn't have both schema and table parts.
14114         */
14115        private String getQualifiedPrefixFromColumn(TObjectName column) {
14116                if (column == null) return null;
14117
14118                // Check if both schema and table tokens are present (3-part name)
14119                String schemaStr = column.getSchemaString();
14120                String tableStr = column.getTableString();
14121
14122                if (schemaStr != null && !schemaStr.isEmpty() &&
14123                        tableStr != null && !tableStr.isEmpty()) {
14124                        return schemaStr + "." + tableStr;
14125                }
14126
14127                // Fallback: parse from toString() for complex cases
14128                String[] splits = column.toString().split("\\.");
14129                if (splits.length >= 3) {
14130                        // Return all parts except the last one (column name)
14131                        StringBuilder prefix = new StringBuilder();
14132                        for (int i = 0; i < splits.length - 1; i++) {
14133                                if (i > 0) prefix.append(".");
14134                                prefix.append(splits[i]);
14135                        }
14136                        return prefix.toString();
14137                }
14138
14139                return null;
14140        }
14141
14142        private String getResultSetType(ResultSet resultSetModel) {
14143                if (resultSetModel instanceof QueryTable) {
14144                        QueryTable table = (QueryTable) resultSetModel;
14145                        if (table.getTableObject().getCTE() != null) {
14146                                return "with_cte";
14147                        }
14148                }
14149
14150                if (resultSetModel instanceof SelectSetResultSet) {
14151                        ESetOperatorType type = ((SelectSetResultSet) resultSetModel).getSetOperatorType();
14152                        return "select_" + type.name();
14153                }
14154
14155                if (resultSetModel instanceof SelectResultSet) {
14156                        if (((SelectResultSet) resultSetModel).getSelectStmt().getParentStmt() instanceof TInsertSqlStatement) {
14157                                return "insert-select";
14158                        }
14159                        if (((SelectResultSet) resultSetModel).getSelectStmt().getParentStmt() instanceof TUpdateSqlStatement) {
14160                                return "update-select";
14161                        }
14162                }
14163
14164                if (resultSetModel.getGspObject() instanceof TMergeUpdateClause) {
14165                        return "merge-update";
14166                }
14167
14168                if (resultSetModel.getGspObject() instanceof TOutputClause) {
14169                        return ResultSetType.output.name();
14170                }
14171
14172                if (resultSetModel.getGspObject() instanceof TMergeInsertClause) {
14173                        return "merge-insert";
14174                }
14175
14176                if (resultSetModel.getGspObject() instanceof TUpdateSqlStatement) {
14177                        return "update-set";
14178                }
14179
14180                if (resultSetModel.getGspObject() instanceof TFunctionCall && ((TFunctionCall)resultSetModel.getGspObject()).getFunctionType() == EFunctionType.array_t) {
14181                        return ResultSetType.array.name();
14182                }
14183
14184                if (resultSetModel.getGspObject() instanceof TFunctionCall && ((TFunctionCall)resultSetModel.getGspObject()).getFunctionType() == EFunctionType.struct_t) {
14185                        return ResultSetType.struct.name();
14186                }
14187
14188                if (resultSetModel.getGspObject() instanceof TFunctionCall || resultSetModel instanceof Function) {
14189                        return ResultSetType.function.name();
14190                }
14191
14192                if (resultSetModel.getGspObject() instanceof TAliasClause) {
14193                        return ResultSetType.alias.name();
14194                }
14195
14196                if (resultSetModel.getGspObject() instanceof TCursorDeclStmt) {
14197                        return ResultSetType.cursor.name();
14198                }
14199
14200                if (resultSetModel instanceof PivotedTable) {
14201                        if (((PivotedTable) resultSetModel).isUnpivoted()) {
14202                                return ResultSetType.unpivot_table.name();
14203                        }
14204                        return ResultSetType.pivot_table.name();
14205                }
14206
14207                return "select_list";
14208        }
14209
14210        private String getTableName(Table tableModel) {
14211                if (modelManager.DISPLAY_NAME.containsKey(tableModel.getId())) {
14212                        return modelManager.DISPLAY_NAME.get(tableModel.getId());
14213                }
14214
14215                String tableName;
14216                if (tableModel.getFullName() != null && tableModel.getFullName().trim().length() > 0) {
14217                        return tableModel.getFullName();
14218                }
14219                if (tableModel.getAlias() != null && tableModel.getAlias().trim().length() > 0) {
14220                        tableName = getResultSetWithId("RESULT_OF_" + tableModel.getAlias());
14221
14222                } else {
14223                        tableName = getResultSetDisplayId("RS");
14224                }
14225                modelManager.DISPLAY_NAME.put(tableModel.getId(), tableName);
14226                return tableName;
14227        }
14228
14229        private String getProcedureName(Procedure procedureModel) {
14230                if (modelManager.DISPLAY_NAME.containsKey(procedureModel.getId())) {
14231                        return modelManager.DISPLAY_NAME.get(procedureModel.getId());
14232                }
14233
14234                String procedureName = procedureModel.getFullName();
14235
14236                modelManager.DISPLAY_NAME.put(procedureModel.getId(), procedureName);
14237                return procedureName;
14238        }
14239
14240        private String getProcessName(Process processModel) {
14241                if (modelManager.DISPLAY_NAME.containsKey(processModel.getId())) {
14242                        return modelManager.DISPLAY_NAME.get(processModel.getId());
14243                } else {
14244                        if (processModel.getCustomType() != null) {
14245                                String name = processModel.getCustomType();
14246                                modelManager.DISPLAY_NAME.put(processModel.getId(), name);
14247                                return name;
14248                        }
14249                        String name = processModel.getType();
14250                        String procedureName = getProcedureParentName(processModel.getGspObject());
14251                        if (procedureName != null) {
14252                                name = getResultSetDisplayId(procedureName + " " + name);
14253                        } else {
14254                                name = getResultSetDisplayId("Query " + name);
14255                        }
14256                        modelManager.DISPLAY_NAME.put(processModel.getId(), name);
14257                        return name;
14258                }
14259        }
14260
14261        private String getDisplayIdByType(String type) {
14262                if (!modelManager.DISPLAY_ID.containsKey(type)) {
14263                        modelManager.DISPLAY_ID.put(type, option.getStartId() + 1);
14264                        return type + "-" + (option.getStartId() + 1);
14265                } else {
14266                        long id = modelManager.DISPLAY_ID.get(type);
14267                        modelManager.DISPLAY_ID.put(type, id + 1);
14268                        return type + "-" + (id + 1);
14269                }
14270        }
14271        
14272        private String getDisplayIdByTypeFromZero(String type) {
14273                if (!modelManager.DISPLAY_ID.containsKey(type)) {
14274                        modelManager.DISPLAY_ID.put(type, option.getStartId());
14275                        if(option.getStartId() == 0) {
14276                                return type;
14277                        }
14278                        return type + "-" + (option.getStartId() + 1);
14279                } else {
14280                        long id = modelManager.DISPLAY_ID.get(type);
14281                        modelManager.DISPLAY_ID.put(type, id + 1);
14282                        return type + "-" + (id + 1);
14283                }
14284        }
14285
14286        private String getConstantName(Table tableModel) {
14287                if (modelManager.DISPLAY_NAME.containsKey(tableModel.getId())) {
14288                        return modelManager.DISPLAY_NAME.get(tableModel.getId());
14289                } else {
14290                        String name = getDisplayIdByType("SQL_CONSTANTS");
14291                        modelManager.DISPLAY_NAME.put(tableModel.getId(), name);
14292                        return name;
14293                }
14294        }
14295        
14296        private String getTempTableName(TTable table) {
14297                if (modelManager.DISPLAY_NAME.containsKey((long)System.identityHashCode(table))) {
14298                        return modelManager.DISPLAY_NAME.get((long)System.identityHashCode(table));
14299                } else {
14300                        String name = getDisplayIdByTypeFromZero(table.getName());
14301                        modelManager.DISPLAY_NAME.put((long)System.identityHashCode(table), name);
14302                        return name;
14303                }
14304        }
14305
14306        private String getResultSetName(ResultSet resultSetModel) {
14307
14308                if (modelManager.DISPLAY_NAME.containsKey(resultSetModel.getId())) {
14309                        return modelManager.DISPLAY_NAME.get(resultSetModel.getId());
14310                }
14311                
14312                if (resultSetModel.getGspObject() instanceof TFunctionCall && ((TFunctionCall)resultSetModel.getGspObject()).getFunctionType() == EFunctionType.array_t) {
14313                        String name = getResultSetDisplayId("ARRAY");
14314                        modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
14315                        if(option.containsResultSetType(ResultSetType.array)) {
14316                                resultSetModel.setTarget(true);
14317                        }
14318                        return name;
14319                }
14320                
14321                if (resultSetModel.getGspObject() instanceof TFunctionCall && ((TFunctionCall)resultSetModel.getGspObject()).getFunctionType() == EFunctionType.struct_t) {
14322                        String name = getResultSetDisplayId("STRUCT");
14323                        modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
14324                        if(option.containsResultSetType(ResultSetType.struct)) {
14325                                resultSetModel.setTarget(true);
14326                        }
14327                        return name;
14328                }
14329
14330                if (resultSetModel instanceof QueryTable) {
14331                        QueryTable table = (QueryTable) resultSetModel;
14332                        if (table.getAlias() != null && table.getAlias().trim().length() > 0) {
14333                                String name = getResultSetWithId("RESULT_OF_" + table.getAlias().trim());
14334                                if (table.getTableObject().getCTE() != null) {
14335                                        name = getResultSetWithId("RESULT_OF_" + table.getTableObject().getCTE().getTableName().toString()
14336                                                        + "_" + table.getAlias().trim());
14337                                }
14338                                modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
14339                                if(option.containsResultSetType(ResultSetType.result_of)) {
14340                                        resultSetModel.setTarget(true);
14341                                }
14342                                return name;
14343                        } else if (table.getTableObject().getCTE() != null) {
14344                                String name = getResultSetWithId("CTE-" + table.getTableObject().getCTE().getTableName().toString());
14345                                modelManager.DISPLAY_NAME.put(table.getId(), name);
14346                                if(option.containsResultSetType(ResultSetType.cte)) {
14347                                        resultSetModel.setTarget(true);
14348                                }
14349                                return name;
14350                        }
14351                }
14352
14353                if (resultSetModel instanceof SelectResultSet) {
14354                        if (((SelectResultSet) resultSetModel).getSelectStmt().getParentStmt() instanceof TInsertSqlStatement) {
14355                                String name = getResultSetDisplayId("INSERT-SELECT");
14356                                modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
14357                                if(option.containsResultSetType(ResultSetType.insert_select)) {
14358                                        resultSetModel.setTarget(true);
14359                                }
14360                                return name;
14361                        }
14362
14363                        if (((SelectResultSet) resultSetModel).getSelectStmt().getParentStmt() instanceof TUpdateSqlStatement) {
14364                                String name = getResultSetDisplayId("UPDATE-SELECT");
14365                                modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
14366                                if(option.containsResultSetType(ResultSetType.update_select)) {
14367                                        resultSetModel.setTarget(true);
14368                                }
14369                                return name;
14370                        }
14371                }
14372
14373                if (resultSetModel instanceof SelectSetResultSet) {
14374                        ESetOperatorType type = ((SelectSetResultSet) resultSetModel).getSetOperatorType();
14375                        String name = getResultSetDisplayId("RESULT_OF_" + type.name().toUpperCase());
14376                        modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
14377                        if(option.containsResultSetType(ResultSetType.result_of)) {
14378                                resultSetModel.setTarget(true);
14379                        }
14380                        return name;
14381                }
14382
14383                if (resultSetModel.getGspObject() instanceof TMergeUpdateClause) {
14384                        String name = getResultSetDisplayId("MERGE-UPDATE");
14385                        modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
14386                        if(option.containsResultSetType(ResultSetType.merge_update)) {
14387                                resultSetModel.setTarget(true);
14388                        }
14389                        return name;
14390                }
14391
14392                if (resultSetModel.getGspObject() instanceof TOutputClause) {
14393                        String name = getResultSetDisplayId("OUTPUT");
14394                        modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
14395                        if(option.containsResultSetType(ResultSetType.output)) {
14396                                resultSetModel.setTarget(true);
14397                        }
14398                        return name;
14399                }
14400
14401                if (resultSetModel.getGspObject() instanceof TMergeInsertClause) {
14402                        String name = getResultSetDisplayId("MERGE-INSERT");
14403                        modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
14404                        if(option.containsResultSetType(ResultSetType.merge_insert)) {
14405                                resultSetModel.setTarget(true);
14406                        }
14407                        return name;
14408                }
14409
14410                if (resultSetModel.getGspObject() instanceof TUpdateSqlStatement) {
14411                        String name = getResultSetDisplayId("UPDATE-SET");
14412                        modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
14413                        if(option.containsResultSetType(ResultSetType.update_set)) {
14414                                resultSetModel.setTarget(true);
14415                        }
14416                        return name;
14417                }
14418
14419                if (resultSetModel.getGspObject() instanceof TCaseExpression) {
14420                        String name = ((Function) resultSetModel).getFunctionName();
14421                        modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
14422                        if (option.containsResultSetType(ResultSetType.case_when) || option.containsResultSetType(ResultSetType.function)) {
14423                                resultSetModel.setTarget(true);
14424                        }
14425                        return name;
14426                }
14427                
14428                if (resultSetModel.getGspObject() instanceof TFunctionCall || resultSetModel instanceof Function) {
14429                        String name = ((Function) resultSetModel).getFunctionName();
14430                        modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
14431                        if(option.containsResultSetType(ResultSetType.function)) {
14432                                resultSetModel.setTarget(true);
14433                        }
14434                        return name;
14435                }
14436
14437                if (resultSetModel instanceof PivotedTable) {
14438                        String name = getResultSetDisplayId("PIVOT-TABLE");
14439                        if (((PivotedTable) resultSetModel).isUnpivoted()) {
14440                                name = getResultSetDisplayId("UNPIVOT-TABLE");
14441                                if(option.containsResultSetType(ResultSetType.unpivot_table)) {
14442                                        resultSetModel.setTarget(true);
14443                                }
14444                        }
14445                        else {
14446                                if(option.containsResultSetType(ResultSetType.pivot_table)) {
14447                                        resultSetModel.setTarget(true);
14448                                }
14449                        }
14450                        modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
14451                        return name;
14452                }
14453
14454                if (resultSetModel instanceof Alias) {
14455                        String name = getResultSetDisplayId("ALIAS");
14456                        modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
14457                        if(option.containsResultSetType(ResultSetType.alias)) {
14458                                resultSetModel.setTarget(true);
14459                        }
14460                        return name;
14461                }
14462
14463                String name = getResultSetDisplayId("RS");
14464                modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
14465                if(option.containsResultSetType(ResultSetType.select_list)) {
14466                        resultSetModel.setTarget(true);
14467                }
14468                return name;
14469        }
14470
14471        private String getResultSetWithId(String type) {
14472                type = DlineageUtil.getIdentifierNormalTableName(type);
14473                if (!modelManager.DISPLAY_ID.containsKey(type)) {
14474                        modelManager.DISPLAY_ID.put(type, option.getStartId() + 1);
14475                        return type + "-" + (option.getStartId() + 1);
14476                } else {
14477                        long id = modelManager.DISPLAY_ID.get(type);
14478                        modelManager.DISPLAY_ID.put(type, id + 1);
14479                        return type + "-" + (id + 1);
14480                }
14481        }
14482
14483        private String getResultSetDisplayId(String type) {
14484                if (!modelManager.DISPLAY_ID.containsKey(type)) {
14485                        modelManager.DISPLAY_ID.put(type, option.getStartId() + 1);
14486                        return type + "-" + (option.getStartId() + 1);
14487                } else {
14488                        long id = modelManager.DISPLAY_ID.get(type);
14489                        modelManager.DISPLAY_ID.put(type, id + 1);
14490                        return type + "-" + (id + 1);
14491                }
14492        }
14493
14494        private void appendViews(dataflow dataflow) {
14495                List<TCustomSqlStatement> views = modelManager.getViews();
14496                for (int i = 0; i < views.size(); i++) {
14497                        Table viewModel = (Table) modelManager.getViewModel(views.get(i));
14498                        if (!tableIds.contains(viewModel.getId())) {
14499                                appendViewModel(dataflow, viewModel);
14500                                tableIds.add(viewModel.getId());
14501                        }
14502                }
14503
14504                List<TTable> tables = modelManager.getBaseTables();
14505                for (int i = 0; i < tables.size(); i++) {
14506                        Object model = modelManager.getModel(tables.get(i));
14507                        if (model instanceof Table) {
14508                                Table tableModel = (Table) model;
14509                                if (tableModel.isView()) {
14510                                        if (!tableIds.contains(tableModel.getId())) {
14511                                                appendViewModel(dataflow, tableModel);
14512                                                tableIds.add(tableModel.getId());
14513                                        }
14514                                }
14515                        }
14516                }
14517
14518                List<Table> tableNames = modelManager.getTablesByName();
14519                for (int i = 0; i < tableNames.size(); i++) {
14520                        Table tableModel = tableNames.get(i);
14521                        if (tableModel.isView()) {
14522                                if (!tableIds.contains(tableModel.getId())) {
14523                                        appendViewModel(dataflow, tableModel);
14524                                        tableIds.add(tableModel.getId());
14525                                }
14526                        }
14527                }
14528        }
14529
14530        private void appendViewModel(dataflow dataflow, Table viewModel) {
14531                table viewElement = new table();
14532                viewElement.setId(String.valueOf(viewModel.getId()));
14533                if (!SQLUtil.isEmpty(viewModel.getDatabase())) {
14534                        viewElement.setDatabase(viewModel.getDatabase());
14535                }
14536                if (!SQLUtil.isEmpty(viewModel.getSchema())) {
14537                        viewElement.setSchema(viewModel.getSchema());
14538                }
14539                viewElement.setServer(viewModel.getServer());
14540                viewElement.setName(viewModel.getName());
14541                viewElement.setType("view");
14542                // Propagate the view sub type (e.g. temp_table for CREATE TEMPORARY VIEW)
14543                // so consumers can tell a temporary view apart from a regular one. Mantis 4538.
14544                if (viewModel.getSubType() != null) {
14545                        viewElement.setSubType(viewModel.getSubType().name());
14546                }
14547                viewElement.setStarStmt(viewModel.getStarStmt());
14548
14549                if(viewModel.isFromDDL()){
14550                        viewElement.setFromDDL(String.valueOf(viewModel.isFromDDL()));
14551                }
14552
14553        if(option.isTraceTablePosition()){
14554            for (Pair<Pair3<Long, Long, String>, Pair3<Long, Long, String>> position:viewModel.getPositions()){
14555                viewElement.setCoordinate(convertCoordinate(position.first)+","+convertCoordinate(position.second));
14556            }
14557        }
14558        else {
14559            viewElement.setCoordinate(convertCoordinate(viewModel.getStartPosition()) + ","
14560                    + convertCoordinate(viewModel.getEndPosition()));
14561        }
14562
14563                if (viewModel.getProcesses() != null) {
14564                        List<String> processIds = new ArrayList<String>();
14565                        for (Process process : viewModel.getProcesses()) {
14566                                processIds.add(String.valueOf(process.getId()));
14567                        }
14568                        viewElement.setProcessIds(processIds);
14569                }
14570                dataflow.getViews().add(viewElement);
14571
14572                List<TableColumn> columns = viewModel.getColumns();
14573
14574                if (containStarColumn(columns)) {
14575                        for (TableColumn column : columns) {
14576                                if (column.getName().endsWith("*")) {
14577                                        for (TableColumn starElement : columns) {
14578                                                if (starElement == column) {
14579                                                        continue;
14580                                                }
14581                                                TObjectName columnObject = starElement.getColumnObject();
14582                                                column.bindStarLinkColumn(columnObject);
14583                                        }
14584                                        if (viewModel.isCreateTable()) {
14585                                                column.setShowStar(false);
14586                                        }
14587                                }
14588                        }
14589                }
14590
14591                for (int j = 0; j < columns.size(); j++) {
14592                        TableColumn columnModel = (TableColumn) columns.get(j);
14593                        if (!columnModel.isPseduo() && columnModel.hasStarLinkColumn()) {
14594                                List<String> starLinkColumnList = columnModel.getStarLinkColumnNames();
14595                                for (int k = 0; k < starLinkColumnList.size(); k++) {
14596                                        column columnElement = new column();
14597                                        columnElement.setId(String.valueOf(columnModel.getId()) + "_" + k);
14598                                        String columnName = starLinkColumnList.get(k);
14599                                        if (containStarColumn(columns, columnName)) {
14600                                                continue;
14601                                        }
14602                                        columnElement.setName(columnName);
14603                                        if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
14604                                                columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
14605                                                                + convertCoordinate(columnModel.getEndPosition()));
14606                                        }
14607                                        viewElement.getColumns().add(columnElement);
14608                                }
14609
14610                                if (columnModel.isShowStar()) {
14611                                        column columnElement = new column();
14612                                        columnElement.setId(String.valueOf(columnModel.getId()));
14613                                        columnElement.setName(columnModel.getName());
14614                                        if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
14615                                                columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
14616                                                                + convertCoordinate(columnModel.getEndPosition()));
14617                                        }
14618                                        viewElement.getColumns().add(columnElement);
14619                                }
14620
14621                        } else {
14622                                column columnElement = new column();
14623                                columnElement.setId(String.valueOf(columnModel.getId()));
14624                                columnElement.setName(columnModel.getName());
14625                                if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
14626                                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
14627                                                        + convertCoordinate(columnModel.getEndPosition()));
14628                                }
14629                                if(columnModel.isPseduo()) {
14630                                        columnElement.setSource("system");
14631                                }
14632                                viewElement.getColumns().add(columnElement);
14633                        }
14634                }
14635
14636                TableRelationRows relationRows = viewModel.getRelationRows();
14637                if (relationRows.hasRelation()) {
14638                        column relationRowsElement = new column();
14639                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
14640                        relationRowsElement.setName(relationRows.getName());
14641                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
14642                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
14643                                                + convertCoordinate(relationRows.getEndPosition()));
14644                        }
14645                        relationRowsElement.setSource("system");
14646                        viewElement.getColumns().add(relationRowsElement);
14647                }
14648        }
14649
14650        private void appendStreamModel(dataflow dataflow, Table streamModel) {
14651                table streamElement = new table();
14652                streamElement.setId(String.valueOf(streamModel.getId()));
14653                if (!SQLUtil.isEmpty(streamModel.getDatabase())) {
14654                        streamElement.setDatabase(streamModel.getDatabase());
14655                }
14656                if (!SQLUtil.isEmpty(streamModel.getSchema())) {
14657                        streamElement.setSchema(streamModel.getSchema());
14658                }
14659                streamElement.setServer(streamModel.getServer());
14660                streamElement.setName(streamModel.getName());
14661                streamElement.setType("stream");
14662                if (streamModel.getFileType() != null) {
14663                        streamElement.setFileType(SQLUtil.trimColumnStringQuote(streamModel.getFileType()));
14664                }
14665
14666                if (streamModel.getStartPosition() != null && streamModel.getEndPosition() != null) {
14667                        streamElement.setCoordinate(convertCoordinate(streamModel.getStartPosition()) + ","
14668                                        + convertCoordinate(streamModel.getEndPosition()));
14669                }
14670
14671                if (streamModel.getProcesses() != null) {
14672                        List<String> processIds = new ArrayList<String>();
14673                        for (Process process : streamModel.getProcesses()) {
14674                                processIds.add(String.valueOf(process.getId()));
14675                        }
14676                        streamElement.setProcessIds(processIds);
14677                }
14678                dataflow.getStreams().add(streamElement);
14679
14680                List<TableColumn> columns = streamModel.getColumns();
14681
14682                for (int j = 0; j < columns.size(); j++) {
14683                        TableColumn columnModel = (TableColumn) columns.get(j);
14684                        column columnElement = new column();
14685                        columnElement.setId(String.valueOf(columnModel.getId()));
14686                        columnElement.setName(SQLUtil.trimColumnStringQuote(columnModel.getName()));
14687                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
14688                                        + convertCoordinate(columnModel.getEndPosition()));
14689                        streamElement.getColumns().add(columnElement);
14690                }
14691
14692                TableRelationRows relationRows = streamModel.getRelationRows();
14693                if (relationRows.hasRelation()) {
14694                        column relationRowsElement = new column();
14695                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
14696                        relationRowsElement.setName(relationRows.getName());
14697                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
14698                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
14699                                                + convertCoordinate(relationRows.getEndPosition()));
14700                        }
14701                        relationRowsElement.setSource("system");
14702                        streamElement.getColumns().add(relationRowsElement);
14703                }
14704        }
14705
14706        private void appendStageModel(dataflow dataflow, Table stageModel) {
14707                table stageElement = new table();
14708                stageElement.setId(String.valueOf(stageModel.getId()));
14709                if (!SQLUtil.isEmpty(stageModel.getDatabase())) {
14710                        stageElement.setDatabase(stageModel.getDatabase());
14711                }
14712                if (!SQLUtil.isEmpty(stageModel.getSchema())) {
14713                        stageElement.setSchema(stageModel.getSchema());
14714                }
14715                stageElement.setServer(stageModel.getServer());
14716                stageElement.setName(stageModel.getName());
14717                stageElement.setType("stage");
14718                stageElement.setLocation(stageModel.getLocation());
14719                if (stageModel.getFileType() != null) {
14720                        stageElement.setFileType(SQLUtil.trimColumnStringQuote(stageModel.getFileType()));
14721                }
14722
14723                if (stageModel.getStartPosition() != null && stageModel.getEndPosition() != null) {
14724                        stageElement.setCoordinate(convertCoordinate(stageModel.getStartPosition()) + ","
14725                                        + convertCoordinate(stageModel.getEndPosition()));
14726                }
14727
14728                if (stageModel.getProcesses() != null) {
14729                        List<String> processIds = new ArrayList<String>();
14730                        for (Process process : stageModel.getProcesses()) {
14731                                processIds.add(String.valueOf(process.getId()));
14732                        }
14733                        stageElement.setProcessIds(processIds);
14734                }
14735                dataflow.getStages().add(stageElement);
14736
14737                List<TableColumn> columns = stageModel.getColumns();
14738
14739                for (int j = 0; j < columns.size(); j++) {
14740                        TableColumn columnModel = (TableColumn) columns.get(j);
14741                        column columnElement = new column();
14742                        columnElement.setId(String.valueOf(columnModel.getId()));
14743                        columnElement.setName(SQLUtil.trimColumnStringQuote(columnModel.getName()));
14744                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
14745                                        + convertCoordinate(columnModel.getEndPosition()));
14746                        stageElement.getColumns().add(columnElement);
14747                }
14748
14749                TableRelationRows relationRows = stageModel.getRelationRows();
14750                if (relationRows.hasRelation()) {
14751                        column relationRowsElement = new column();
14752                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
14753                        relationRowsElement.setName(relationRows.getName());
14754                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
14755                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
14756                                                + convertCoordinate(relationRows.getEndPosition()));
14757                        }
14758                        relationRowsElement.setSource("system");
14759                        stageElement.getColumns().add(relationRowsElement);
14760                }
14761        }
14762
14763        private void appendSequenceModel(dataflow dataflow, Table sequenceModel) {
14764                table sequenceElement = new table();
14765                sequenceElement.setId(String.valueOf(sequenceModel.getId()));
14766                if (!SQLUtil.isEmpty(sequenceModel.getDatabase())) {
14767                        sequenceElement.setDatabase(sequenceModel.getDatabase());
14768                }
14769                if (!SQLUtil.isEmpty(sequenceModel.getSchema())) {
14770                        sequenceElement.setSchema(sequenceModel.getSchema());
14771                }
14772                sequenceElement.setServer(sequenceModel.getServer());
14773                sequenceElement.setName(sequenceModel.getName());
14774                sequenceElement.setType("sequence");
14775                sequenceElement.setLocation(sequenceModel.getLocation());
14776                if (sequenceModel.getFileType() != null) {
14777                        sequenceElement.setFileType(SQLUtil.trimColumnStringQuote(sequenceModel.getFileType()));
14778                }
14779
14780                if (sequenceModel.getStartPosition() != null && sequenceModel.getEndPosition() != null) {
14781                        sequenceElement.setCoordinate(convertCoordinate(sequenceModel.getStartPosition()) + ","
14782                                        + convertCoordinate(sequenceModel.getEndPosition()));
14783                }
14784
14785                if (sequenceModel.getProcesses() != null) {
14786                        List<String> processIds = new ArrayList<String>();
14787                        for (Process process : sequenceModel.getProcesses()) {
14788                                processIds.add(String.valueOf(process.getId()));
14789                        }
14790                        sequenceElement.setProcessIds(processIds);
14791                }
14792                dataflow.getSequences().add(sequenceElement);
14793
14794                List<TableColumn> columns = sequenceModel.getColumns();
14795
14796                for (int j = 0; j < columns.size(); j++) {
14797                        TableColumn columnModel = (TableColumn) columns.get(j);
14798                        column columnElement = new column();
14799                        columnElement.setId(String.valueOf(columnModel.getId()));
14800                        columnElement.setName(SQLUtil.trimColumnStringQuote(columnModel.getName()));
14801                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
14802                                        + convertCoordinate(columnModel.getEndPosition()));
14803                        sequenceElement.getColumns().add(columnElement);
14804                }
14805
14806                TableRelationRows relationRows = sequenceModel.getRelationRows();
14807                if (relationRows.hasRelation()) {
14808                        column relationRowsElement = new column();
14809                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
14810                        relationRowsElement.setName(relationRows.getName());
14811                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
14812                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
14813                                                + convertCoordinate(relationRows.getEndPosition()));
14814                        }
14815                        relationRowsElement.setSource("system");
14816                        sequenceElement.getColumns().add(relationRowsElement);
14817                }
14818        }
14819
14820        private void appendDataSourceModel(dataflow dataflow, Table datasourceModel) {
14821                table datasourceElement = new table();
14822                datasourceElement.setId(String.valueOf(datasourceModel.getId()));
14823                if (!SQLUtil.isEmpty(datasourceModel.getDatabase())) {
14824                        datasourceElement.setDatabase(datasourceModel.getDatabase());
14825                }
14826                if (!SQLUtil.isEmpty(datasourceModel.getSchema())) {
14827                        datasourceElement.setSchema(datasourceModel.getSchema());
14828                }
14829                datasourceElement.setServer(datasourceModel.getServer());
14830                datasourceElement.setName(datasourceModel.getName());
14831                datasourceElement.setType("datasource");
14832                datasourceElement.setLocation(datasourceModel.getLocation());
14833                if (datasourceModel.getFileType() != null) {
14834                        datasourceElement.setFileType(SQLUtil.trimColumnStringQuote(datasourceModel.getFileType()));
14835                }
14836
14837                if (datasourceModel.getStartPosition() != null && datasourceModel.getEndPosition() != null) {
14838                        datasourceElement.setCoordinate(convertCoordinate(datasourceModel.getStartPosition()) + ","
14839                                        + convertCoordinate(datasourceModel.getEndPosition()));
14840                }
14841
14842                if (datasourceModel.getProcesses() != null) {
14843                        List<String> processIds = new ArrayList<String>();
14844                        for (Process process : datasourceModel.getProcesses()) {
14845                                processIds.add(String.valueOf(process.getId()));
14846                        }
14847                        datasourceElement.setProcessIds(processIds);
14848                }
14849                dataflow.getDatasources().add(datasourceElement);
14850
14851                List<TableColumn> columns = datasourceModel.getColumns();
14852
14853                for (int j = 0; j < columns.size(); j++) {
14854                        TableColumn columnModel = (TableColumn) columns.get(j);
14855                        column columnElement = new column();
14856                        columnElement.setId(String.valueOf(columnModel.getId()));
14857                        columnElement.setName(SQLUtil.trimColumnStringQuote(columnModel.getName()));
14858                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
14859                                        + convertCoordinate(columnModel.getEndPosition()));
14860                        datasourceElement.getColumns().add(columnElement);
14861                }
14862
14863                TableRelationRows relationRows = datasourceModel.getRelationRows();
14864                if (relationRows.hasRelation()) {
14865                        column relationRowsElement = new column();
14866                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
14867                        relationRowsElement.setName(relationRows.getName());
14868                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
14869                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
14870                                                + convertCoordinate(relationRows.getEndPosition()));
14871                        }
14872                        relationRowsElement.setSource("system");
14873                        datasourceElement.getColumns().add(relationRowsElement);
14874                }
14875        }
14876
14877        private void appendDatabaseModel(dataflow dataflow, Table databaseModel) {
14878                table databaseElement = new table();
14879                databaseElement.setId(String.valueOf(databaseModel.getId()));
14880                if (!SQLUtil.isEmpty(databaseModel.getDatabase())) {
14881                        databaseElement.setDatabase(databaseModel.getDatabase());
14882                }
14883                if (!SQLUtil.isEmpty(databaseModel.getSchema())) {
14884                        databaseElement.setSchema(databaseModel.getSchema());
14885                }
14886                databaseElement.setServer(databaseModel.getServer());
14887                databaseElement.setName(databaseModel.getName());
14888                databaseElement.setType("database");
14889                if (databaseModel.getFileType() != null) {
14890                        databaseElement.setFileType(SQLUtil.trimColumnStringQuote(databaseModel.getFileType()));
14891                }
14892
14893                if (databaseModel.getStartPosition() != null && databaseModel.getEndPosition() != null) {
14894                        databaseElement.setCoordinate(convertCoordinate(databaseModel.getStartPosition()) + ","
14895                                        + convertCoordinate(databaseModel.getEndPosition()));
14896                }
14897
14898                if (databaseModel.getProcesses() != null) {
14899                        List<String> processIds = new ArrayList<String>();
14900                        for (Process process : databaseModel.getProcesses()) {
14901                                processIds.add(String.valueOf(process.getId()));
14902                        }
14903                        databaseElement.setProcessIds(processIds);
14904                }
14905                dataflow.getDatabases().add(databaseElement);
14906
14907                List<TableColumn> columns = databaseModel.getColumns();
14908
14909                for (int j = 0; j < columns.size(); j++) {
14910                        TableColumn columnModel = (TableColumn) columns.get(j);
14911                        column columnElement = new column();
14912                        columnElement.setId(String.valueOf(columnModel.getId()));
14913                        columnElement.setName(SQLUtil.trimColumnStringQuote(columnModel.getName()));
14914                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
14915                                        + convertCoordinate(columnModel.getEndPosition()));
14916                        databaseElement.getColumns().add(columnElement);
14917                }
14918
14919                TableRelationRows relationRows = databaseModel.getRelationRows();
14920                if (relationRows.hasRelation()) {
14921                        column relationRowsElement = new column();
14922                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
14923                        relationRowsElement.setName(relationRows.getName());
14924                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
14925                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
14926                                                + convertCoordinate(relationRows.getEndPosition()));
14927                        }
14928                        relationRowsElement.setSource("system");
14929                        databaseElement.getColumns().add(relationRowsElement);
14930                }
14931        }
14932
14933        private void appendSchemaModel(dataflow dataflow, Table schemaModel) {
14934                table schemaElement = new table();
14935                schemaElement.setId(String.valueOf(schemaModel.getId()));
14936                if (!SQLUtil.isEmpty(schemaModel.getDatabase())) {
14937                        schemaElement.setDatabase(schemaModel.getDatabase());
14938                }
14939                if (!SQLUtil.isEmpty(schemaModel.getSchema())) {
14940                        schemaElement.setSchema(schemaModel.getSchema());
14941                }
14942                schemaElement.setServer(schemaModel.getServer());
14943                schemaElement.setName(schemaModel.getName());
14944                schemaElement.setType("schema");
14945                if (schemaModel.getFileType() != null) {
14946                        schemaElement.setFileType(SQLUtil.trimColumnStringQuote(schemaModel.getFileType()));
14947                }
14948
14949                if (schemaModel.getStartPosition() != null && schemaModel.getEndPosition() != null) {
14950                        schemaElement.setCoordinate(convertCoordinate(schemaModel.getStartPosition()) + ","
14951                                        + convertCoordinate(schemaModel.getEndPosition()));
14952                }
14953
14954                if (schemaModel.getProcesses() != null) {
14955                        List<String> processIds = new ArrayList<String>();
14956                        for (Process process : schemaModel.getProcesses()) {
14957                                processIds.add(String.valueOf(process.getId()));
14958                        }
14959                        schemaElement.setProcessIds(processIds);
14960                }
14961                dataflow.getSchemas().add(schemaElement);
14962
14963                List<TableColumn> columns = schemaModel.getColumns();
14964
14965                for (int j = 0; j < columns.size(); j++) {
14966                        TableColumn columnModel = (TableColumn) columns.get(j);
14967                        column columnElement = new column();
14968                        columnElement.setId(String.valueOf(columnModel.getId()));
14969                        columnElement.setName(SQLUtil.trimColumnStringQuote(columnModel.getName()));
14970                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
14971                                        + convertCoordinate(columnModel.getEndPosition()));
14972                        schemaElement.getColumns().add(columnElement);
14973                }
14974
14975                TableRelationRows relationRows = schemaModel.getRelationRows();
14976                if (relationRows.hasRelation()) {
14977                        column relationRowsElement = new column();
14978                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
14979                        relationRowsElement.setName(relationRows.getName());
14980                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
14981                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
14982                                                + convertCoordinate(relationRows.getEndPosition()));
14983                        }
14984                        relationRowsElement.setSource("system");
14985                        schemaElement.getColumns().add(relationRowsElement);
14986                }
14987        }
14988
14989        private void appendPathModel(dataflow dataflow, Table pathModel) {
14990                table pathElement = new table();
14991                pathElement.setId(String.valueOf(pathModel.getId()));
14992                if (!SQLUtil.isEmpty(pathModel.getDatabase())) {
14993                        pathElement.setDatabase(pathModel.getDatabase());
14994                }
14995                if (!SQLUtil.isEmpty(pathModel.getSchema())) {
14996                        pathElement.setSchema(pathModel.getSchema());
14997                }
14998                pathElement.setServer(pathModel.getServer());
14999                pathElement.setName(pathModel.getName());
15000                pathElement.setType("path");
15001                if (pathModel.getFileFormat() != null) {
15002                        pathElement.setFileFormat(SQLUtil.trimColumnStringQuote(pathModel.getFileFormat()));
15003                }
15004
15005                if (pathModel.getStartPosition() != null && pathModel.getEndPosition() != null) {
15006                        pathElement.setCoordinate(convertCoordinate(pathModel.getStartPosition()) + ","
15007                                        + convertCoordinate(pathModel.getEndPosition()));
15008                }
15009
15010                if (pathModel.getProcesses() != null) {
15011                        List<String> processIds = new ArrayList<String>();
15012                        for (Process process : pathModel.getProcesses()) {
15013                                processIds.add(String.valueOf(process.getId()));
15014                        }
15015                        pathElement.setProcessIds(processIds);
15016                }
15017
15018                pathElement.setUri(pathModel.getName());
15019
15020                dataflow.getPaths().add(pathElement);
15021
15022                List<TableColumn> columns = pathModel.getColumns();
15023
15024                for (int j = 0; j < columns.size(); j++) {
15025                        TableColumn columnModel = (TableColumn) columns.get(j);
15026                        column columnElement = new column();
15027                        columnElement.setId(String.valueOf(columnModel.getId()));
15028                        columnElement.setName(SQLUtil.trimColumnStringQuote(columnModel.getName()));
15029                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
15030                                        + convertCoordinate(columnModel.getEndPosition()));
15031                        pathElement.getColumns().add(columnElement);
15032                }
15033
15034                TableRelationRows relationRows = pathModel.getRelationRows();
15035                if (relationRows.hasRelation()) {
15036                        column relationRowsElement = new column();
15037                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
15038                        relationRowsElement.setName(relationRows.getName());
15039                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
15040                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
15041                                                + convertCoordinate(relationRows.getEndPosition()));
15042                        }
15043                        relationRowsElement.setSource("system");
15044                        pathElement.getColumns().add(relationRowsElement);
15045                }
15046        }
15047
15048        private void appendVariableModel(dataflow dataflow, Table variableModel) {
15049                table variableElement = new table();
15050                variableElement.setId(String.valueOf(variableModel.getId()));
15051                if (!SQLUtil.isEmpty(variableModel.getDatabase())) {
15052                        variableElement.setDatabase(variableModel.getDatabase());
15053                }
15054                if (!SQLUtil.isEmpty(variableModel.getSchema())) {
15055                        variableElement.setSchema(variableModel.getSchema());
15056                }
15057                variableElement.setServer(variableModel.getServer());
15058                variableElement.setName(variableModel.getName());
15059                variableElement.setType("variable");
15060                variableElement.setParent(variableModel.getParent());
15061                if (variableModel.getSubType() != null) {
15062                        variableElement.setSubType(variableModel.getSubType().name());
15063                }
15064
15065                if (variableModel.getStartPosition() != null && variableModel.getEndPosition() != null) {
15066                        variableElement.setCoordinate(convertCoordinate(variableModel.getStartPosition()) + ","
15067                                        + convertCoordinate(variableModel.getEndPosition()));
15068                }
15069                dataflow.getVariables().add(variableElement);
15070
15071                List<TableColumn> columns = variableModel.getColumns();
15072
15073                if (containStarColumn(columns)) {
15074                        for (TableColumn column : columns) {
15075                                if (column.getName().endsWith("*")) {
15076                                        for (TableColumn starElement : columns) {
15077                                                if (starElement == column) {
15078                                                        continue;
15079                                                }
15080                                                TObjectName columnObject = starElement.getColumnObject();
15081                                                column.bindStarLinkColumn(columnObject);
15082                                        }
15083//                                      column.setShowStar(false);
15084                                }
15085                        }
15086                }
15087
15088                for (int j = 0; j < columns.size(); j++) {
15089                        TableColumn columnModel = columns.get(j);
15090                        if (columnModel.hasStarLinkColumn()) {
15091                                List<String> starLinkColumnList = columnModel.getStarLinkColumnNames();
15092                                for (int k = 0; k < starLinkColumnList.size(); k++) {
15093                                        column columnElement = new column();
15094                                        columnElement.setId(columnModel.getId() + "_" + k);
15095                                        String columnName = starLinkColumnList.get(k);
15096                                        if (containStarColumn(columns, columnName)) {
15097                                                continue;
15098                                        }
15099                                        columnElement.setName(columnName);
15100                                        if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
15101                                                columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
15102                                                                + convertCoordinate(columnModel.getEndPosition()));
15103                                        }
15104                                        variableElement.getColumns().add(columnElement);
15105                                }
15106
15107                                if (columnModel.isShowStar()) {
15108                                        column columnElement = new column();
15109                                        columnElement.setId(String.valueOf(columnModel.getId()));
15110                                        columnElement.setName(columnModel.getName());
15111                                        if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
15112                                                columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
15113                                                                + convertCoordinate(columnModel.getEndPosition()));
15114                                        }
15115                                        variableElement.getColumns().add(columnElement);
15116                                }
15117
15118                        } else {
15119                                column columnElement = new column();
15120                                columnElement.setId(String.valueOf(columnModel.getId()));
15121                                columnElement.setName(columnModel.getName());
15122                                if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
15123                                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
15124                                                        + convertCoordinate(columnModel.getEndPosition()));
15125                                }
15126                                variableElement.getColumns().add(columnElement);
15127                        }
15128                }
15129
15130                TableRelationRows relationRows = variableModel.getRelationRows();
15131                if (relationRows.hasRelation()) {
15132                        column relationRowsElement = new column();
15133                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
15134                        relationRowsElement.setName(relationRows.getName());
15135                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
15136                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
15137                                                + convertCoordinate(relationRows.getEndPosition()));
15138                        }
15139                        relationRowsElement.setSource("system");
15140                        variableElement.getColumns().add(relationRowsElement);
15141                }
15142        }
15143
15144        private void appendCursorModel(dataflow dataflow, Table cursorModel) {
15145                table cursorElement = new table();
15146                cursorElement.setId(String.valueOf(cursorModel.getId()));
15147                if (!SQLUtil.isEmpty(cursorModel.getDatabase())) {
15148                        cursorElement.setDatabase(cursorModel.getDatabase());
15149                }
15150                if (!SQLUtil.isEmpty(cursorModel.getSchema())) {
15151                        cursorElement.setSchema(cursorModel.getSchema());
15152                }
15153                cursorElement.setServer(cursorModel.getServer());
15154                cursorElement.setName(cursorModel.getName());
15155                cursorElement.setType("variable");
15156                if (cursorElement.getSubType() != null) {
15157                        cursorElement.setSubType(cursorModel.getSubType().name());
15158                }
15159
15160                if (cursorModel.getStartPosition() != null && cursorModel.getEndPosition() != null) {
15161                        cursorElement.setCoordinate(convertCoordinate(cursorModel.getStartPosition()) + ","
15162                                        + convertCoordinate(cursorModel.getEndPosition()));
15163                }
15164                dataflow.getVariables().add(cursorElement);
15165
15166                List<TableColumn> columns = cursorModel.getColumns();
15167
15168                if (containStarColumn(columns)) {
15169                        for (TableColumn column : columns) {
15170                                if (column.getName().endsWith("*")) {
15171                                        for (TableColumn starElement : columns) {
15172                                                if (starElement == column) {
15173                                                        continue;
15174                                                }
15175                                                TObjectName columnObject = starElement.getColumnObject();
15176                                                column.bindStarLinkColumn(columnObject);
15177                                        }
15178                                        column.setShowStar(false);
15179                                }
15180                        }
15181                }
15182
15183                for (int j = 0; j < columns.size(); j++) {
15184                        TableColumn columnModel = (TableColumn) columns.get(j);
15185                        if (columnModel.hasStarLinkColumn()) {
15186                                List<String> starLinkColumnList = columnModel.getStarLinkColumnNames();
15187                                for (int k = 0; k < starLinkColumnList.size(); k++) {
15188                                        column columnElement = new column();
15189                                        columnElement.setId(String.valueOf(columnModel.getId()) + "_" + k);
15190                                        String columnName = starLinkColumnList.get(k);
15191                                        if (containStarColumn(columns, columnName)) {
15192                                                continue;
15193                                        }
15194                                        columnElement.setName(columnName);
15195                                        if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
15196                                                columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
15197                                                                + convertCoordinate(columnModel.getEndPosition()));
15198                                        }
15199                                        cursorElement.getColumns().add(columnElement);
15200                                }
15201
15202                                if (columnModel.isShowStar()) {
15203                                        column columnElement = new column();
15204                                        columnElement.setId(String.valueOf(columnModel.getId()));
15205                                        columnElement.setName(columnModel.getName());
15206                                        if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
15207                                                columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
15208                                                                + convertCoordinate(columnModel.getEndPosition()));
15209                                        }
15210                                        cursorElement.getColumns().add(columnElement);
15211                                }
15212
15213                        } else {
15214                                column columnElement = new column();
15215                                columnElement.setId(String.valueOf(columnModel.getId()));
15216                                columnElement.setName(columnModel.getName());
15217                                if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
15218                                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
15219                                                        + convertCoordinate(columnModel.getEndPosition()));
15220                                }
15221                                cursorElement.getColumns().add(columnElement);
15222                        }
15223                }
15224
15225                TableRelationRows relationRows = cursorModel.getRelationRows();
15226                if (relationRows.hasRelation()) {
15227                        column relationRowsElement = new column();
15228                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
15229                        relationRowsElement.setName(relationRows.getName());
15230                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
15231                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
15232                                                + convertCoordinate(relationRows.getEndPosition()));
15233                        }
15234                        relationRowsElement.setSource("system");
15235                        cursorElement.getColumns().add(relationRowsElement);
15236                }
15237        }
15238
15239        private void appendErrors(dataflow dataflow) {
15240                List<ErrorInfo> errorInfos = this.getErrorMessages();
15241                if (metadataErrors != null) {
15242                        for (int i = 0; i < metadataErrors.size(); i++) {
15243                                errorInfos.add(i, metadataErrors.get(i));
15244                        }
15245                }
15246
15247                for (int i = 0; i < errorInfos.size(); ++i) {
15248                        ErrorInfo errorInfo = errorInfos.get(i);
15249                        error error = new error();
15250                        if (!SQLUtil.isEmpty(errorInfo.getErrorMessage())) {
15251                                error.setErrorMessage(errorInfo.getErrorMessage());
15252                        }
15253                        if (!SQLUtil.isEmpty(errorInfo.getErrorType())) {
15254                                error.setErrorType(errorInfo.getErrorType());
15255                        }
15256                        if (errorInfo.getStartPosition() != null && errorInfo.getEndPosition() != null) {
15257                                error.setCoordinate(convertCoordinate(errorInfo.getStartPosition()) + ","
15258                                                + convertCoordinate(errorInfo.getEndPosition()));
15259                        }
15260                        if (!SQLUtil.isEmpty(errorInfo.getFileName())) {
15261                                error.setFile(errorInfo.getFileName());
15262                        }
15263                        if (errorInfo.getOriginStartPosition() != null && errorInfo.getOriginEndPosition() != null) {
15264                                error.setOriginCoordinate(errorInfo.getOriginStartPosition() + "," + errorInfo.getOriginEndPosition());
15265                        }
15266                        dataflow.getErrors().add(error);
15267                }
15268        }
15269
15270        private void appendOraclePackages(dataflow dataflow) {
15271                List<OraclePackage> packages = this.modelManager.getOraclePackageModels();
15272
15273                for (int i = 0; i < packages.size(); ++i) {
15274                        OraclePackage model = packages.get(i);
15275                        oraclePackage oraclePackage = new oraclePackage();
15276                        oraclePackage.setId(String.valueOf(model.getId()));
15277                        if (!SQLUtil.isEmpty(model.getDatabase())) {
15278                                oraclePackage.setDatabase(model.getDatabase());
15279                        }
15280                        if (!SQLUtil.isEmpty(model.getSchema())) {
15281                                oraclePackage.setSchema(model.getSchema());
15282                        }
15283                        oraclePackage.setServer(model.getServer());
15284                        oraclePackage.setName(model.getName());
15285                        if (model.getType() != null) {
15286                                oraclePackage.setType(model.getType().name().replace("sst", ""));
15287                        }
15288                        if (model.getStartPosition() != null && model.getEndPosition() != null) {
15289                                oraclePackage.setCoordinate(
15290                                                convertCoordinate(model.getStartPosition()) + "," + convertCoordinate(model.getEndPosition()));
15291                        }
15292
15293                        dataflow.getPackages().add(oraclePackage);
15294
15295                        List<Argument> arguments = model.getArguments();
15296
15297                        for (int j = 0; j < arguments.size(); ++j) {
15298                                Argument argumentModel = (Argument) arguments.get(j);
15299                                argument argumentElement = new argument();
15300                                argumentElement.setId(String.valueOf(argumentModel.getId()));
15301                                argumentElement.setName(argumentModel.getName());
15302                                if (argumentModel.getStartPosition() != null && argumentModel.getEndPosition() != null) {
15303                                        argumentElement.setCoordinate(convertCoordinate(argumentModel.getStartPosition()) + ","
15304                                                        + convertCoordinate(argumentModel.getEndPosition()));
15305                                }
15306
15307                                argumentElement.setDatatype(argumentModel.getDataType().getDataTypeName());
15308                                argumentElement.setInout(argumentModel.getMode().name());
15309                                oraclePackage.getArguments().add(argumentElement);
15310                        }
15311
15312                        for (int j = 0; j < model.getProcedures().size(); j++) {
15313                                Procedure procedureModel = model.getProcedures().get(j);
15314                                procedure procedure = new procedure();
15315                                procedure.setId(String.valueOf(procedureModel.getId()));
15316                                if (!SQLUtil.isEmpty(procedureModel.getDatabase())) {
15317                                        procedure.setDatabase(procedureModel.getDatabase());
15318                                }
15319                                if (!SQLUtil.isEmpty(procedureModel.getSchema())) {
15320                                        procedure.setSchema(procedureModel.getSchema());
15321                                }
15322                                procedure.setServer(procedureModel.getServer());
15323                                procedure.setName(procedureModel.getName());
15324                                if (procedureModel.getType() != null) {
15325                                        procedure.setType(procedureModel.getType().name().replace("sst", ""));
15326                                }
15327                                if (procedureModel.getStartPosition() != null && procedureModel.getEndPosition() != null) {
15328                                        procedure.setCoordinate(convertCoordinate(procedureModel.getStartPosition()) + ","
15329                                                        + convertCoordinate(procedureModel.getEndPosition()));
15330                                }
15331
15332                                oraclePackage.getProcedures().add(procedure);
15333
15334                                List<Argument> procedureArguments = procedureModel.getArguments();
15335
15336                                for (int k = 0; k < procedureArguments.size(); ++k) {
15337                                        Argument argumentModel = (Argument) procedureArguments.get(k);
15338                                        argument argumentElement = new argument();
15339                                        argumentElement.setId(String.valueOf(argumentModel.getId()));
15340                                        argumentElement.setName(argumentModel.getName());
15341                                        if (argumentModel.getStartPosition() != null && argumentModel.getEndPosition() != null) {
15342                                                argumentElement.setCoordinate(convertCoordinate(argumentModel.getStartPosition()) + ","
15343                                                                + convertCoordinate(argumentModel.getEndPosition()));
15344                                        }
15345
15346                                        argumentElement.setDatatype(argumentModel.getDataType().getDataTypeName());
15347                                        argumentElement.setInout(argumentModel.getMode().name());
15348                                        procedure.getArguments().add(argumentElement);
15349                                }
15350                        }
15351                }
15352        }
15353
15354        private void appendProcedures(dataflow dataflow) {
15355                List<Procedure> procedures = this.modelManager.getProcedureModels();
15356
15357                for (int i = 0; i < procedures.size(); ++i) {
15358                        Procedure model = procedures.get(i);
15359                        if (model.getParentPackage() != null) {
15360                                continue;
15361                        }
15362                        procedure procedure = new procedure();
15363                        procedure.setId(String.valueOf(model.getId()));
15364                        if (!SQLUtil.isEmpty(model.getDatabase())) {
15365                                procedure.setDatabase(model.getDatabase());
15366                        }
15367                        if (!SQLUtil.isEmpty(model.getSchema())) {
15368                                procedure.setSchema(model.getSchema());
15369                        }
15370                        procedure.setServer(model.getServer());
15371                        procedure.setName(model.getName());
15372                        if (model.getType() != null) {
15373                                procedure.setType(model.getType().name().replace("sst", ""));
15374                        }
15375                        if (model.getStartPosition() != null && model.getEndPosition() != null) {
15376                                procedure.setCoordinate(
15377                                                convertCoordinate(model.getStartPosition()) + "," + convertCoordinate(model.getEndPosition()));
15378                        }
15379
15380                        dataflow.getProcedures().add(procedure);
15381
15382                        List<Argument> arguments = model.getArguments();
15383
15384                        for (int j = 0; j < arguments.size(); ++j) {
15385                                Argument argumentModel = (Argument) arguments.get(j);
15386                                argument argumentElement = new argument();
15387                                argumentElement.setId(String.valueOf(argumentModel.getId()));
15388                                argumentElement.setName(argumentModel.getName());
15389                                if (argumentModel.getStartPosition() != null && argumentModel.getEndPosition() != null) {
15390                                        argumentElement.setCoordinate(convertCoordinate(argumentModel.getStartPosition()) + ","
15391                                                        + convertCoordinate(argumentModel.getEndPosition()));
15392                                }
15393
15394                                argumentElement.setDatatype(argumentModel.getDataType().getDataTypeName());
15395                                argumentElement.setInout(argumentModel.getMode().name());
15396                                procedure.getArguments().add(argumentElement);
15397                        }
15398                }
15399        }
15400
15401        private void appendProcesses(dataflow dataflow) {
15402                List<Process> processes = this.modelManager.getProcessModels();
15403
15404                for (int i = 0; i < processes.size(); ++i) {
15405                        Process model = processes.get(i);
15406                        process process = new process();
15407                        process.setId(String.valueOf(model.getId()));
15408                        if (!SQLUtil.isEmpty(model.getDatabase())) {
15409                                process.setDatabase(model.getDatabase());
15410                        }
15411                        if (!SQLUtil.isEmpty(model.getSchema())) {
15412                                process.setSchema(model.getSchema());
15413                        }
15414                        process.setServer(model.getServer());
15415                        process.setName(getProcessName(model));
15416                        if (!SQLUtil.isEmpty(model.getProcedureName())) {
15417                                process.setProcedureName(model.getProcedureName());
15418                        }
15419                        if (model.getProcedureId() != null) {
15420                                process.setProcedureId(String.valueOf(model.getProcedureId()));
15421                        }
15422                        if (!SQLUtil.isEmpty(model.getQueryHashId())) {
15423                                process.setQueryHashId(model.getQueryHashId());
15424                        }
15425                        if (model.getGspObject() != null) {
15426                                process.setType(model.getGspObject().sqlstatementtype.name());
15427                        }
15428                        if (model.getStartPosition() != null && model.getEndPosition() != null) {
15429                                process.setCoordinate(
15430                                                convertCoordinate(model.getStartPosition()) + "," + convertCoordinate(model.getEndPosition()));
15431                        }
15432                        if (model.getTransforms() != null && !model.getTransforms().isEmpty()) {
15433                                for (Transform transformItem : model.getTransforms()) {
15434                                        process.addTransform(transformItem);
15435                                }
15436                        }
15437                        dataflow.getProcesses().add(process);
15438                }
15439        }
15440
15441        private void appendTables(dataflow dataflow) {
15442                List<TTable> tables = modelManager.getBaseTables();
15443                Map<String, table> tableMap = new HashMap<String, table>();
15444                Set<Long> tableModelIds = new HashSet<Long>();
15445                for (int i = 0; i < tables.size(); i++) {
15446                        Object model = modelManager.getModel(tables.get(i));
15447                        if (model instanceof Table) {
15448                                Table tableModel = (Table) model;
15449                                if(tableModelIds.contains(tableModel.getId())) {
15450                                        continue;
15451                                }
15452                                else {
15453                                        tableModelIds.add(tableModel.getId());
15454                                }
15455                                if (tableModel.isView()) {
15456                                        continue;
15457                                }
15458                                if (tableModel.isStage()) {
15459                                        appendStageModel(dataflow, tableModel);
15460                                        continue;
15461                                }
15462                                if (tableModel.isSequence()) {
15463                                        appendSequenceModel(dataflow, tableModel);
15464                                        continue;
15465                                }
15466                                if (tableModel.isDataSource()) {
15467                                        appendDataSourceModel(dataflow, tableModel);
15468                                        continue;
15469                                }
15470                                if (tableModel.isDatabase()) {
15471                                        appendDatabaseModel(dataflow, tableModel);
15472                                        continue;
15473                                }
15474                                if (tableModel.isSchema()) {
15475                                        appendSchemaModel(dataflow, tableModel);
15476                                        continue;
15477                                }
15478                                if (tableModel.isStream()) {
15479                                        appendStreamModel(dataflow, tableModel);
15480                                        continue;
15481                                }
15482                                if (tableModel.isPath()) {
15483                                        appendPathModel(dataflow, tableModel);
15484                                        continue;
15485                                }
15486                                if (tableModel.isVariable() && !tableModel.isCursor()) {
15487                                        appendVariableModel(dataflow, tableModel);
15488                                        continue;
15489                                }
15490                                if (tableModel.isCursor()) {
15491                                        appendCursorModel(dataflow, tableModel);
15492                                        continue;
15493                                }
15494                                if (tableModel.isConstant()) {
15495                                        appendConstantModel(dataflow, tableModel);
15496                                        continue;
15497                                }
15498                                if (!tableIds.contains(tableModel.getId())) {
15499                                        appendTableModel(dataflow, tableModel, tableMap);
15500                                        tableIds.add(tableModel.getId());
15501                                }
15502                        } else if (model instanceof QueryTable) {
15503                                QueryTable queryTable = (QueryTable) model;
15504                                if (!tableIds.contains(queryTable.getId())) {
15505                                        appendResultSet(dataflow, queryTable);
15506                                        tableIds.add(queryTable.getId());
15507                                }
15508                        }
15509                }
15510
15511                List<Table> tableNames = modelManager.getTablesByName();
15512                tableNames.addAll(modelManager.getDropTables());
15513                
15514                for (int i = 0; i < tableNames.size(); i++) {
15515                        Table tableModel = tableNames.get(i);
15516                        if(tableModelIds.contains(tableModel.getId())) {
15517                                continue;
15518                        }
15519                        else {
15520                                tableModelIds.add(tableModel.getId());
15521                        }
15522                        if (tableModel.isView()) {
15523                                continue;
15524                        }
15525                        if (tableModel.isDatabase()) {
15526                                appendDatabaseModel(dataflow, tableModel);
15527                                continue;
15528                        }
15529                        if (tableModel.isSchema()) {
15530                                appendSchemaModel(dataflow, tableModel);
15531                                continue;
15532                        }
15533                        if (tableModel.isStage()) {
15534                                appendStageModel(dataflow, tableModel);
15535                                continue;
15536                        }
15537                        if (tableModel.isSequence()) {
15538                                appendSequenceModel(dataflow, tableModel);
15539                                continue;
15540                        }
15541                        if (tableModel.isDataSource()) {
15542                                appendDataSourceModel(dataflow, tableModel);
15543                                continue;
15544                        }
15545                        if (tableModel.isStream()) {
15546                                appendStreamModel(dataflow, tableModel);
15547                                continue;
15548                        }
15549                        if (tableModel.isPath()) {
15550                                appendPathModel(dataflow, tableModel);
15551                                continue;
15552                        }
15553                        if (tableModel.isVariable()) {
15554                                appendVariableModel(dataflow, tableModel);
15555                                continue;
15556                        }
15557                        if (tableModel.isCursor()) {
15558                                appendCursorModel(dataflow, tableModel);
15559                                continue;
15560                        }
15561                        if (tableModel.isConstant()) {
15562                                appendConstantModel(dataflow, tableModel);
15563                                continue;
15564                        }
15565                        if (!tableIds.contains(tableModel.getId())) {
15566                                appendTableModel(dataflow, tableModel, tableMap);
15567                                tableIds.add(tableModel.getId());
15568                        }
15569                }
15570        }
15571
15572        private void appendConstantModel(dataflow dataflow, Table tableModel) {
15573                table constantElement = new table();
15574                constantElement.setId(String.valueOf(tableModel.getId()));
15575                if (!SQLUtil.isEmpty(tableModel.getDatabase())) {
15576                        constantElement.setDatabase(tableModel.getDatabase());
15577                }
15578                if (!SQLUtil.isEmpty(tableModel.getSchema())) {
15579                        constantElement.setSchema(tableModel.getSchema());
15580                }
15581                constantElement.setServer(tableModel.getServer());
15582                constantElement.setName(getConstantName(tableModel));
15583                constantElement.setType("constantTable");
15584
15585                if (tableModel.getStartPosition() != null && tableModel.getEndPosition() != null) {
15586                        constantElement.setCoordinate(convertCoordinate(tableModel.getStartPosition()) + ","
15587                                        + convertCoordinate(tableModel.getEndPosition()));
15588                }
15589                dataflow.getTables().add(constantElement);
15590
15591                List<TableColumn> columns = tableModel.getColumns();
15592                for (int j = 0; j < columns.size(); j++) {
15593                        TableColumn columnModel = (TableColumn) columns.get(j);
15594                        column columnElement = new column();
15595                        columnElement.setId(String.valueOf(columnModel.getId()));
15596                        columnElement.setName(columnModel.getName());
15597                        if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
15598                                columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
15599                                                + convertCoordinate(columnModel.getEndPosition()));
15600                        }
15601                        constantElement.getColumns().add(columnElement);
15602                }
15603        }
15604
15605        /**
15606         * Copies the authoritative endpoint classification from the internal model onto the
15607         * XML element, emitting each field ONLY when it carries information (kind/intro not
15608         * UNKNOWN, createdInSql only when true, normalizedName only when it differs from the
15609         * raw name). This keeps existing XML byte-identical for endpoints that aren't
15610         * classified, honouring the strictly-additive contract (§9 of
15611         * docs/tmp/dlineage-authoritative-endpoint-classification.md).
15612         */
15613        private void appendEndpointClassification(table tableElement, Table tableModel) {
15614                boolean created = tableModel.isCreatedInSql();
15615                EndpointKind kind = tableModel.getEndpointKind();
15616                // Emit endpointKind only when it carries signal: a non-catalog special kind
15617                // (temp / global-temp / table-variable / tempdb) always, or CATALOG_OBJECT only
15618                // when the object is created here. A plainly-referenced catalog table is the
15619                // default case and carries no attribute, so existing golden XML for ordinary
15620                // table references stays byte-identical.
15621                if (kind != null && kind != EndpointKind.UNKNOWN
15622                                && (created || kind != EndpointKind.CATALOG_OBJECT)) {
15623                        tableElement.setEndpointKind(kind.name());
15624                }
15625                EndpointIntroduction intro = tableModel.getEndpointIntroduction();
15626                if (intro != null && intro != EndpointIntroduction.UNKNOWN) {
15627                        tableElement.setEndpointIntroduction(intro.name());
15628                }
15629                if (created) {
15630                        tableElement.setCreatedInSql("true");
15631                }
15632                // Emit normalizedName only when the raw name actually carries delimiters
15633                // (brackets / quotes / backticks) — that is when the normalized form is useful
15634                // and not trivially derivable. Avoids case-only churn on every plain name.
15635                String rawName = tableModel.getName();
15636                if (rawName != null && hasIdentifierDelimiter(rawName)) {
15637                        String normalized = tableModel.getNormalizedName();
15638                        if (normalized != null && !normalized.equals(rawName)) {
15639                                tableElement.setNormalizedName(normalized);
15640                        }
15641                }
15642        }
15643
15644        private static boolean hasIdentifierDelimiter(String name) {
15645                return name.indexOf('[') != -1 || name.indexOf(']') != -1
15646                                || name.indexOf('`') != -1 || name.indexOf('"') != -1;
15647        }
15648
15649        private void appendTableModel(dataflow dataflow, Table tableModel, Map<String, table> tableMap) {
15650                if(tableModel.getSubType() == SubType.unnest) {}
15651                table tableElement = new table();
15652                tableElement.setId(String.valueOf(tableModel.getId()));
15653                // A one-part name that is ambiguous across two or more schemas (with no object
15654                // in the default schema) must NOT be bound to a fabricated default-schema
15655                // object: surface it unqualified and expose the candidate set instead.
15656                boolean ambiguousUnqualified = tableModel.isAmbiguousUnqualifiedTable()
15657                                && tableModel.getCandidateTables() != null
15658                                && !tableModel.getCandidateTables().isEmpty();
15659                if (!ambiguousUnqualified) {
15660                        if (!SQLUtil.isEmpty(tableModel.getDatabase())) {
15661                                tableElement.setDatabase(tableModel.getDatabase());
15662                        }
15663                        if (!SQLUtil.isEmpty(tableModel.getSchema())) {
15664                                tableElement.setSchema(tableModel.getSchema());
15665                        }
15666                }
15667                tableElement.setServer(tableModel.getServer());
15668                if (ambiguousUnqualified) {
15669                        tableElement.setName(DlineageUtil.getSimpleTableName(tableModel.getName()));
15670                        tableElement.setCandidateTables(new ArrayList<String>(tableModel.getCandidateTables()));
15671                } else {
15672                        tableElement.setName(tableModel.getName());
15673                }
15674                tableElement.setDisplayName(tableModel.getDisplayName());
15675                tableElement.setStarStmt(tableModel.getStarStmt());
15676                if(tableModel.isFromDDL()) {
15677                        tableElement.setFromDDL(String.valueOf(tableModel.isFromDDL()));
15678                }
15679
15680                // Authoritative endpoint classification (additive). Emit only non-default
15681                // values so existing XML stays byte-identical for unclassified endpoints.
15682                // See docs/tmp/dlineage-authoritative-endpoint-classification.md.
15683                appendEndpointClassification(tableElement, tableModel);
15684
15685                if (tableModel.isPseudo()) {
15686                        tableElement.setType("pseudoTable");
15687                } else {
15688                        tableElement.setType("table");
15689                }
15690
15691                if (tableModel.getSubType() != null) {
15692                        if (tableModel.getSubType() == SubType.unnest) {
15693                                tableElement.setType(SubType.unnest.name());
15694                        }
15695                        else {
15696                                tableElement.setSubType(tableModel.getSubType().name());
15697                        }
15698                }
15699                if (tableModel.getParent() != null) {
15700                        tableElement.setParent(tableModel.getParent());
15701                }
15702                if (tableModel.getAlias() != null && tableModel.getAlias().trim().length() > 0) {
15703                        tableElement.setAlias(tableModel.getAlias());
15704                }
15705                if (tableModel.getStartPosition() != null && tableModel.getEndPosition() != null) {
15706            if(option.isTraceTablePosition()){
15707                for (Pair<Pair3<Long, Long, String>, Pair3<Long, Long, String>> position:tableModel.getPositions()){
15708                    tableElement.appendCoordinate(convertCoordinate(position.first)+","+convertCoordinate(position.second));
15709                }
15710            }
15711            else {
15712                tableElement.setCoordinate(convertCoordinate(tableModel.getStartPosition()) + ","
15713                    + convertCoordinate(tableModel.getEndPosition()));
15714            }
15715                }
15716                if (tableModel.getProcesses() != null) {
15717                        List<String> processIds = new ArrayList<String>();
15718                        for (Process process : tableModel.getProcesses()) {
15719                                processIds.add(String.valueOf(process.getId()));
15720                        }
15721                        tableElement.setProcessIds(processIds);
15722                }
15723
15724                table oldTableElement = null;
15725                String tableFullName = DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getQualifiedTableName(tableElement));
15726                
15727                if(tableMap.containsKey(tableFullName)) {
15728                        oldTableElement = tableMap.get(tableFullName);
15729                }
15730                else {
15731                        tableMap.put(tableFullName, tableElement);
15732                }
15733                
15734                if (tableModel.getSubType() == SubType.unnest) {
15735                        dataflow.getResultsets().add(tableElement);
15736                } else {
15737                        dataflow.getTables().add(tableElement);
15738                }
15739
15740                List<TableColumn> columns = tableModel.getColumns();
15741
15742                if (containStarColumn(columns)) {
15743                        for (TableColumn column : columns) {
15744                                if (column.getName().endsWith("*")) {
15745                                        for (TableColumn starElement : columns) {
15746                                                if (starElement == column) {
15747                                                        continue;
15748                                                }
15749                                                if (starElement.isNotBindStarLinkColumn()) {
15750                                                        continue;
15751                                                }
15752                                                TObjectName columnObject = starElement.getColumnObject();
15753                                                column.bindStarLinkColumn(columnObject);
15754                                        }
15755                                        if (tableModel.isCreateTable() && column.isExpandStar()) {
15756                                                column.setShowStar(false);
15757                                        }
15758                                }
15759                        }
15760                }
15761
15762                for (int j = 0; j < columns.size(); j++) {
15763                        TableColumn columnModel = columns.get(j);
15764
15765                        if(oldTableElement != null){
15766                                if(!CollectionUtil.isEmpty(oldTableElement.getColumns())){
15767                                        for(column oldColumnElement: oldTableElement.getColumns()){
15768                                                if(oldColumnElement.getName().equalsIgnoreCase(columnModel.getName())){
15769                                                        if(columnModel.getDataType() == null && oldColumnElement.getDataType() != null){
15770                                                                columnModel.setDataType(oldColumnElement.getDataType());
15771                                                        }
15772                                                        if(columnModel.getPrimaryKey() == null && oldColumnElement.isPrimaryKey() != null){
15773                                                                columnModel.setPrimaryKey(oldColumnElement.isPrimaryKey());
15774                                                        }
15775                                                        if(columnModel.getIndexKey() == null && oldColumnElement.isIndexKey() != null){
15776                                                                columnModel.setIndexKey(oldColumnElement.isIndexKey());
15777                                                        }
15778                                                        if(columnModel.getUnqiueKey() == null && oldColumnElement.isUnqiueKey() != null){
15779                                                                columnModel.setUnqiueKey(oldColumnElement.isUnqiueKey());
15780                                                        }
15781                                                        if(columnModel.getForeignKey() == null && oldColumnElement.isForeignKey() != null){
15782                                                                columnModel.setForeignKey(oldColumnElement.isForeignKey());
15783                                                        }
15784
15785                                                        if(oldColumnElement.getDataType() == null && columnModel.getDataType() != null){
15786                                                                oldColumnElement.setDataType(columnModel.getDataType());
15787                                                        }
15788                                                        if(oldColumnElement.isPrimaryKey() == null && columnModel.getPrimaryKey() != null){
15789                                                                oldColumnElement.setPrimaryKey(columnModel.getPrimaryKey());
15790                                                        }
15791                                                        if(oldColumnElement.isIndexKey() == null && columnModel.getIndexKey() != null){
15792                                                                oldColumnElement.setIndexKey(columnModel.getIndexKey());
15793                                                        }
15794                                                        if(oldColumnElement.isUnqiueKey() == null && columnModel.getUnqiueKey() != null){
15795                                                                oldColumnElement.setUnqiueKey(columnModel.getUnqiueKey());
15796                                                        }
15797                                                        if(oldColumnElement.isForeignKey() == null && columnModel.getForeignKey() != null){
15798                                                                oldColumnElement.setForeignKey(columnModel.getForeignKey());
15799                                                        }
15800                                                        break;
15801                                                }
15802                                        }
15803                                }
15804
15805                        }
15806
15807                        if (!columnModel.isPseduo() && columnModel.hasStarLinkColumn() && !columnModel.isVariant()) {
15808                                List<String> starLinkColumnList = columnModel.getStarLinkColumnNames();
15809                                for (int k = 0; k < starLinkColumnList.size(); k++) {
15810                                        column columnElement = new column();
15811                                        columnElement.setId(String.valueOf(columnModel.getId()) + "_" + k);
15812                                        String columnName = starLinkColumnList.get(k);
15813                                        if (containStarColumn(columns, columnName)) {
15814                                                continue;
15815                                        }
15816                                        columnElement.setName(columnName);
15817                                        if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
15818                                                columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
15819                                                                + convertCoordinate(columnModel.getEndPosition()));
15820                                        }
15821                                        if (columnModel.getForeignKey()) {
15822                                                columnElement.setForeignKey(columnModel.getForeignKey());
15823                                        }
15824                                        if (columnModel.getIndexKey()) {
15825                                                columnElement.setIndexKey(columnModel.getIndexKey());
15826                                        }
15827                                        if (columnModel.getPrimaryKey()) {
15828                                                columnElement.setPrimaryKey(columnModel.getPrimaryKey());
15829                                        }
15830                                        if (columnModel.getUnqiueKey()) {
15831                                                columnElement.setUnqiueKey(columnModel.getUnqiueKey());
15832                                        }
15833                                        if (columnModel.getDataType() != null) {
15834                                                columnElement.setDataType(columnModel.getDataType());
15835                                        }
15836                                        tableElement.getColumns().add(columnElement);
15837                                }
15838                                if (columnModel.isShowStar()) {
15839                                        column columnElement = new column();
15840                                        columnElement.setId(String.valueOf(columnModel.getId()));
15841                                        columnElement.setName(columnModel.getName());
15842                                        if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
15843                                                columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
15844                                                                + convertCoordinate(columnModel.getEndPosition()));
15845                                        }
15846                                        if (columnModel.getForeignKey()) {
15847                                                columnElement.setForeignKey(columnModel.getForeignKey());
15848                                        }
15849                                        if (columnModel.getIndexKey()) {
15850                                                columnElement.setIndexKey(columnModel.getIndexKey());
15851                                        }
15852                                        if (columnModel.getPrimaryKey()) {
15853                                                columnElement.setPrimaryKey(columnModel.getPrimaryKey());
15854                                        }
15855                                        if (columnModel.getUnqiueKey()) {
15856                                                columnElement.setUnqiueKey(columnModel.getUnqiueKey());
15857                                        }
15858                                        if (columnModel.getDataType() != null) {
15859                                                columnElement.setDataType(columnModel.getDataType());
15860                                        }
15861                                        tableElement.getColumns().add(columnElement);
15862                                }
15863                        } else {
15864                                column columnElement = new column();
15865                                columnElement.setId(String.valueOf(columnModel.getId()));
15866                                columnElement.setName(columnModel.getName());
15867                                columnElement.setDisplayName(columnModel.getDisplayName());
15868                                if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
15869                                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
15870                                                        + convertCoordinate(columnModel.getEndPosition()));
15871                                }
15872                                if (columnModel.isPseduo()) {
15873                                        columnElement.setSource("system");
15874                                }
15875                                if (columnModel.getForeignKey()) {
15876                                        columnElement.setForeignKey(columnModel.getForeignKey());
15877                                }
15878                                if (columnModel.getIndexKey()) {
15879                                        columnElement.setIndexKey(columnModel.getIndexKey());
15880                                }
15881                                if (columnModel.getPrimaryKey()) {
15882                                        columnElement.setPrimaryKey(columnModel.getPrimaryKey());
15883                                }
15884                                if (columnModel.getUnqiueKey()) {
15885                                        columnElement.setUnqiueKey(columnModel.getUnqiueKey());
15886                                }
15887                                if (columnModel.getDataType() != null) {
15888                                        columnElement.setDataType(columnModel.getDataType());
15889                                }
15890                                tableElement.getColumns().add(columnElement);
15891                        }
15892                }
15893
15894                TableRelationRows relationRows = tableModel.getRelationRows();
15895                if (relationRows.hasRelation()) {
15896                        column relationRowsElement = new column();
15897                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
15898                        relationRowsElement.setName(relationRows.getName());
15899                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
15900                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
15901                                                + convertCoordinate(relationRows.getEndPosition()));
15902                        }
15903                        relationRowsElement.setSource("system");
15904                        tableElement.getColumns().add(relationRowsElement);
15905                }
15906        }
15907
15908        private boolean containStarColumn(List<TableColumn> columns, String qualifiedColumnName) {
15909                for (TableColumn tableColumn : columns) {
15910                        if (DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()).equals(qualifiedColumnName)) {
15911                                return true;
15912                        }
15913                }
15914                return false;
15915        }
15916
15917        private TableColumn searchTableColumn(List<TableColumn> columns, String qualifiedColumnName) {
15918                for (TableColumn tableColumn : columns) {
15919                        if (DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()).equals(qualifiedColumnName)) {
15920                                return tableColumn;
15921                        }
15922                }
15923                return null;
15924        }
15925
15926        private boolean containStarColumn(Collection<RelationshipElement<?>> elements, String qualifiedColumnName) {
15927                for (RelationshipElement element : elements) {
15928                        if (element.getElement() instanceof TableColumn) {
15929                                if (DlineageUtil.getIdentifierNormalColumnName(((TableColumn) element.getElement()).getName())
15930                                                .equals(qualifiedColumnName)) {
15931                                        return true;
15932                                }
15933                        } else if (element.getElement() instanceof ResultColumn) {
15934                                if (DlineageUtil.getIdentifierNormalColumnName(((ResultColumn) element.getElement()).getName())
15935                                                .equals(qualifiedColumnName)) {
15936                                        return true;
15937                                }
15938                        }
15939                }
15940                return false;
15941        }
15942
15943        /**
15944         * Returns true if there is a non-star source element matching the column name
15945         * whose parent (table/resultset) does NOT also contribute a star source in
15946         * the same relationship. This identifies "definitive" explicit sources like
15947         * subquery columns (e.g., "coalesce(...) as col_a") vs incidental references
15948         * (e.g., "aTab.id" where aTab.* is also a source).
15949         */
15950        private boolean hasDefinitiveNonStarSource(Collection<RelationshipElement<?>> elements, String qualifiedColumnName) {
15951                // First, collect parent IDs of all star (*) sources
15952                Set<Long> starSourceParentIds = new HashSet<Long>();
15953                for (RelationshipElement<?> element : elements) {
15954                        if (element.getElement() instanceof TableColumn) {
15955                                TableColumn tc = (TableColumn) element.getElement();
15956                                if ("*".equals(tc.getName()) && tc.getTable() != null) {
15957                                        starSourceParentIds.add(tc.getTable().getId());
15958                                }
15959                        } else if (element.getElement() instanceof ResultColumn) {
15960                                ResultColumn rc = (ResultColumn) element.getElement();
15961                                if ("*".equals(rc.getName()) && rc.getResultSet() != null) {
15962                                        starSourceParentIds.add(rc.getResultSet().getId());
15963                                }
15964                        }
15965                }
15966
15967                // Then check if any non-star source matching the column name has a parent
15968                // that does NOT also have a star source
15969                for (RelationshipElement<?> element : elements) {
15970                        if (element.getElement() instanceof TableColumn) {
15971                                TableColumn tc = (TableColumn) element.getElement();
15972                                if (!"*".equals(tc.getName())
15973                                                && DlineageUtil.getIdentifierNormalColumnName(tc.getName()).equals(qualifiedColumnName)) {
15974                                        if (tc.getTable() != null && !starSourceParentIds.contains(tc.getTable().getId())) {
15975                                                return true;
15976                                        }
15977                                }
15978                        } else if (element.getElement() instanceof ResultColumn) {
15979                                ResultColumn rc = (ResultColumn) element.getElement();
15980                                if (!"*".equals(rc.getName())
15981                                                && DlineageUtil.getIdentifierNormalColumnName(rc.getName()).equals(qualifiedColumnName)) {
15982                                        if (rc.getResultSet() != null && !starSourceParentIds.contains(rc.getResultSet().getId())) {
15983                                                return true;
15984                                        }
15985                                }
15986                        }
15987                }
15988                return false;
15989        }
15990
15991        private void analyzeSelectStmt(TSelectSqlStatement stmt) {
15992                if (!accessedSubqueries.contains(stmt)) {
15993                        accessedSubqueries.add(stmt);
15994                } else {
15995                        if (modelManager.getModel(stmt) != null) {
15996                                return;
15997                        }
15998                }
15999
16000                if (stmt.getParentStmt() == null && stmt.getIntoClause() == null && stmt.getIntoTableClause() == null) {
16001                        if(option.isIgnoreTopSelect() && (option.isIgnoreRecordSet() || option.isSimpleOutput())){
16002                                if(option.getAnalyzeMode() == null || option.getAnalyzeMode() == AnalyzeMode.dataflow){
16003                                        return;
16004                                }
16005                        }
16006                }
16007
16008                if (stmt.getSetOperatorType() != ESetOperatorType.none) {
16009
16010                        // Iteratively analyze all descendant UNION branches before processing this node.
16011                        // Uses iterative post-order traversal to preserve the original left-right-self
16012                        // processing order, avoiding StackOverflow with deeply nested UNION trees.
16013                        {
16014                                Deque<TSelectSqlStatement> stack1 = new ArrayDeque<>();
16015                                List<TSelectSqlStatement> postOrder = new ArrayList<>();
16016                                stack1.push(stmt);
16017                                while (!stack1.isEmpty()) {
16018                                        TSelectSqlStatement node = stack1.pop();
16019                                        postOrder.add(node);
16020                                        if (node.getSetOperatorType() != ESetOperatorType.none) {
16021                                                // Push left first, then right, so that after reversal left comes first
16022                                                if (node.getLeftStmt() != null) stack1.push(node.getLeftStmt());
16023                                                if (node.getRightStmt() != null) stack1.push(node.getRightStmt());
16024                                        }
16025                                }
16026                                Collections.reverse(postOrder);
16027                                // Process all descendants in post-order, skip stmt itself (processed below)
16028                                for (int pi = 0; pi < postOrder.size() - 1; pi++) {
16029                                        TSelectSqlStatement node = postOrder.get(pi);
16030                                        if (!accessedStatements.contains(node)) {
16031                                                accessedStatements.add(node);
16032                                                analyzeSelectStmt(node);
16033                                        }
16034                                }
16035                        }
16036
16037                        stmtStack.push(stmt);
16038                        SelectSetResultSet resultSet = modelFactory.createSelectSetResultSet(stmt);
16039
16040                        ResultSet leftResultSetModel = (ResultSet) modelManager.getModel(stmt.getLeftStmt());
16041                        if (leftResultSetModel != null && leftResultSetModel != resultSet
16042                                        && !leftResultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
16043                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
16044                                impactRelation.setEffectType(EffectType.select);
16045                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
16046                                                leftResultSetModel.getRelationRows()));
16047                                impactRelation.setTarget(
16048                                                new RelationRowsRelationshipElement<ResultSetRelationRows>(resultSet.getRelationRows()));
16049                        }
16050                        
16051                        ResultSet rightResultSetModel = (ResultSet) modelManager.getModel(stmt.getRightStmt());
16052                        if (rightResultSetModel != null && rightResultSetModel != resultSet
16053                                        && !rightResultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
16054                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
16055                                impactRelation.setEffectType(EffectType.select);
16056                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
16057                                                rightResultSetModel.getRelationRows()));
16058                                impactRelation.setTarget(
16059                                                new RelationRowsRelationshipElement<ResultSetRelationRows>(resultSet.getRelationRows()));
16060                        }
16061                        
16062                        if ((leftResultSetModel != null && leftResultSetModel.isDetermined())
16063                                        || (rightResultSetModel != null && rightResultSetModel.isDetermined())) {
16064                                resultSet.setDetermined(true);
16065                        }
16066                        
16067                        if (resultSet.getColumns() == null || resultSet.getColumns().isEmpty()) {
16068                                if (getResultColumnList(stmt.getLeftStmt()) != null) {
16069                                        createSelectSetResultColumns(resultSet, stmt.getLeftStmt());
16070                                } else if (getResultColumnList(stmt.getRightStmt()) != null) {
16071                                        createSelectSetResultColumns(resultSet, stmt.getRightStmt());
16072                                }
16073                        }
16074
16075                        List<ResultColumn> columns = resultSet.getColumns();
16076                        for (int i = 0; i < columns.size(); i++) {
16077                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16078                                relation.setEffectType(EffectType.select);
16079                                relation.setTarget(new ResultColumnRelationshipElement(columns.get(i)));
16080
16081                                if (!stmt.getLeftStmt().isCombinedQuery()) {
16082                                        ResultSet sourceResultSet = (ResultSet) modelManager
16083                                                        .getModel(stmt.getLeftStmt().getResultColumnList());
16084                                        if (sourceResultSet!=null && sourceResultSet.getColumns().size() > i) {
16085                                                if (columns.get(i).getName().endsWith("*")) {
16086                                                        for (ResultColumn column : sourceResultSet.getColumns()) {
16087                                                                relation.addSource(new ResultColumnRelationshipElement(column));
16088                                                        }
16089                                                } else {
16090                                                        relation.addSource(
16091                                                                        new ResultColumnRelationshipElement(sourceResultSet.getColumns().get(i)));
16092                                                }
16093                                        }
16094                                } else {
16095                                        ResultSet sourceResultSet = (ResultSet) modelManager.getModel(stmt.getLeftStmt());
16096                                        if (sourceResultSet != null && sourceResultSet.getColumns().size() > i) {
16097                                                if (columns.get(i).getName().endsWith("*")) {
16098                                                        for (ResultColumn column : sourceResultSet.getColumns()) {
16099                                                                relation.addSource(new ResultColumnRelationshipElement(column));
16100                                                        }
16101                                                } else {
16102                                                        relation.addSource(
16103                                                                        new ResultColumnRelationshipElement(sourceResultSet.getColumns().get(i)));
16104                                                }
16105                                        }
16106                                }
16107
16108                                if (!stmt.getRightStmt().isCombinedQuery()) {
16109                                        ResultSet sourceResultSet = (ResultSet) modelManager
16110                                                        .getModel(stmt.getRightStmt().getResultColumnList());
16111                                        if (sourceResultSet != null && sourceResultSet.getColumns().size() > i) {
16112                                                if (columns.get(i).getName().endsWith("*")) {
16113                                                        for (ResultColumn column : sourceResultSet.getColumns()) {
16114                                                                relation.addSource(new ResultColumnRelationshipElement(column));
16115                                                        }
16116                                                } else {
16117                                                        relation.addSource(
16118                                                                        new ResultColumnRelationshipElement(sourceResultSet.getColumns().get(i)));
16119                                                }
16120                                        } else if (sourceResultSet != null) {
16121                                                for (ResultColumn column : sourceResultSet.getColumns()) {
16122                                                        if (column.hasStarLinkColumn()) {
16123                                                                relation.addSource(new ResultColumnRelationshipElement(column));
16124                                                        }
16125                                                }
16126                                        }
16127                                } else {
16128                                        ResultSet sourceResultSet = (ResultSet) modelManager.getModel(stmt.getRightStmt());
16129                                        if (sourceResultSet != null && sourceResultSet.getColumns().size() > i) {
16130                                                relation.addSource(new ResultColumnRelationshipElement(sourceResultSet.getColumns().get(i)));
16131                                        } else if (sourceResultSet != null) {
16132                                                for (ResultColumn column : sourceResultSet.getColumns()) {
16133                                                        if (column.hasStarLinkColumn()) {
16134                                                                relation.addSource(new ResultColumnRelationshipElement(column));
16135                                                        }
16136                                                }
16137                                        }
16138                                }
16139                        }
16140                        
16141                        analyzeSelectIntoClause(stmt);
16142                        
16143                        stmtStack.pop();
16144                } else {
16145
16146                        // handle hive stmt, issue_id I3SGZB
16147                        if (stmt.getHiveBodyList() != null && stmt.getHiveBodyList().size() > 0) {
16148                                stmtStack.push(stmt);
16149                                hiveFromTables = stmt.tables;
16150                                if (hiveFromTables != null) {
16151                                        for (int i = 0; i < hiveFromTables.size(); i++) {
16152                                                modelFactory.createTable(hiveFromTables.getTable(i));
16153                                        }
16154                                }
16155                                for (int i = 0; i < stmt.getHiveBodyList().size(); i++) {
16156                                        analyzeCustomSqlStmt(stmt.getHiveBodyList().get(i));
16157                                }
16158                                stmtStack.pop();
16159                                return;
16160                        }
16161                        
16162                        if (stmt.getTransformClause() != null) {
16163                                analyzeHiveTransformClause(stmt, stmt.getTransformClause());
16164                                return;
16165                        }
16166
16167                        if (stmt.getResultColumnList() == null) {
16168                                return;
16169                        }
16170
16171                        stmtStack.push(stmt);
16172
16173                        TTableList fromTables = stmt.tables;
16174                        if ((fromTables == null || fromTables.size() == 0)
16175                                        && (hiveFromTables != null && hiveFromTables.size() > 0)) {
16176                                fromTables = hiveFromTables;
16177                        }
16178
16179                        for (int i = 0; i < fromTables.size(); i++) {
16180                                TTable table = fromTables.getTable(i);
16181                                if (table.getLateralViewList() != null && !table.getLateralViewList().isEmpty()) {
16182                                        analyzeTableSubquery(table);
16183                                        analyzeLateralView(stmt, table, table.getLateralViewList());
16184                                        stmtStack.pop();
16185                                        return;
16186                                }
16187                                if (table.getUnnestClause() != null && option.getVendor() == EDbVendor.dbvpresto) {
16188                                        analyzeTableSubquery(table);
16189                                        analyzePrestoUnnest(stmt, table);
16190                                        stmtStack.pop();
16191                                        return;
16192                                }
16193                                if (table.getUnnestClause() != null && option.getVendor() == EDbVendor.dbvbigquery) {
16194                                        analyzeBigQueryUnnest(stmt, table);
16195                                }
16196                        }
16197                        
16198                        //用来获取 pivotedTable 对应的columns
16199                        TPivotedTable pivotedTable = null; 
16200                        if (stmt.getJoins() != null && stmt.getJoins().size() > 0) {
16201                                for (TJoin join : stmt.getJoins()) {
16202                                        if (join.getTable() != null && join.getTable().getPivotedTable() != null) {
16203                                                if (isUnPivotedTable(join.getTable().getPivotedTable())) {
16204                                                        analyzeUnPivotedTable(stmt, join.getTable().getPivotedTable());
16205                                                        pivotedTable = join.getTable().getPivotedTable();
16206                                                } else {
16207                                                        analyzePivotedTable(stmt, join.getTable().getPivotedTable());
16208                                                        pivotedTable = join.getTable().getPivotedTable();
16209                                                }
16210                                        }
16211                                }
16212                        }
16213
16214                        for (int i = 0; i < fromTables.size(); i++) {
16215                                TTable table = fromTables.getTable(i);
16216                                // Handle TABLE(func()) which has tableType=tableExpr and funcCall=null
16217                                if (table.getTableType() == ETableSource.tableExpr && table.getFuncCall() == null && pipelinedAnalyzer != null) {
16218                                        try {
16219                                                pipelinedAnalyzer.tryStitchTableExpr(table);
16220                                        } catch (Exception e) {
16221                                                // Don't let pipelined stitching failure break main flow
16222                                        }
16223                                }
16224
16225                                if (table.getFuncCall() != null || (table.getTableExpr()!=null && table.getTableExpr().getFunctionCall()!=null)) {
16226                                        TFunctionCall functionCall = table.getFuncCall();
16227                                        if (functionCall == null) {
16228                                                functionCall = table.getTableExpr().getFunctionCall();
16229                                        }
16230                                        Procedure callee = modelManager.getProcedureByName(
16231                                                        DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
16232                                        if (callee == null && procedureDDLMap.containsKey(DlineageUtil.getFunctionNameWithArgNum(functionCall))) {
16233                                                analyzeCustomSqlStmt(procedureDDLMap.get(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
16234                                                callee = modelManager.getProcedureByName(
16235                                                                DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
16236                                        }
16237
16238                                        if(callee!=null) {
16239                                                if (callee.getArguments() != null) {
16240                                                        for (int j = 0; j < callee.getArguments().size(); j++) {
16241                                                                Argument argument = callee.getArguments().get(j);
16242                                                                Variable variable = modelFactory.createVariable(callee, argument.getName(), false);
16243                                                                if(variable!=null) {
16244                                                                        if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
16245                                                                                Transform transform = new Transform();
16246                                                                                transform.setType(Transform.FUNCTION);
16247                                                                                transform.setCode(functionCall);
16248                                                                                variable.getColumns().get(0).setTransform(transform);
16249                                                                        }
16250                                                                        Process process = modelFactory.createProcess(functionCall);
16251                                                                        variable.addProcess(process);
16252                                                                        analyzeFunctionArgumentsDataFlowRelation(variable.getColumns().get(0), functionCall, j, process);
16253                                                                }
16254                                                        }
16255                                                }
16256                                                Set<Object> functionTableModelObjs = modelManager.getFunctionTable(DlineageUtil
16257                                                                .getIdentifierNormalTableName(functionCall.getFunctionName().toString()));
16258                                                if (functionTableModelObjs != null) {
16259                                                        modelManager.bindModel(table, functionTableModelObjs.iterator().next());
16260                                                }
16261                                                // Try pipelined function stitching
16262                                                if (pipelinedAnalyzer != null) {
16263                                                        try {
16264                                                                pipelinedAnalyzer.tryStitchCallSite(table, functionCall);
16265                                                        } catch (Exception e) {
16266                                                                // Don't let pipelined stitching failure break main flow
16267                                                        }
16268                                                }
16269                                                continue;
16270                                        } else {
16271                                                Set<Object> functionTableModelObjs = modelManager.getFunctionTable(DlineageUtil
16272                                                                .getIdentifierNormalTableName(functionCall.getFunctionName().toString()));
16273                                                if (functionTableModelObjs == null) {
16274//                                                      Procedure procedure = modelManager.getProcedureByName(DlineageUtil
16275//                                                                      .getIdentifierNormalTableName(functionCall.getFunctionName().toString()));
16276//                                                      if (procedure != null) {
16277                                                                createFunction(functionCall);
16278                                                                continue;
16279//                                                      }
16280                                                }
16281                                                if (functionTableModelObjs!=null && functionTableModelObjs.iterator().next() instanceof Table) {
16282                                                        Table functionTableModel = (Table) functionTableModelObjs.iterator().next();
16283                                                        if (functionTableModel.getColumns() != null) {
16284                                                                Table functionTable = modelFactory.createTableFromCreateDDL(table, true);
16285                                                                if (functionTable == functionTableModel)
16286                                                                        continue;
16287                                                                for (int j = 0; j < functionTableModel.getColumns().size(); j++) {
16288                                                                        TableColumn column = modelFactory.createTableColumn(functionTable,
16289                                                                                        functionTableModel.getColumns().get(j).getColumnObject(), true);
16290                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16291                                                                        relation.setEffectType(EffectType.select);
16292                                                                        relation.setTarget(new TableColumnRelationshipElement(column));
16293                                                                        relation.addSource(
16294                                                                                        new TableColumnRelationshipElement(functionTableModel.getColumns().get(j)));
16295                                                                }
16296                                                        }
16297                                                } else if (functionTableModelObjs!=null && functionTableModelObjs.iterator().next() instanceof ResultSet) {
16298                                                        ResultSet functionTableModel = (ResultSet) functionTableModelObjs.iterator().next();
16299                                                        if (functionTableModel.getColumns() != null) {
16300                                                                Table functionTable = modelFactory.createTableFromCreateDDL(table, true);
16301                                                                for (int j = 0; j < functionTableModel.getColumns().size(); j++) {
16302                                                                        TParseTreeNode columnObj = functionTableModel.getColumns().get(j).getColumnObject();
16303                                                                        if (columnObj instanceof TObjectName) {
16304                                                                                TableColumn column = modelFactory.createTableColumn(functionTable,
16305                                                                                                (TObjectName) columnObj, true);
16306                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16307                                                                                relation.setEffectType(EffectType.select);
16308                                                                                relation.setTarget(new TableColumnRelationshipElement(column));
16309                                                                                relation.addSource(new ResultColumnRelationshipElement(
16310                                                                                                functionTableModel.getColumns().get(j)));
16311                                                                        } else if (columnObj instanceof TResultColumn) {
16312                                                                                TableColumn column = modelFactory.createTableColumn(functionTable,
16313                                                                                                (TResultColumn) columnObj);
16314                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16315                                                                                relation.setEffectType(EffectType.select);
16316                                                                                relation.setTarget(new TableColumnRelationshipElement(column));
16317                                                                                relation.addSource(new ResultColumnRelationshipElement(
16318                                                                                                functionTableModel.getColumns().get(j)));
16319                                                                        }
16320                                                                }
16321                                                        }
16322                                                }
16323                                                // Try pipelined function stitching for unresolved function calls
16324                                                if (pipelinedAnalyzer != null) {
16325                                                        try {
16326                                                                pipelinedAnalyzer.tryStitchCallSite(table, functionCall);
16327                                                        } catch (Exception e) {
16328                                                                // Don't let pipelined stitching failure break main flow
16329                                                        }
16330                                                }
16331                                        }
16332                                }
16333
16334                                if (table.getPartitionExtensionClause() != null
16335                                                && table.getPartitionExtensionClause().getKeyValues() != null) {
16336                                        TExpressionList values = table.getPartitionExtensionClause().getKeyValues();
16337                                        Table tableModel = modelFactory.createTable(table);
16338                                        for (TExpression value : values) {
16339                                                if (value.getExpressionType() == EExpressionType.simple_constant_t) {
16340                                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
16341                                                        impactRelation.setEffectType(EffectType.select);
16342                                                        Table constantTable = modelFactory.createConstantsTable(stmtStack.peek());
16343                                                        TableColumn constantColumn = modelFactory.createTableColumn(constantTable, value.getConstantOperand());
16344                                                        impactRelation.addSource(new TableColumnRelationshipElement(constantColumn));
16345                                                        impactRelation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(
16346                                                                        tableModel.getRelationRows()));
16347                                                }
16348                                        }
16349                                }
16350
16351                                if (table.getSubquery() != null) {
16352                                        QueryTable queryTable = modelFactory.createQueryTable(table);
16353                                        TSelectSqlStatement subquery = table.getSubquery();
16354                                        analyzeSelectStmt(subquery);
16355
16356                                        ResultSet resultSetModel = (ResultSet) modelManager.getModel(subquery);
16357                                        if (resultSetModel != null && resultSetModel.isDetermined()) {
16358                                                queryTable.setDetermined(true);
16359                                        }
16360
16361                                        if (resultSetModel != null && resultSetModel != queryTable
16362                                                        && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
16363                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
16364                                                impactRelation.setEffectType(EffectType.select);
16365                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
16366                                                                resultSetModel.getRelationRows()));
16367                                                impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
16368                                                                queryTable.getRelationRows()));
16369                                        }
16370
16371                                        if (resultSetModel != null && resultSetModel != queryTable
16372                                                        && queryTable.getTableObject().getAliasClause() != null
16373                                                        && queryTable.getTableObject().getAliasClause().getColumns() != null) {
16374                                                for (int j = 0; j < queryTable.getColumns().size()
16375                                                                && j < resultSetModel.getColumns().size(); j++) {
16376                                                        ResultColumn sourceColumn = resultSetModel.getColumns().get(j);
16377                                                        ResultColumn targetColumn = queryTable.getColumns().get(j);
16378
16379                                                        DataFlowRelationship queryRalation = modelFactory.createDataFlowRelation();
16380                                                        queryRalation.setEffectType(EffectType.select);
16381                                                        queryRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
16382                                                        queryRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
16383                                                }
16384                                        } else if (subquery.getSetOperatorType() != ESetOperatorType.none) {
16385                                                SelectSetResultSet selectSetResultSetModel = (SelectSetResultSet) modelManager
16386                                                                .getModel(subquery);
16387                                                for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) {
16388                                                        ResultColumn sourceColumn = selectSetResultSetModel.getColumns().get(j);
16389                                                        ResultColumn targetColumn = modelFactory.createSelectSetResultColumn(queryTable,
16390                                                                        sourceColumn);
16391                                                        for (TObjectName starLinkColumn : sourceColumn.getStarLinkColumnList()) {
16392                                                                targetColumn.bindStarLinkColumn(starLinkColumn);
16393                                                        }
16394                                                        DataFlowRelationship selectSetRalation = modelFactory.createDataFlowRelation();
16395                                                        selectSetRalation.setEffectType(EffectType.select);
16396                                                        selectSetRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
16397                                                        selectSetRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
16398                                                }
16399                                        }
16400                                } else if (table.getOutputMerge() != null) {
16401                                        QueryTable queryTable = modelFactory.createQueryTable(table);
16402                                        TMergeSqlStatement subquery = table.getOutputMerge();
16403                                        analyzeMergeStmt(subquery);
16404
16405                                        for (TResultColumn column : subquery.getOutputClause().getSelectItemList()) {
16406                                                modelFactory.createResultColumn(queryTable, column);
16407                                                analyzeResultColumn(column, EffectType.select);
16408                                        }
16409
16410                                        ResultSet resultSetModel = (ResultSet) modelManager
16411                                                        .getModel(subquery.getOutputClause().getSelectItemList());
16412
16413                                        if (resultSetModel != null && resultSetModel != queryTable
16414                                                        && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
16415                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
16416                                                impactRelation.setEffectType(EffectType.select);
16417                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
16418                                                                resultSetModel.getRelationRows()));
16419                                                impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
16420                                                                queryTable.getRelationRows()));
16421                                        }
16422
16423                                        if (resultSetModel != null && resultSetModel != queryTable
16424                                                        && queryTable.getTableObject().getAliasClause() != null
16425                                                        && queryTable.getTableObject().getAliasClause().getColumns() != null) {
16426                                                for (int j = 0; j < queryTable.getColumns().size()
16427                                                                && j < resultSetModel.getColumns().size(); j++) {
16428                                                        ResultColumn sourceColumn = resultSetModel.getColumns().get(j);
16429                                                        ResultColumn targetColumn = queryTable.getColumns().get(j);
16430
16431                                                        DataFlowRelationship queryRalation = modelFactory.createDataFlowRelation();
16432                                                        queryRalation.setEffectType(EffectType.select);
16433                                                        queryRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
16434                                                        queryRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
16435                                                }
16436                                        }
16437                                } else if (table.getTableExpr() != null && table.getTableExpr().getSubQuery() != null) {
16438                                        QueryTable queryTable = modelFactory.createQueryTable(table);
16439                                        TSelectSqlStatement subquery = table.getTableExpr().getSubQuery();
16440                                        analyzeSelectStmt(subquery);
16441
16442                                        ResultSet resultSetModel = (ResultSet) modelManager.getModel(subquery);
16443
16444                                        if (resultSetModel != null && resultSetModel != queryTable
16445                                                        && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
16446                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
16447                                                impactRelation.setEffectType(EffectType.select);
16448                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
16449                                                                resultSetModel.getRelationRows()));
16450                                                impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
16451                                                                queryTable.getRelationRows()));
16452                                        }
16453
16454                                        if (resultSetModel != null && resultSetModel != queryTable
16455                                                        && queryTable.getTableObject().getAliasClause() != null) {
16456                                                for (int j = 0; j < resultSetModel.getColumns().size(); j++) {
16457                                                        ResultColumn sourceColumn = resultSetModel.getColumns().get(j);
16458                                                        ResultColumn targetColumn = modelFactory.createSelectSetResultColumn(queryTable,
16459                                                                        sourceColumn);
16460
16461                                                        DataFlowRelationship queryRalation = modelFactory.createDataFlowRelation();
16462                                                        queryRalation.setEffectType(EffectType.select);
16463                                                        queryRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
16464                                                        queryRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
16465                                                }
16466                                        } else if (subquery.getSetOperatorType() != ESetOperatorType.none) {
16467                                                SelectSetResultSet selectSetResultSetModel = (SelectSetResultSet) modelManager
16468                                                                .getModel(subquery);
16469                                                for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) {
16470                                                        ResultColumn sourceColumn = selectSetResultSetModel.getColumns().get(j);
16471                                                        ResultColumn targetColumn = modelFactory.createSelectSetResultColumn(queryTable,
16472                                                                        sourceColumn);
16473                                                        for (TObjectName starLinkColumn : sourceColumn.getStarLinkColumnList()) {
16474                                                                targetColumn.bindStarLinkColumn(starLinkColumn);
16475                                                        }
16476                                                        DataFlowRelationship selectSetRalation = modelFactory.createDataFlowRelation();
16477                                                        selectSetRalation.setEffectType(EffectType.select);
16478                                                        selectSetRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
16479                                                        selectSetRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
16480                                                }
16481                                        }
16482                                } else if (table.getCTE() != null) {
16483                                        QueryTable queryTable = modelFactory.createQueryTable(table);
16484
16485                                        TObjectNameList cteColumns = table.getCTE().getColumnList();
16486                                        if (cteColumns != null) {
16487                                                for (int j = 0; j < cteColumns.size(); j++) {
16488                                                        modelFactory.createResultColumn(queryTable, cteColumns.getObjectName(j));
16489                                                }
16490                                                queryTable.setDetermined(true);
16491                                        }
16492                                        TSelectSqlStatement subquery = table.getCTE().getSubquery();
16493                                        if (subquery != null && !stmtStack.contains(subquery) && subquery.getResultColumnList() != null) {
16494                                                analyzeSelectStmt(subquery);
16495
16496                                                ResultSet resultSetModel = (ResultSet) modelManager.getModel(subquery);
16497                                                if (resultSetModel != null && resultSetModel != queryTable
16498                                                                && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
16499                                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
16500                                                        impactRelation.setEffectType(EffectType.select);
16501                                                        impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
16502                                                                        resultSetModel.getRelationRows()));
16503                                                        impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
16504                                                                        queryTable.getRelationRows()));
16505                                                }
16506
16507                                                if (subquery.getSetOperatorType() != ESetOperatorType.none) {
16508                                                        SelectSetResultSet selectSetResultSetModel = (SelectSetResultSet) modelManager
16509                                                                        .getModel(subquery);
16510                                                        int x = 0;
16511                                                        for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) {
16512                                                                ResultColumn sourceColumn = selectSetResultSetModel.getColumns().get(j);
16513                                                                ResultColumn targetColumn = null;
16514                                                                if (cteColumns != null) {
16515                                                                        if (queryTable.getColumns().size() <= j) {
16516                                                                                for (int k = 0; k < queryTable.getColumns().size(); k++) {
16517                                                                                        if (queryTable.getColumns().get(k).getName().equals(sourceColumn.getName())
16518                                                                                                        || queryTable.getColumns().get(k).getName()
16519                                                                                                                        .equals(sourceColumn.getAlias())) {
16520                                                                                                targetColumn = queryTable.getColumns().get(k);
16521                                                                                        }
16522                                                                                }
16523                                                                        } else {
16524                                                                                if (x < j) {
16525                                                                                        x = j;
16526                                                                                }
16527                                                                                targetColumn = queryTable.getColumns().get(x);
16528                                                                                x++;
16529                                                                                if (resultSetModel.getColumns().get(j).getName().contains("*")
16530                                                                                                && x < cteColumns.size()) {
16531                                                                                        j--;
16532                                                                                }
16533                                                                        }
16534                                                                } else {
16535                                                                        targetColumn = modelFactory.createSelectSetResultColumn(queryTable, sourceColumn);
16536                                                                }
16537                                                                for (TObjectName starLinkColumn : sourceColumn.getStarLinkColumnList()) {
16538                                                                        targetColumn.bindStarLinkColumn(starLinkColumn);
16539                                                                }
16540                                                                DataFlowRelationship selectSetRalation = modelFactory.createDataFlowRelation();
16541                                                                selectSetRalation.setEffectType(EffectType.select);
16542                                                                selectSetRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
16543                                                                selectSetRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
16544                                                        }
16545                                                        if (!queryTable.isDetermined()) {
16546                                                                queryTable.setDetermined(selectSetResultSetModel.isDetermined());
16547                                                        }
16548                                                } else {
16549                                                        int x = 0;
16550                                                        for (int j = 0; j < resultSetModel.getColumns().size(); j++) {
16551                                                                ResultColumn sourceColumn = resultSetModel.getColumns().get(j);
16552                                                                ResultColumn targetColumn = null;
16553                                                                if (cteColumns != null) {
16554                                                                        if (queryTable.getColumns().size() <= j) {
16555                                                                                for (int k = 0; k < queryTable.getColumns().size(); k++) {
16556                                                                                        if (queryTable.getColumns().get(k).getName().equals(sourceColumn.getName())
16557                                                                                                        || queryTable.getColumns().get(k).getName()
16558                                                                                                                        .equals(sourceColumn.getAlias())) {
16559                                                                                                targetColumn = queryTable.getColumns().get(k);
16560                                                                                        }
16561                                                                                }
16562                                                                        } else {
16563                                                                                if (x < j) {
16564                                                                                        x = j;
16565                                                                                }
16566                                                                                targetColumn = queryTable.getColumns().get(x);
16567                                                                                x++;
16568                                                                                if (resultSetModel.getColumns().get(j).getName().contains("*")
16569                                                                                                && x < cteColumns.size()) {
16570                                                                                        j--;
16571                                                                                }
16572                                                                        }
16573                                                                } else {
16574                                                                        targetColumn = modelFactory.createSelectSetResultColumn(queryTable, sourceColumn);
16575                                                                }
16576                                                                for (TObjectName starLinkColumn : sourceColumn.getStarLinkColumnList()) {
16577                                                                        targetColumn.bindStarLinkColumn(starLinkColumn);
16578                                                                }
16579                                                                DataFlowRelationship selectSetRalation = modelFactory.createDataFlowRelation();
16580                                                                selectSetRalation.setEffectType(EffectType.select);
16581                                                                selectSetRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
16582                                                                selectSetRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
16583                                                        }
16584                                                        if (!queryTable.isDetermined()) {
16585                                                                queryTable.setDetermined(resultSetModel.isDetermined());
16586                                                        }
16587                                                }
16588                                        } else if (table.getCTE().getUpdateStmt() != null) {
16589                                                analyzeCustomSqlStmt(table.getCTE().getUpdateStmt());
16590                                        } else if (table.getCTE().getInsertStmt() != null) {
16591                                                analyzeCustomSqlStmt(table.getCTE().getInsertStmt());
16592                                        } else if (table.getCTE().getDeleteStmt() != null) {
16593                                                analyzeCustomSqlStmt(table.getCTE().getDeleteStmt());
16594                                        }
16595                                } else if (table.getTableType().name().startsWith("open")) {
16596                                        continue;
16597                                } else if (table.getTableType() == ETableSource.jsonTable) {
16598                                        Table functionTable = modelFactory.createJsonTable(table);
16599                                        TJsonTable jsonTable = table.getJsonTable();
16600                                        TColumnDefinitionList definitions = jsonTable.getColumnDefinitions();
16601                                        if (definitions != null) {
16602                                                for (int j = 0; j < definitions.size(); j++) {
16603                            TColumnDefinitionList nestDefinitions = definitions.getColumn(j).getNestedTableColumns();
16604                                                        if(nestDefinitions!=null) {
16605                                                                for(int k=0;k<nestDefinitions.size();k++){
16606                                                                        TableColumn column = modelFactory.createTableColumn(functionTable,
16607                                            nestDefinitions.getColumn(k).getColumnName(), true);
16608                                    if (nestDefinitions.getColumn(k).getColumnPath() != null) {
16609                                        column.setDisplayName(
16610                                                column.getName() + ":" + nestDefinitions.getColumn(k).getColumnPath());
16611                                    }
16612                                                                }
16613                                                        }
16614                                                        else {
16615                                TableColumn column = modelFactory.createTableColumn(functionTable,
16616                                        definitions.getColumn(j).getColumnName(), true);
16617                                if (definitions.getColumn(j).getColumnPath() != null) {
16618                                    column.setDisplayName(
16619                                            column.getName() + ":" + definitions.getColumn(j).getColumnPath());
16620                                }
16621                            }
16622                                                }
16623                                        } else {
16624                                                TObjectName keyColumn = new TObjectName();
16625                                                keyColumn.setString("key");
16626                                                modelFactory.createJsonTableColumn(functionTable, keyColumn);
16627                                                TObjectName valueColumn = new TObjectName();
16628                                                valueColumn.setString("value");
16629                                                modelFactory.createJsonTableColumn(functionTable, valueColumn);
16630                                                TObjectName typeColumn = new TObjectName();
16631                                                typeColumn.setString("type");
16632                                                modelFactory.createJsonTableColumn(functionTable, typeColumn);
16633                                        }
16634
16635                                        functionTable.setCreateTable(true);
16636                                        functionTable.setSubType(SubType.function);
16637                                        modelManager.bindCreateModel(table, functionTable);
16638
16639                                        if (jsonTable.getJsonExpression() == null) {
16640                                                ErrorInfo errorInfo = new ErrorInfo();
16641                                                errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
16642                                                errorInfo.setErrorMessage("Can't handle the json table: " + jsonTable.toString());
16643                                                errorInfo.setStartPosition(new Pair3<Long, Long, String>(jsonTable.getStartToken().lineNo,
16644                                                                jsonTable.getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
16645                                                errorInfo.setEndPosition(new Pair3<Long, Long, String>(jsonTable.getEndToken().lineNo,
16646                                                                jsonTable.getEndToken().columnNo + jsonTable.getEndToken().getAstext().length(),
16647                                                                ModelBindingManager.getGlobalHash()));
16648                                                errorInfo.fillInfo(this);
16649                                                errorInfos.add(errorInfo);
16650                                        } else {
16651                                                String jsonName = jsonTable.getJsonExpression().toString();
16652                                                if (!jsonName.startsWith("@")) {
16653                                                        columnsInExpr visitor = new columnsInExpr();
16654                                                        jsonTable.getJsonExpression().inOrderTraverse(visitor);
16655                                                        List<TObjectName> objectNames = visitor.getObjectNames();
16656                                                        List<TParseTreeNode> functions = visitor.getFunctions();
16657                                                        List<TParseTreeNode> constants = visitor.getConstants();
16658                                                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
16659
16660                                                        for (int j = 0; j < functionTable.getColumns().size(); j++) {
16661                                                                TableColumn tableColumn = functionTable.getColumns().get(j);
16662                                                                if (functions != null && !functions.isEmpty()) {
16663                                                                        analyzeFunctionDataFlowRelation(tableColumn, functions, EffectType.function);
16664                                                                }
16665                                                                if (subquerys != null && !subquerys.isEmpty()) {
16666                                                                        analyzeSubqueryDataFlowRelation(tableColumn, subquerys, EffectType.select);
16667                                                                }
16668                                                                if (objectNames != null && !objectNames.isEmpty()) {
16669                                                                        analyzeDataFlowRelation(tableColumn, objectNames, EffectType.select, functions);
16670                                                                }
16671                                                                if (constants != null && !constants.isEmpty()) {
16672                                                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select,
16673                                                                                        functions);
16674                                                                }
16675                                                        }
16676                                                } else {
16677                                                        TStatementList stmts = stmt.getGsqlparser().getSqlstatements();
16678                                                        for (int j = 0; j < stmts.size(); j++) {
16679                                                                TCustomSqlStatement item = stmts.get(j);
16680                                                                if (item instanceof TMssqlDeclare) {
16681                                                                        if (analyzeMssqlJsonDeclare((TMssqlDeclare) item, jsonName, functionTable)) {
16682                                                                                break;
16683                                                                        }
16684                                                                }
16685                                                        }
16686                                                }
16687                                        }
16688                                } else if (table.getTableType() == ETableSource.xmltable) {
16689                                        Table functionTable = modelFactory.createXmlTable(table);
16690                                        TXmlTable xmlTable = table.getXmlTable();
16691                                        TXmlTableParameter param = xmlTable != null ? xmlTable.getArg() : null;
16692
16693                                        // Output columns from COLUMNS clause
16694                                        if (param != null && param.getXmlTableColumns() != null) {
16695                                                TColumnDefinitionList defs = param.getXmlTableColumns();
16696                                                for (int j = 0; j < defs.size(); j++) {
16697                                                        TColumnDefinition def = defs.getColumn(j);
16698                                                        TableColumn column = modelFactory.createTableColumn(
16699                                                                        functionTable, def.getColumnName(), true);
16700                                                        if (def.getXmlTableColumnPath() != null) {
16701                                                                column.setDisplayName(
16702                                                                                column.getName() + ":" + def.getXmlTableColumnPath());
16703                                                        }
16704                                                }
16705                                        }
16706                                        functionTable.setCreateTable(true);
16707                                        functionTable.setSubType(SubType.function);
16708                                        modelManager.bindCreateModel(table, functionTable);
16709
16710                                        // Source columns from PASSING (v1: many-to-many)
16711                                        TResultColumnList passing = null;
16712                                        if (param != null && param.getXmlPassingClause() != null) {
16713                                                passing = param.getXmlPassingClause().getPassingList();
16714                                        }
16715                                        if (passing != null && passing.size() > 0) {
16716                                                columnsInExpr visitor = new columnsInExpr();
16717                                                for (int p = 0; p < passing.size(); p++) {
16718                                                        TExpression e = passing.getResultColumn(p).getExpr();
16719                                                        if (e != null) e.inOrderTraverse(visitor);
16720                                                }
16721                                                List<TObjectName> objectNames = visitor.getObjectNames();
16722                                                List<TParseTreeNode> functions = visitor.getFunctions();
16723                                                List<TParseTreeNode> constants = visitor.getConstants();
16724                                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
16725
16726                                                for (int j = 0; j < functionTable.getColumns().size(); j++) {
16727                                                        TableColumn tc = functionTable.getColumns().get(j);
16728                                                        if (functions != null && !functions.isEmpty())
16729                                                                analyzeFunctionDataFlowRelation(tc, functions, EffectType.function);
16730                                                        if (subquerys != null && !subquerys.isEmpty())
16731                                                                analyzeSubqueryDataFlowRelation(tc, subquerys, EffectType.select);
16732                                                        if (objectNames != null && !objectNames.isEmpty())
16733                                                                analyzeDataFlowRelation(tc, objectNames, EffectType.select, functions);
16734                                                        if (constants != null && !constants.isEmpty())
16735                                                                analyzeConstantDataFlowRelation(tc, constants, EffectType.select, functions);
16736                                                }
16737                                        } else {
16738                                                ErrorInfo errorInfo = new ErrorInfo();
16739                                                errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
16740                                                errorInfo.setErrorMessage("Can't analyze XMLTABLE: missing PASSING clause.");
16741                                                errorInfos.add(errorInfo);
16742                                        }
16743                                } else if (getTableLinkedColumns(table) != null && getTableLinkedColumns(table).size() > 0) {
16744                                        if (table.getTableType() == ETableSource.rowList && table.getRowList() != null
16745                                                        && table.getRowList().size() > 0) {
16746                                                Table tableModel = modelFactory.createTable(table);
16747                                                for (int j = 0; j < table.getRowList().size(); j++) {
16748                                                        TMultiTarget rowList = table.getRowList().getMultiTarget(j);
16749                                                        for (int k = 0; k < rowList.getColumnList().size(); k++) {
16750                                                                TResultColumn column = rowList.getColumnList().getResultColumn(k);
16751                                                                if (column.getFieldAttr() == null)
16752                                                                        continue;
16753                                                                TableColumn tableColumn = modelFactory.createTableColumn(tableModel,
16754                                                                                column.getFieldAttr(), true);
16755
16756                                                                columnsInExpr visitor = new columnsInExpr();
16757                                                                column.getExpr().inOrderTraverse(visitor);
16758                                                                List<TObjectName> objectNames = visitor.getObjectNames();
16759                                                                List<TParseTreeNode> functions = visitor.getFunctions();
16760                                                                List<TParseTreeNode> constants = visitor.getConstants();
16761                                                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
16762
16763                                                                if (functions != null && !functions.isEmpty()) {
16764                                                                        analyzeFunctionDataFlowRelation(tableColumn, functions, EffectType.function);
16765                                                                }
16766                                                                if (subquerys != null && !subquerys.isEmpty()) {
16767                                                                        analyzeSubqueryDataFlowRelation(tableColumn, subquerys, EffectType.select);
16768                                                                }
16769                                                                if (objectNames != null && !objectNames.isEmpty()) {
16770                                                                        analyzeDataFlowRelation(tableColumn, objectNames, EffectType.select, functions);
16771                                                                }
16772                                                                if (constants != null && !constants.isEmpty()) {
16773                                                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select,
16774                                                                                        functions);
16775                                                                }
16776                                                        }
16777                                                }
16778                                        } else {
16779                                                if(table.getTableType() == ETableSource.pivoted_table) {
16780                                                        continue;
16781                                                }
16782                                                if (table.getTableType() == ETableSource.rowList) {
16783                                                        List<TResultColumnList> rowList = table.getValueClause().getRows();
16784                                                        
16785                                                        QueryTable tableModel = modelFactory.createQueryTable(table);
16786                                                        TAliasClause aliasClause = table.getAliasClause();
16787                                                        if (aliasClause != null && aliasClause.getColumns() != null) {
16788                                                                for (TObjectName column : aliasClause.getColumns()) {
16789                                                                        modelFactory.createResultColumn(tableModel, column, true);
16790                                                                }       
16791                                                        } else {        
16792                                                                int columnCount = rowList.get(0).size();
16793                                                                for (int j = 1; j <= columnCount; j++) {
16794                                                                        TObjectName columnName = new TObjectName();
16795                                                                        columnName.setString("column" + j);
16796                                                                        modelFactory.createResultColumn(tableModel, columnName, true);
16797                                                                }
16798                                                        }
16799                                                        tableModel.setDetermined(true);
16800                                                        
16801                                                        for (TResultColumnList resultColumnList : rowList) {
16802                                                                for (int j = 0; j < resultColumnList.size(); j++) {
16803                                                                        TResultColumn resultColumn = resultColumnList.getResultColumn(j);
16804                                                                        analyzeValueColumn(tableModel.getColumns().get(j), resultColumn, EffectType.select);
16805                                                                }
16806                                                        }
16807                                                        
16808                                                } else {
16809                                                        Table tableModel = modelFactory.createTable(table);
16810                                                        for (int j = 0; j < getTableLinkedColumns(table).size(); j++) {
16811                                                                TObjectName object = getTableLinkedColumns(table).getObjectName(j);
16812
16813                                                                if (object.getDbObjectType() == EDbObjectType.variable) {
16814                                                                        continue;
16815                                                                }
16816
16817                                                                if (object.getColumnNameOnly().startsWith("@")
16818                                                                                && (option.getVendor() == EDbVendor.dbvmssql
16819                                                                                                || option.getVendor() == EDbVendor.dbvazuresql)) {
16820                                                                        continue;
16821                                                                }
16822
16823                                                                if (object.getColumnNameOnly().startsWith(":")
16824                                                                                && (option.getVendor() == EDbVendor.dbvhana
16825                                                                                                || option.getVendor() == EDbVendor.dbvteradata)) {
16826                                                                        continue;
16827                                                                }
16828
16829                                                                if (isBuiltInFunctionName(object) && isFromFunction(object)) {
16830                                                                        continue;
16831                                                                }
16832
16833                                                                if (!"*".equals(getColumnName(object))) {
16834                                                                        if (isStructColumn(object)) {
16835
16836                                                                        } else {
16837                                                                                if (pivotedTable != null) {
16838                                                                                        ResultColumn resultColumn = getPivotedTableColumn(pivotedTable, object);
16839                                                                                        if (resultColumn != null) {
16840                                                                                                continue;
16841                                                                                        }
16842                                                                                }
16843                                                                                TableColumn tableColumn = modelFactory.createTableColumn(tableModel, object,
16844                                                                                                false);
16845                                                                                if(tableColumn == null) {
16846                                                                                        continue;
16847                                                                                }
16848                                                                                if (table.getUnnestClause() != null
16849                                                                                                && table.getUnnestClause().getArrayExpr() != null) {
16850                                                                                        columnsInExpr visitor = new columnsInExpr();
16851                                                                                        table.getUnnestClause().getArrayExpr().inOrderTraverse(visitor);
16852
16853                                                                                        List<TObjectName> objectNames = visitor.getObjectNames();
16854                                                                                        List<TParseTreeNode> functions = visitor.getFunctions();
16855
16856                                                                                        if (functions != null && !functions.isEmpty()) {
16857                                                                                                analyzeFunctionDataFlowRelation(tableColumn, functions,
16858                                                                                                                EffectType.select);
16859
16860                                                                                        }
16861
16862                                                                                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
16863                                                                                        if (subquerys != null && !subquerys.isEmpty()) {
16864                                                                                                analyzeSubqueryDataFlowRelation(tableColumn, subquerys,
16865                                                                                                                EffectType.select);
16866                                                                                        }
16867
16868                                                                                        analyzeDataFlowRelation(tableColumn, objectNames, EffectType.select,
16869                                                                                                        functions);
16870
16871                                                                                        List<TParseTreeNode> constants = visitor.getConstants();
16872                                                                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select,
16873                                                                                                        functions);
16874                                                                                }
16875                                                                        }
16876                                                                } else {
16877                                                                        boolean flag = false;
16878                                                                        for (TObjectName column : getTableLinkedColumns(table)) {
16879                                                                                if ("*".equals(getColumnName(column))) {
16880                                                                                        continue;
16881                                                                                }
16882                                                                                if (column.getLocation() != ESqlClause.where
16883                                                                                                && column.getLocation() != ESqlClause.joinCondition) {
16884                                                                                        flag = true;
16885                                                                                }
16886                                                                        }
16887                                                                        if (!flag) {
16888                                                                                TableColumn tableColumn = modelFactory.createTableColumn(tableModel, object,
16889                                                                                                false);
16890                                                                                if (tableColumn != null && !tableModel.hasSQLEnv()) {
16891                                                                                        tableColumn.setShowStar(true);
16892                                                                                }
16893                                                                        }
16894                                                                }
16895
16896                                                        }
16897                                                }
16898                                        }
16899                                }
16900                                else {
16901                                        modelFactory.createTable(table);
16902                                }
16903                        }
16904
16905                        if (pivotedTable!=null) {
16906                                for (TJoin join : stmt.getJoins()) {
16907                                        if (join.getTable() != null && join.getTable().getPivotedTable() != null) {
16908                                                if (isUnPivotedTable(join.getTable().getPivotedTable())) {
16909                                                        analyzeUnPivotedTable(stmt, join.getTable().getPivotedTable());
16910                                                } else {
16911                                                        analyzePivotedTable(stmt, join.getTable().getPivotedTable());
16912                                                }
16913                                                stmtStack.pop();
16914                                                return;
16915                                        }
16916                                }
16917                        }
16918
16919                        if (!stmt.isCombinedQuery()) {
16920                                Object queryModel = modelManager.getModel(stmt.getResultColumnList());
16921
16922                                if (queryModel == null) {
16923                                        TSelectSqlStatement parentStmt = getParentSetSelectStmt(stmt);
16924                                        if (isTopResultSet(stmt) || parentStmt == null) {
16925                                                ResultSet resultSetModel = modelFactory.createResultSet(stmt,
16926                                                                isTopResultSet(stmt) && isShowTopSelectResultSet() && stmt.getIntoClause() == null);
16927
16928                                                createPseudoImpactRelation(stmt, resultSetModel, EffectType.select);
16929
16930                                                boolean isDetermined = true;
16931                                                Map<String, AtomicInteger> keyMap = new HashMap<String, AtomicInteger>();
16932                                                Set<String> columnNames = new HashSet<String>();
16933                                                for (int i = 0; i < stmt.getResultColumnList().size(); i++) {
16934                                                        TResultColumn column = stmt.getResultColumnList().getResultColumn(i);
16935
16936                                                        if (column.getExpr().getComparisonType() == EComparisonType.equals
16937                                                                        && column.getExpr().getLeftOperand().getObjectOperand() != null) {
16938                                                                TObjectName columnObject = column.getExpr().getLeftOperand().getObjectOperand();
16939
16940                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel,
16941                                                                                columnObject);
16942                                                                if (columnObject.getDbObjectType() == EDbObjectType.variable) {
16943                                                                        Table variable = modelManager
16944                                                                                        .getTableByName(DlineageUtil.getTableFullName(columnObject.toString()));
16945                                                                        if (variable != null) {
16946                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16947                                                                                relation.setEffectType(EffectType.select);
16948                                                                                TableColumn columnModel = variable.getColumns().get(0);
16949                                                                                relation.setTarget(new TableColumnRelationshipElement(columnModel));
16950                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
16951                                                                        } else {
16952                                                                                variable = modelFactory.createVariable(columnObject);
16953                                                                                variable.setCreateTable(true);
16954                                                                                variable.setSubType(SubType.record);
16955                                                                                TObjectName variableProperties = new TObjectName();
16956                                                                                variableProperties.setString("*");
16957                                                                                TableColumn variableProperty = modelFactory.createTableColumn(variable,
16958                                                                                                variableProperties, true);
16959                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16960                                                                                relation.setEffectType(EffectType.select);
16961                                                                                relation.setTarget(new TableColumnRelationshipElement(variableProperty));
16962                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
16963                                                                        }
16964                                                                }
16965
16966                                                                columnsInExpr visitor = new columnsInExpr();
16967                                                                column.getExpr().getRightOperand().inOrderTraverse(visitor);
16968
16969                                                                List<TObjectName> objectNames = visitor.getObjectNames();
16970                                                                List<TParseTreeNode> functions = visitor.getFunctions();
16971
16972                                                                if (functions != null && !functions.isEmpty()) {
16973                                                                        analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.select);
16974
16975                                                                }
16976
16977                                                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
16978                                                                if (subquerys != null && !subquerys.isEmpty()) {
16979                                                                        analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.select);
16980                                                                }
16981
16982                                                                analyzeDataFlowRelation(resultColumn, objectNames, EffectType.select, functions);
16983
16984                                                                List<TParseTreeNode> constants = visitor.getConstants();
16985                                                                analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.select, functions);
16986                                                        } else {
16987                                                                if (column.getFieldAttr() != null && isStructColumn(column.getFieldAttr())) {
16988                                                                        Table table = modelFactory.createTable(column.getFieldAttr().getSourceTable());
16989                                                                        for (int j = 0; j < table.getColumns().size(); j++) {
16990                                                                                TObjectName columnName = new TObjectName();
16991                                                                                if (table.getColumns().get(j).getName().equals(table.getAlias())) {
16992                                                                                        if (!SQLUtil.isEmpty(column.getColumnAlias())) {
16993                                                                                                columnName.setString(column.getColumnAlias());
16994                                                                                        } else {
16995                                                                                                columnName.setString(table.getColumns().get(j).getName());
16996                                                                                        }
16997                                                                                } else {
16998                                                                                        columnName.setString(
16999                                                                                                        table.getAlias() + "." + table.getColumns().get(j).getName());
17000                                                                                }
17001                                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel,
17002                                                                                                columnName);
17003                                                                                DataFlowRelationship relationship = modelFactory.createDataFlowRelation();
17004                                                                                relationship.setTarget(new ResultColumnRelationshipElement(resultColumn));
17005                                                                                relationship.addSource(
17006                                                                                                new TableColumnRelationshipElement(table.getColumns().get(j)));
17007                                                                        }
17008                                                                } else if (column.getExpr().getFunctionCall() != null && column.getExpr().getFunctionCall().getFunctionType() == EFunctionType.struct_t) {
17009                                                                        Function function = (Function) createFunction(column.getExpr().getFunctionCall());
17010                                                                        String functionName = getResultSetName(function);
17011                                                                        for (int j = 0; j < function.getColumns().size(); j++) {
17012                                                                                TObjectName columnName = new TObjectName();
17013                                                                                if (column.getAliasClause() != null) {
17014                                                                                        columnName.setString(column.getAliasClause() + "."
17015                                                                                                        + function.getColumns().get(j).getName());
17016                                                                                }
17017                                                                                else {
17018                                                                                        columnName.setString(functionName + "."
17019                                                                                                        + function.getColumns().get(j).getName());
17020                                                                                }
17021                                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel,
17022                                                                                                columnName);
17023                                                                                resultColumn.setStruct(true);
17024                                                                                DataFlowRelationship relationship = modelFactory.createDataFlowRelation();
17025                                                                                relationship.setTarget(new ResultColumnRelationshipElement(resultColumn));
17026                                                                                relationship.addSource(
17027                                                                                                new ResultColumnRelationshipElement(function.getColumns().get(j)));
17028                                                                        }
17029                                                                } else if (column.getExpr().getFunctionCall() != null && column.getExpr().getFunctionCall().getFunctionType() == EFunctionType.array_t) {
17030                                                                        Function function = (Function) createFunction(column.getExpr().getFunctionCall());
17031                                                                        String functionName = getResultSetName(function);
17032                                                                        if (function.getColumns().size() == 1) {
17033                                                                                // Scalar result inside ARRAY() - use just alias as column name
17034                                                                                TObjectName columnName = new TObjectName();
17035                                                                                if (column.getAliasClause() != null) {
17036                                                                                        columnName.setString(column.getAliasClause().toString());
17037                                                                                } else {
17038                                                                                        columnName.setString(functionName);
17039                                                                                }
17040                                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel,
17041                                                                                                columnName);
17042                                                                                DataFlowRelationship relationship = modelFactory.createDataFlowRelation();
17043                                                                                relationship.setTarget(new ResultColumnRelationshipElement(resultColumn));
17044                                                                                relationship.addSource(
17045                                                                                                new ResultColumnRelationshipElement(function.getColumns().get(0)));
17046                                                                        } else {
17047                                                                                for (int j = 0; j < function.getColumns().size(); j++) {
17048                                                                                        TObjectName columnName = new TObjectName();
17049                                                                                        if (column.getAliasClause() != null) {
17050                                                                                                columnName.setString(column.getAliasClause() + "."
17051                                                                                                                + getColumnNameOnly(function.getColumns().get(j).getName()));
17052                                                                                        }
17053                                                                                        else {
17054                                                                                                columnName.setString(functionName + "."
17055                                                                                                                + getColumnNameOnly(function.getColumns().get(j).getName()));
17056                                                                                        }
17057                                                                                        ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel,
17058                                                                                                        columnName);
17059                                                                                        resultColumn.setStruct(true);
17060                                                                                        DataFlowRelationship relationship = modelFactory.createDataFlowRelation();
17061                                                                                        relationship.setTarget(new ResultColumnRelationshipElement(resultColumn));
17062                                                                                        relationship.addSource(
17063                                                                                                        new ResultColumnRelationshipElement(function.getColumns().get(j)));
17064                                                                                }
17065                                                                        }
17066                                                                } else if (column.getExpr().getExprList() != null && column.getExpr().getExpressionType() == EExpressionType.array_t) {
17067                                                                        if(column.getExpr().getObjectOperand()!=null) {
17068                                                                                TObjectName exprColumnName = column.getExpr().getObjectOperand();
17069                                                                                Table table = modelFactory.createTable(exprColumnName.getSourceTable());
17070                                                                                for (int z = 0; z < table.getColumns().size(); z++) {
17071                                                                                        TableColumn tableColumn = table.getColumns().get(z);
17072                                                                                        if(getColumnName(exprColumnName.toString()).equals(getColumnName(tableColumn.getName()))) {
17073                                                                                                TObjectName columnName = new TObjectName();
17074                                                                                                if (column.getAliasClause() != null) {
17075                                                                                                        columnName.setString(column.getAliasClause().toString());
17076                                                                                                } else {
17077                                                                                                        columnName
17078                                                                                                                        .setString(getColumnNameOnly(columnName.toString()));
17079                                                                                                }
17080                                                                                                ResultColumn resultColumn = modelFactory
17081                                                                                                                .createResultColumn(resultSetModel, columnName);
17082                                                                                                resultColumn.setStruct(true);
17083                                                                                                DataFlowRelationship relationship = modelFactory
17084                                                                                                                .createDataFlowRelation();
17085                                                                                                relationship.setTarget(
17086                                                                                                                new ResultColumnRelationshipElement(resultColumn));
17087                                                                                                relationship.addSource(new TableColumnRelationshipElement(
17088                                                                                                                tableColumn));
17089                                                                                        }
17090                                                                                }
17091                                                                        }
17092                                                                        else {
17093                                                                                for (int j = 0; j < column.getExpr().getExprList().size(); j++) {
17094                                                                                        TExpression expression = column.getExpr().getExprList().getExpression(j);
17095                                                                                        if(expression.getExpressionType() == EExpressionType.function_t) {
17096                                                                                                Function function = (Function)createFunction(expression.getFunctionCall());
17097                                                                                                String functionName = getResultSetName(function);
17098                                                                                                if (function != null && function.getColumns() != null) {
17099                                                                                                        if (function.getColumns().size() == 1) {
17100                                                                                                                // Scalar function inside ARRAY[] (e.g., ARRAY[TO_JSON_STRING(col)])
17101                                                                                                                // Use just the alias as column name, not alias.functionName
17102                                                                                                                TObjectName columnName = new TObjectName();
17103                                                                                                                if (column.getAliasClause() != null) {
17104                                                                                                                        columnName.setString(column.getAliasClause().toString());
17105                                                                                                                } else {
17106                                                                                                                        columnName.setString(functionName);
17107                                                                                                                }
17108                                                                                                                ResultColumn resultColumn = modelFactory
17109                                                                                                                                .createResultColumn(resultSetModel, columnName);
17110                                                                                                                DataFlowRelationship relationship = modelFactory
17111                                                                                                                                .createDataFlowRelation();
17112                                                                                                                relationship.setTarget(
17113                                                                                                                                new ResultColumnRelationshipElement(resultColumn));
17114                                                                                                                relationship.addSource(new ResultColumnRelationshipElement(
17115                                                                                                                                function.getColumns().get(0)));
17116                                                                                                        } else {
17117                                                                                                                // Multi-column (struct-like) function - use dotted naming
17118                                                                                                                for (int x = 0; x < function.getColumns().size(); x++) {
17119                                                                                                                        TObjectName columnName = new TObjectName();
17120                                                                                                                        if (column.getAliasClause() != null) {
17121                                                                                                                                columnName.setString(column.getAliasClause() + "."
17122                                                                                                                                                + getColumnNameOnly(
17123                                                                                                                                                function.getColumns().get(x).getName()));
17124                                                                                                                        } else {
17125                                                                                                                                columnName.setString(
17126                                                                                                                                                functionName + "." + getColumnNameOnly(
17127                                                                                                                                                                function.getColumns().get(x).getName()));
17128                                                                                                                        }
17129                                                                                                                        ResultColumn resultColumn = modelFactory
17130                                                                                                                                        .createResultColumn(resultSetModel, columnName);
17131                                                                                                                        resultColumn.setStruct(true);
17132                                                                                                                        DataFlowRelationship relationship = modelFactory
17133                                                                                                                                        .createDataFlowRelation();
17134                                                                                                                        relationship.setTarget(
17135                                                                                                                                        new ResultColumnRelationshipElement(resultColumn));
17136                                                                                                                        relationship.addSource(new ResultColumnRelationshipElement(
17137                                                                                                                                        function.getColumns().get(x)));
17138                                                                                                                }
17139                                                                                                        }
17140                                                                                                }
17141                                                                                        }
17142                                                                                        else if(expression.getExpressionType() == EExpressionType.simple_object_name_t) {
17143                                                                                                TObjectName exprColumnName = expression.getObjectOperand();
17144                                                                                                Table table = modelFactory.createTable(exprColumnName.getSourceTable());
17145                                                                                                for (int z = 0; z < table.getColumns().size(); z++) {
17146                                                                                                        TableColumn tableColumn = table.getColumns().get(z);
17147                                                                                                        if(getColumnName(exprColumnName.toString()).equals(getColumnName(tableColumn.getName()))) {
17148                                                                                                                TObjectName columnName = new TObjectName();
17149                                                                                                                if (column.getAliasClause() != null) {
17150                                                                                                                        columnName.setString(column.getAliasClause().toString());
17151                                                                                                                } else {
17152                                                                                                                        columnName
17153                                                                                                                                        .setString(getColumnNameOnly(columnName.toString()));
17154                                                                                                                }
17155                                                                                                                ResultColumn resultColumn = modelFactory
17156                                                                                                                                .createResultColumn(resultSetModel, columnName);
17157                                                                                                                resultColumn.setStruct(true);
17158                                                                                                                DataFlowRelationship relationship = modelFactory
17159                                                                                                                                .createDataFlowRelation();
17160                                                                                                                relationship.setTarget(
17161                                                                                                                                new ResultColumnRelationshipElement(resultColumn));
17162                                                                                                                relationship.addSource(new TableColumnRelationshipElement(
17163                                                                                                                                tableColumn));
17164                                                                                                        }
17165                                                                                                }
17166                                                                                        }
17167                                                                                }
17168                                                                        }
17169                                                                } else {
17170                                                                        if ("*".equals(column.getColumnNameOnly())) {
17171                                                                                Map<String, Pair<String, TExpression>> replaceAsIdentifierMap = new HashMap<String, Pair<String, TExpression>>();
17172                                                                                Map<String, TObjectName> replaceColumnMap = new HashMap<String, TObjectName>();
17173                                                                                if(column.getReplaceExprAsIdentifiers()!=null && column.getReplaceExprAsIdentifiers().size()>0) {
17174                                                                                        for(TReplaceExprAsIdentifier replace: column.getReplaceExprAsIdentifiers()) {
17175                                                                                                replaceAsIdentifierMap.put(replace.getIdentifier().toString(), new Pair<String, TExpression>(column.getExpr().getExceptReplaceClause().toString(), replace.getExpr()));
17176                                                                                                replaceColumnMap.put(replace.getIdentifier().toString(), replace.getIdentifier());
17177                                                                                        }
17178                                                                                }
17179                                                                                
17180                                                                                TObjectName columnObject = column.getFieldAttr();
17181                                                                                List<TTable> sourceTables = columnObject.getSourceTableList();
17182                                                                                if (sourceTables != null && !sourceTables.isEmpty()) {
17183                                                                                        boolean[] determine = new boolean[sourceTables.size()];
17184                                                                                        for (int k = 0; k < sourceTables.size(); k++) {
17185                                                                                                TTable sourceTable = sourceTables.get(k);
17186                                                                                                Object tableModel = modelManager.getModel(sourceTable);
17187                                                                                                if (tableModel instanceof Table && ((Table) tableModel).isCreateTable()) {
17188                                                                                                        Table table = (Table) tableModel;
17189                                                                                                        for (int j = 0; j < table.getColumns().size(); j++) {
17190                                                                                                                TableColumn tableColumn = table.getColumns().get(j);
17191                                                                                                                if (column.getExceptColumnList() != null) {
17192                                                                                                                        boolean except = false;
17193                                                                                                                        for (TObjectName objectName : column.getExceptColumnList()) {
17194                                                                                                                                if(getColumnName(objectName.toString()).equals(getColumnName(tableColumn.getName()))) {
17195                                                                                                                                        except = true;
17196                                                                                                                                        break;
17197                                                                                                                                }
17198                                                                                                                        }
17199                                                                                                                        if (!except && tableColumn.isStruct()) {
17200                                                                                                                                List<String> names = SQLUtil
17201                                                                                                                                                .parseNames(tableColumn.getName());
17202                                                                                                                                for (String name : names) {
17203                                                                                                                                        for (TObjectName objectName : column
17204                                                                                                                                                        .getExceptColumnList()) {
17205                                                                                                                                                if (getColumnName(objectName.toString())
17206                                                                                                                                                                .equals(getColumnName(name))) {
17207                                                                                                                                                        except = true;
17208                                                                                                                                                        break;
17209                                                                                                                                                }
17210                                                                                                                                        }
17211                                                                                                                                        if (except) {
17212                                                                                                                                                break;
17213                                                                                                                                        }
17214                                                                                                                                }
17215                                                                                                                        }
17216                                                                                                                        if(except){
17217                                                                                                                                continue;
17218                                                                                                                        }
17219                                                                                                                }
17220                                                                                                                
17221                                                                                                                if (replaceAsIdentifierMap.containsKey(tableColumn.getName())) {
17222                                                                                                                        Pair<String, TExpression> expr = replaceAsIdentifierMap.get(tableColumn.getName());
17223                                                                                                                        ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel, replaceColumnMap.get(tableColumn.getName()));
17224                                                                                                                        Transform transform = new Transform();
17225                                                                                                                        transform.setType(Transform.EXPRESSION);
17226                                                                                                                        TObjectName expression = new TObjectName();
17227                                                                                                                        expression.setString(expr.first);
17228                                                                                                                transform.setCode(expression);
17229                                                                                                                        resultColumn.setTransform(transform);
17230                                                                                                                        analyzeResultColumnExpressionRelation(resultColumn, expr.second);
17231                                                                                                                }
17232                                                                                                                else {
17233                                                                                                                        String columnName = DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName());
17234                                                                                                                        //bigquery
17235                                                                                                                        boolean exist = false;
17236                                                                                                                        if (!columnName.matches("(?i)f\\d+_")) {
17237                                                                                                                                if (!keyMap.containsKey(columnName)) {
17238                                                                                                                                        keyMap.put(columnName, new AtomicInteger(0));
17239                                                                                                                                } else {
17240                                                                                                                                        while (columnNames.contains(columnName)) {
17241                                                                                                                                                if(columnName.indexOf("*") == -1 && stmt.toString().matches("(?is).*using\\s*\\(\\s*"+columnName+"\\s*\\).*")) {
17242                                                                                                                                                        exist = true;
17243                                                                                                                                                        break;
17244                                                                                                                                                } else if (keyMap.containsKey(columnName)) {
17245                                                                                                                                                        int index = keyMap.get(columnName)
17246                                                                                                                                                                        .incrementAndGet();
17247                                                                                                                                                        columnName = columnName + index;
17248                                                                                                                                                }
17249                                                                                                                                        }
17250                                                                                                                                }
17251                                                                                                                                columnNames.add(columnName);
17252                                                                                                                        }
17253                                                                                                                        if (exist) {
17254                                                                                                                                String targetColumn = columnName;
17255                                                                                                                                DataFlowRelationship relation = modelFactory
17256                                                                                                                                                .createDataFlowRelation();
17257                                                                                                                                relation.setEffectType(EffectType.select);
17258                                                                                                                                relation.setTarget(new ResultColumnRelationshipElement(
17259                                                                                                                                                resultSetModel.getColumns().stream()
17260                                                                                                                                                                .filter(t -> t.getName()
17261                                                                                                                                                                                .equalsIgnoreCase(targetColumn))
17262                                                                                                                                                                .findFirst().get()));
17263                                                                                                                                relation.addSource(new TableColumnRelationshipElement(
17264                                                                                                                                                tableColumn));
17265                                                                                                                                continue;
17266                                                                                                                        }
17267                                                                                                                        if (DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName())
17268                                                                                                                                        .equalsIgnoreCase(DlineageUtil.getIdentifierNormalColumnName(columnName))) {
17269                                                                                                                                columnName = tableColumn.getName();
17270                                                                                                                        }
17271                                                                                                                        ResultColumn resultColumn = modelFactory.createStarResultColumn(resultSetModel, column, columnName);
17272                                                                                                                        resultColumn.setStruct(tableColumn.isStruct());
17273                                                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17274                                                                                                                        relation.setEffectType(EffectType.select);
17275                                                                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
17276                                                                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
17277                                                                                                                }
17278                                                                                                        }
17279                                                                                                        determine[k] = true;
17280                                                                                                        continue;
17281                                                                                                }
17282                                                                                                else if (tableModel instanceof ResultSet && ((ResultSet) tableModel).isDetermined()) {
17283                                                                                                        ResultSet table = (ResultSet) tableModel;
17284                                                                                                        for (int j = 0; j < table.getColumns().size(); j++) {
17285                                                                                                                ResultColumn tableColumn = table.getColumns().get(j);
17286                                                                                                                if (column.getExceptColumnList() != null) {
17287                                                                                                                        boolean except = false;
17288                                                                                                                        for (TObjectName objectName : column.getExceptColumnList()) {
17289                                                                                                                                if(getColumnName(objectName.toString()).equals(getColumnName(tableColumn.getName()))) {
17290                                                                                                                                        except = true;
17291                                                                                                                                        break;
17292                                                                                                                                }
17293                                                                                                                        }
17294                                                                                                                        if (!except && tableColumn.isStruct()) {
17295                                                                                                                                List<String> names = SQLUtil
17296                                                                                                                                                .parseNames(tableColumn.getName());
17297                                                                                                                                for (String name : names) {
17298                                                                                                                                        for (TObjectName objectName : column
17299                                                                                                                                                        .getExceptColumnList()) {
17300                                                                                                                                                if (getColumnName(objectName.toString())
17301                                                                                                                                                                .equals(getColumnName(name))) {
17302                                                                                                                                                        except = true;
17303                                                                                                                                                        break;
17304                                                                                                                                                }
17305                                                                                                                                        }
17306                                                                                                                                        if (except) {
17307                                                                                                                                                break;
17308                                                                                                                                        }
17309                                                                                                                                }
17310                                                                                                                        }
17311                                                                                                                        if(except){
17312                                                                                                                                continue;
17313                                                                                                                        }
17314                                                                                                                }
17315                                                                                                                if (replaceAsIdentifierMap.containsKey(tableColumn.getName())) {
17316                                                                                                                        Pair<String, TExpression> expr = replaceAsIdentifierMap.get(tableColumn.getName());
17317                                                                                                                        ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel, replaceColumnMap.get(tableColumn.getName()));
17318                                                                                                                        Transform transform = new Transform();
17319                                                                                                                        transform.setType(Transform.EXPRESSION);
17320                                                                                                                        TObjectName expression = new TObjectName();
17321                                                                                                                        expression.setString(expr.first);
17322                                                                                                                transform.setCode(expression);
17323                                                                                                                        resultColumn.setTransform(transform);
17324                                                                                                                        analyzeResultColumnExpressionRelation(resultColumn, expr.second);
17325                                                                                                                }
17326                                                                                                                else if(tableColumn.getRefColumnName()!=null) {
17327                                                                                                                        ResultColumn resultColumn = modelFactory.createStarResultColumn(resultSetModel, column, tableColumn.getRefColumnName());
17328                                                                                                                        if(tableColumn.isStruct()) {
17329                                                                                                                                resultColumn.setStruct(true);
17330                                                                                                                        }
17331                                                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17332                                                                                                                        relation.setEffectType(EffectType.select);
17333                                                                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
17334                                                                                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn));
17335                                                                                                                }
17336                                                                                                                else {
17337                                                                                                                        String columnName = DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName());
17338                                                                                                                        //bigquery
17339                                                                                                                        if (!columnName.matches("(?i)f\\d+_")) {
17340                                                                                                                                if (!keyMap.containsKey(columnName)) {
17341                                                                                                                                        keyMap.put(columnName, new AtomicInteger(0));
17342                                                                                                                                } else {
17343                                                                                                                                        while (columnNames.contains(columnName)) {
17344                                                                                                                                                int index = keyMap.get(columnName)
17345                                                                                                                                                                .incrementAndGet();
17346                                                                                                                                                columnName = columnName + index;
17347                                                                                                                                        }
17348                                                                                                                                }
17349                                                                                                                                columnNames.add(columnName);
17350                                                                                                                        }
17351                                                                                                                        if (DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName())
17352                                                                                                                                        .equalsIgnoreCase(DlineageUtil.getIdentifierNormalColumnName(columnName))) {
17353                                                                                                                                columnName = tableColumn.getName();
17354                                                                                                                        }
17355                                                                                                                        if (modelManager.getModel(column) instanceof ResultColumn) {
17356                                                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17357                                                                                                                                relation.setEffectType(EffectType.select);
17358                                                                                                                                relation.setTarget(new ResultColumnRelationshipElement((ResultColumn)modelManager.getModel(column)));
17359                                                                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
17360                                                                                                                        }
17361                                                                                                                        else {
17362                                                                                                                                ResultColumn resultColumn = modelFactory.createStarResultColumn(resultSetModel, column, columnName);
17363                                                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17364                                                                                                                                relation.setEffectType(EffectType.select);
17365                                                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
17366                                                                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
17367                                                                                                                        }
17368                                                                                                                }
17369                                                                                                        }
17370                                                                                                        determine[k] = true;
17371                                                                                                        continue;
17372                                                                                                }
17373                                                                                                else {
17374                                                                                                        ResultColumn resultColumn = modelFactory
17375                                                                                                                        .createResultColumn(resultSetModel, column);
17376                                                                                                        if (tableModel instanceof Function) {
17377                                                                                                
17378                                                                                                        } else {
17379                                                                                                                TObjectName[] columns = modelManager
17380                                                                                                                                .getTableColumns(sourceTable);
17381                                                                                                                for (int j = 0; j < columns.length; j++) {
17382                                                                                                                        TObjectName columnName = columns[j];
17383                                                                                                                        if (columnName == null
17384                                                                                                                                        || "*".equals(getColumnName(columnName))) {
17385                                                                                                                                continue;
17386                                                                                                                        }
17387                                                                                                                        if (isStructColumn(columnName)) {
17388                                                                                                                                continue;
17389                                                                                                                        }
17390
17391                                                                                                                        resultColumn.bindStarLinkColumn(columnName);
17392                                                                                                                        if (column.getExceptColumnList() != null) {
17393                                                                                                                                for (TObjectName objectName : column
17394                                                                                                                                                .getExceptColumnList()) {
17395                                                                                                                                        resultColumn.unbindStarLinkColumn(objectName);
17396                                                                                                                                }
17397                                                                                                                        }
17398                                                                                                                }
17399                                                                                                                if (tableModel instanceof ResultSet) {
17400                                                                                                                        ResultSet queryTable = (ResultSet) tableModel;
17401                                                                                                                        if (!containStarColumn(queryTable)) {
17402                                                                                                                                resultColumn.setShowStar(false);
17403                                                                                                                        }
17404                                                                                                                }
17405                                                                                                                if (tableModel instanceof Table) {
17406                                                                                                                        Table table = (Table) tableModel;
17407                                                                                                                        if (table.isCreateTable()) {
17408                                                                                                                                resultColumn.setShowStar(false);
17409                                                                                                                        }
17410                                                                                                                }
17411                                                                                                        }
17412                                                                                                }
17413                                                                                        }
17414                                                                                        if(!Arrays.toString(determine).contains("false")) {
17415                                                                                                continue;
17416                                                                                        }
17417                                                                                } else {
17418                                                                                        ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel, column);
17419                                                                                        TTableList tables = stmt.getTables();
17420                                                                                        for (int k = 0; k < tables.size(); k++) {
17421                                                                                                TTable table = tables.getTable(k);
17422                                                                                                TObjectName[] columns = modelManager.getTableColumns(table);
17423                                                                                                for (int j = 0; j < columns.length; j++) {
17424                                                                                                        TObjectName columnName = columns[j];
17425                                                                                                        if (columnName == null) {
17426                                                                                                                continue;
17427                                                                                                        }
17428                                                                                                        if ("*".equals(getColumnName(columnName))) {
17429                                                                                                                if (modelManager.getModel(table) instanceof Table) {
17430                                                                                                                        Table tableModel = (Table) modelManager.getModel(table);
17431                                                                                                                        if (tableModel != null
17432                                                                                                                                        && !tableModel.getColumns().isEmpty()) {
17433                                                                                                                                for (TableColumn item : tableModel.getColumns()) {
17434                                                                                                                                        resultColumn
17435                                                                                                                                                        .bindStarLinkColumn(item.getColumnObject());
17436                                                                                                                                        if (table.getSubquery() == null
17437                                                                                                                                                        && table.getCTE() == null
17438                                                                                                                                                        && !tableModel.isCreateTable()) {
17439                                                                                                                                                resultColumn.setShowStar(true);
17440                                                                                                                                        }
17441                                                                                                                                }
17442                                                                                                                        }
17443                                                                                                                } else if (modelManager.getModel(table) instanceof QueryTable) {
17444                                                                                                                        QueryTable tableModel = (QueryTable) modelManager
17445                                                                                                                                        .getModel(table);
17446                                                                                                                        if (tableModel != null
17447                                                                                                                                        && !tableModel.getColumns().isEmpty()) {
17448                                                                                                                                for (ResultColumn item : tableModel.getColumns()) {
17449                                                                                                                                        if (item.hasStarLinkColumn()) {
17450                                                                                                                                                for (TObjectName starLinkColumn : item
17451                                                                                                                                                                .getStarLinkColumnList()) {
17452                                                                                                                                                        resultColumn
17453                                                                                                                                                                        .bindStarLinkColumn(starLinkColumn);
17454                                                                                                                                                }
17455                                                                                                                                        } else if (item
17456                                                                                                                                                        .getColumnObject() instanceof TObjectName) {
17457                                                                                                                                                resultColumn.bindStarLinkColumn(
17458                                                                                                                                                                (TObjectName) item.getColumnObject());
17459                                                                                                                                        } else if (item
17460                                                                                                                                                        .getColumnObject() instanceof TResultColumn) {
17461                                                                                                                                                TResultColumn queryTableColumn = (TResultColumn) item
17462                                                                                                                                                                .getColumnObject();
17463                                                                                                                                                TObjectName tableColumnObject = queryTableColumn
17464                                                                                                                                                                .getFieldAttr();
17465                                                                                                                                                if (tableColumnObject != null) {
17466                                                                                                                                                        resultColumn.bindStarLinkColumn(
17467                                                                                                                                                                        tableColumnObject);
17468                                                                                                                                                } else if (queryTableColumn
17469                                                                                                                                                                .getAliasClause() != null && !item.isStruct()) {
17470                                                                                                                                                        resultColumn.bindStarLinkColumn(
17471                                                                                                                                                                        queryTableColumn.getAliasClause()
17472                                                                                                                                                                                        .getAliasName());
17473                                                                                                                                                }
17474                                                                                                                                        }
17475                                                                                                                                }
17476                                                                                                                        }
17477                                                                                                                }
17478                                                                                                                continue;
17479                                                                                                        }
17480                                                                                                        resultColumn.bindStarLinkColumn(columnName);
17481                                                                                                }
17482                                                                                        }
17483                                                                                }
17484                                                                                isDetermined = false;
17485                                                                        }
17486                                                                        else {
17487                                                                                if(column.getAliasClause()!=null && column.getAliasClause().getColumns()!=null) {
17488                                                                                        for(TObjectName aliasColumn: column.getAliasClause().getColumns()) {
17489                                                                                                modelFactory.createResultColumn(resultSetModel, aliasColumn);
17490                                                                                        }
17491                                                                                }
17492                                                                                else {
17493                                                                                        modelFactory.createResultColumn(resultSetModel, column);
17494                                                                                }
17495                                                                        }
17496                                                                        analyzeResultColumn(column, EffectType.select);
17497                                                                }
17498                                                        }
17499                                                }
17500                                                if (isDetermined) {
17501                                                        resultSetModel.setDetermined(isDetermined);
17502                                                }
17503                                        }
17504
17505                                        TSelectSqlStatement parent = getParentSetSelectStmt(stmt);
17506                                        if (parent != null && parent.getSetOperatorType() != ESetOperatorType.none) {
17507                                                ResultSet resultSetModel = modelFactory.createResultSet(stmt, false);
17508                                                if(queryModel == null) {
17509                                                        queryModel = resultSetModel;
17510                                                } 
17511
17512                                                createPseudoImpactRelation(stmt, resultSetModel, EffectType.select);
17513
17514                                                boolean isDetermined = true;
17515                                                for (int i = 0; i < stmt.getResultColumnList().size(); i++) {
17516                                                        TResultColumn column = stmt.getResultColumnList().getResultColumn(i);
17517                                                        if ("*".equals(column.getColumnNameOnly())) {
17518                                                                
17519                                                                Map<String, Pair<String, TExpression>> replaceAsIdentifierMap = new HashMap<String, Pair<String, TExpression>>();
17520                                                                Map<String, TObjectName> replaceColumnMap = new HashMap<String, TObjectName>();
17521                                                                if(column.getReplaceExprAsIdentifiers()!=null && column.getReplaceExprAsIdentifiers().size()>0) {
17522                                                                        for(TReplaceExprAsIdentifier replace: column.getReplaceExprAsIdentifiers()) {
17523                                                                                replaceAsIdentifierMap.put(replace.getIdentifier().toString(), new Pair<String, TExpression>(column.getExpr().getExceptReplaceClause().toString(), replace.getExpr()));
17524                                                                                replaceColumnMap.put(replace.getIdentifier().toString(), replace.getIdentifier());
17525                                                                        }
17526                                                                }
17527                                                                
17528                                                                TObjectName columnObject = column.getFieldAttr();
17529                                                                TTable sourceTable = columnObject.getSourceTable();
17530                                                                if (sourceTable != null) {
17531                                                                        Object tableModel = modelManager.getModel(sourceTable);
17532                                                                        if (tableModel instanceof Table && ((Table) tableModel).isCreateTable()) {
17533                                                                                Table table = (Table) tableModel;
17534                                                                                for (int j = 0; j < table.getColumns().size(); j++) {
17535                                                                                        TableColumn tableColumn = table.getColumns().get(j);
17536                                                                                        if (column.getExceptColumnList() != null) {
17537                                                                                                boolean except = false;
17538                                                                                                for (TObjectName objectName : column.getExceptColumnList()) {
17539                                                                                                        if (getColumnName(objectName.toString())
17540                                                                                                                        .equals(getColumnName(tableColumn.getName()))) {
17541                                                                                                                except = true;
17542                                                                                                                break;
17543                                                                                                        }
17544                                                                                                }
17545                                                                                                if (!except && tableColumn.isStruct()) {
17546                                                                                                        List<String> names = SQLUtil
17547                                                                                                                        .parseNames(tableColumn.getName());
17548                                                                                                        for (String name : names) {
17549                                                                                                                for (TObjectName objectName : column
17550                                                                                                                                .getExceptColumnList()) {
17551                                                                                                                        if (getColumnName(objectName.toString())
17552                                                                                                                                        .equals(getColumnName(name))) {
17553                                                                                                                                except = true;
17554                                                                                                                                break;
17555                                                                                                                        }
17556                                                                                                                }
17557                                                                                                                if (except) {
17558                                                                                                                        break;
17559                                                                                                                }
17560                                                                                                        }
17561                                                                                                }
17562                                                                                                if (except) {
17563                                                                                                        continue;
17564                                                                                                }
17565                                                                                        }
17566                                                                                        
17567                                                                                        if (replaceAsIdentifierMap.containsKey(tableColumn.getName())) {
17568                                                                                                Pair<String, TExpression> expr = replaceAsIdentifierMap.get(tableColumn.getName());
17569                                                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel, replaceColumnMap.get(tableColumn.getName()));
17570                                                                                                Transform transform = new Transform();
17571                                                                                                transform.setType(Transform.EXPRESSION);
17572                                                                                                TObjectName expression = new TObjectName();
17573                                                                                                expression.setString(expr.first);
17574                                                                                        transform.setCode(expression);
17575                                                                                                resultColumn.setTransform(transform);
17576                                                                                                analyzeResultColumnExpressionRelation(resultColumn, expr.second);
17577                                                                                        }
17578                                                                                        else {
17579                                                                                                ResultColumn resultColumn = modelFactory.createStarResultColumn(
17580                                                                                                                resultSetModel, column, tableColumn.getName());
17581                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17582                                                                                                relation.setEffectType(EffectType.select);
17583                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
17584                                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
17585                                                                                        }
17586                                                                                }
17587                                                                                continue;
17588                                                                        } else if (tableModel instanceof ResultSet
17589                                                                                        && ((ResultSet) tableModel).isDetermined()) {
17590                                                                                ResultSet table = (ResultSet) tableModel;
17591                                                                                for (int j = 0; j < table.getColumns().size(); j++) {
17592                                                                                        ResultColumn tableColumn = table.getColumns().get(j);
17593                                                                                        if (column.getExceptColumnList() != null) {
17594                                                                                                boolean except = false;
17595                                                                                                for (TObjectName objectName : column.getExceptColumnList()) {
17596                                                                                                        if (getColumnName(objectName.toString())
17597                                                                                                                        .equals(getColumnName(tableColumn.getName()))) {
17598                                                                                                                except = true;
17599                                                                                                                break;
17600                                                                                                        }
17601                                                                                                }
17602                                                                                                if (!except && tableColumn.isStruct()) {
17603                                                                                                        List<String> names = SQLUtil
17604                                                                                                                        .parseNames(tableColumn.getName());
17605                                                                                                        for (String name : names) {
17606                                                                                                                for (TObjectName objectName : column
17607                                                                                                                                .getExceptColumnList()) {
17608                                                                                                                        if (getColumnName(objectName.toString())
17609                                                                                                                                        .equals(getColumnName(name))) {
17610                                                                                                                                except = true;
17611                                                                                                                                break;
17612                                                                                                                        }
17613                                                                                                                }
17614                                                                                                                if (except) {
17615                                                                                                                        break;
17616                                                                                                                }
17617                                                                                                        }
17618                                                                                                }
17619                                                                                                if (except) {
17620                                                                                                        continue;
17621                                                                                                }
17622                                                                                        }
17623                                                                                        
17624                                                                                        if (replaceAsIdentifierMap.containsKey(tableColumn.getName())) {
17625                                                                                                Pair<String, TExpression> expr = replaceAsIdentifierMap.get(tableColumn.getName());
17626                                                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel, replaceColumnMap.get(tableColumn.getName()));
17627                                                                                                Transform transform = new Transform();
17628                                                                                                transform.setType(Transform.EXPRESSION);
17629                                                                                                TObjectName expression = new TObjectName();
17630                                                                                                expression.setString(expr.first);
17631                                                                                        transform.setCode(expression);
17632                                                                                                resultColumn.setTransform(transform);
17633                                                                                                analyzeResultColumnExpressionRelation(resultColumn, expr.second);
17634                                                                                        }
17635                                                                                        else if (tableColumn.getRefColumnName() != null) {
17636                                                                                                ResultColumn resultColumn = modelFactory.createStarResultColumn(
17637                                                                                                                (ResultSet)queryModel, column, tableColumn.getRefColumnName());
17638                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17639                                                                                                relation.setEffectType(EffectType.select);
17640                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
17641                                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
17642                                                                                        } else {
17643                                                                                                ResultColumn resultColumn = modelFactory.createStarResultColumn(
17644                                                                                                                resultSetModel, column, tableColumn.getName());
17645                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17646                                                                                                relation.setEffectType(EffectType.select);
17647                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
17648                                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
17649                                                                                        }
17650                                                                                }
17651                                                                                continue;
17652                                                                        }
17653                                                                        else {
17654                                                                                isDetermined = false;
17655                                                                        }
17656                                                                }
17657                                                                
17658                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel, column);
17659                                                                
17660                                                                if (columnObject.getTableToken() != null && sourceTable != null) {
17661                                                                        TObjectName[] columns = modelManager.getTableColumns(sourceTable);
17662                                                                        for (int j = 0; j < columns.length; j++) {
17663                                                                                TObjectName columnName = columns[j];
17664                                                                                if (columnName == null || "*".equals(getColumnName(columnName))) {
17665                                                                                        continue;
17666                                                                                }
17667                                                                                resultColumn.bindStarLinkColumn(columnName);
17668                                                                        }
17669
17670                                                                        if (modelManager.getModel(sourceTable) instanceof Table) {
17671                                                                                Table tableModel = (Table) modelManager.getModel(sourceTable);
17672                                                                                if (tableModel != null && !tableModel.getColumns().isEmpty()) {
17673                                                                                        for (TableColumn item : tableModel.getColumns()) {
17674                                                                                                if ("*".equals(getColumnName(item.getColumnObject()))) {
17675                                                                                                        continue;
17676                                                                                                }
17677                                                                                                resultColumn.bindStarLinkColumn(item.getColumnObject());
17678                                                                                        }
17679                                                                                }
17680                                                                        } else if (modelManager.getModel(sourceTable) instanceof QueryTable) {
17681                                                                                QueryTable tableModel = (QueryTable) modelManager.getModel(sourceTable);
17682                                                                                if (tableModel != null && !tableModel.getColumns().isEmpty()) {
17683                                                                                        for (ResultColumn item : tableModel.getColumns()) {
17684                                                                                                if (item.hasStarLinkColumn()) {
17685                                                                                                        for (TObjectName starLinkColumn : item.getStarLinkColumnList()) {
17686                                                                                                                if ("*".equals(getColumnName(starLinkColumn))) {
17687                                                                                                                        continue;
17688                                                                                                                }
17689                                                                                                                resultColumn.bindStarLinkColumn(starLinkColumn);
17690                                                                                                        }
17691                                                                                                } else if (item.getColumnObject() instanceof TObjectName) {
17692                                                                                                        TObjectName starLinkColumn = (TObjectName) item.getColumnObject();
17693                                                                                                        if ("*".equals(getColumnName(starLinkColumn))) {
17694                                                                                                                continue;
17695                                                                                                        }
17696                                                                                                        resultColumn.bindStarLinkColumn(starLinkColumn);
17697                                                                                                }
17698                                                                                        }
17699                                                                                }
17700                                                                        }
17701
17702                                                                } else {
17703                                                                        TTableList tables = stmt.getTables();
17704                                                                        for (int k = 0; k < tables.size(); k++) {
17705                                                                                TTable table = tables.getTable(k);
17706                                                                                TObjectName[] columns = modelManager.getTableColumns(table);
17707                                                                                for (int j = 0; j < columns.length; j++) {
17708                                                                                        TObjectName columnName = columns[j];
17709                                                                                        if (columnName == null) {
17710                                                                                                continue;
17711                                                                                        }
17712                                                                                        if ("*".equals(getColumnName(columnName))) {
17713                                                                                                if (modelManager.getModel(table) instanceof Table) {
17714                                                                                                        Table tableModel = (Table) modelManager.getModel(table);
17715                                                                                                        if (tableModel != null && !tableModel.getColumns().isEmpty()) {
17716                                                                                                                for (TableColumn item : tableModel.getColumns()) {
17717                                                                                                                        resultColumn.bindStarLinkColumn(item.getColumnObject());
17718                                                                                                                }
17719                                                                                                        }
17720                                                                                                } else if (modelManager.getModel(table) instanceof QueryTable) {
17721                                                                                                        QueryTable tableModel = (QueryTable) modelManager.getModel(table);
17722                                                                                                        if (tableModel != null && !tableModel.getColumns().isEmpty()) {
17723                                                                                                                for (ResultColumn item : tableModel.getColumns()) {
17724                                                                                                                        if (item.hasStarLinkColumn()) {
17725                                                                                                                                for (TObjectName starLinkColumn : item
17726                                                                                                                                                .getStarLinkColumnList()) {
17727                                                                                                                                        resultColumn.bindStarLinkColumn(starLinkColumn);
17728                                                                                                                                }
17729                                                                                                                        } else if (item.getColumnObject() instanceof TObjectName) {
17730                                                                                                                                resultColumn.bindStarLinkColumn(
17731                                                                                                                                                (TObjectName) item.getColumnObject());
17732                                                                                                                        } else if (item
17733                                                                                                                                        .getColumnObject() instanceof TResultColumn) {
17734                                                                                                                                TResultColumn queryTableColumn = (TResultColumn) item
17735                                                                                                                                                .getColumnObject();
17736                                                                                                                                TObjectName tableColumnObject = queryTableColumn
17737                                                                                                                                                .getFieldAttr();
17738                                                                                                                                if (tableColumnObject != null) {
17739                                                                                                                                        resultColumn.bindStarLinkColumn(tableColumnObject);
17740                                                                                                                                } else if (queryTableColumn.getAliasClause() != null) {
17741                                                                                                                                        resultColumn.bindStarLinkColumn(queryTableColumn
17742                                                                                                                                                        .getAliasClause().getAliasName());
17743                                                                                                                                }
17744                                                                                                                        }
17745                                                                                                                }
17746                                                                                                        }
17747                                                                                                }
17748
17749                                                                                                continue;
17750                                                                                        }
17751                                                                                        resultColumn.bindStarLinkColumn(columnName);
17752                                                                                }
17753                                                                        }
17754                                                                }
17755                                                        }
17756                                                        else {
17757                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel, column);
17758                                                        }
17759                                                        analyzeResultColumn(column, EffectType.select);
17760
17761                                                }
17762                                                
17763                                                resultSetModel.setDetermined(isDetermined);
17764                                        }
17765                                } else {
17766                                        for (int i = 0; i < stmt.getResultColumnList().size(); i++) {
17767                                                TResultColumn column = stmt.getResultColumnList().getResultColumn(i);
17768
17769                                                if (!(queryModel instanceof ResultSet)) {
17770                                                        continue;
17771                                                }
17772                                                
17773                                                ResultSet resultSetModel = (ResultSet)queryModel;
17774                                                        
17775                                                if ("*".equals(column.getColumnNameOnly())) {
17776                                                        TObjectName columnObject = column.getFieldAttr();
17777                                                        TTable sourceTable = columnObject.getSourceTable();
17778                                                        if (column.toString().indexOf(".") == -1 && stmt.getTables().size() > 1) {
17779                                                                sourceTable = null;
17780                                                        }
17781                                                        if (sourceTable != null) {
17782                                                                {
17783                                                                        Object tableModel = modelManager.getModel(sourceTable);
17784                                                                        if (tableModel instanceof Table && ((Table) tableModel).isCreateTable()) {
17785                                                                                Table table = (Table) tableModel;
17786                                                                                for (int j = 0; j < table.getColumns().size(); j++) {
17787                                                                                        TableColumn tableColumn = table.getColumns().get(j);
17788                                                                                        if (column.getExceptColumnList() != null) {
17789                                                                                                boolean except = false;
17790                                                                                                for (TObjectName objectName : column.getExceptColumnList()) {
17791                                                                                                        if (getColumnName(objectName.toString())
17792                                                                                                                        .equals(getColumnName(tableColumn.getName()))) {
17793                                                                                                                except = true;
17794                                                                                                                break;
17795                                                                                                        }
17796                                                                                                }
17797                                                                                                if (!except && tableColumn.isStruct()) {
17798                                                                                                        List<String> names = SQLUtil
17799                                                                                                                        .parseNames(tableColumn.getName());
17800                                                                                                        for (String name : names) {
17801                                                                                                                for (TObjectName objectName : column
17802                                                                                                                                .getExceptColumnList()) {
17803                                                                                                                        if (getColumnName(objectName.toString())
17804                                                                                                                                        .equals(getColumnName(name))) {
17805                                                                                                                                except = true;
17806                                                                                                                                break;
17807                                                                                                                        }
17808                                                                                                                }
17809                                                                                                                if (except) {
17810                                                                                                                        break;
17811                                                                                                                }
17812                                                                                                        }
17813                                                                                                }
17814                                                                                                if (except) {
17815                                                                                                        continue;
17816                                                                                                }
17817                                                                                        }
17818                                                                                        ResultColumn resultColumn = modelFactory.createStarResultColumn(
17819                                                                                                        resultSetModel, column, tableColumn.getName());
17820                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17821                                                                                        relation.setEffectType(EffectType.select);
17822                                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
17823                                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
17824                                                                                }
17825                                                                                if(stmt.getResultColumnList().size() == 1) {
17826                                                                                        resultSetModel.setDetermined(true);
17827                                                                                }
17828                                                                                else {
17829                                                                                        int starCount = 0;
17830                                                                                        for (int j = 0; j < stmt.getResultColumnList().size(); j++) {
17831                                                                                                if (stmt.getResultColumnList().getResultColumn(j).getColumnNameOnly()
17832                                                                                                                .endsWith("*")) {
17833                                                                                                        starCount += 1;
17834                                                                                                }
17835                                                                                        }
17836                                                                                        if (starCount <= 1) {
17837                                                                                                resultSetModel.setDetermined(true);
17838                                                                                        }
17839                                                                                }
17840                                                                                continue;
17841                                                                        } else if (tableModel instanceof ResultSet
17842                                                                                        && ((ResultSet) tableModel).isDetermined()) {
17843                                                                                ResultSet table = (ResultSet) tableModel;
17844                                                                                for (int j = 0; j < table.getColumns().size(); j++) {
17845                                                                                        ResultColumn tableColumn = table.getColumns().get(j);
17846                                                                                        if (column.getExceptColumnList() != null) {
17847                                                                                                boolean except = false;
17848                                                                                                for (TObjectName objectName : column.getExceptColumnList()) {
17849                                                                                                        if (getColumnName(objectName.toString())
17850                                                                                                                        .equals(getColumnName(tableColumn.getName()))) {
17851                                                                                                                except = true;
17852                                                                                                                break;
17853                                                                                                        }
17854                                                                                                }
17855                                                                                                if (!except && tableColumn.isStruct()) {
17856                                                                                                        List<String> names = SQLUtil
17857                                                                                                                        .parseNames(tableColumn.getName());
17858                                                                                                        for (String name : names) {
17859                                                                                                                for (TObjectName objectName : column
17860                                                                                                                                .getExceptColumnList()) {
17861                                                                                                                        if (getColumnName(objectName.toString())
17862                                                                                                                                        .equals(getColumnName(name))) {
17863                                                                                                                                except = true;
17864                                                                                                                                break;
17865                                                                                                                        }
17866                                                                                                                }
17867                                                                                                                if (except) {
17868                                                                                                                        break;
17869                                                                                                                }
17870                                                                                                        }
17871                                                                                                }
17872                                                                                                if (except) {
17873                                                                                                        continue;
17874                                                                                                }
17875                                                                                        }
17876                                                                                        if (tableColumn.getRefColumnName() != null) {
17877                                                                                                ResultColumn resultColumn = modelFactory.createStarResultColumn(
17878                                                                                                                (ResultSet)queryModel, column, tableColumn.getRefColumnName());
17879                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17880                                                                                                relation.setEffectType(EffectType.select);
17881                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
17882                                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
17883                                                                                        } else {
17884                                                                                                ResultColumn resultColumn = modelFactory.createStarResultColumn(
17885                                                                                                                resultSetModel, column, tableColumn.getName());
17886                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
17887                                                                                                relation.setEffectType(EffectType.select);
17888                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
17889                                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
17890                                                                                        }
17891                                                                                }
17892                                                                                if(stmt.getResultColumnList().size() == 1) {
17893                                                                                        resultSetModel.setDetermined(true);
17894                                                                                }
17895                                                                                else {
17896                                                                                        int starCount = 0;
17897                                                                                        for (int j = 0; j < stmt.getResultColumnList().size(); j++) {
17898                                                                                                if (stmt.getResultColumnList().getResultColumn(j).getColumnNameOnly()
17899                                                                                                                .endsWith("*")) {
17900                                                                                                        starCount += 1;
17901                                                                                                }
17902                                                                                        }
17903                                                                                        if (starCount <= 1) {
17904                                                                                                resultSetModel.setDetermined(true);
17905                                                                                        }
17906                                                                                }
17907                                                                                continue;
17908                                                                        }
17909                                                                }
17910                                                                
17911                                                                ResultColumn resultColumn  = modelFactory.createResultColumn(resultSetModel, column);   
17912                                                                if (modelManager.getModel(sourceTable) instanceof Table) {
17913                                                                        Table tableModel = (Table) modelManager.getModel(sourceTable);
17914                                                                        if (tableModel != null) {
17915                                                                                modelFactory.createTableColumn(tableModel, columnObject, false);
17916                                                                        }
17917                                                                        TObjectName[] columns = modelManager.getTableColumns(sourceTable);
17918                                                                        for (int j = 0; j < columns.length; j++) {
17919                                                                                TObjectName columnName = columns[j];
17920                                                                                if (columnName == null || "*".equals(getColumnName(columnName))) {
17921                                                                                        continue;
17922                                                                                }
17923                                                                                resultColumn.bindStarLinkColumn(columnName);
17924                                                                        }
17925
17926                                                                        if (tableModel.getColumns() != null) {
17927                                                                                for (int j = 0; j < tableModel.getColumns().size(); j++) {
17928                                                                                        TableColumn tableColumn = tableModel.getColumns().get(j);
17929                                                                                        TObjectName columnName = tableColumn.getColumnObject();
17930                                                                                        if (columnName == null || "*".equals(getColumnName(columnName))) {
17931                                                                                                continue;
17932                                                                                        }
17933                                                                                        resultColumn.bindStarLinkColumn(columnName);
17934                                                                                }
17935                                                                        }
17936                                                                } else if (modelManager.getModel(sourceTable) instanceof QueryTable) {
17937                                                                        QueryTable tableModel = (QueryTable) modelManager.getModel(sourceTable);
17938                                                                        if (tableModel != null && !tableModel.getColumns().isEmpty()) {
17939                                                                                for (ResultColumn item : tableModel.getColumns()) {
17940                                                                                        if (item.hasStarLinkColumn()) {
17941                                                                                                for (TObjectName starLinkColumn : item.getStarLinkColumnList()) {
17942                                                                                                        resultColumn.bindStarLinkColumn(starLinkColumn);
17943                                                                                                }
17944                                                                                        } else if (item.getColumnObject() instanceof TObjectName) {
17945                                                                                                resultColumn.bindStarLinkColumn((TObjectName) item.getColumnObject());
17946                                                                                        } else if (item.getColumnObject() instanceof TResultColumn) {
17947                                                                                                TResultColumn queryTableColumn = (TResultColumn) item.getColumnObject();
17948                                                                                                TObjectName tableColumnObject = queryTableColumn.getFieldAttr();
17949                                                                                                if (tableColumnObject != null) {
17950                                                                                                        resultColumn.bindStarLinkColumn(tableColumnObject);
17951                                                                                                } else if (queryTableColumn.getAliasClause() != null) {
17952                                                                                                        resultColumn.bindStarLinkColumn(
17953                                                                                                                        queryTableColumn.getAliasClause().getAliasName());
17954                                                                                                }
17955                                                                                        }
17956                                                                                }
17957                                                                        }
17958                                                                }
17959                                                        } else {
17960                                                                ResultColumn resultColumn  = modelFactory.createResultColumn(resultSetModel, column);   
17961                                                                TTableList tables = stmt.getTables();
17962                                                                for (int k = 0; k < tables.size(); k++) {
17963                                                                        TTable table = tables.getTable(k);
17964                                                                        TObjectName[] columns = modelManager.getTableColumns(table);
17965                                                                        for (int j = 0; j < columns.length; j++) {
17966                                                                                TObjectName columnName = columns[j];
17967                                                                                if (columnName == null) {
17968                                                                                        continue;
17969                                                                                }
17970                                                                                if ("*".equals(getColumnName(columnName))) {
17971                                                                                        if (modelManager.getModel(table) instanceof Table) {
17972                                                                                                Table tableModel = (Table) modelManager.getModel(table);
17973                                                                                                if (tableModel != null) {
17974                                                                                                        modelFactory.createTableColumn(tableModel, columnName, false);
17975                                                                                                }
17976                                                                                                if (tableModel != null && !tableModel.getColumns().isEmpty()) {
17977                                                                                                        for (int z = 0; z < tableModel.getColumns().size(); z++) {
17978                                                                                                                resultColumn.bindStarLinkColumn(
17979                                                                                                                                tableModel.getColumns().get(z).getColumnObject());
17980                                                                                                        }
17981                                                                                                }
17982                                                                                        } else if (modelManager.getModel(table) instanceof QueryTable) {
17983                                                                                                QueryTable tableModel = (QueryTable) modelManager.getModel(table);
17984                                                                                                if (tableModel != null && !tableModel.getColumns().isEmpty()) {
17985                                                                                                        for (ResultColumn item : tableModel.getColumns()) {
17986                                                                                                                if (item.hasStarLinkColumn()) {
17987                                                                                                                        for (TObjectName starLinkColumn : item
17988                                                                                                                                        .getStarLinkColumnList()) {
17989                                                                                                                                resultColumn.bindStarLinkColumn(starLinkColumn);
17990                                                                                                                        }
17991                                                                                                                } else if (item.getColumnObject() instanceof TObjectName) {
17992                                                                                                                        resultColumn.bindStarLinkColumn(
17993                                                                                                                                        (TObjectName) item.getColumnObject());
17994                                                                                                                } else if (item.getColumnObject() instanceof TResultColumn) {
17995                                                                                                                        TResultColumn queryTableColumn = (TResultColumn) item
17996                                                                                                                                        .getColumnObject();
17997                                                                                                                        TObjectName tableColumnObject = queryTableColumn
17998                                                                                                                                        .getFieldAttr();
17999                                                                                                                        if (tableColumnObject != null) {
18000                                                                                                                                resultColumn.bindStarLinkColumn(tableColumnObject);
18001                                                                                                                        } else if (queryTableColumn.getAliasClause() != null) {
18002                                                                                                                                resultColumn.bindStarLinkColumn(queryTableColumn
18003                                                                                                                                                .getAliasClause().getAliasName());
18004                                                                                                                        }
18005                                                                                                                }
18006                                                                                                        }
18007                                                                                                }
18008                                                                                        }
18009                                                                                        continue;
18010                                                                                }
18011                                                                                resultColumn.bindStarLinkColumn(columnName);
18012                                                                        }
18013                                                                }
18014                                                        }
18015                                                }
18016                                                else {
18017                                                        ResultColumn resultColumn  = modelFactory.createResultColumn(resultSetModel, column);   
18018                                                }
18019                                                
18020                                                analyzeResultColumn(column, EffectType.select);
18021
18022                                        }
18023                                        
18024                                        if (queryModel instanceof ResultSet) {
18025                                                boolean isDetermined = true;
18026                                                ResultSet resultSet = (ResultSet) queryModel;
18027                                                for (ResultColumn column : resultSet.getColumns()) {
18028                                                        if (column.getName().endsWith("*")) {
18029                                                                isDetermined = false;
18030                                                                break;
18031                                                        }
18032                                                }
18033                                                if (isDetermined) {
18034                                                        resultSet.setDetermined(isDetermined);
18035                                                }
18036                                        }
18037                                }
18038                        }
18039
18040                
18041                        analyzeSelectIntoClause(stmt);
18042                
18043
18044                        if (stmt.getJoins() != null && stmt.getJoins().size() > 0) {
18045                                for (int i = 0; i < stmt.getJoins().size(); i++) {
18046                                        TJoin join = stmt.getJoins().getJoin(i);
18047                                        ResultSet topResultSet = (ResultSet) modelManager.getModel(stmt);
18048                                        if (join.getJoinItems() != null && join.getJoinItems().size() > 0) {
18049                                                for (int k = 0; k < join.getJoinItems().size(); k++) {
18050                                                        TTable table = join.getJoinItems().getJoinItem(k).getTable();
18051                                                        if (table != null && table.getSubquery() != null) {
18052                                                                
18053                                                                ResultSet joinResultSet = (ResultSet) modelManager.getModel(table.getSubquery());
18054                                                                for (int x = 0; x < joinResultSet.getColumns().size(); x++) {
18055                                                                        ResultColumn sourceColumn = joinResultSet.getColumns().get(x);
18056                                                                        ResultColumn resultColumn = matchResultColumn(topResultSet.getColumns(),
18057                                                                                        sourceColumn);
18058                                                                        if (resultColumn != null
18059                                                                                        && resultColumn.getColumnObject() instanceof TResultColumn) {
18060                                                                                TResultColumn column = (TResultColumn) resultColumn.getColumnObject();
18061                                                                                if (column.getAliasClause() == null && column.getFieldAttr() != null) {
18062                                                                                        TObjectName resultObject = column.getFieldAttr();
18063                                                                                        if (resultObject.getSourceTable() == null
18064                                                                                                        || resultObject.getSourceTable().equals(table)) {
18065                                                                                                DataFlowRelationship combinedQueryRelation = modelFactory
18066                                                                                                                .createDataFlowRelation();
18067                                                                                                combinedQueryRelation.setEffectType(EffectType.select);
18068                                                                                                combinedQueryRelation
18069                                                                                                                .setTarget(new ResultColumnRelationshipElement(resultColumn));
18070                                                                                                combinedQueryRelation
18071                                                                                                                .addSource(new ResultColumnRelationshipElement(sourceColumn));
18072                                                                                        }
18073                                                                                }
18074                                                                        }
18075                                                                }
18076                                                        }
18077                                                        
18078                                                        if(join.getJoinItems().getJoinItem(k).getJoin()!=null) {
18079                                                                analyzeJoin(join.getJoinItems().getJoinItem(k).getJoin(), EffectType.select);
18080                                                        }
18081                                                }
18082                                        }
18083                                        analyzeJoin(join, EffectType.select);
18084                                }
18085                        }
18086
18087                        if (stmt.getWhereClause() != null) {
18088                                TExpression expr = stmt.getWhereClause().getCondition();
18089                                if (expr != null) {
18090                                        analyzeFilterCondition(null, expr, null, JoinClauseType.where, EffectType.select);
18091                                }
18092                        }
18093
18094                        stmtStack.pop();
18095                }
18096        }
18097
18098        protected TObjectNameList getTableLinkedColumns(TTable table) {
18099                if(structObjectMap.containsKey(table)) {
18100                        return structObjectMap.get(table);
18101                }
18102                return table.getLinkedColumns();
18103        }
18104
18105        protected boolean isTopResultSet(TSelectSqlStatement stmt) {
18106                TCustomSqlStatement parent = stmt.getParentStmt();
18107                if (parent == null)
18108                        return true;
18109                if (parent instanceof TMssqlReturn) {
18110                        return true;
18111                }
18112                if (parent instanceof TReturnStmt) {
18113                        return true;
18114                }
18115                if (parent instanceof TCommonBlock) {
18116                        TCommonBlock block = (TCommonBlock) parent;
18117                        if (block.getStatements() != null) {
18118                                for (int i = 0; i < block.getStatements().size(); i++) {
18119                                        TCustomSqlStatement child = block.getStatements().get(i);
18120                                        if(stmt == child) {
18121                                                return true;
18122                                        }
18123                                }
18124                        }
18125                }
18126                if (parent instanceof TMssqlBlock) {
18127                        TMssqlBlock block = (TMssqlBlock) parent;
18128                        if (block.getStatements() != null) {
18129                                for (int i = 0; i < block.getStatements().size(); i++) {
18130                                        TCustomSqlStatement child = block.getStatements().get(i);
18131                                        if(stmt == child) {
18132                                                return true;
18133                                        }
18134                                }
18135                        }
18136                }
18137                if (parent instanceof TStoredProcedureSqlStatement) {
18138                        TStoredProcedureSqlStatement block = (TStoredProcedureSqlStatement) parent;
18139                        if (block.getStatements() != null) {
18140                                for (int i = 0; i < block.getStatements().size(); i++) {
18141                                        TCustomSqlStatement child = block.getStatements().get(i);
18142                                        if(child == stmt) {
18143                                                return true;
18144                                        }
18145                                        if (child instanceof TReturnStmt) {
18146                                                TReturnStmt returnStmt = (TReturnStmt) child;
18147                                                if (returnStmt.getStatements() != null) {
18148                                                        for (int j = 0; j < returnStmt.getStatements().size(); j++) {
18149                                                                TCustomSqlStatement child1 = returnStmt.getStatements().get(j);
18150                                                                if(child1 == stmt) {
18151                                                                        return true;
18152                                                                }
18153                                                        }
18154                                                }
18155                                        }
18156                                        if (child instanceof TMssqlReturn) {
18157                                                TMssqlReturn returnStmt = (TMssqlReturn) child;
18158                                                if (returnStmt.getStatements() != null) {
18159                                                        for (int j = 0; j < returnStmt.getStatements().size(); j++) {
18160                                                                TCustomSqlStatement child1 = returnStmt.getStatements().get(j);
18161                                                                if(child1 == stmt) {
18162                                                                        return true;
18163                                                                }
18164                                                        }
18165                                                }
18166                                        }
18167                                }
18168                        }
18169                }
18170                return false;
18171        }
18172
18173        protected void analyzeTableSubquery(TTable table) {
18174                if(table.getSubquery()!=null) {
18175                        QueryTable queryTable = modelFactory.createQueryTable(table);
18176                        TSelectSqlStatement subquery = table.getSubquery();
18177                        analyzeSelectStmt(subquery);
18178
18179                        ResultSet resultSetModel = (ResultSet) modelManager.getModel(subquery);
18180
18181                        if (resultSetModel != null && resultSetModel != queryTable
18182                                        && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
18183                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
18184                                impactRelation.setEffectType(EffectType.select);
18185                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
18186                                                resultSetModel.getRelationRows()));
18187                                impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
18188                                                queryTable.getRelationRows()));
18189                        }
18190
18191                        if (resultSetModel != null && resultSetModel != queryTable
18192                                        && queryTable.getTableObject().getAliasClause() != null
18193                                        && queryTable.getTableObject().getAliasClause().getColumns() != null) {
18194                                for (int j = 0; j < queryTable.getColumns().size()
18195                                                && j < resultSetModel.getColumns().size(); j++) {
18196                                        ResultColumn sourceColumn = resultSetModel.getColumns().get(j);
18197                                        ResultColumn targetColumn = queryTable.getColumns().get(j);
18198
18199                                        DataFlowRelationship queryRalation = modelFactory.createDataFlowRelation();
18200                                        queryRalation.setEffectType(EffectType.select);
18201                                        queryRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
18202                                        queryRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
18203                                }
18204                        } else if (subquery.getSetOperatorType() != ESetOperatorType.none) {
18205                                SelectSetResultSet selectSetResultSetModel = (SelectSetResultSet) modelManager
18206                                                .getModel(subquery);
18207                                for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) {
18208                                        ResultColumn sourceColumn = selectSetResultSetModel.getColumns().get(j);
18209                                        ResultColumn targetColumn = modelFactory.createSelectSetResultColumn(queryTable,
18210                                                        sourceColumn);
18211                                        for (TObjectName starLinkColumn : sourceColumn.getStarLinkColumnList()) {
18212                                                targetColumn.bindStarLinkColumn(starLinkColumn);
18213                                        }
18214                                        DataFlowRelationship selectSetRalation = modelFactory.createDataFlowRelation();
18215                                        selectSetRalation.setEffectType(EffectType.select);
18216                                        selectSetRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
18217                                        selectSetRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
18218                                }
18219                        }
18220                }
18221        }
18222
18223        private ResultColumn getPivotedTableColumn(TPivotedTable pivotedTable, TObjectName columnName) {
18224                List<TPivotClause> pivotClauses = new ArrayList<TPivotClause>();
18225                if (pivotedTable.getPivotClause() != null) {
18226                        pivotClauses.add(pivotedTable.getPivotClause());
18227                }
18228                if (pivotedTable.getPivotClauseList() != null) {
18229                        for (int i = 0; i < pivotedTable.getPivotClauseList().size(); i++) {
18230                                pivotClauses.add(pivotedTable.getPivotClauseList().getElement(i));
18231                        }
18232                }
18233                for (TPivotClause clause : pivotClauses) {
18234                        Object model = modelManager.getModel(clause);
18235                        if (model instanceof PivotedTable) {
18236                                PivotedTable pivotedTableModel = (PivotedTable) model;
18237                                if (pivotedTableModel.getColumns() != null) {
18238                                        for (ResultColumn column : pivotedTableModel.getColumns()) {
18239                                                if (DlineageUtil.compareColumnIdentifier(getColumnName(columnName),
18240                                                                getColumnName(SQLUtil.trimColumnStringQuote(column.getName())))) {
18241                                                        return column;
18242                                                }
18243                                        }
18244                                }
18245                        }
18246                }
18247                return null;
18248        }
18249
18250        private void analyzeHiveTransformClause(TSelectSqlStatement stmt, THiveTransformClause transformClause) {
18251                Table mapSourceTable = null;
18252                QueryTable mapQueryTable = null;
18253                if(stmt.getTables()!=null) {
18254                        for(int i=0;i<stmt.getTables().size();i++) {
18255                                TTable table = stmt.getTables().getTable(i);
18256                                if (table.getSubquery() != null) {
18257                                        if (transformClause.getTransformType() == ETransformType.ettReduce) {
18258                                                mapQueryTable = modelFactory.createQueryTable(table);
18259                                        }
18260                                        analyzeSelectStmt(table.getSubquery());
18261                                }
18262                                else {
18263                                        mapSourceTable = modelFactory.createTable(table);
18264                                }
18265                        }
18266                }
18267                
18268                if (transformClause.getTransformType() == ETransformType.ettReduce) {
18269                        modelFactory.createResultSet(stmt, false);
18270                }
18271                
18272                List<TableColumn> mapTableColumns = new ArrayList<TableColumn>();
18273                List<ResultColumn> mapResultSetColumns = new ArrayList<ResultColumn>();
18274                List<ResultColumn> redueResultSetColumns = new ArrayList<ResultColumn>();
18275                
18276                if(transformClause.getExpressionList()!=null) {
18277                        for(TExpression expression: transformClause.getExpressionList()) {
18278                                if(expression.getObjectOperand()!=null) {
18279                                        if (transformClause.getTransformType() == ETransformType.ettMap || transformClause.getTransformType() == ETransformType.ettSelect) {
18280                                                if (mapSourceTable != null) {
18281                                                        TableColumn tableColumn = modelFactory.createTableColumn(mapSourceTable,
18282                                                                        expression.getObjectOperand(), false);
18283                                                        if (tableColumn != null) {
18284                                                                mapTableColumns.add(tableColumn);
18285                                                        }
18286                                                }
18287                                        }
18288                                        else if (transformClause.getTransformType() == ETransformType.ettReduce) {
18289                                                if (mapQueryTable != null) {
18290                                                        ResultColumn resultColumn = modelFactory.createResultColumn(mapQueryTable,
18291                                                                        expression.getObjectOperand(), false);
18292                                                        if (resultColumn != null) {
18293                                                                mapResultSetColumns.add(resultColumn);
18294                                                        }
18295                                                }
18296                                        }
18297                                }
18298                        }
18299                }
18300                
18301                if (transformClause.getAliasClause() != null) {
18302                        Object model = modelManager.getModel(stmt);
18303                        if (model instanceof ResultSet) {
18304                                ResultSet result = (ResultSet) model;
18305                                if (result!=null && transformClause.getAliasClause().getColumns() != null) {
18306                                        for (TObjectName column : transformClause.getAliasClause().getColumns()) {
18307                                                ResultColumn resultColumn = modelFactory.createResultColumn(result, column);
18308                                                if (resultColumn != null) {
18309                                                        if (transformClause.getTransformType() == ETransformType.ettMap 
18310                                                                        || transformClause.getTransformType() == ETransformType.ettSelect) {
18311                                                                mapResultSetColumns.add(resultColumn);
18312                                                        }
18313                                                        else if (transformClause.getTransformType() == ETransformType.ettReduce) {
18314                                                                redueResultSetColumns.add(resultColumn);
18315                                                        }
18316                                                }
18317                                        }
18318                                }
18319                        }
18320                }
18321                
18322                if (transformClause.getTransformType() == ETransformType.ettMap 
18323                                || transformClause.getTransformType() == ETransformType.ettSelect) {
18324                        if (!mapTableColumns.isEmpty() && !mapResultSetColumns.isEmpty()) {
18325                                for (ResultColumn resultColumn : mapResultSetColumns) {
18326                                        for (TableColumn tableColumn : mapTableColumns) {
18327                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18328                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
18329                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
18330                                                relation.setEffectType(EffectType.select);
18331                                        }
18332                                }
18333                        }
18334                }
18335                else if (transformClause.getTransformType() == ETransformType.ettReduce) {
18336                        if (!redueResultSetColumns.isEmpty() && !mapResultSetColumns.isEmpty()) {
18337                                for (ResultColumn reduceResultColumn : redueResultSetColumns) {
18338                                        for (ResultColumn mapResultColumn : mapResultSetColumns) {
18339                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18340                                                relation.setTarget(new ResultColumnRelationshipElement(reduceResultColumn));
18341                                                relation.addSource(new ResultColumnRelationshipElement(mapResultColumn));
18342                                                relation.setEffectType(EffectType.select);
18343                                        }
18344                                }
18345                        }
18346                }
18347        }
18348
18349        protected boolean isStructColumn(TObjectName columnName) {
18350                return columnName.getSourceTable() != null && columnName.getSourceTable().getAliasClause() != null
18351                                && columnName.getSourceTable().getUnnestClause() != null
18352                                && DlineageUtil.compareColumnIdentifier(getColumnName(columnName),
18353                                                getColumnName(columnName.getSourceTable().getAliasClause().getAliasName()));
18354        }
18355
18356        private TObjectName getObjectName(ResultColumn column) {
18357                if (column.getColumnObject() instanceof TResultColumn) {
18358                        TResultColumn resultColumn = (TResultColumn) column.getColumnObject();
18359                        if (resultColumn.getAliasClause() != null && resultColumn.getAliasClause().getAliasName() != null) {
18360                                return resultColumn.getAliasClause().getAliasName();
18361                        }
18362                        if (resultColumn.getFieldAttr() != null) {
18363                                return resultColumn.getFieldAttr();
18364                        }
18365                        if (resultColumn.getExpr() != null
18366                                        && resultColumn.getExpr().getExpressionType() == EExpressionType.simple_object_name_t) {
18367                                return resultColumn.getExpr().getObjectOperand();
18368                        }
18369                } else if (column.getColumnObject() instanceof TObjectName) {
18370                        return (TObjectName) column.getColumnObject();
18371                }
18372                return null;
18373        }
18374
18375        private boolean isShowTopSelectResultSet() {
18376                if (option.isSimpleOutput() && !option.isSimpleShowTopSelectResultSet())
18377                        return false;
18378                return true;
18379        }
18380
18381        private void analyzeSelectIntoClause(TSelectSqlStatement stmt) {
18382                if (stmt.getParentStmt() instanceof TSelectSqlStatement) {
18383                        return;
18384                }
18385                
18386                TableColumn oracleIntoTableColumn = null;
18387                
18388                TIntoClause intoClause = stmt.getIntoClause();
18389                
18390                TSelectSqlStatement leftStmt = DlineageUtil.getLeftStmt(stmt);
18391                
18392                if (intoClause == null && leftStmt != null) {
18393                        intoClause = leftStmt.getIntoClause();
18394                }
18395                
18396                if (intoClause != null) {
18397                        List<TObjectName> tableNames = new ArrayList<TObjectName>();
18398                        if (intoClause.getExprList() != null) {
18399                                for (int j = 0; j < intoClause.getExprList().size(); j++) {
18400                                        TObjectName tableName = intoClause.getExprList().getExpression(j).getObjectOperand();
18401                                        if (tableName != null) {
18402                                                if (tableName.toString().startsWith(":") && option.getVendor() == EDbVendor.dbvoracle
18403                                                                && tableName.getDbObjectType() == EDbObjectType.column) {
18404                                                        TObjectName tableAlias = new TObjectName();
18405                                                        tableAlias.setString(tableName.getTableString());
18406                                                        tableNames.add(tableAlias);
18407                                                        TTable sourceTable = tableName.getSourceTable();
18408                                                        Table sourceTableModel = modelFactory.createTable(sourceTable, tableAlias);
18409                                                        oracleIntoTableColumn = modelFactory.createTableColumn(sourceTableModel, tableName, true);
18410                                                } else {
18411                                                        if (tableName != null) {
18412                                                                tableNames.add(tableName);
18413                                                        }
18414                                                }
18415                                        }
18416                                        else if(intoClause.getExprList().getExpression(j).getFunctionCall()!=null) {
18417                                                TObjectName variableName = intoClause.getExprList().getExpression(j).getFunctionCall().getFunctionName();
18418                                                tableNames.add(variableName);
18419                                                Variable variable = modelFactory.createVariable(variableName);
18420                                                variable.setSubType(SubType.record);
18421                                                TObjectName variableProperties = new TObjectName();
18422                                                variableProperties.setString("*");
18423                                                modelFactory.createTableColumn(variable, variableProperties, true);
18424                                        }
18425                                }
18426                        } else if (intoClause.getVariableList() != null) {
18427                                for (int j = 0; j < intoClause.getVariableList().size(); j++) {
18428                                        TObjectName tableName = intoClause.getVariableList().getObjectName(j);
18429                                        if (tableName != null) {
18430                                                tableNames.add(tableName);
18431                                        }
18432                                }
18433                        } else if (intoClause.getIntoName() != null) {
18434                                tableNames.add(intoClause.getIntoName());
18435                        }
18436
18437                        ResultSet queryModel = (ResultSet) modelManager.getModel(stmt.getResultColumnList());
18438                        if (stmt.getSetOperatorType() != ESetOperatorType.none) {
18439                                queryModel = (ResultSet) modelManager.getModel(stmt);
18440                        }
18441                        for (int j = 0; j < tableNames.size(); j++) {
18442                                TObjectName tableName = tableNames.get(j);
18443                                if (tableName.getColumnNameOnly().startsWith("@")
18444                                                && (option.getVendor() == EDbVendor.dbvmssql || option.getVendor() == EDbVendor.dbvazuresql)) {
18445                                        continue;
18446                                }
18447
18448                                if (tableName.getColumnNameOnly().startsWith(":")
18449                                                && (option.getVendor() == EDbVendor.dbvhana || option.getVendor() == EDbVendor.dbvteradata)) {
18450                                        continue;
18451                                }
18452
18453                                Table tableModel;
18454                                TableColumn variableColumn = null;
18455
18456                                if (tableName.getDbObjectType() == EDbObjectType.variable) {
18457                                        if (tableName.toString().indexOf(".") != -1) {
18458                                                List<String> splits = SQLUtil.parseNames(tableName.toString());
18459                                                tableModel = modelFactory.createVariable(splits.get(splits.size() - 2));
18460                                        } else {
18461                                                tableModel = modelFactory.createVariable(tableName);
18462                                        }
18463                                        if (tableModel.getSubType() == null) {
18464                                                tableModel.setSubType(SubType.record);
18465                                        }
18466                                        if(tableModel.getColumns() == null || tableModel.getColumns().isEmpty()) {
18467                                                TObjectName variableProperties = new TObjectName();
18468                                                variableProperties.setString("*");
18469                                                variableColumn = modelFactory.createTableColumn(tableModel, variableProperties, true);
18470                                        }
18471                                } else {
18472                                        tableModel = modelFactory.createTableByName(tableName);
18473                                        // SELECT ... INTO <table> creates the target. Parse fact, set
18474                                        // regardless of isDetermined() (unlike the gated setCreateTable
18475                                        // below). See dlineage-authoritative-endpoint-classification.md.
18476                                        tableModel.setCreatedInSql(true);
18477                                        tableModel.setEndpointIntroduction(EndpointIntroduction.SELECT_INTO);
18478                                }
18479
18480                                if (queryModel instanceof ResultSet && (stmt.getWhereClause() != null || hasJoin(stmt))) {
18481                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
18482                                        impactRelation.setEffectType(EffectType.insert);
18483                                        impactRelation.addSource(
18484                                                        new RelationRowsRelationshipElement<ResultSetRelationRows>(((ResultSet)queryModel).getRelationRows()));
18485                                        impactRelation.setTarget(
18486                                                        new RelationRowsRelationshipElement<TableRelationRows>(tableModel.getRelationRows()));
18487                                }
18488                                
18489                                Process process = modelFactory.createProcess(stmt);
18490                                tableModel.addProcess(process);
18491                                
18492                                if (stmt.getSetOperatorType() != ESetOperatorType.none) {
18493                                        for (ResultColumn resultColumn : queryModel.getColumns()) {
18494                                                TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
18495                                                                resultColumn.getName());
18496
18497                                                if (DlineageUtil.isTempTable(tableModel, option.getVendor()) && sqlenv != null
18498                                                                && tableModel.getDatabase() != null && tableModel.getSchema() != null) {
18499                                                        TSQLSchema schema = sqlenv
18500                                                                        .getSQLSchema(tableModel.getDatabase() + "." + tableModel.getSchema(), true);
18501                                                        if (schema != null) {
18502                                                                TSQLTable tempTable = schema
18503                                                                                .createTable(DlineageUtil.getSimpleTableName(tableModel.getName()));
18504                                                                tempTable.addColumn(tableColumn.getName());
18505                                                        }
18506                                                }
18507
18508                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18509                                                relation.setEffectType(EffectType.insert);
18510                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
18511                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
18512                                                relation.setProcess(process);
18513                                        }
18514                                        
18515                                        tableModel.setDetermined(queryModel.isDetermined());
18516                                        if(queryModel.isDetermined() && DlineageUtil.isTempTable(tableModel, option.getVendor())) {
18517                                                tableModel.setCreateTable(true, false);
18518                                        }
18519                                        return;
18520                                }
18521
18522                                boolean isDetermined = true;
18523                                for (int i = 0; i < stmt.getResultColumnList().size(); i++) {
18524                                        if (tableNames.size() > 1 && tableName.getDbObjectType() == EDbObjectType.variable) {
18525                                                if (i != j) {
18526                                                        continue;
18527                                                }
18528                                        }
18529                                        TResultColumn column = stmt.getResultColumnList().getResultColumn(i);
18530
18531                                        if ("*".equals(column.getColumnNameOnly()) && column.getFieldAttr() != null
18532                                                        && column.getFieldAttr().getSourceTable() != null) {
18533                                                Object model = modelManager.getModel(column);
18534                                                if(model instanceof LinkedHashMap) {
18535                                                        LinkedHashMap<String, ResultColumn> columns = (LinkedHashMap<String, ResultColumn>)model;
18536                                                        for(String key: columns.keySet()) {
18537                                                                ResultColumn sourceColumn = columns.get(key);
18538                                                                TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
18539                                                                                sourceColumn.getName());
18540                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18541                                                                relation.setEffectType(EffectType.insert);
18542                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
18543                                                                relation.addSource(new ResultColumnRelationshipElement(sourceColumn));
18544                                                                relation.setProcess(process);
18545                                                        }
18546                                                }
18547                                                else if(model instanceof ResultColumn) {
18548                                                        isDetermined = false;
18549                                                        ResultColumn resultColumn = (ResultColumn) model;
18550                                                        List<TObjectName> columns = resultColumn.getStarLinkColumnList();
18551                                                        if (columns.size() > 0) {
18552                                                                for (int k = 0; k < columns.size(); k++) {
18553        
18554                                                                        TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
18555                                                                                        columns.get(k));
18556        
18557                                                                        if (DlineageUtil.isTempTable(tableModel, option.getVendor()) && sqlenv != null
18558                                                                                        && tableModel.getDatabase() != null && tableModel.getSchema() != null) {
18559                                                                                TSQLSchema schema = sqlenv.getSQLSchema(
18560                                                                                                tableModel.getDatabase() + "." + tableModel.getSchema(), true);
18561                                                                                if (schema != null) {
18562                                                                                        TSQLTable tempTable = schema.createTable(DlineageUtil.getSimpleTableName(tableModel.getName()));
18563                                                                                        tempTable.addColumn(tableColumn.getName());
18564                                                                                }
18565                                                                        }
18566        
18567                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18568                                                                        relation.setEffectType(EffectType.insert);
18569                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
18570                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
18571                                                                        relation.setProcess(process);
18572                                                                }
18573                                                                if (resultColumn.isShowStar()) {
18574                                                                        TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
18575                                                                                        column.getFieldAttr());
18576                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18577                                                                        relation.setEffectType(EffectType.insert);
18578                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
18579                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
18580                                                                        relation.setProcess(process);
18581                                                                }
18582                                                        } else {
18583                                                                TObjectName columnObject = column.getFieldAttr();
18584                                                                if (column.getAliasClause() != null) {
18585                                                                        columnObject = column.getAliasClause().getAliasName();
18586                                                                }
18587                                                                if (columnObject != null) {
18588                                                                        TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
18589                                                                                        columnObject);
18590                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18591                                                                        relation.setEffectType(EffectType.insert);
18592                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
18593                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
18594                                                                        relation.setProcess(process);
18595                                                                } else if (!SQLUtil.isEmpty(column.getColumnAlias())) {
18596                                                                        TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
18597                                                                                        column.getAliasClause().getAliasName());
18598                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18599                                                                        relation.setEffectType(EffectType.insert);
18600                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
18601                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
18602                                                                        relation.setProcess(process);
18603                                                                }
18604                                                        }
18605                                                }
18606                                        } else {
18607                                                ResultColumn resultColumn = null;
18608
18609                                                if (queryModel instanceof QueryTable) {
18610                                                        resultColumn = (ResultColumn) modelManager.getModel(column);
18611                                                } else if (queryModel instanceof ResultSet) {
18612                                                        resultColumn = (ResultColumn) modelManager.getModel(column);
18613                                                } else {
18614                                                        continue;
18615                                                }
18616
18617                                                if (resultColumn == null && column.getAliasClause() != null) {
18618                                                        resultColumn = (ResultColumn) modelManager.getModel(column.getAliasClause().getAliasName());
18619                                                }
18620
18621                                                if (resultColumn != null) {
18622                                                        TObjectName columnObject = column.getFieldAttr();
18623                                                        if (column.getAliasClause() != null) {
18624                                                                columnObject = column.getAliasClause().getAliasName();
18625                                                        }
18626                                                        TableColumn tableColumn = null;
18627                                                        if (columnObject != null) {
18628                                                                if (tableModel.isVariable()) {
18629                                                                        if (variableColumn != null) {
18630                                                                                tableColumn = variableColumn;
18631                                                                        } else {
18632                                                                                tableColumn = tableModel.getColumns().get(0);
18633                                                                        }
18634                                                                } 
18635                                                                else if (oracleIntoTableColumn != null) {
18636                                                                        tableColumn = oracleIntoTableColumn;
18637                                                                }
18638                                                                else {
18639                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel, columnObject);
18640                                                                        if (containStarColumn(queryModel)) {
18641                                                                                tableColumn.notBindStarLinkColumn(true);
18642                                                                        }
18643                                                                }
18644                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18645                                                                relation.setEffectType(EffectType.insert);
18646                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
18647                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
18648                                                                relation.setProcess(process);
18649                                                        } else if (!SQLUtil.isEmpty(column.getColumnAlias())) {
18650                                                                if (tableModel.isVariable()) {
18651                                                                        if (variableColumn != null) {
18652                                                                                tableColumn = variableColumn;
18653                                                                        } else {
18654                                                                                tableColumn = tableModel.getColumns().get(0);
18655                                                                        }
18656                                                                } else {
18657                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
18658                                                                                        column.getAliasClause().getAliasName());
18659                                                                }
18660                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18661                                                                relation.setEffectType(EffectType.insert);
18662                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
18663                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
18664                                                                relation.setProcess(process);
18665                                                        } else {
18666                                                                if (tableModel.isVariable()) {
18667                                                                        if (variableColumn != null) {
18668                                                                                tableColumn = variableColumn;
18669                                                                        } else {
18670                                                                                tableColumn = tableModel.getColumns().get(0);
18671                                                                        }
18672                                                                }
18673                                                                if (tableColumn != null) {
18674                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18675                                                                        relation.setEffectType(EffectType.insert);
18676                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
18677                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
18678                                                                        relation.setProcess(process);
18679                                                                }
18680                                                        }
18681                                                }
18682                                        }
18683                                }
18684                                tableModel.setDetermined(isDetermined);
18685                                if(isDetermined && DlineageUtil.isTempTable(tableModel, option.getVendor())) {
18686                                        tableModel.setCreateTable(true, false);
18687                                }
18688                        }
18689                }
18690        }
18691
18692        private boolean isUnPivotedTable(TPivotedTable pivotedTable) {
18693                if (pivotedTable.getPivotClauseList() != null && pivotedTable.getPivotClauseList().size() > 0) {
18694                        return pivotedTable.getPivotClauseList().getElement(0).getType() == TPivotClause.unpivot;
18695                } else {
18696                        TPivotClause pivotClause = pivotedTable.getPivotClause();
18697                        return pivotClause.getType() == TPivotClause.unpivot;
18698                }
18699        }
18700
18701        private void analyzeUnPivotedTable(TSelectSqlStatement stmt, TPivotedTable pivotedTable) {
18702                List<Object> tables = new ArrayList<Object>();
18703                Set<Object> pivotedColumns = new HashSet<Object>();
18704                TTable fromTable = pivotedTable.getTableSource();
18705                Object table = modelManager.getModel(fromTable);
18706                List<TPivotClause> pivotClauses = new ArrayList<TPivotClause>();
18707                if (pivotedTable.getPivotClauseList() != null && pivotedTable.getPivotClauseList().size() > 0) {
18708                        for (int i = 0; i < pivotedTable.getPivotClauseList().size(); i++) {
18709                                pivotClauses.add(pivotedTable.getPivotClauseList().getElement(i));
18710                        }
18711                } else {
18712                        TPivotClause pivotClause = pivotedTable.getPivotClause();
18713                        pivotClauses.add(pivotClause);
18714                }
18715
18716                for (int y = 0; y < pivotClauses.size(); y++) {
18717                        TPivotClause pivotClause = pivotClauses.get(y);
18718                        PivotedTable pivotTable = modelFactory.createPivotdTable(pivotClause);
18719                        pivotTable.setUnpivoted(true);
18720
18721                        if (pivotClause.getValueColumnList() != null) {
18722                                for (int j = 0; j < pivotClause.getValueColumnList().size(); j++) {
18723                                        modelFactory.createResultColumn(pivotTable, pivotClause.getValueColumnList().getObjectName(j));
18724                                }
18725                        }
18726                        if (pivotClause.getPivotColumnList() != null) {
18727                                for (int j = 0; j < pivotClause.getPivotColumnList().size(); j++) {
18728                                        modelFactory.createResultColumn(pivotTable, pivotClause.getPivotColumnList().getObjectName(j));
18729                                }
18730                        }
18731                        if (pivotClause.getUnpivotInClause()!=null && pivotClause.getUnpivotInClause().getItems() != null) {
18732                                for (int j = 0; j < pivotClause.getUnpivotInClause().getItems().size(); j++) {
18733                                        TObjectName columnName = pivotClause.getUnpivotInClause().getItems().getElement(j).getColumn();
18734                                        if (columnName != null) {
18735                                                if (table instanceof QueryTable) {
18736                                                        for (ResultColumn tableColumn : ((QueryTable) table).getColumns()) {
18737                                                                if (getColumnName(columnName)
18738                                                                                .equals(DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
18739                                                                        for (ResultColumn resultColumn : pivotTable.getColumns()) {
18740                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18741                                                                                relation.setEffectType(EffectType.select);
18742                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
18743                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
18744                                                                                pivotedColumns.add(tableColumn);
18745                                                                        }
18746                                                                        break;
18747                                                                }
18748                                                        }
18749                                                } else if (table instanceof Table) {
18750                                                        for (TableColumn tableColumn : ((Table) table).getColumns()) {
18751                                                                if (getColumnName(columnName)
18752                                                                                .equals(DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
18753                                                                        for (ResultColumn resultColumn : pivotTable.getColumns()) {
18754                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18755                                                                                relation.setEffectType(EffectType.select);
18756                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
18757                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
18758                                                                                pivotedColumns.add(tableColumn);
18759                                                                        }
18760                                                                        break;
18761                                                                }
18762                                                        }
18763                                                }
18764                                        } else {
18765                                                TObjectNameList columnNames = pivotClause.getUnpivotInClause().getItems().getElement(j)
18766                                                                .getColumnList();
18767                                                for (TObjectName columnName1 : columnNames) {
18768                                                        if (table instanceof QueryTable) {
18769                                                                for (ResultColumn tableColumn : ((QueryTable) table).getColumns()) {
18770                                                                        if (getColumnName(columnName1).equals(
18771                                                                                        DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
18772                                                                                for (ResultColumn resultColumn : pivotTable.getColumns()) {
18773                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18774                                                                                        relation.setEffectType(EffectType.select);
18775                                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
18776                                                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn));
18777                                                                                        pivotedColumns.add(tableColumn);
18778                                                                                }
18779                                                                                break;
18780                                                                        }
18781                                                                }
18782                                                        } else if (table instanceof Table) {
18783                                                                for (TableColumn tableColumn : ((Table) table).getColumns()) {
18784                                                                        if (getColumnName(columnName1).equals(
18785                                                                                        DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
18786                                                                                for (ResultColumn resultColumn : pivotTable.getColumns()) {
18787                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18788                                                                                        relation.setEffectType(EffectType.select);
18789                                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
18790                                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
18791                                                                                        pivotedColumns.add(tableColumn);
18792                                                                                }
18793                                                                                break;
18794                                                                        }
18795                                                                }
18796                                                        }
18797                                                }
18798                                        }
18799                                }
18800                        }
18801                        tables.add(pivotTable);
18802                        tables.add(table);
18803                }
18804
18805                ResultSet resultSet = modelFactory.createResultSet(stmt,
18806                                isTopResultSet(stmt) && isShowTopSelectResultSet());
18807                TResultColumnList columnList = stmt.getResultColumnList();
18808                for (int i = 0; i < columnList.size(); i++) {
18809                        TResultColumn column = columnList.getResultColumn(i);
18810                        ResultColumn resultColumn = modelFactory.createAndBindingSelectSetResultColumn(resultSet, column, i);
18811                        if (resultColumn.getColumnObject() instanceof TResultColumn) {
18812                                TResultColumn columnObject = (TResultColumn) resultColumn.getColumnObject();
18813                                if (columnObject.getFieldAttr() != null) {
18814                                        if ("*".equals(getColumnName(columnObject.getFieldAttr()))) {
18815                                                resultColumn.setShowStar(false);
18816                                                int index = 0;
18817                                                for (int k = 0; k < tables.size(); k++) {
18818                                                        Object tableItem = tables.get(k);
18819                                                        if (tableItem instanceof ResultSet) {
18820                                                                for (int x = 0; x < ((ResultSet) tableItem).getColumns().size(); x++) {
18821                                                                        ResultColumn tableColumn = ((ResultSet) tableItem).getColumns().get(x);
18822                                                                        if (pivotedColumns.contains(tableColumn)) {
18823                                                                                continue;
18824                                                                        }
18825                                                                        if (tableColumn.getColumnObject() instanceof TObjectName) {
18826                                                                                resultColumn.bindStarLinkColumn((TObjectName) tableColumn.getColumnObject());
18827                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18828                                                                                relation.setEffectType(EffectType.select);
18829                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn, (TObjectName) tableColumn.getColumnObject()));
18830                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
18831                                                                        } else if (tableColumn.getColumnObject() instanceof TResultColumn) {
18832                                                                                if (((TResultColumn) tableColumn.getColumnObject()).getFieldAttr() != null) {
18833                                                                                        if (tableColumn.hasStarLinkColumn()) {
18834                                                                                                for (int z = 0; z < tableColumn.getStarLinkColumnList().size(); z++) {
18835                                                                                                        ResultColumn resultColumn1 = modelFactory.createResultColumn(
18836                                                                                                                        (ResultSet) tableItem,
18837                                                                                                                        tableColumn.getStarLinkColumnList().get(z));
18838                                                                                                        DataFlowRelationship relation = modelFactory
18839                                                                                                                        .createDataFlowRelation();
18840                                                                                                        relation.setEffectType(EffectType.select);
18841                                                                                                        relation.setTarget(
18842                                                                                                                        new ResultColumnRelationshipElement(resultColumn));
18843                                                                                                        relation.addSource(
18844                                                                                                                        new ResultColumnRelationshipElement(resultColumn1));
18845                                                                                                        tableColumn.getStarLinkColumns().remove(
18846                                                                                                                        getColumnName(tableColumn.getStarLinkColumnList().get(z)));
18847                                                                                                        z--;
18848                                                                                                }
18849                                                                                        } else {
18850                                                                                                resultColumn.bindStarLinkColumn(
18851                                                                                                                ((TResultColumn) tableColumn.getColumnObject()).getFieldAttr());
18852                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18853                                                                                                relation.setEffectType(EffectType.select);
18854                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn, ((TResultColumn) tableColumn.getColumnObject()).getFieldAttr()));
18855                                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
18856                                                                                        }
18857                                                                                } else if (((TResultColumn) tableColumn.getColumnObject()).getExpr() != null) {
18858                                                                                        TExpression expr = ((TResultColumn) tableColumn.getColumnObject())
18859                                                                                                        .getExpr();
18860                                                                                        if (expr.getExpressionType() == EExpressionType.simple_constant_t) {
18861                                                                                                TObjectName columnName = new TObjectName();
18862                                                                                                columnName.setString(expr.toString());
18863                                                                                                resultColumn.bindStarLinkColumn(columnName);
18864                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18865                                                                                                relation.setEffectType(EffectType.select);
18866                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn, columnName));
18867                                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
18868                                                                                        }
18869                                                                                }
18870                                                                        }
18871                                                                }
18872                                                        } else if (tableItem instanceof Table) {
18873                                                                for (TableColumn tableColumn : ((Table) tableItem).getColumns()) {
18874                                                                        if (pivotedColumns.contains(tableColumn)) {
18875                                                                                continue;
18876                                                                        }
18877                                                                        resultColumn.bindStarLinkColumn((TObjectName) tableColumn.getColumnObject(), index);
18878                                                                        index++;
18879                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18880                                                                        relation.setEffectType(EffectType.select);
18881                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn, (TObjectName) tableColumn.getColumnObject()));
18882                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
18883                                                                }
18884                                                        }
18885                                                }
18886                                        } else {
18887                                                for (int k = 0; k < tables.size(); k++) {
18888                                                        Object tableItem = tables.get(k);
18889                                                        if (tableItem instanceof ResultSet) {
18890                                                                for (ResultColumn tableColumn : ((ResultSet) tableItem).getColumns()) {
18891                                                                        if (getColumnName(columnObject.getFieldAttr()).equals(
18892                                                                                        DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
18893                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18894                                                                                relation.setEffectType(EffectType.select);
18895                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
18896                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
18897                                                                                break;
18898                                                                        }
18899                                                                }
18900                                                        } else if (tableItem instanceof Table) {
18901                                                                for (TableColumn tableColumn : ((Table) tableItem).getColumns()) {
18902                                                                        if (getColumnName(columnObject.getFieldAttr()).equals(
18903                                                                                        DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
18904                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18905                                                                                relation.setEffectType(EffectType.select);
18906                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
18907                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
18908                                                                                break;
18909                                                                        }
18910                                                                }
18911                                                        }
18912                                                }
18913                                        }
18914                                } else if (columnObject.getExpr() != null) {
18915                                        analyzeResultColumn(column, EffectType.select);
18916                                }
18917                        }
18918                }
18919        }
18920
18921        private void analyzePivotedTable(TSelectSqlStatement stmt, TPivotedTable pivotedTable) {
18922                List<Object> tables = new ArrayList<Object>();
18923                Set<Object> pivotedColumns = new HashSet<Object>();
18924                TTable fromTable = pivotedTable.getTableSource();
18925                Object table = modelManager.getModel(fromTable);
18926                if(table == null && fromTable.getSubquery()!=null) {
18927                        table = modelFactory.createQueryTable(fromTable);
18928                        TSelectSqlStatement subquery = fromTable.getSubquery();
18929                        analyzeSelectStmt(subquery);
18930                }
18931                List<TPivotClause> pivotClauses = new ArrayList<TPivotClause>();
18932                if (pivotedTable.getPivotClauseList() != null && pivotedTable.getPivotClauseList().size() > 0) {
18933                        for (int i = 0; i < pivotedTable.getPivotClauseList().size(); i++) {
18934                                pivotClauses.add(pivotedTable.getPivotClauseList().getElement(i));
18935                        }
18936                } else {
18937                        TPivotClause pivotClause = pivotedTable.getPivotClause();
18938                        pivotClauses.add(pivotClause);
18939                }
18940
18941                for (int y = 0; y < pivotClauses.size(); y++) {
18942                        TPivotClause pivotClause = pivotClauses.get(y);
18943                        List<TFunctionCall> functionCalls = new ArrayList<TFunctionCall>();
18944                        if (pivotClause.getAggregation_function() != null || pivotClause.getAggregation_function_list() != null) {
18945                                if (pivotClause.getAggregation_function() != null) {
18946                                        functionCalls.add((TFunctionCall) pivotClause.getAggregation_function());
18947                                } else if (pivotClause.getAggregation_function_list() != null) {
18948                                        for (int i = 0; i < pivotClause.getAggregation_function_list().size(); i++) {
18949                                                functionCalls.add((TFunctionCall) pivotClause.getAggregation_function_list().getResultColumn(i)
18950                                                                .getExpr().getFunctionCall());
18951                                        }
18952                                }
18953
18954                                if (functionCalls.isEmpty()) {
18955                                        return;
18956                                }
18957
18958                                if (pivotClause.getPivotColumnList() == null) {
18959                                        return;
18960                                }
18961
18962                                if (pivotClause.getPivotInClause() == null) {
18963                                        return;
18964                                }
18965
18966                                for (int x = 0; x < functionCalls.size(); x++) {
18967                                        TFunctionCall functionCall = functionCalls.get(x);
18968                                        Function function = modelFactory.createFunction(functionCall);
18969                                        ResultColumn column = modelFactory.createFunctionResultColumn(function,
18970                                                        ((TFunctionCall) functionCall).getFunctionName());
18971
18972                                        List<TExpression> expressions = new ArrayList<TExpression>();
18973                                        getFunctionExpressions(expressions, new ArrayList<TExpression>(), functionCall);
18974
18975                                        for (int j = 0; j < expressions.size(); j++) {
18976                                                columnsInExpr visitor = new columnsInExpr();
18977                                                expressions.get(j).inOrderTraverse(visitor);
18978                                                List<TObjectName> objectNames = visitor.getObjectNames();
18979                                                if (objectNames == null) {
18980                                                        continue;
18981                                                }
18982                                                for (TObjectName columnName : objectNames) {
18983                                                        if (table instanceof QueryTable) {
18984                                                                for (int i = 0; i < ((QueryTable) table).getColumns().size(); i++) {
18985                                                                        boolean find = false;
18986                                                                        ResultColumn tableColumn = ((QueryTable) table).getColumns().get(i);
18987                                                                        if (tableColumn.hasStarLinkColumn()) {
18988                                                                                for (int k = 0; k < tableColumn.getStarLinkColumnList().size(); k++) {
18989                                                                                        TObjectName objectName = tableColumn.getStarLinkColumnList().get(k);
18990                                                                                        if (getColumnName(columnName).equals(getColumnName(objectName))) {
18991                                                                                                ResultColumn resultColumn = modelFactory
18992                                                                                                                .createResultColumn((QueryTable) table, objectName);
18993                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
18994                                                                                                relation.setEffectType(EffectType.select);
18995                                                                                                relation.setTarget(new ResultColumnRelationshipElement(column));
18996                                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
18997                                                                                                pivotedColumns.add(resultColumn);
18998                                                                                                tableColumn.getStarLinkColumns()
18999                                                                                                                .remove(DlineageUtil.getColumnName(objectName));
19000                                                                                                find = true;
19001                                                                                                break;
19002                                                                                        }
19003                                                                                }
19004                                                                        } else if (getColumnName(columnName).equals(
19005                                                                                        DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
19006                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19007                                                                                relation.setEffectType(EffectType.select);
19008                                                                                relation.setTarget(new ResultColumnRelationshipElement(column));
19009                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
19010                                                                                pivotedColumns.add(tableColumn);
19011                                                                                find = true;
19012                                                                                break;
19013                                                                        } 
19014                                                                        
19015                                                                        if (!find && tableColumn.getName().endsWith("*")) {
19016                                                                                QueryTable queryTable = (QueryTable)table;
19017                                                                                ResultColumn resultColumn = modelFactory.createResultColumn(queryTable, columnName);
19018                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19019                                                                                relation.setEffectType(EffectType.select);
19020                                                                                relation.setTarget(new ResultColumnRelationshipElement(column));
19021                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
19022                                                                                tableColumn.bindStarLinkColumn(columnName);
19023                                                                        }
19024                                                                }
19025                                                        } else if (table instanceof Table) {
19026                                                                for (TableColumn tableColumn : ((Table) table).getColumns()) {
19027                                                                        if (getColumnName(columnName).equals(
19028                                                                                        DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
19029                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19030                                                                                relation.setEffectType(EffectType.select);
19031                                                                                relation.setTarget(new ResultColumnRelationshipElement(column));
19032                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
19033                                                                                pivotedColumns.add(tableColumn);
19034                                                                                break;
19035                                                                        }
19036                                                                }
19037                                                        }
19038                                                }
19039                                        }
19040
19041                                        PivotedTable pivotTable = modelFactory.createPivotdTable(pivotClause);
19042                                        pivotTable.setUnpivoted(false);
19043
19044                                        if (pivotClause.getPivotInClause().getItems() != null) {
19045                                                for (int j = 0; j < pivotClause.getPivotInClause().getItems().size(); j++) {
19046                                                        ResultColumn resultColumn = null;
19047                                                        if (pivotClause.getAggregation_function_list() != null
19048                                                                        && pivotClause.getAggregation_function_list().size() > 1) {
19049                                                                TResultColumn functionColumn = pivotClause.getAggregation_function_list()
19050                                                                                .getResultColumn(x);
19051                                                                TObjectName tableColumn = new TObjectName();
19052                                                                if (option.getVendor() == EDbVendor.dbvbigquery) {
19053                                                                        tableColumn.setString(getResultColumnString(functionColumn) + "_"
19054                                                                                        + SQLUtil.trimColumnStringQuote(getResultColumnString(
19055                                                                                                        pivotClause.getPivotInClause().getItems().getResultColumn(j))));
19056                                                                }
19057                                                                else {
19058                                                                        tableColumn.setString(SQLUtil
19059                                                                                        .trimColumnStringQuote(getResultColumnString(
19060                                                                                                        pivotClause.getPivotInClause().getItems().getResultColumn(j)))
19061                                                                                        + "_" + getResultColumnString(functionColumn));
19062                                                                }
19063                                                                resultColumn = modelFactory.createResultColumn(pivotTable, tableColumn);
19064                                                        } else {
19065                                                                resultColumn = modelFactory.createSelectSetResultColumn(pivotTable,
19066                                                                                pivotClause.getPivotInClause().getItems().getResultColumn(j), j);
19067                                                        }
19068                                                        {
19069                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19070                                                                relation.setEffectType(EffectType.select);
19071                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
19072                                                                relation.addSource(new ResultColumnRelationshipElement(column));
19073                                                        }
19074                                                        {
19075                                                                for (TObjectName columnName : pivotClause.getPivotColumnList()) {
19076                                                                        if (table instanceof QueryTable) {
19077                                                                                for (int i = 0; i < ((QueryTable) table).getColumns().size(); i++) {
19078                                                                                        ResultColumn tableColumn = ((QueryTable) table).getColumns().get(i);
19079                                                                                        if (tableColumn.hasStarLinkColumn()) {
19080                                                                                                for (int k = 0; k < tableColumn.getStarLinkColumnList().size(); k++) {
19081                                                                                                        TObjectName objectName = tableColumn.getStarLinkColumnList().get(k);
19082                                                                                                        if (getColumnName(columnName).equals(getColumnName(objectName))) {
19083                                                                                                                ResultColumn resultColumn1 = modelFactory
19084                                                                                                                                .createResultColumn((QueryTable) table, objectName);
19085                                                                                                                DataFlowRelationship relation = modelFactory
19086                                                                                                                                .createDataFlowRelation();
19087                                                                                                                relation.setEffectType(EffectType.select);
19088                                                                                                                relation.setTarget(new ResultColumnRelationshipElement(column));
19089                                                                                                                relation.addSource(
19090                                                                                                                                new ResultColumnRelationshipElement(resultColumn1));
19091                                                                                                                pivotedColumns.add(resultColumn1);
19092                                                                                                                tableColumn.getStarLinkColumns()
19093                                                                                                                                .remove(DlineageUtil.getColumnName(objectName));
19094                                                                                                                break;
19095                                                                                                        }
19096                                                                                                }
19097                                                                                        } else if (getColumnName(columnName).equals(DlineageUtil
19098                                                                                                        .getIdentifierNormalColumnName(tableColumn.getName()))) {
19099                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19100                                                                                                relation.setEffectType(EffectType.select);
19101                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
19102                                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
19103                                                                                                pivotedColumns.add(tableColumn);
19104                                                                                                break;
19105                                                                                        }
19106                                                                                }
19107                                                                        } else if (table instanceof Table) {
19108                                                                                for (TableColumn tableColumn : ((Table) table).getColumns()) {
19109                                                                                        if (getColumnName(columnName).equals(DlineageUtil
19110                                                                                                        .getIdentifierNormalColumnName(tableColumn.getName()))) {
19111                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19112                                                                                                relation.setEffectType(EffectType.select);
19113                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
19114                                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
19115                                                                                                pivotedColumns.add(tableColumn);
19116                                                                                                break;
19117                                                                                        }
19118                                                                                }
19119                                                                        }
19120                                                                }
19121                                                        }
19122                                                }
19123                                        } else if (pivotClause.getPivotInClause().getSubQuery() != null) {
19124                                                TSelectSqlStatement subquery = pivotClause.getPivotInClause().getSubQuery();
19125                                                analyzeSelectStmt(subquery);
19126                                                ResultSet selectSetResultSetModel = (ResultSet) modelManager.getModel(subquery);
19127                                                for (int j = 0; j < subquery.getResultColumnList().size(); j++) {
19128                                                        ResultColumn resultColumn = modelFactory.createSelectSetResultColumn(pivotTable,
19129                                                                        subquery.getResultColumnList().getResultColumn(j), j);
19130                                                        {
19131                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19132                                                                relation.setEffectType(EffectType.select);
19133                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
19134                                                                relation.addSource(new ResultColumnRelationshipElement(column));
19135                                                                relation.addSource(new ResultColumnRelationshipElement(
19136                                                                                selectSetResultSetModel.getColumns().get(j)));
19137                                                        }
19138                                                        {
19139                                                                for (TObjectName columnName : pivotClause.getPivotColumnList()) {
19140                                                                        if (table instanceof QueryTable) {
19141                                                                                for (ResultColumn tableColumn : ((QueryTable) table).getColumns()) {
19142                                                                                        if (getColumnName(columnName).equals(DlineageUtil
19143                                                                                                        .getIdentifierNormalColumnName(tableColumn.getName()))) {
19144                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19145                                                                                                relation.setEffectType(EffectType.select);
19146                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
19147                                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
19148                                                                                                pivotedColumns.add(tableColumn);
19149                                                                                                break;
19150                                                                                        }
19151                                                                                }
19152                                                                        } else if (table instanceof Table) {
19153                                                                                for (TableColumn tableColumn : ((Table) table).getColumns()) {
19154                                                                                        if (getColumnName(columnName).equals(DlineageUtil
19155                                                                                                        .getIdentifierNormalColumnName(tableColumn.getName()))) {
19156                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19157                                                                                                relation.setEffectType(EffectType.select);
19158                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
19159                                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
19160                                                                                                pivotedColumns.add(tableColumn);
19161                                                                                                break;
19162                                                                                        }
19163                                                                                }
19164                                                                        }
19165                                                                }
19166                                                        }
19167                                                }
19168                                        }
19169                                        tables.add(pivotTable);
19170                                        tables.add(table);
19171                                }
19172                        }
19173                }
19174
19175                TPivotClause pivotClause = pivotClauses.get(0);
19176                boolean hasAlias = pivotClause.getAliasClause() != null && pivotClause.getAliasClause().getColumns() != null
19177                                && pivotClause.getAliasClause().getColumns().size() > 0;
19178                if (hasAlias) {
19179                        Alias alias = modelFactory.createAlias(pivotClause.getAliasClause());
19180                        List<TObjectName> aliasColumns = new ArrayList<TObjectName>();
19181                        int index = 0;
19182                        for (int k = 0; k < tables.size(); k++) {
19183                                Object tableItem = tables.get(k);
19184                                if (tableItem instanceof ResultSet) {
19185                                        for (ResultColumn tableColumn : ((ResultSet) tableItem).getColumns()) {
19186                                                if (pivotedColumns.contains(tableColumn)) {
19187                                                        continue;
19188                                                }
19189                                                if (tableColumn.getColumnObject() instanceof TObjectName) {
19190                                                        aliasColumns.add((TObjectName) tableColumn.getColumnObject());
19191                                                } else if (tableColumn.getColumnObject() instanceof TResultColumn) {
19192                                                        if (((TResultColumn) tableColumn.getColumnObject()).getFieldAttr() != null) {
19193                                                                aliasColumns.add(((TResultColumn) tableColumn.getColumnObject()).getFieldAttr());
19194                                                        } else {
19195                                                                TExpression expr = ((TResultColumn) tableColumn.getColumnObject()).getExpr();
19196                                                                if (expr.getExpressionType() == EExpressionType.simple_constant_t) {
19197                                                                        TObjectName columnName = new TObjectName();
19198                                                                        columnName.setString(expr.toString());
19199                                                                        aliasColumns.add(columnName);
19200                                                                }
19201                                                        }
19202                                                }
19203                                        }
19204                                } else if (tableItem instanceof Table) {
19205                                        for (TableColumn tableColumn : ((Table) tableItem).getColumns()) {
19206                                                if (pivotedColumns.contains(tableColumn)) {
19207                                                        continue;
19208                                                }
19209                                                aliasColumns.add(index, (TObjectName) tableColumn.getColumnObject());
19210                                                index++;
19211                                        }
19212                                }
19213                        }
19214
19215                        IndexedLinkedHashMap<String, ResultColumn> aliasColumnMap = new IndexedLinkedHashMap<String, ResultColumn>();
19216                        int diffCount = pivotClause.getAliasClause().getColumns().size() -  aliasColumns.size();
19217                        for (int k = 0; k < pivotClause.getAliasClause().getColumns().size(); k++) {
19218                                if (pivotClause.getAliasClause().getColumns().size() > aliasColumns.size()) {
19219                                        if (k < diffCount) {
19220                                                continue;
19221                                        }
19222                                        ResultColumn resultColumn = modelFactory.createResultColumn(alias,
19223                                                        pivotClause.getAliasClause().getColumns().getObjectName(k));
19224                                        if ((k - diffCount) < aliasColumns.size()) {
19225                                                aliasColumnMap.put(aliasColumns.get(k - diffCount).toString(), resultColumn);
19226                                        }
19227                                } else {
19228                                        ResultColumn resultColumn = modelFactory.createResultColumn(alias,
19229                                                        pivotClause.getAliasClause().getColumns().getObjectName(k));
19230                                        if (k < aliasColumns.size()) {
19231                                                aliasColumnMap.put(aliasColumns.get(k).toString(), resultColumn);
19232                                        }
19233                                }
19234                        }
19235
19236                        for (int k = 0; k < tables.size(); k++) {
19237                                Object tableItem = tables.get(k);
19238                                if (tableItem instanceof ResultSet) {
19239                                        int resultColumnSize = ((ResultSet) tableItem).getColumns().size();
19240                                        for (int x = 0; x < resultColumnSize; x++) {
19241                                                ResultColumn tableColumn = ((ResultSet) tableItem).getColumns().get(x);
19242                                                if (pivotedColumns.contains(tableColumn)) {
19243                                                        continue;
19244                                                }
19245                                                if (tableColumn.getColumnObject() instanceof TObjectName) {
19246                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19247                                                        relation.setEffectType(EffectType.select);
19248                                                        relation.setTarget(new ResultColumnRelationshipElement(
19249                                                                        aliasColumnMap.get(tableColumn.getColumnObject().toString())));
19250                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn));
19251                                                } else if (tableColumn.getColumnObject() instanceof TResultColumn) {
19252                                                        if (((TResultColumn) tableColumn.getColumnObject()).getFieldAttr() != null) {
19253                                                                ResultColumn targetColumn = aliasColumnMap.get(
19254                                                                                ((TResultColumn) tableColumn.getColumnObject()).getFieldAttr().toString());
19255                                                                if(targetColumn == null) {
19256                                                                        continue;
19257                                                                }
19258                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19259                                                                relation.setEffectType(EffectType.select);
19260                                                                relation.setTarget(new ResultColumnRelationshipElement(targetColumn));
19261                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
19262                                                        } else if (((TResultColumn) tableColumn.getColumnObject()).getExpr() != null) {
19263                                                                TExpression expr = ((TResultColumn) tableColumn.getColumnObject()).getExpr();
19264                                                                if (expr.getExpressionType() == EExpressionType.simple_constant_t) {
19265                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19266                                                                        relation.setEffectType(EffectType.select);
19267                                                                        ResultColumn targetColumn = (ResultColumn) aliasColumnMap
19268                                                                                        .getValueAtIndex(aliasColumnMap.size() - resultColumnSize + x);
19269                                                                        relation.setTarget(new ResultColumnRelationshipElement(targetColumn));
19270                                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn));
19271                                                                }
19272                                                        }
19273                                                }
19274                                        }
19275                                } else if (tableItem instanceof Table) {
19276                                        for (TableColumn tableColumn : ((Table) tableItem).getColumns()) {
19277                                                if (pivotedColumns.contains(tableColumn)) {
19278                                                        continue;
19279                                                }
19280                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19281                                                relation.setEffectType(EffectType.select);
19282                                                relation.setTarget(new ResultColumnRelationshipElement(
19283                                                                aliasColumnMap.get(tableColumn.getColumnObject().toString())));
19284                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
19285                                        }
19286                                }
19287                        }
19288
19289                        ResultSet resultSet = modelFactory.createResultSet(stmt,
19290                                        isTopResultSet(stmt) && isShowTopSelectResultSet());
19291                        TResultColumnList columnList = stmt.getResultColumnList();
19292                        for (int i = 0; i < columnList.size(); i++) {
19293                                TResultColumn column = columnList.getResultColumn(i);
19294                                ResultColumn resultColumn = modelFactory.createAndBindingSelectSetResultColumn(resultSet, column, i);
19295                                if (resultColumn.getColumnObject() instanceof TResultColumn) {
19296                                        TResultColumn columnObject = (TResultColumn) resultColumn.getColumnObject();
19297                                        if (columnObject.getFieldAttr() != null) {
19298                                                if ("*".equals(getColumnName(columnObject.getFieldAttr()))) {
19299                                                        resultColumn.setShowStar(false);
19300                                                        for (ResultColumn tableColumn : ((ResultSet) alias).getColumns()) {
19301                                                                resultColumn.bindStarLinkColumn((TObjectName) tableColumn.getColumnObject());
19302                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19303                                                                relation.setEffectType(EffectType.select);
19304                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn, (TObjectName) tableColumn.getColumnObject()));
19305                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
19306                                                        }
19307                                                } else {
19308                                                        for (ResultColumn tableColumn : ((ResultSet) alias).getColumns()) {
19309                                                                if (getColumnName(columnObject.getFieldAttr())
19310                                                                                .equals(DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
19311                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19312                                                                        relation.setEffectType(EffectType.select);
19313                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
19314                                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn));
19315                                                                        break;
19316                                                                }
19317                                                        }
19318                                                }
19319                                        } else if (columnObject.getExpr() != null && columnObject.getExpr()
19320                                                        .getExpressionType() == EExpressionType.sqlserver_proprietary_column_alias_t) {
19321                                                for (ResultColumn tableColumn : ((ResultSet) alias).getColumns()) {
19322                                                        if (getColumnName(columnObject.getExpr().getRightOperand().getObjectOperand())
19323                                                                        .equals(DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
19324                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19325                                                                relation.setEffectType(EffectType.select);
19326                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
19327                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
19328                                                                break;
19329                                                        }
19330                                                }
19331                                        } else if (columnObject.getExpr() != null && columnObject.getExpr()
19332                                                        .getExpressionType() == EExpressionType.function_t) {
19333                                                Function function = (Function) createFunction(columnObject.getExpr().getFunctionCall());
19334                                                for (ResultColumn arg : function.getColumns()) {
19335                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19336                                                        relation.setEffectType(EffectType.select);
19337                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
19338                                                        relation.addSource(new ResultColumnRelationshipElement(arg));
19339                                                }
19340                                        }
19341                                }
19342                        }
19343                } else {
19344                        ResultSet resultSet = modelFactory.createResultSet(stmt, isTopResultSet(stmt));
19345                        TResultColumnList columnList = stmt.getResultColumnList();
19346                        for (int i = 0; i < columnList.size(); i++) {
19347                                TResultColumn column = columnList.getResultColumn(i);
19348                                ResultColumn resultColumn = modelFactory.createAndBindingSelectSetResultColumn(resultSet, column, i);
19349                                if (resultColumn.getColumnObject() instanceof TResultColumn) {
19350                                        boolean fromFunction = false;
19351                                        TResultColumn columnObject = (TResultColumn) resultColumn.getColumnObject();
19352                                        TObjectName resultColumnFieldAttr = columnObject.getFieldAttr();
19353                                        List<TObjectName> resultColumnNames = new ArrayList<TObjectName>();
19354                                        if (resultColumnFieldAttr != null) {
19355                                                resultColumnNames.add(resultColumnFieldAttr);
19356                                        } else if (columnObject.getExpr() != null
19357                                                        && column.getExpr().getExpressionType() == EExpressionType.function_t) {
19358                                                extractFunctionObjectNames(column.getExpr().getFunctionCall(), resultColumnNames);
19359                                                fromFunction = true;
19360                                        }
19361                                        
19362                                        if (!resultColumnNames.isEmpty()) {
19363                                                for (TObjectName resultColumnName : resultColumnNames) {
19364                                                        if ("*".equals(getColumnName(resultColumnName))) {
19365                                                                resultColumn.setShowStar(false);
19366                                                                int index = 0;
19367                                                                for (int k = 0; k < tables.size(); k++) {
19368                                                                        Object tableItem = tables.get(k);
19369                                                                        if (tableItem instanceof ResultSet && !(tableItem instanceof QueryTable)) {
19370                                                                                for (int x = 0; x < ((ResultSet) tableItem).getColumns().size(); x++) {
19371                                                                                        ResultColumn tableColumn = ((ResultSet) tableItem).getColumns().get(x);
19372                                                                                        if (pivotedColumns.contains(tableColumn)) {
19373                                                                                                continue;
19374                                                                                        }
19375                                                                                        if (tableColumn.getColumnObject() instanceof TObjectName) {
19376                                                                                                resultColumn.bindStarLinkColumn(
19377                                                                                                                (TObjectName) tableColumn.getColumnObject());
19378                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19379                                                                                                relation.setEffectType(EffectType.select);
19380                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn, (TObjectName) tableColumn.getColumnObject()));
19381                                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
19382                                                                                        } else if (tableColumn.getColumnObject() instanceof TResultColumn) {
19383                                                                                                if (((TResultColumn) tableColumn.getColumnObject())
19384                                                                                                                .getFieldAttr() != null) {
19385                                                                                                        if (tableColumn.hasStarLinkColumn()) {
19386                                                                                                                for (int z = 0; z < tableColumn.getStarLinkColumnList()
19387                                                                                                                                .size(); z++) {
19388                                                                                                                        ResultColumn resultColumn1 = modelFactory
19389                                                                                                                                        .createResultColumn((ResultSet) tableItem,
19390                                                                                                                                                        tableColumn.getStarLinkColumnList().get(z));
19391                                                                                                                        DataFlowRelationship relation = modelFactory
19392                                                                                                                                        .createDataFlowRelation();
19393                                                                                                                        relation.setEffectType(EffectType.select);
19394                                                                                                                        relation.setTarget(
19395                                                                                                                                        new ResultColumnRelationshipElement(resultColumn));
19396                                                                                                                        relation.addSource(
19397                                                                                                                                        new ResultColumnRelationshipElement(resultColumn1));
19398                                                                                                                        tableColumn.getStarLinkColumns().remove(getColumnName(
19399                                                                                                                                        tableColumn.getStarLinkColumnList().get(z)));
19400                                                                                                                        z--;
19401                                                                                                                }
19402                                                                                                        } else {
19403                                                                                                                resultColumn.bindStarLinkColumn(
19404                                                                                                                                ((TResultColumn) tableColumn.getColumnObject())
19405                                                                                                                                                .getFieldAttr());
19406                                                                                                                DataFlowRelationship relation = modelFactory
19407                                                                                                                                .createDataFlowRelation();
19408                                                                                                                relation.setEffectType(EffectType.select);
19409                                                                                                                relation.setTarget(
19410                                                                                                                                new ResultColumnRelationshipElement(resultColumn, ((TResultColumn) tableColumn.getColumnObject())
19411                                                                                                                                                .getFieldAttr()));
19412                                                                                                                relation.addSource(
19413                                                                                                                                new ResultColumnRelationshipElement(tableColumn));
19414                                                                                                        }
19415                                                                                                } else if (((TResultColumn) tableColumn.getColumnObject())
19416                                                                                                                .getExpr() != null) {
19417                                                                                                        TExpression expr = ((TResultColumn) tableColumn.getColumnObject())
19418                                                                                                                        .getExpr();
19419                                                                                                        if (expr.getExpressionType() == EExpressionType.simple_constant_t) {
19420                                                                                                                TObjectName columnName = new TObjectName();
19421                                                                                                                columnName.setString(expr.toString());
19422                                                                                                                resultColumn.bindStarLinkColumn(columnName);
19423                                                                                                                DataFlowRelationship relation = modelFactory
19424                                                                                                                                .createDataFlowRelation();
19425                                                                                                                relation.setEffectType(EffectType.select);
19426                                                                                                                relation.setTarget(
19427                                                                                                                                new ResultColumnRelationshipElement(resultColumn, columnName));
19428                                                                                                                relation.addSource(
19429                                                                                                                                new ResultColumnRelationshipElement(tableColumn));
19430                                                                                                        }
19431                                                                                                }
19432                                                                                        }
19433                                                                                }
19434                                                                        } else if (tableItem instanceof Table) {
19435                                                                                for (TableColumn tableColumn : ((Table) tableItem).getColumns()) {
19436                                                                                        if (pivotedColumns.contains(tableColumn)) {
19437                                                                                                continue;
19438                                                                                        }
19439                                                                                        resultColumn.bindStarLinkColumn((TObjectName) tableColumn.getColumnObject(),
19440                                                                                                        index);
19441                                                                                        index++;
19442                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19443                                                                                        relation.setEffectType(EffectType.select);
19444                                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn, (TObjectName) tableColumn.getColumnObject()));
19445                                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
19446                                                                                }
19447                                                                        } else if (tableItem instanceof QueryTable) {
19448                                                                                for (ResultColumn tableColumn : ((QueryTable) tableItem).getColumns()) {
19449                                                                                        if (pivotedColumns.contains(tableColumn)) {
19450                                                                                                continue;
19451                                                                                        }
19452                                                                                        TObjectName column1 = new TObjectName();
19453                                                                                        column1.setString(tableColumn.getName());
19454                                                                                        resultColumn.bindStarLinkColumn(column1, index);
19455                                                                                        index++;
19456                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19457                                                                                        relation.setEffectType(EffectType.select);
19458                                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn, column1));
19459                                                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn), false);
19460                                                                                }
19461                                                                        }
19462                                                                }
19463                                                        } else {
19464                                                                ResultColumn pivotedTableColumn = getPivotedTableColumn(pivotedTable, resultColumnName);
19465                                                                        if (pivotedTableColumn != null) {
19466                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19467                                                                                relation.setEffectType(EffectType.select);
19468                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
19469                                                                                relation.addSource(new ResultColumnRelationshipElement(pivotedTableColumn));
19470                                                                        } else {
19471                                                                                for (int k = 0; k < tables.size(); k++) {
19472                                                                                        Object tableItem = tables.get(k);
19473                                                                                        if (tableItem instanceof ResultSet) {
19474                                                                                                for (ResultColumn tableColumn : ((ResultSet) tableItem).getColumns()) {
19475                                                                                                        if (DlineageUtil
19476                                                                                                                        .getIdentifierNormalColumnName(tableColumn.getName()).equals(getColumnName(resultColumnName))) {
19477                                                                                                                if (fromFunction) {
19478                                                                                                                        Function function = (Function)createPivotedFunction(column.getExpr().getFunctionCall(), tableColumn);
19479                                                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19480                                                                                                                        relation.setEffectType(EffectType.select);
19481                                                                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
19482                                                                                                                        if (function.getColumns() != null && !function.getColumns().isEmpty()) {
19483                                                                                                                                for (ResultColumn functionColumn : function.getColumns()) {
19484                                                                                                                                        relation.addSource(new ResultColumnRelationshipElement(functionColumn));
19485                                                                                                                                }
19486                                                                                                                        }
19487                                                                                                                } else {
19488                                                                                                                        DataFlowRelationship relation = modelFactory
19489                                                                                                                                        .createDataFlowRelation();
19490                                                                                                                        relation.setEffectType(EffectType.select);
19491                                                                                                                        relation.setTarget(
19492                                                                                                                                        new ResultColumnRelationshipElement(resultColumn));
19493                                                                                                                        relation.addSource(
19494                                                                                                                                        new ResultColumnRelationshipElement(tableColumn));
19495                                                                                                                }
19496                                                                                                                break;
19497                                                                                                        }
19498                                                                                                }
19499                                                                                        } else if (tableItem instanceof Table) {
19500                                                                                                for (TableColumn tableColumn : ((Table) tableItem).getColumns()) {
19501                                                                                                        if (getColumnName(resultColumnName).equals(DlineageUtil
19502                                                                                                                        .getIdentifierNormalColumnName(tableColumn.getName()))) {
19503                                                                                                                DataFlowRelationship relation = modelFactory
19504                                                                                                                                .createDataFlowRelation();
19505                                                                                                                relation.setEffectType(EffectType.select);
19506                                                                                                                relation.setTarget(
19507                                                                                                                                new ResultColumnRelationshipElement(resultColumn));
19508                                                                                                                relation.addSource(
19509                                                                                                                                new TableColumnRelationshipElement(tableColumn));
19510                                                                                                                break;
19511                                                                                                        }
19512                                                                                                }
19513                                                                                        }
19514                                                                                }
19515                                                                        }
19516                                                                }
19517                                                }
19518                                        } else if (columnObject.getExpr() != null && columnObject.getExpr()
19519                                                        .getExpressionType() == EExpressionType.sqlserver_proprietary_column_alias_t) {
19520                                                for (int k = 0; k < tables.size(); k++) {
19521                                                        Object tableItem = tables.get(k);
19522                                                        if (tableItem instanceof ResultSet) {
19523                                                                for (ResultColumn tableColumn : ((ResultSet) tableItem).getColumns()) {
19524                                                                        if (columnObject.getExpr().getRightOperand().getObjectOperand() != null
19525                                                                                        && getColumnName(
19526                                                                                                        columnObject.getExpr().getRightOperand().getObjectOperand())
19527                                                                                                                        .equals(DlineageUtil.getIdentifierNormalColumnName(
19528                                                                                                                                        tableColumn.getName()))) {
19529                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19530                                                                                relation.setEffectType(EffectType.select);
19531                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
19532                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
19533                                                                                break;
19534                                                                        } else if (columnObject.getExpr().getRightOperand()
19535                                                                                        .getExpressionType() == EExpressionType.function_t) {
19536                                                                                List<TExpression> expressions = new ArrayList<TExpression>();
19537                                                                                getFunctionExpressions(expressions, new ArrayList<TExpression>(),
19538                                                                                                columnObject.getExpr().getRightOperand().getFunctionCall());
19539                                                                                for (int j = 0; j < expressions.size(); j++) {
19540                                                                                        columnsInExpr visitor = new columnsInExpr();
19541                                                                                        expressions.get(j).inOrderTraverse(visitor);
19542                                                                                        List<TObjectName> objectNames = visitor.getObjectNames();
19543                                                                                        if (objectNames == null) {
19544                                                                                                continue;
19545                                                                                        }
19546                                                                                        for (TObjectName columnName : objectNames) {
19547                                                                                                if (getColumnName(columnName).equals(DlineageUtil
19548                                                                                                                .getIdentifierNormalColumnName(tableColumn.getName()))) {
19549                                                                                                        DataFlowRelationship relation = modelFactory
19550                                                                                                                        .createDataFlowRelation();
19551                                                                                                        relation.setEffectType(EffectType.select);
19552                                                                                                        relation.setTarget(
19553                                                                                                                        new ResultColumnRelationshipElement(resultColumn));
19554                                                                                                        relation.addSource(
19555                                                                                                                        new ResultColumnRelationshipElement(tableColumn));
19556                                                                                                        break;
19557                                                                                                }
19558                                                                                        }
19559                                                                                }
19560                                                                        }
19561                                                                }
19562                                                        } else if (tableItem instanceof Table) {
19563                                                                for (TableColumn tableColumn : ((Table) tableItem).getColumns()) {
19564                                                                        if (getColumnName(columnObject.getExpr().getRightOperand().getObjectOperand())
19565                                                                                        .equals(DlineageUtil
19566                                                                                                        .getIdentifierNormalColumnName(tableColumn.getName()))) {
19567                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19568                                                                                relation.setEffectType(EffectType.select);
19569                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
19570                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
19571                                                                                break;
19572                                                                        }
19573                                                                }
19574                                                        }
19575                                                }
19576                                        }
19577                                }
19578                        }
19579                }
19580
19581                analyzeSelectIntoClause(stmt);
19582        }
19583
19584        private Function createPivotedFunction(TFunctionCall functionCall, ResultColumn sourceColumn) {
19585                Function function = modelFactory.createFunction((TFunctionCall) functionCall);
19586                ResultColumn column = modelFactory.createFunctionResultColumn(function,
19587                                ((TFunctionCall) functionCall).getFunctionName());
19588                if ("COUNT".equalsIgnoreCase(((TFunctionCall) functionCall).getFunctionName().toString())) {
19589                        // @see https://e.gitee.com/gudusoft/issues/list?issue=I40NUP
19590                        // COUNT特殊处理，不和参数关联
19591                        if (option.isShowCountTableColumn()) {
19592                                analyzePivotedFunctionArgumentsDataFlowRelation(column, functionCall, sourceColumn);
19593                        }
19594                } else {
19595                        analyzePivotedFunctionArgumentsDataFlowRelation(column, functionCall, sourceColumn);
19596                        Set<Object> functionTableModelObjs = modelManager.getFunctionTable(getIdentifiedFunctionName(function));
19597                        if (functionTableModelObjs!=null && functionTableModelObjs.iterator().next() instanceof ResultSet) {
19598                                ResultSet functionTableModel = (ResultSet) functionTableModelObjs.iterator().next();
19599                                if (functionTableModel.getColumns() != null) {
19600                                        for (int j = 0; j < functionTableModel.getColumns().size(); j++) {
19601                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19602                                                        relation.setEffectType(EffectType.select);
19603                                                        relation.setTarget(new ResultColumnRelationshipElement(column));
19604                                                        relation.addSource(new ResultColumnRelationshipElement(
19605                                                                        functionTableModel.getColumns().get(j)));
19606                                        }
19607                                }
19608                        }
19609                }
19610                return function;
19611        }
19612
19613        private void analyzePivotedFunctionArgumentsDataFlowRelation(ResultColumn column, TFunctionCall functionCall,
19614                        ResultColumn sourceColumn) {
19615                List<TExpression> directExpressions = new ArrayList<TExpression>();
19616                List<TExpression> indirectExpressions = new ArrayList<TExpression>();
19617
19618                getFunctionExpressions(directExpressions, indirectExpressions, functionCall);
19619
19620                for (int j = 0; j < directExpressions.size(); j++) {
19621                        columnsInExpr visitor = new columnsInExpr();
19622                        directExpressions.get(j).inOrderTraverse(visitor);
19623
19624                        List<TObjectName> objectNames = visitor.getObjectNames();
19625                        List<TParseTreeNode> constants = visitor.getConstants();
19626
19627                        if (objectNames != null) {
19628                                for (TObjectName name : objectNames) {
19629                                        if (DlineageUtil.compareColumnIdentifier(name.toString(), sourceColumn.getName())) {
19630                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19631                                                relation.setEffectType(EffectType.select);
19632                                                relation.setTarget(new ResultColumnRelationshipElement(column));
19633                                                relation.addSource(new ResultColumnRelationshipElement(sourceColumn));
19634                                        }
19635                                }
19636                        }
19637                        if (constants != null) {
19638                                for (TParseTreeNode name : constants) {
19639                                        if (name.toString().equals(sourceColumn.getName())) {
19640                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19641                                                relation.setEffectType(EffectType.select);
19642                                                relation.setTarget(new ResultColumnRelationshipElement(column));
19643                                                relation.addSource(new ResultColumnRelationshipElement(sourceColumn));
19644                                        }
19645                                }
19646                        }
19647                }
19648        }
19649        
19650        private void extractFunctionObjectNames(TFunctionCall functionCall, List<TObjectName> resultColumnNames) {
19651                List<TExpression> directExpressions = new ArrayList<TExpression>();
19652                List<TExpression> indirectExpressions = new ArrayList<TExpression>();
19653
19654                getFunctionExpressions(directExpressions, indirectExpressions, functionCall);
19655
19656                for (int j = 0; j < directExpressions.size(); j++) {
19657                        columnsInExpr visitor = new columnsInExpr();
19658                        directExpressions.get(j).inOrderTraverse(visitor);
19659
19660                        List<TObjectName> objectNames = visitor.getObjectNames();
19661                        List<TParseTreeNode> functions = visitor.getFunctions();
19662                        List<TParseTreeNode> constants = visitor.getConstants(); 
19663
19664                        if (objectNames != null) {
19665                                resultColumnNames.addAll(objectNames);
19666                        }
19667                        
19668                        if (constants != null) {
19669                                for(TParseTreeNode item: constants) {
19670                                        if(item instanceof TConstant && ((TConstant) item).getLiteralType().getText().equals(ELiteralType.string_et.getText())) {
19671                                                TObjectName object = new TObjectName();
19672                                                object.setString(item.toString());
19673                                                resultColumnNames.add(object);
19674                                        }
19675                                }
19676                        }
19677
19678                        if (functions != null && !functions.isEmpty()) {
19679                                for (TParseTreeNode function : functions) {
19680                                        if (function instanceof TFunctionCall) {
19681                                                extractFunctionObjectNames((TFunctionCall) function, resultColumnNames);
19682                                        }
19683                                }
19684                        }
19685                }
19686        }
19687
19688        private String getResultColumnString(TResultColumn resultColumn) {
19689                if (resultColumn.getAliasClause() != null) {
19690                        return resultColumn.getAliasClause().toString();
19691                }
19692                return resultColumn.toString();
19693        }
19694
19695        private void analyzeBigQueryUnnest(TSelectSqlStatement stmt, TTable table) {
19696                Table unnestTable = modelFactory.createTableFromCreateDDL(table, false, getTempTableName(table));
19697                unnestTable.setSubType(SubType.unnest);
19698                TUnnestClause clause = table.getUnnestClause();
19699                TExpression arrayExpr = clause.getArrayExpr();
19700                if (arrayExpr == null){
19701                        if (clause.getColumns() != null) {
19702                                for (TObjectName column : clause.getColumns()) {
19703                                        if (clause.getDerivedColumnList() != null) {
19704                                                unnestTable.setCreateTable(true);
19705                                                for (int i = 0; i < clause.getDerivedColumnList().size(); i++) {
19706                                                        TObjectName columnName = new TObjectName();
19707                                                        columnName.setString(column.getColumnNameOnly() + "."
19708                                                                        + clause.getDerivedColumnList().getObjectName(i).getColumnNameOnly());
19709                                                        TableColumn tableColumn = modelFactory.createTableColumn(unnestTable, columnName, true);
19710                                                        List<TObjectName> columns = new ArrayList<TObjectName>();
19711                                                        columns.add(column);
19712                                                        analyzeDataFlowRelation(tableColumn, columns, EffectType.select, null);
19713                                                }
19714                                        }
19715                                        else {
19716                                                unnestTable.setCreateTable(true);
19717                                                boolean find = false;
19718                                                if (column.getSourceTable() != null && modelManager.getModel(column.getSourceTable()) instanceof Table) {
19719                                                        Table sourceTable = (Table)modelManager.getModel(column.getSourceTable());
19720                                                        if(sourceTable!=null) {
19721                                                                for(TableColumn tableColumn: sourceTable.getColumns()) {
19722                                                                        if(tableColumn.isStruct()) {
19723                                                                                List<String> names = SQLUtil.parseNames(tableColumn.getName());
19724                                                                                if (names.get(0).equalsIgnoreCase(column.getColumnNameOnly())) {
19725                                                                                        TObjectName columnName = new TObjectName();
19726                                                                                        columnName.setString(tableColumn.getName());
19727                                                                                        TableColumn unnestTableColumn = modelFactory.createTableColumn(unnestTable,
19728                                                                                                        columnName, true);
19729                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19730                                                                                        relation.setEffectType(EffectType.select);
19731                                                                                        relation.setTarget(new TableColumnRelationshipElement(unnestTableColumn));
19732                                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
19733                                                                                        find = true;
19734                                                                                }
19735                                                                        }
19736                                                                }
19737                                                        }
19738                                                }
19739                                                if (!find) {
19740                                                        TObjectName colName = column;
19741                                                        if (table.getAliasClause() != null && table.getAliasClause().getAliasName() != null) {
19742                                                                colName = table.getAliasClause().getAliasName();
19743                                                        }
19744                                                        TableColumn tableColumn = modelFactory.createTableColumn(unnestTable, colName, true);
19745                                                        List<TObjectName> columns = new ArrayList<TObjectName>();
19746                                                        columns.add(column);
19747                                                        analyzeDataFlowRelation(tableColumn, columns, EffectType.select, null);
19748                                                }
19749                                        }
19750                                }
19751                        }
19752                        return;
19753                }
19754                List<TExpression> expressions = new ArrayList<TExpression>();
19755                TExpressionList values = arrayExpr.getExprList();
19756                if (values == null) {
19757                        expressions.add(arrayExpr);
19758                }
19759                else {
19760                        for(TExpression value: values) {
19761                                expressions.add(value);
19762                        }
19763                }
19764                for (TExpression value : expressions) {
19765                        unnestTable.setCreateTable(true);
19766                        if (value.getExpressionType() == EExpressionType.simple_object_name_t) {
19767                                if (table.getAliasClause() != null) {
19768                                        TableColumn tableColumn = modelFactory.createTableColumn(unnestTable,
19769                                                        table.getAliasClause().getAliasName(), true);
19770                                        columnsInExpr visitor = new columnsInExpr();
19771                                        value.inOrderTraverse(visitor);
19772                                        List<TObjectName> columns = visitor.getObjectNames();
19773                                        analyzeDataFlowRelation(tableColumn, columns, EffectType.select, null);
19774                                } else {
19775                                        TResultColumnList resultColumnList = stmt.getResultColumnList();
19776                                        for (int i = 0; i < resultColumnList.size(); i++) {
19777                                                TObjectName firstColumn = new TObjectName();
19778                                                firstColumn.setString(resultColumnList.getResultColumn(0).getColumnNameOnly());
19779                                                TableColumn tableColumn = modelFactory.createTableColumn(unnestTable, firstColumn, true);
19780                                                columnsInExpr visitor = new columnsInExpr();
19781                                                value.inOrderTraverse(visitor);
19782                                                List<TObjectName> columns = visitor.getObjectNames();
19783                                                analyzeDataFlowRelation(tableColumn, columns, EffectType.select, null);
19784                                        }
19785                                }
19786                        } else if (value.getExpressionType() == EExpressionType.function_t) {
19787                                if (table.getAliasClause() != null) {
19788                                        TableColumn tableColumn = modelFactory.createTableColumn(unnestTable,
19789                                                        table.getAliasClause().getAliasName(), true);
19790                                        columnsInExpr visitor = new columnsInExpr();
19791                                        value.inOrderTraverse(visitor);
19792                                        List<TParseTreeNode> functions = visitor.getFunctions();
19793                                        analyzeFunctionDataFlowRelation(tableColumn, functions, EffectType.select, null);
19794                                } else {
19795                                        TResultColumnList resultColumnList = stmt.getResultColumnList();
19796                                        for (int i = 0; i < resultColumnList.size(); i++) {
19797                                                TObjectName firstColumn = new TObjectName();
19798                                                firstColumn.setString(resultColumnList.getResultColumn(0).getColumnNameOnly());
19799                                                TableColumn tableColumn = modelFactory.createTableColumn(unnestTable, firstColumn, true);
19800                                                columnsInExpr visitor = new columnsInExpr();
19801                                                value.inOrderTraverse(visitor);
19802                                                List<TObjectName> columns = visitor.getObjectNames();
19803                                                analyzeDataFlowRelation(tableColumn, columns, EffectType.select, null);
19804                                        }
19805                                }
19806                        } else if (value.getExpressionType() == EExpressionType.simple_constant_t) {
19807                                if (table.getAliasClause() != null) {
19808                                        TableColumn tableColumn = modelFactory.createTableColumn(unnestTable,
19809                                                        table.getAliasClause().getAliasName(), true);
19810                                        columnsInExpr visitor = new columnsInExpr();
19811                                        value.inOrderTraverse(visitor);
19812                                        List<TParseTreeNode> constants = visitor.getConstants();
19813                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select, null);
19814                                } else {
19815                                        TObjectName firstColumn = new TObjectName();
19816                                        firstColumn.setString("f0_");
19817                                        TableColumn tableColumn = modelFactory.createTableColumn(unnestTable, firstColumn, true);
19818                                        columnsInExpr visitor = new columnsInExpr();
19819                                        value.inOrderTraverse(visitor);
19820                                        List<TParseTreeNode> constants = visitor.getConstants();
19821                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select, null);
19822                                }
19823                        } else if (value.getExpressionType() == EExpressionType.list_t) {
19824                                if (arrayExpr.getTypeName() != null && arrayExpr.getTypeName().getColumnDefList() != null) {
19825                                        for (int i = 0; i < arrayExpr.getTypeName().getColumnDefList().size(); i++) {
19826                                                TColumnDefinition column = arrayExpr.getTypeName().getColumnDefList().getColumn(i);
19827                                                if (column != null && column.getColumnName() != null) {
19828                                                        TableColumn tableColumn = modelFactory.createTableColumn(unnestTable,
19829                                                                        column.getColumnName(), true);
19830                                                        columnsInExpr visitor = new columnsInExpr();
19831                                                        value.getExprList().getExpression(i).inOrderTraverse(visitor);
19832                                                        List<TParseTreeNode> constants = visitor.getConstants();
19833                                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select, null);
19834                                                }
19835                                        }
19836                                } else {
19837                                        for (int i = 0; i < value.getExprList().size(); i++) {
19838                                                TObjectName firstColumn = new TObjectName();
19839                                                firstColumn.setString("f" + i + "_");
19840                                                TableColumn tableColumn = modelFactory.createTableColumn(unnestTable, firstColumn, true);
19841                                                columnsInExpr visitor = new columnsInExpr();
19842                                                value.getExprList().getExpression(i).inOrderTraverse(visitor);
19843                                                List<TParseTreeNode> constants = visitor.getConstants();
19844                                                analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select, null);
19845                                        }
19846                                }
19847                        } else if (value.getExpressionType() == EExpressionType.subquery_t) {
19848                                analyzeSelectStmt(value.getSubQuery());
19849                                ResultSet resultSet = (ResultSet) modelManager.getModel(value.getSubQuery());
19850                                if (resultSet != null) {
19851                                        for (int i = 0; i < resultSet.getColumns().size(); i++) {
19852                                                TObjectName columnName =  new TObjectName();
19853                                                if(resultSet.getColumns().get(i).getAlias()!=null) {
19854                                                        columnName.setString(resultSet.getColumns().get(i).getAlias());
19855                                                }
19856                                                else {
19857                                                        columnName.setString(getColumnName(resultSet.getColumns().get(i).getName()));
19858                                                }
19859                                                TableColumn tableColumn = modelFactory.createTableColumn(unnestTable, columnName, true);
19860                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19861                                                relation.setEffectType(EffectType.select);
19862                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
19863                                                relation.addSource(
19864                                                                new ResultColumnRelationshipElement(resultSet.getColumns().get(i)));
19865                                        }
19866                                }
19867                        }
19868                }
19869        }
19870
19871        private void analyzePrestoUnnest(TSelectSqlStatement stmt, TTable table) {
19872                List<Table> tables = new ArrayList<Table>();
19873                TTable targetTable = stmt.getTables().getTable(0);
19874                Table stmtTable = modelFactory.createTable(targetTable);
19875                Object sourceTable = null;
19876                if (targetTable.getSubquery() != null) {
19877                        analyzeSelectStmt(targetTable.getSubquery());
19878                        if (targetTable.getSubquery().isCombinedQuery()) {
19879                                ResultSet sourceResultSet = (ResultSet) modelManager.getModel(targetTable.getSubquery());
19880                                sourceTable = sourceResultSet;
19881                        } else if (targetTable.getSubquery().getResultColumnList() != null) {
19882                                ResultSet sourceResultSet = (ResultSet) modelManager
19883                                                .getModel(targetTable.getSubquery().getResultColumnList());
19884                                sourceTable = sourceResultSet;
19885                        } else if (targetTable.getSubquery().getValueClause() != null) {
19886                                List<TResultColumnList> rowList = targetTable.getSubquery().getValueClause().getRows();
19887                                if (rowList != null && rowList.size() > 0) {
19888                                        Table valuesTable = modelFactory.createTableByName("Values-Table", true);
19889                                        int columnCount = rowList.get(0).size();
19890                                        for (int j = 1; j <= columnCount; j++) {
19891                                                TObjectName columnName = new TObjectName();
19892                                                TResultColumn columnObject = rowList.get(0).getResultColumn(j - 1);
19893                                                if (columnObject.getExpr().getExpressionType() == EExpressionType.typecast_t) {
19894                                                        columnName.setString(columnObject.getExpr().getLeftOperand().toString());
19895                                                } else {
19896                                                        columnName.setString(columnObject.getExpr().toString());
19897                                                }
19898                                                modelFactory.createTableColumn(valuesTable, columnName, true);
19899                                        }
19900                                        valuesTable.setCreateTable(true);
19901                                        valuesTable.setSubType(SubType.values_table);
19902                                        sourceTable = valuesTable;
19903                                }
19904                        }
19905                }
19906                tables.add(stmtTable);
19907                Table unnestTable = modelFactory.createTable(table);
19908                unnestTable.setSubType(SubType.unnest);
19909                tables.add(unnestTable);
19910                if (table.getAliasClause() != null && table.getAliasClause().getColumns() != null
19911                                && table.getUnnestClause().getColumns() != null) {
19912                        int unnestTableSize = table.getUnnestClause().getColumns().size();
19913                        for (int i = 0; i < table.getAliasClause().getColumns().size(); i++) {
19914                                TableColumn sourceColumn = null;
19915                                if (unnestTableSize > i) {
19916                                        sourceColumn = modelFactory.createTableColumn(stmtTable,
19917                                                        table.getUnnestClause().getColumns().getObjectName(i), true);
19918                                        if (sourceTable instanceof Table) {
19919                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19920                                                relation.setEffectType(EffectType.select);
19921                                                relation.setTarget(new TableColumnRelationshipElement(sourceColumn));
19922                                                relation.addSource(
19923                                                                new TableColumnRelationshipElement(((Table) sourceTable).getColumns().get(i)));
19924                                        } else if (sourceTable instanceof ResultSet) {
19925                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19926                                                relation.setEffectType(EffectType.select);
19927                                                relation.setTarget(new TableColumnRelationshipElement(sourceColumn));
19928                                                relation.addSource(
19929                                                                new ResultColumnRelationshipElement(((ResultSet) sourceTable).getColumns().get(i)));
19930                                        }
19931                                } else {
19932                                        sourceColumn = modelFactory.createTableColumn(stmtTable,
19933                                                        table.getUnnestClause().getColumns().getObjectName(unnestTableSize - 1), true);
19934                                }
19935                                TableColumn targetColumn = modelFactory.createTableColumn(unnestTable,
19936                                                table.getAliasClause().getColumns().getObjectName(i), true);
19937                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19938                                relation.setEffectType(EffectType.select);
19939                                relation.setTarget(new TableColumnRelationshipElement(targetColumn));
19940                                relation.addSource(new TableColumnRelationshipElement(sourceColumn));
19941                        }
19942                }
19943
19944                ResultSet resultSet = modelFactory.createResultSet(stmt,
19945                                isTopResultSet(stmt) && isShowTopSelectResultSet());
19946                TResultColumnList columnList = stmt.getResultColumnList();
19947                for (int i = 0; i < columnList.size(); i++) {
19948                        TResultColumn column = columnList.getResultColumn(i);
19949                        ResultColumn resultColumn = modelFactory.createSelectSetResultColumn(resultSet, column, i);
19950                        if (resultColumn.getColumnObject() instanceof TResultColumn) {
19951                                TResultColumn columnObject = (TResultColumn) resultColumn.getColumnObject();
19952                                if (columnObject.getFieldAttr() != null) {
19953                                        if ("*".equals(getColumnName(columnObject.getFieldAttr()))) {
19954                                                for (int k = 0; k < tables.size(); k++) {
19955                                                        Table tableItem = tables.get(k);
19956                                                        for (TableColumn tableColumn : tableItem.getColumns()) {
19957                                                                resultColumn.bindStarLinkColumn(tableColumn.getColumnObject());
19958                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19959                                                                relation.setEffectType(EffectType.select);
19960                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn, tableColumn.getColumnObject()));
19961                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
19962                                                        }
19963                                                }
19964                                        } else {
19965                                                boolean match = false;
19966                                                for (int k = 0; k < tables.size(); k++) {
19967                                                        Table tableItem = tables.get(k);
19968                                                        for (TableColumn tableColumn : tableItem.getColumns()) {
19969                                                                if (getColumnName(columnObject.getFieldAttr())
19970                                                                                .equals(DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
19971                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19972                                                                        relation.setEffectType(EffectType.select);
19973                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
19974                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
19975                                                                        match = true;
19976                                                                }
19977                                                        }
19978                                                }
19979                                                if (!match) {
19980                                                        TableColumn tableColumn = modelFactory.createTableColumn(stmtTable,
19981                                                                        columnObject.getFieldAttr(), false);
19982                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
19983                                                        relation.setEffectType(EffectType.select);
19984                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
19985                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
19986                                                }
19987                                        }
19988                                }
19989                        }
19990                }
19991        }
19992
19993        private void analyzeLateralView(TSelectSqlStatement stmt, TTable table, ArrayList<TLateralView> lateralViews) {
19994                List<Object> tables = new ArrayList<Object>();
19995                Object stmtTable = null;
19996                if(table.getSubquery()!=null) {
19997                        stmtTable = modelFactory.createQueryTable(table);
19998                        tables.add(stmtTable);
19999                }
20000                else {
20001                        stmtTable = modelFactory.createTable(table);
20002                        tables.add(stmtTable);
20003                }
20004                for (int i = 0; i < lateralViews.size(); i++) {
20005                        TLateralView lateralView = lateralViews.get(i);
20006                        TFunctionCall functionCall = lateralView.getUdtf();
20007                        List<TExpression> expressions = new ArrayList<TExpression>();
20008                        if (functionCall == null) {
20009                                continue;
20010                        }
20011                        Function function = modelFactory.createFunction(functionCall);
20012                        ResultColumn column = modelFactory.createFunctionResultColumn(function,
20013                                        ((TFunctionCall) functionCall).getFunctionName());
20014
20015                        Table lateralTable = null;
20016                        if (lateralView.getTableAlias() != null) {
20017                                lateralTable = modelFactory.createTableByName(lateralView.getTableAlias().getAliasName(), true);
20018                        } else {
20019                                lateralTable = modelFactory.createTableByName(functionCall.toString(), true);
20020                        }
20021
20022                        for (int j = 0; j < lateralView.getColumnAliasList().size(); j++) {
20023                                TObjectName viewColumn = lateralView.getColumnAliasList().getObjectName(j);
20024                                TableColumn tableColumn = modelFactory.createTableColumn(lateralTable, viewColumn, true);
20025
20026                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
20027                                relation.setEffectType(EffectType.select);
20028                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
20029                                relation.addSource(new ResultColumnRelationshipElement(column));
20030                        }
20031
20032                        getFunctionExpressions(expressions, new ArrayList<TExpression>(), functionCall);
20033                        for (int j = 0; j < expressions.size(); j++) {
20034                                columnsInExpr visitor = new columnsInExpr();
20035                                expressions.get(j).inOrderTraverse(visitor);
20036                                List<TObjectName> objectNames = visitor.getObjectNames();
20037                                if (objectNames == null) {
20038                                        continue;
20039                                }
20040                                for (TObjectName columnName : objectNames) {
20041                                        boolean match = false;
20042                                        for (int k = 0; k < tables.size(); k++) {
20043                                                Object item = tables.get(k);
20044                                                if(item instanceof Table) {
20045                                                        Table tableItem = (Table)item;
20046                                                        for (TableColumn tableColumn : tableItem.getColumns()) {
20047                                                                if (getColumnName(columnName)
20048                                                                                .equals(DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
20049                                                                        match = true;
20050                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
20051                                                                        relation.setEffectType(EffectType.select);
20052                                                                        relation.setTarget(new ResultColumnRelationshipElement(column));
20053                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
20054                                                                }
20055                                                        }
20056                                                }
20057                                                else {
20058                                                        ResultSet tableItem = (ResultSet)item;
20059                                                        for (ResultColumn tableColumn : tableItem.getColumns()) {
20060                                                                if (getColumnName(columnName)
20061                                                                                .equals(DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
20062                                                                        match = true;
20063                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
20064                                                                        relation.setEffectType(EffectType.select);
20065                                                                        relation.setTarget(new ResultColumnRelationshipElement(column));
20066                                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn));
20067                                                                }
20068                                                        }
20069                                                }
20070                                        }
20071                                        if (!match) {
20072                                                if(stmtTable instanceof Table) {
20073                                                        TableColumn tableColumn = modelFactory.createTableColumn((Table)stmtTable, columnName, false);
20074                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
20075                                                        relation.setEffectType(EffectType.select);
20076                                                        relation.setTarget(new ResultColumnRelationshipElement(column));
20077                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
20078                                                }
20079                                                else {
20080                                                        ResultColumn tableColumn = modelFactory.createResultColumn((ResultSet)stmtTable, columnName, false);
20081                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
20082                                                        relation.setEffectType(EffectType.select);
20083                                                        relation.setTarget(new ResultColumnRelationshipElement(column));
20084                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn));
20085                                                }
20086                                        }
20087                                }
20088                                List<TParseTreeNode> constants = visitor.getConstants();
20089                                if (!constants.isEmpty()) {
20090                                        if (option.isShowConstantTable()) {
20091                                                Table constantTable = modelFactory.createConstantsTable(stmtStack.peek());
20092                                                for (TParseTreeNode constant : constants) {
20093                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
20094                                                        relation.setEffectType(EffectType.select);
20095                                                        relation.setTarget(new ResultColumnRelationshipElement(column));
20096                                                        if (constant instanceof TConstant) {
20097                                                                TableColumn constantColumn = modelFactory.createTableColumn(constantTable,
20098                                                                                (TConstant) constant);
20099                                                                relation.addSource(new ConstantRelationshipElement(constantColumn));
20100                                                        } else if (constant instanceof TObjectName) {
20101                                                                TableColumn constantColumn = modelFactory.createTableColumn(constantTable,
20102                                                                                (TObjectName) constant, false);
20103                                                                relation.addSource(new ConstantRelationshipElement(constantColumn));
20104                                                        }
20105                                                }
20106                                        }
20107                                }
20108                                
20109                                List<TParseTreeNode> functions = visitor.getFunctions();
20110                                if (functions != null && !functions.isEmpty()) {
20111                                        analyzeFunctionDataFlowRelation(column, functions, EffectType.function);
20112                                }
20113                        }
20114                        tables.add(lateralTable);
20115                }
20116
20117                ResultSet resultSet = modelFactory.createResultSet(stmt,
20118                                isTopResultSet(stmt) && isShowTopSelectResultSet());
20119                TResultColumnList columnList = stmt.getResultColumnList();
20120                for (int i = 0; i < columnList.size(); i++) {
20121                        TResultColumn column = columnList.getResultColumn(i);
20122                        ResultColumn resultColumn = modelFactory.createSelectSetResultColumn(resultSet, column, i);
20123                        if (resultColumn.getColumnObject() instanceof TResultColumn) {
20124                                TResultColumn columnObject = (TResultColumn) resultColumn.getColumnObject();
20125                                if (columnObject.getFieldAttr() != null) {
20126                                        if ("*".equals(getColumnName(columnObject.getFieldAttr()))) {
20127                                                for (int k = 0; k < tables.size(); k++) {
20128                                                        Object item = tables.get(k);
20129                                                        if(item instanceof Table) {
20130                                                                Table tableItem = (Table)item;
20131                                                                for (TableColumn tableColumn : tableItem.getColumns()) {
20132                                                                        resultColumn.bindStarLinkColumn(tableColumn.getColumnObject());
20133                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
20134                                                                        relation.setEffectType(EffectType.select);
20135                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn, tableColumn.getColumnObject()));
20136                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
20137                                                                }
20138                                                        }
20139                                                        else {
20140                                                                ResultSet tableItem = (ResultSet)item;
20141                                                                for (ResultColumn tableColumn : tableItem.getColumns()) {
20142                                                                        TObjectName linkColumn = new TObjectName();
20143                                                                        linkColumn.setString(tableColumn.getName());
20144                                                                        resultColumn.bindStarLinkColumn(linkColumn);
20145                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
20146                                                                        relation.setEffectType(EffectType.select);
20147                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn, linkColumn));
20148                                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn));
20149                                                                }
20150                                                        }
20151                                                }
20152                                        } else {
20153                                                boolean match = false;
20154                                                for (int k = 0; k < tables.size(); k++) {
20155                                                        Object item = tables.get(k);
20156                                                        if(item instanceof Table) {
20157                                                                Table tableItem = (Table)item;
20158                                                                for (TableColumn tableColumn : tableItem.getColumns()) {
20159                                                                        if (getColumnName(columnObject.getFieldAttr())
20160                                                                                        .equals(DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
20161                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
20162                                                                                relation.setEffectType(EffectType.select);
20163                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
20164                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
20165                                                                                match = true;
20166                                                                        }
20167                                                                }
20168                                                        }
20169                                                        else {
20170                                                                ResultSet tableItem = (ResultSet)item;
20171                                                                for (ResultColumn tableColumn : tableItem.getColumns()) {
20172                                                                        if (getColumnName(columnObject.getFieldAttr())
20173                                                                                        .equals(DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
20174                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
20175                                                                                relation.setEffectType(EffectType.select);
20176                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
20177                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
20178                                                                                match = true;
20179                                                                        }
20180                                                                }
20181                                                        }
20182                                                }
20183                                                if (!match) {
20184                                                        if(stmtTable instanceof Table) {
20185                                                                TableColumn tableColumn = modelFactory.createTableColumn((Table)stmtTable,
20186                                                                                columnObject.getFieldAttr(), false);
20187                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
20188                                                                relation.setEffectType(EffectType.select);
20189                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
20190                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
20191                                                        }
20192                                                        else {
20193                                                                ResultColumn tableColumn = modelFactory.createResultColumn((ResultSet)stmtTable,
20194                                                                                columnObject.getFieldAttr(), false);
20195                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
20196                                                                relation.setEffectType(EffectType.select);
20197                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
20198                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
20199                                                        }
20200                                                }
20201                                        }
20202                                }
20203                                else if (columnObject.getExpr() != null
20204                                                && columnObject.getExpr().getExpressionType() == EExpressionType.function_t) {
20205                                        analyzeResultColumn(column, EffectType.select);
20206                                        for (int k = 0; k < tables.size(); k++) {
20207                                                Object item = tables.get(k);
20208                                                if (item instanceof Table) {
20209                                                        Table tableItem = (Table) item;
20210                                                        for (TableColumn tableColumn : tableItem.getColumns()) {
20211                                                                if (getColumnName(resultColumn.getName()).equals(
20212                                                                                DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
20213                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
20214                                                                        relation.setEffectType(EffectType.select);
20215                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
20216                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
20217                                                                }
20218                                                        }
20219                                                } else {
20220                                                        ResultSet tableItem = (ResultSet) item;
20221                                                        for (ResultColumn tableColumn : tableItem.getColumns()) {
20222                                                                if (getColumnName(resultColumn.getName()).equals(
20223                                                                                DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
20224                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
20225                                                                        relation.setEffectType(EffectType.select);
20226                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
20227                                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn));
20228                                                                }
20229                                                        }
20230                                                }
20231                                        }
20232                                }
20233                        }
20234                }
20235        }
20236
20237        private boolean isFromFunction(TObjectName object) {
20238
20239                Stack<TParseTreeNode> nodes = object.getStartToken().getNodesStartFromThisToken();
20240                if (nodes != null) {
20241                        for (int i = 0; i < nodes.size(); i++) {
20242                                if (nodes.get(i) instanceof TFunctionCall) {
20243                                        return true;
20244                                }
20245                        }
20246                }
20247                return false;
20248        }
20249
20250        private TResultColumnList getResultColumnList(TSelectSqlStatement stmt) {
20251                // Iterative DFS (left-first) to find the first non-combined query's result column list.
20252                // Avoids StackOverflow with deeply nested UNION trees.
20253                Deque<TSelectSqlStatement> stack = new ArrayDeque<>();
20254                stack.push(stmt);
20255                while (!stack.isEmpty()) {
20256                        TSelectSqlStatement current = stack.pop();
20257                        if (current.isCombinedQuery()) {
20258                                // Push right first so left is processed first (stack is LIFO)
20259                                if (current.getRightStmt() != null) stack.push(current.getRightStmt());
20260                                if (current.getLeftStmt() != null) stack.push(current.getLeftStmt());
20261                        } else {
20262                                if (current.getResultColumnList() != null) {
20263                                        return current.getResultColumnList();
20264                                }
20265                        }
20266                }
20267                return null;
20268        }
20269
20270        private void createPseudoImpactRelation(TCustomSqlStatement stmt, ResultSet resultSetModel, EffectType effectType) {
20271                if (stmt.getTables() != null) {
20272                        for (int i = 0; i < stmt.getTables().size(); i++) {
20273                                TTable table = stmt.getTables().getTable(i);
20274                                if (modelManager.getModel(table) instanceof ResultSet) {
20275                                        ResultSet tableModel = (ResultSet) modelManager.getModel(table);
20276                                        if (tableModel != resultSetModel && !tableModel.getRelationRows().getHoldRelations().isEmpty()) {
20277                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
20278                                                impactRelation.setEffectType(effectType);
20279                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
20280                                                                tableModel.getRelationRows()));
20281                                                impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
20282                                                                resultSetModel.getRelationRows()));
20283                                        }
20284                                } else if (modelManager.getModel(table) instanceof Table) {
20285                                        Table tableModel = (Table) modelManager.getModel(table);
20286                                        if (!tableModel.getRelationRows().getHoldRelations().isEmpty()) {
20287                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
20288                                                impactRelation.setEffectType(effectType);
20289                                                impactRelation.addSource(
20290                                                                new RelationRowsRelationshipElement<TableRelationRows>(tableModel.getRelationRows()));
20291                                                impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
20292                                                                resultSetModel.getRelationRows()));
20293                                        }
20294                                }
20295                        }
20296                }
20297        }
20298
20299        private void analyzeFunctionDataFlowRelation(Object gspObject, List<TParseTreeNode> functions,
20300                        EffectType effectType) {
20301                for (int i = 0; i < functions.size(); i++) {
20302                        TParseTreeNode functionCall = functions.get(i);
20303                        if (functionCall instanceof TFunctionCall) {
20304                                String functionName = DlineageUtil.getIdentifierNormalTableName(
20305                                                DlineageUtil.getFunctionNameWithArgNum((TFunctionCall) functionCall));
20306                                Procedure procedure = modelManager.getProcedureByName(functionName);
20307                                if (procedure != null) {
20308                                        String procedureParent = getProcedureParentName(stmtStack.peek());
20309                                        if (procedureParent != null) {
20310                                                Procedure caller = modelManager
20311                                                                .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
20312                                                if (caller != null) {
20313                                                        CallRelationship callRelation = modelFactory.createCallRelation();
20314                                                        callRelation.setCallObject(functionCall);
20315                                                        callRelation.setTarget(new ProcedureRelationshipElement(caller));
20316                                                        callRelation.addSource(new ProcedureRelationshipElement(procedure));
20317                                                        if (isBuiltInFunctionName(((TFunctionCall)functionCall).getFunctionName())||isKeyword(((TFunctionCall)functionCall).getFunctionName())) {
20318                                                                callRelation.setBuiltIn(true);
20319                                                        }
20320                                                }
20321                                        }
20322                                        if (procedure.getProcedureObject() instanceof TCreateFunctionStmt) {
20323                                                TCreateFunctionStmt createFunction = (TCreateFunctionStmt)procedure.getProcedureObject();
20324                                                TTypeName dataType = createFunction.getReturnDataType();
20325                                                if (dataType!=null && dataType.getTypeOfList() != null && dataType.getTypeOfList().getColumnDefList() != null) {
20326                                                        Object modelObject = modelManager.getModel(gspObject);
20327                                                        if(modelObject instanceof ResultColumn) {
20328                                                                ResultColumn resultColumn = (ResultColumn)modelObject;
20329                                                                ResultSet resultSet = resultColumn.getResultSet();
20330                                                                for (int j = 0; j < dataType.getTypeOfList().getColumnDefList().size(); j++) {
20331                                                                        TObjectName columnName = new TObjectName();
20332                                                                        if( dataType.getDataType() == EDataType.array_t) {
20333//                                                                              columnName.setString(resultColumn.getName() + ".array."
20334//                                                                                              + dataType.getTypeOfList().getColumnDefList().getColumn(j)
20335//                                                                                                              .getColumnName().getColumnNameOnly());
20336                                                                                columnName.setString(resultColumn.getName() + "."
20337                                                                                                + dataType.getTypeOfList().getColumnDefList().getColumn(j)
20338                                                                                                .getColumnName().getColumnNameOnly());
20339                                                                        }
20340                                                                        else {
20341                                                                                columnName.setString(resultColumn.getName() + "."
20342                                                                                                + dataType.getTypeOfList().getColumnDefList().getColumn(j)
20343                                                                                                                .getColumnName().getColumnNameOnly());
20344                                                                        }
20345                                                                        ResultColumn sturctColumn = modelFactory.createResultColumn(resultSet, columnName,
20346                                                                                        true);
20347                                                                        sturctColumn.setStruct(true);
20348                                                                        Function sourceFunction = (Function)createFunction(functionCall);
20349                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
20350                                                                        relation.setEffectType(effectType);
20351                                                                        relation.setTarget(new ResultColumnRelationshipElement(sturctColumn));
20352                                                                        if (sourceFunction.getColumns() != null && !sourceFunction.getColumns().isEmpty()) {
20353                                                                                for (ResultColumn column : sourceFunction.getColumns()) {
20354                                                                                        relation.addSource(new ResultColumnRelationshipElement(column));
20355                                                                                }
20356                                                                        }
20357                                                                }
20358                                                                resultSet.getColumns().remove(resultColumn);
20359                                                        }
20360                                                        return;
20361                                                }
20362                                        }
20363                                }
20364                        }
20365                        
20366                        if(gspObject instanceof TResultColumn) {
20367                                TResultColumn resultColumn = (TResultColumn)gspObject;
20368                                if(resultColumn.getAliasClause()!=null && resultColumn.getAliasClause().getColumns()!=null) {
20369                                        for(TObjectName columnName: resultColumn.getAliasClause().getColumns()) {
20370                                                analyzeFunctionDataFlowRelation(columnName, Arrays.asList(functionCall), effectType, null);
20371                                        }
20372                                        return;
20373                                }
20374                        }
20375                        analyzeFunctionDataFlowRelation(gspObject, Arrays.asList(functionCall), effectType, null);
20376                }
20377        }
20378
20379        private void analyzeFunctionDataFlowRelation(Object gspObject, List<TParseTreeNode> functions,
20380                        EffectType effectType, Process process) {
20381
20382                Object modelObject = modelManager.getModel(gspObject);
20383                if (modelObject == null) {
20384                        if (gspObject instanceof ResultColumn || gspObject instanceof TableColumn) {
20385                                modelObject = gspObject;
20386                        }
20387                }
20388
20389                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
20390                relation.setEffectType(effectType);
20391                relation.setProcess(process);
20392
20393                if (modelObject instanceof ResultColumn) {
20394                        relation.setTarget(new ResultColumnRelationshipElement((ResultColumn) modelObject));
20395
20396                } else if (modelObject instanceof TableColumn) {
20397                        relation.setTarget(new TableColumnRelationshipElement((TableColumn) modelObject));
20398
20399                } else {
20400                        throw new UnsupportedOperationException();
20401                }
20402
20403                for (int i = 0; i < functions.size(); i++) {
20404                        TParseTreeNode functionCall = functions.get(i);
20405
20406                        if (functionCall instanceof TFunctionCall) {
20407                                TFunctionCall call = (TFunctionCall) functionCall;
20408                                Procedure callee = modelManager.getProcedureByName(
20409                                                DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(call)));
20410                                if (callee == null && procedureDDLMap.containsKey(DlineageUtil.getFunctionNameWithArgNum(call))) {
20411                                        analyzeCustomSqlStmt(procedureDDLMap.get(DlineageUtil.getFunctionNameWithArgNum(call)));
20412                                        callee = modelManager.getProcedureByName(
20413                                                        DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(call)));
20414                                }
20415
20416                                if (callee != null) {
20417                                        String procedureParent = getProcedureParentName(stmtStack.peek());
20418                                        if (procedureParent != null) {
20419                                                Procedure caller = modelManager
20420                                                                .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
20421                                                if (caller != null) {
20422                                                        CallRelationship callRelation = modelFactory.createCallRelation();
20423                                                        callRelation.setCallObject(functionCall);
20424                                                        callRelation.setTarget(new ProcedureRelationshipElement(caller));
20425                                                        callRelation.addSource(new ProcedureRelationshipElement(callee));
20426                                                        if (isBuiltInFunctionName(call.getFunctionName()) || isKeyword(call.getFunctionName())) {
20427                                                                callRelation.setBuiltIn(true);
20428                                                        }
20429                                                }
20430                                        }
20431                                        if (callee.getArguments() != null) {
20432                                                for (int j = 0; j < callee.getArguments().size(); j++) {
20433                                                        Argument argument = callee.getArguments().get(j);
20434                                                        Variable variable = modelFactory.createVariable(callee, argument.getName(), false);
20435                                                        if(variable!=null) {
20436                                                                if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
20437                                                                        Transform transform = new Transform();
20438                                                                        transform.setType(Transform.FUNCTION);
20439                                                                        transform.setCode(call);
20440                                                                        variable.getColumns().get(0).setTransform(transform);
20441                                                                }
20442                                                                Process callProcess = modelFactory.createProcess(call);
20443                                                                variable.addProcess(callProcess);
20444                                                                analyzeFunctionArgumentsDataFlowRelation(variable.getColumns().get(0), call, j, callProcess);
20445                                                        }
20446                                                }
20447                                        }
20448                                        Set<Object> functionTableModelObjs = modelManager.getFunctionTable(DlineageUtil
20449                                                        .getIdentifierNormalTableName(call.getFunctionName().toString()));
20450                                        if (functionTableModelObjs != null) {
20451                                                modelManager.bindModel(call, functionTableModelObjs.iterator().next());
20452                                                for (Object functionTableModelObj : functionTableModelObjs) {
20453                                                        if (functionTableModelObj instanceof ResultSet) {
20454                                                                ResultSet resultSet = (ResultSet) functionTableModelObj;
20455                                                                for (ResultColumn column : resultSet.getColumns()) {
20456                                                                        relation.addSource(new ResultColumnRelationshipElement(column));
20457                                                                }
20458                                                        }
20459                                                }
20460                                        }
20461                                        continue;
20462                                }
20463                        }
20464
20465
20466                        Object functionModel = createFunction(functionCall);
20467                        if (functionModel instanceof Function) {
20468                                Function sourceFunction = (Function)functionModel;
20469                                if (sourceFunction.getColumns() != null && !sourceFunction.getColumns().isEmpty()) {
20470                                        for (ResultColumn column : sourceFunction.getColumns()) {
20471                                                relation.addSource(new ResultColumnRelationshipElement(column));
20472                                        }
20473                                }
20474                                else if (functionCall instanceof TFunctionCall) {
20475                                        relation.addSource(new ResultColumnRelationshipElement((FunctionResultColumn) modelManager
20476                                                        .getModel(((TFunctionCall) functionCall).getFunctionName())));
20477                                } else if (functionCall instanceof TCaseExpression) {
20478                                        relation.addSource(new ResultColumnRelationshipElement((FunctionResultColumn) modelManager
20479                                                        .getModel(((TCaseExpression) functionCall).getWhenClauseItemList())));
20480                                }
20481                                
20482                                if (sourceFunction != null && !sourceFunction.getRelationRows().getHoldRelations().isEmpty()) {
20483                                        boolean find = false;
20484                                        if (modelObject instanceof ResultColumn) {
20485                                                ResultSetRelationRows targetRelationRows = ((ResultColumn) modelObject).getResultSet().getRelationRows();
20486                                                if(targetRelationRows.hasRelation()) {
20487                                                        for(Relationship relationship: targetRelationRows.getHoldRelations()) {
20488                                                                if(relationship.getSources().contains(sourceFunction.getRelationRows())) {
20489                                                                        find = true;
20490                                                                        break;
20491                                                                }
20492                                                        }
20493                                                }
20494                                        }
20495                                        else if (modelObject instanceof TableColumn) {
20496                                                TableRelationRows targetRelationRows = ((TableColumn) modelObject).getTable().getRelationRows();
20497                                                if(targetRelationRows.hasRelation()) {
20498                                                        for(Relationship relationship: targetRelationRows.getHoldRelations()) {
20499                                                                if(relationship.getSources().contains(sourceFunction.getRelationRows())) {
20500                                                                        find = true;
20501                                                                        break;
20502                                                                }
20503                                                        }
20504                                                }
20505                                        }
20506                                        
20507                                        if (!find) {
20508                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
20509                                                impactRelation.setEffectType(EffectType.select);
20510                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
20511                                                                sourceFunction.getRelationRows()));
20512                                                if (modelObject instanceof ResultColumn) {
20513                                                        impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
20514                                                                        ((ResultColumn) modelObject).getResultSet().getRelationRows()));
20515                                                } else if (modelObject instanceof TableColumn) {
20516                                                        impactRelation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(
20517                                                                        ((TableColumn) modelObject).getTable().getRelationRows()));
20518                                                }
20519                                        }
20520                                }
20521                        } else if (functionModel instanceof Table) {
20522                                TFunctionCall call = (TFunctionCall) functionCall;
20523                                String functionName = call.getFunctionName().toString();
20524                                boolean flag = false;
20525                                if (functionName.indexOf(".") != -1) {
20526                                        String columnName = functionName.substring(functionName.indexOf(".") + 1);
20527                                        for (TableColumn tableColumn : ((Table) functionModel).getColumns()) {
20528                                                if (getColumnName(tableColumn.getName()).equalsIgnoreCase(columnName)) {
20529                                                        TableColumnRelationshipElement element = new TableColumnRelationshipElement(tableColumn);
20530                                                        relation.addSource(element);
20531                                                        flag = true;
20532                                                        break;
20533                                                }
20534                                        }
20535                                }
20536
20537                                if (!flag) {
20538                                        TableColumn tableColumn = modelFactory.createTableColumn((Table) functionModel,
20539                                                        ((TFunctionCall) functionCall));
20540                                        TableColumnRelationshipElement element = new TableColumnRelationshipElement(tableColumn);
20541                                        relation.addSource(element);
20542                                }
20543                        }
20544                }
20545
20546        }
20547
20548        private void analyzeSubqueryDataFlowRelation(Object gspObject, List<TSelectSqlStatement> subquerys,
20549                        EffectType effectType) {
20550                analyzeSubqueryDataFlowRelation(gspObject, subquerys, effectType, null);
20551        }
20552
20553        private void analyzeSubqueryDataFlowRelation(Object gspObject, List<TSelectSqlStatement> subquerys,
20554                        EffectType effectType, Process process) {
20555
20556                Object modelObject = modelManager.getModel(gspObject);
20557                if (modelObject == null) {
20558                        if (gspObject instanceof ResultColumn || gspObject instanceof TableColumn) {
20559                                modelObject = gspObject;
20560                        }
20561                }
20562
20563                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
20564                relation.setEffectType(effectType);
20565                relation.setProcess(process);
20566
20567                if (modelObject instanceof ResultColumn) {
20568                        relation.setTarget(new ResultColumnRelationshipElement((ResultColumn) modelObject));
20569
20570                } else if (modelObject instanceof TableColumn) {
20571                        relation.setTarget(new TableColumnRelationshipElement((TableColumn) modelObject));
20572
20573                } else {
20574                        throw new UnsupportedOperationException();
20575                }
20576
20577                for (int i = 0; i < subquerys.size(); i++) {
20578                        TSelectSqlStatement subquery = subquerys.get(i);
20579                        ResultSet resultSetModel = (ResultSet) modelManager.getModel(subquery);
20580                        if (resultSetModel != null && resultSetModel.getColumns() != null) {
20581                                for (ResultColumn column : resultSetModel.getColumns()) {
20582                                        relation.addSource(new ResultColumnRelationshipElement(column));
20583                                }
20584                        }
20585                        
20586                        if (resultSetModel != null && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
20587                                boolean find = false;
20588                                if (modelObject instanceof ResultColumn) {
20589                                        ResultSetRelationRows targetRelationRows = ((ResultColumn) modelObject).getResultSet().getRelationRows();
20590                                        if(targetRelationRows.hasRelation()) {
20591                                                for(Relationship relationship: targetRelationRows.getHoldRelations()) {
20592                                                        if(relationship.getSources().contains(resultSetModel.getRelationRows())) {
20593                                                                find = true;
20594                                                                break;
20595                                                        }
20596                                                }
20597                                        }
20598                                }
20599                                else if (modelObject instanceof TableColumn) {
20600                                        TableRelationRows targetRelationRows = ((TableColumn) modelObject).getTable().getRelationRows();
20601                                        if(targetRelationRows.hasRelation()) {
20602                                                for(Relationship relationship: targetRelationRows.getHoldRelations()) {
20603                                                        if(relationship.getSources().contains(resultSetModel.getRelationRows())) {
20604                                                                find = true;
20605                                                                break;
20606                                                        }
20607                                                }
20608                                        }
20609                                }
20610                                
20611                                if (!find) {
20612                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
20613                                        impactRelation.setEffectType(EffectType.select);
20614                                        impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
20615                                                        resultSetModel.getRelationRows()));
20616                                        if (modelObject instanceof ResultColumn) {
20617                                                impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
20618                                                                ((ResultColumn) modelObject).getResultSet().getRelationRows()));
20619                                        } else if (modelObject instanceof TableColumn) {
20620                                                impactRelation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(
20621                                                                ((TableColumn) modelObject).getTable().getRelationRows()));
20622                                        }
20623                                }
20624                        }
20625                }
20626
20627        }
20628
20629        private Object createFunction(TParseTreeNode functionCall) {
20630                if (functionCall instanceof TFunctionCall) {
20631                        TFunctionCall functionObj = (TFunctionCall) functionCall;
20632
20633                        // Generic structured-dataflow dispatch. When a vendor adapter (e.g.
20634                        // Spark from_json+explode) returns a descriptor, model the function
20635                        // as a structured generator: per-field result columns linked to
20636                        // exact structured source paths such as nodes[*].key. When no
20637                        // adapter matches, fall through to the existing function logic
20638                        // unchanged.
20639                        StructuredAdapterContext sctx = new StructuredAdapterContext(option.getVendor());
20640                        StructuredDataflowDescriptor sdescriptor =
20641                                        StructuredDataflowRegistry.defaultRegistry().describe(functionObj, sctx);
20642                        if (sdescriptor != null) {
20643                                Function structuredFn = createStructuredDataflowFunction(sdescriptor);
20644                                if (structuredFn != null) {
20645                                        return structuredFn;
20646                                }
20647                        }
20648
20649                        if (!isBuiltInFunctionName(functionObj.getFunctionName())) {
20650                                TCustomSqlStatement stmt = stmtStack.peek();
20651                                String procedureParent = getProcedureParentName(stmt);
20652                                if (procedureParent != null) {
20653                                        Procedure procedureCallee = modelManager.getProcedureByName(DlineageUtil
20654                                                        .getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionObj)));
20655                                        if (procedureCallee != null) {
20656                                                if (procedureParent != null) {
20657                                                        Procedure caller = modelManager
20658                                                                        .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
20659                                                        if (caller != null) {
20660                                                                CallRelationship callRelation = modelFactory.createCallRelation();
20661                                                                callRelation.setCallObject(functionCall);
20662                                                                callRelation.setTarget(new ProcedureRelationshipElement(caller));
20663                                                                callRelation.addSource(new ProcedureRelationshipElement(procedureCallee));
20664                                                                if(isBuiltInFunctionName(functionObj.getFunctionName()) || isKeyword(functionObj.getFunctionName())){
20665                                                                        callRelation.setBuiltIn(true);
20666                                                                }
20667                                                        }
20668                                                }
20669                                                if (procedureCallee.getArguments() != null) {
20670                                                        for (int j = 0; j < procedureCallee.getArguments().size(); j++) {
20671                                                                Argument argument = procedureCallee.getArguments().get(j);
20672                                                                Variable variable = modelFactory.createVariable(procedureCallee, argument.getName(), false);
20673                                                                if(variable!=null) {
20674                                                                        if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
20675                                                                                Transform transform = new Transform();
20676                                                                                transform.setType(Transform.FUNCTION);
20677                                                                                transform.setCode(functionObj);
20678                                                                                variable.getColumns().get(0).setTransform(transform);
20679                                                                        }
20680                                                                        Process process = modelFactory.createProcess(functionObj);
20681                                                                        variable.addProcess(process);
20682                                                                        analyzeFunctionArgumentsDataFlowRelation(variable.getColumns().get(0), functionObj, j, process);
20683                                                                }
20684                                                        }
20685                                                }
20686                                        } else {
20687                                                TFunctionCall call = (TFunctionCall)functionCall;
20688                                                String functionName = call.getFunctionName().toString();
20689                                                if (functionName.indexOf(".") != -1) {
20690                                                        Table functionTable = modelManager
20691                                                                        .getTableByName(functionName.substring(0, functionName.indexOf(".")));
20692                                                        if (functionTable != null) {
20693                                                                String columnName = functionName.substring(functionName.indexOf(".") + 1);
20694                                                                for (TableColumn tableColumn : functionTable.getColumns()) {
20695                                                                        if (getColumnName(tableColumn.getName()).equalsIgnoreCase(columnName)) {
20696                                                                                return functionTable;
20697                                                                        }
20698                                                                }
20699                                                        }
20700                                                }
20701                                                Function function = modelFactory.createFunction(call);
20702                                                if (procedureParent != null) {
20703                                                        Procedure caller = modelManager
20704                                                                        .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
20705                                                        if (caller != null) {
20706                                                                CallRelationship callRelation = modelFactory.createCallRelation();
20707                                                                callRelation.setCallObject(functionCall);
20708                                                                callRelation.setTarget(new ProcedureRelationshipElement(caller));
20709                                                                callRelation.addSource(new FunctionRelationshipElement(function));
20710                                                                if(isBuiltInFunctionName(functionObj.getFunctionName()) || isKeyword(functionObj.getFunctionName())){
20711                                                                        callRelation.setBuiltIn(true);
20712                                                                }
20713                                                        }
20714                                                }
20715                                        }
20716                                }
20717                        } else if (isConstantFunction(functionObj.getFunctionName())
20718                                        && (functionObj.getArgs() == null || functionObj.getArgs().size() == 0)) {
20719                                if (option.isShowConstantTable()) {
20720                                        Table constantTable = modelFactory.createConstantsTable(stmtStack.peek());
20721                                        modelFactory.createTableColumn(constantTable, functionObj);
20722                                        return constantTable;
20723                                } else {
20724                                        return null;
20725                                }
20726                        }  
20727                        
20728                        if (functionObj.getFunctionType() == EFunctionType.struct_t) {
20729                                Function function = modelFactory.createFunction((TFunctionCall) functionCall);
20730                                if(functionObj instanceof TTableFunction) {
20731                                        TTableFunction tableFunction = (TTableFunction) functionObj;
20732                                        if (tableFunction.getFieldValues() != null) {
20733                                                for (int i = 0; i < tableFunction.getFieldValues().size(); i++) {
20734                                                        TResultColumn resultColumn = tableFunction.getFieldValues().getResultColumn(i);
20735                                                        if (resultColumn.getAliasClause() != null) {
20736                                                                ResultColumn column = modelFactory.createFunctionResultColumn(function,
20737                                                                                resultColumn.getAliasClause().getAliasName());
20738                                                                columnsInExpr visitor = new columnsInExpr();
20739                                                                resultColumn.getExpr().inOrderTraverse(visitor);
20740                                                                List<TObjectName> objectNames = visitor.getObjectNames();
20741                                                                List<TParseTreeNode> functions = visitor.getFunctions();
20742                                                                if (functions != null && !functions.isEmpty()) {
20743                                                                        analyzeFunctionDataFlowRelation(column, functions, EffectType.function);
20744                                                                }
20745                                                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
20746                                                                if (subquerys != null && !subquerys.isEmpty()) {
20747                                                                        analyzeSubqueryDataFlowRelation(column, subquerys, EffectType.function);
20748                                                                }
20749                                                                analyzeDataFlowRelation(column, objectNames, EffectType.function, functions);
20750                                                                List<TParseTreeNode> constants = visitor.getConstants();
20751                                                                analyzeConstantDataFlowRelation(column, constants, EffectType.function, functions);
20752                                                        } else if (resultColumn.getFieldAttr() != null) {
20753                                                                ResultColumn column = modelFactory.createFunctionResultColumn(function,
20754                                                                                resultColumn.getFieldAttr());
20755                                                                analyzeDataFlowRelation(column, Arrays.asList(resultColumn.getFieldAttr()), EffectType.function, null);
20756                                                        } else if (resultColumn.getExpr() != null) {
20757                                                                if (resultColumn.getExpr().getFunctionCall() != null) {
20758                                                                        Function resultColumnFunction = (Function) createFunction(
20759                                                                                        resultColumn.getExpr().getFunctionCall());
20760                                                                        String functionName = getResultSetName(function);
20761                                                                        for (int j = 0; j < resultColumnFunction.getColumns().size(); j++) {
20762                                                                                TObjectName columnName = new TObjectName();
20763                                                                                if (resultColumn.getAliasClause() != null) {
20764                                                                                        columnName.setString(resultColumn.getAliasClause() + "." + getColumnNameOnly(
20765                                                                                                        resultColumnFunction.getColumns().get(j).getName()));
20766                                                                                } else {
20767                                                                                        columnName.setString(functionName + "." + getColumnNameOnly(
20768                                                                                                        resultColumnFunction.getColumns().get(j).getName()));
20769                                                                                }
20770                                                                                ResultColumn functionResultColumn = modelFactory.createResultColumn(function,
20771                                                                                                columnName);
20772                                                                                DataFlowRelationship relationship = modelFactory.createDataFlowRelation();
20773                                                                                relationship.setTarget(new ResultColumnRelationshipElement(functionResultColumn));
20774                                                                                relationship.addSource(new ResultColumnRelationshipElement(
20775                                                                                                resultColumnFunction.getColumns().get(j)));
20776                                                                        }
20777                                                                }
20778                                                                else if (resultColumn.getExpr().getCaseExpression() != null) {
20779                                                                        function = modelFactory.createFunction(resultColumn.getExpr().getCaseExpression());
20780                                                                        ResultColumn column = modelFactory.createFunctionResultColumn(function,
20781                                                                                        ((TCaseExpression) resultColumn.getExpr().getCaseExpression()).getWhenClauseItemList());
20782                                                                        analyzeFunctionArgumentsDataFlowRelation(column, functionCall);
20783                                                                }
20784                                                        }
20785                                                }
20786                                                return function;
20787                                        }
20788                                }
20789                                else if(functionObj instanceof TFunctionCall) {
20790                                        
20791                                }
20792                        }
20793                        
20794                        // BigQuery array_agg(table_alias) row-reference expansion
20795                        if (functionObj.getFunctionType() == EFunctionType.array_agg_t
20796                                        && option.getVendor() == EDbVendor.dbvbigquery
20797                                        && functionObj.getArgs() != null
20798                                        && functionObj.getArgs().size() == 1) {
20799
20800                                TExpression arg0 = functionObj.getArgs().getExpression(0);
20801                                if (arg0 != null
20802                                                && arg0.getExpressionType() == EExpressionType.simple_object_name_t) {
20803
20804                                        TObjectName on = arg0.getObjectOperand();
20805                                        if (isArrayAggRowReference(on)) {
20806                                                Function function = (Function) modelManager.getModel(functionCall);
20807                                                if (function == null) {
20808                                                        function = modelFactory.createFunction((TFunctionCall) functionCall);
20809                                                }
20810
20811                                                // Only bind once (avoid duplicate processing when
20812                                                // createFunction is called from analyzeFunctionDataFlowRelation)
20813                                                if (function.getColumns() == null || function.getColumns().isEmpty()) {
20814                                                        TTable srcTable = on.getSourceTable();
20815                                                        String tableAlias = srcTable.getAliasName() != null
20816                                                                        ? srcTable.getAliasName().toString() : null;
20817                                                        
20818                                                        Table sourceTable = (Table) modelManager.getModel(srcTable);
20819                                                        sourceTable.removeColumn(tableAlias);
20820
20821                                                        // Collect inferred column names from ORDER BY (in function)
20822                                                        // and GROUP BY (in enclosing SELECT). Excludes the table alias.
20823                                                        List<TObjectName> inferredColumns = collectRowReferenceInferredColumns(
20824                                                                        functionObj, tableAlias);
20825
20826                                                        // Create a single * function result column carrying star-link
20827                                                        // metadata for the inferred columns and direct source edges
20828                                                        // for * and each inferred column.
20829                                                        createRowReferenceStarColumn(function, srcTable, inferredColumns);
20830                                                }
20831
20832                                                return function;
20833                                        }
20834                                }
20835                        }
20836
20837                        if (functionObj.getFunctionType() == EFunctionType.array_t || functionObj.getFunctionType() == EFunctionType.array_agg_t) {
20838                                Function function = modelFactory.createFunction((TFunctionCall) functionCall);
20839                                if(functionObj.getArgs()!=null) {
20840                                        if(functionObj.getArgs().getExpression(0).getSubQuery()!=null) {
20841                                                TSelectSqlStatement stmt = functionObj.getArgs().getExpression(0).getSubQuery();
20842                                                analyzeSelectStmt(stmt);
20843                                                ResultSet resultset = (ResultSet) modelManager.getModel(stmt);
20844                                                for (int i = 0; i < resultset.getColumns().size(); i++) {
20845                                                        ResultColumn sourceColumn = resultset.getColumns().get(i);
20846                                                        TObjectName columnName = new TObjectName();
20847                                                        columnName.setString(sourceColumn.getName());
20848                                                        ResultColumn resultColumn = modelFactory.createFunctionResultColumn(function,
20849                                                                        columnName);
20850                                                        DataFlowRelationship relationship = modelFactory.createDataFlowRelation();
20851                                                        relationship.setTarget(new ResultColumnRelationshipElement(resultColumn));
20852                                                        relationship.addSource(
20853                                                                        new ResultColumnRelationshipElement(sourceColumn));
20854                                                }
20855                                                return function;
20856                                        }
20857                                        else if (functionObj.getArgs().getExpression(0).getExpressionType() == EExpressionType.function_t) {
20858                                                Object functionTableModelObj = createFunction(functionObj.getArgs().getExpression(0).getFunctionCall());
20859                                                if (functionTableModelObj instanceof ResultSet) {
20860                                                        ResultSet resultset = (ResultSet) functionTableModelObj;
20861                                                        for (int i = 0; i < resultset.getColumns().size(); i++) {
20862                                                                ResultColumn sourceColumn = resultset.getColumns().get(i);
20863                                                                TObjectName columnName = new TObjectName();
20864                                                                columnName.setString(sourceColumn.getName());
20865                                                                ResultColumn resultColumn = modelFactory.createFunctionResultColumn(function, columnName);
20866                                                                DataFlowRelationship relationship = modelFactory.createDataFlowRelation();
20867                                                                relationship.setTarget(new ResultColumnRelationshipElement(resultColumn));
20868                                                                relationship.addSource(new ResultColumnRelationshipElement(sourceColumn));
20869                                                        }
20870                                                        return function;
20871                                                }
20872                                        }
20873                                }
20874                        }
20875                        
20876                        Function function = modelFactory.createFunction((TFunctionCall) functionCall);
20877                        ResultColumn column = modelFactory.createFunctionResultColumn(function,
20878                                        ((TFunctionCall) functionCall).getFunctionName());
20879                        if ("COUNT".equalsIgnoreCase(((TFunctionCall) functionCall).getFunctionName().toString())) {
20880                                // @see https://e.gitee.com/gudusoft/issues/list?issue=I40NUP
20881                                // COUNT特殊处理，不和参数关联
20882                                if (option.isShowCountTableColumn()) {
20883                                        analyzeFunctionArgumentsDataFlowRelation(column, functionCall);
20884                                }
20885                        } else {
20886                                boolean isCustomFunction = analyzeCustomFunctionCall((TFunctionCall)functionCall);
20887//                              if(!isCustomFunction) 
20888                                {
20889                                        analyzeFunctionArgumentsDataFlowRelation(column, functionCall);
20890                                }
20891                                Set<Object> functionTableModelObjs = modelManager.getFunctionTable(getIdentifiedFunctionName(function));
20892                                if(functionTableModelObjs!=null) {
20893                                        for(Object functionTableModelObj: functionTableModelObjs) {
20894                                                if (functionTableModelObj instanceof ResultSet) {
20895                                                        ResultSet functionTableModel = (ResultSet) functionTableModelObj;
20896                                                        if (functionTableModel.getColumns() != null) {
20897                                                                for (int j = 0; j < functionTableModel.getColumns().size(); j++) {
20898                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
20899                                                                        relation.setEffectType(EffectType.select);
20900                                                                        relation.setTarget(new ResultColumnRelationshipElement(column));
20901                                                                        relation.addSource(new ResultColumnRelationshipElement(
20902                                                                                        functionTableModel.getColumns().get(j)));
20903                                                                }
20904                                                        }
20905                                                }
20906                                        }
20907                                }
20908                        }
20909                        return function;
20910                } else if (functionCall instanceof TCaseExpression) {
20911                        Function function = modelFactory.createFunction((TCaseExpression) functionCall);
20912                        ResultColumn column = modelFactory.createFunctionResultColumn(function,
20913                                        ((TCaseExpression) functionCall).getWhenClauseItemList());
20914                        analyzeFunctionArgumentsDataFlowRelation(column, functionCall);
20915                        return function;
20916                } else if (functionCall instanceof TObjectName) {
20917                        Function function = modelFactory.createFunction((TObjectName) functionCall);
20918                        TObjectName columnName = new TObjectName();
20919                        columnName.setString(function.getFunctionName());
20920                        ResultColumn column = modelFactory.createResultColumn(function,
20921                                        columnName);
20922                        analyzeFunctionArgumentsDataFlowRelation(column, functionCall);
20923                        return function;
20924                }
20925                return null;
20926        }
20927
20928        protected String getIdentifiedFunctionName(Function function) {
20929                return DlineageUtil.getIdentifierNormalFunctionName(function.getFunctionName());
20930        }
20931        
20932        private boolean isConstantFunction(TObjectName functionName) {
20933                boolean result = CONSTANT_BUILTIN_FUNCTIONS.contains(functionName.toString().toUpperCase());
20934                if (result) {
20935                        return true;
20936                }
20937                return false;
20938        }
20939
20940        private void analyzeFunctionArgumentsDataFlowRelation(Object resultColumn, TParseTreeNode gspObject) {
20941                List<TExpression> directExpressions = new ArrayList<TExpression>();
20942                List<TExpression> indirectExpressions = new ArrayList<TExpression>();
20943                List<TExpression> conditionExpressions = new ArrayList<TExpression>();
20944                if (gspObject instanceof TFunctionCall) {
20945                        TFunctionCall functionCall = (TFunctionCall) gspObject;
20946                        getFunctionExpressions(directExpressions, indirectExpressions, functionCall);
20947                } else if (gspObject instanceof TCaseExpression) {
20948                        TCaseExpression expr = (TCaseExpression) gspObject;
20949                        TExpression inputExpr = expr.getInput_expr();
20950                        if (inputExpr != null) {
20951                                if(option.isShowCaseWhenAsDirect()){
20952                                        directExpressions.add(inputExpr);
20953                                }
20954                                else {
20955                                        conditionExpressions.add(inputExpr);
20956                                }
20957                        }
20958                        TExpression defaultExpr = expr.getElse_expr();
20959                        if (defaultExpr != null) {
20960                                directExpressions.add(defaultExpr);
20961                        }
20962                        TWhenClauseItemList list = expr.getWhenClauseItemList();
20963                        for (int i = 0; i < list.size(); i++) {
20964                                TWhenClauseItem element = list.getWhenClauseItem(i);
20965                                if(option.isShowCaseWhenAsDirect()){
20966                                        directExpressions.add(element.getComparison_expr());
20967                                }
20968                                else {
20969                                        conditionExpressions.add(element.getComparison_expr());
20970                                }
20971                                directExpressions.add(element.getReturn_expr());
20972                        }
20973                }
20974
20975                for (int j = 0; j < directExpressions.size(); j++) {
20976                        columnsInExpr visitor = new columnsInExpr();
20977                        directExpressions.get(j).inOrderTraverse(visitor);
20978
20979                        List<TObjectName> objectNames = visitor.getObjectNames();
20980                        List<TParseTreeNode> functions = visitor.getFunctions();
20981
20982                        if (functions != null && !functions.isEmpty()) {
20983                                analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function);
20984                        }
20985
20986                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
20987                        if (subquerys != null && !subquerys.isEmpty()) {
20988                                analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.function);
20989                        }
20990
20991                        analyzeDataFlowRelation(resultColumn, objectNames, EffectType.function, functions);
20992
20993                        List<TParseTreeNode> constants = visitor.getConstants();
20994                        analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.function, functions);
20995                }
20996
20997                conditionExpressions.addAll(indirectExpressions);
20998                for (int j = 0; j < conditionExpressions.size(); j++) {
20999                        analyzeFilterCondition(resultColumn, conditionExpressions.get(j), null, null, EffectType.function);
21000                }
21001        }
21002        
21003        private void analyzeFunctionArgumentsDataFlowRelation(Object resultColumn, TCallStatement callStatment, int argumentIndex, Process process) {
21004                List<TExpression> directExpressions = new ArrayList<TExpression>();
21005                List<TExpression> indirectExpressions = new ArrayList<TExpression>();
21006                List<TExpression> conditionExpressions = new ArrayList<TExpression>();
21007
21008                getFunctionExpressions(directExpressions, indirectExpressions, callStatment, argumentIndex);
21009
21010                for (int j = 0; j < directExpressions.size(); j++) {
21011                        columnsInExpr visitor = new columnsInExpr();
21012                        directExpressions.get(j).inOrderTraverse(visitor);
21013
21014                        List<TObjectName> objectNames = visitor.getObjectNames();
21015                        List<TParseTreeNode> functions = visitor.getFunctions();
21016
21017                        if (functions != null && !functions.isEmpty()) {
21018                                analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function);
21019                        }
21020
21021                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
21022                        if (subquerys != null && !subquerys.isEmpty()) {
21023                                analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.function);
21024                        }
21025
21026                        DataFlowRelationship relation = analyzeDataFlowRelation(resultColumn, objectNames, EffectType.function, functions);
21027                        if (relation != null) {
21028                                relation.setProcess(process);
21029                        }
21030
21031                        List<TParseTreeNode> constants = visitor.getConstants();
21032                        analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.function, functions);
21033                }
21034
21035                conditionExpressions.addAll(indirectExpressions);
21036                for (int j = 0; j < conditionExpressions.size(); j++) {
21037                        analyzeFilterCondition(resultColumn, conditionExpressions.get(j), null, null, EffectType.function);
21038                }
21039        }
21040
21041        private void analyzeFunctionArgumentsDataFlowRelation(Object resultColumn, TDb2CallStmt callStatment, int argumentIndex, Process process) {
21042                List<TExpression> directExpressions = new ArrayList<TExpression>();
21043                List<TExpression> indirectExpressions = new ArrayList<TExpression>();
21044                List<TExpression> conditionExpressions = new ArrayList<TExpression>();
21045
21046                getFunctionExpressions(directExpressions, indirectExpressions, callStatment, argumentIndex);
21047
21048                for (int j = 0; j < directExpressions.size(); j++) {
21049                        columnsInExpr visitor = new columnsInExpr();
21050                        directExpressions.get(j).inOrderTraverse(visitor);
21051
21052                        List<TObjectName> objectNames = visitor.getObjectNames();
21053                        List<TParseTreeNode> functions = visitor.getFunctions();
21054
21055                        if (functions != null && !functions.isEmpty()) {
21056                                analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function);
21057                        }
21058
21059                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
21060                        if (subquerys != null && !subquerys.isEmpty()) {
21061                                analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.function);
21062                        }
21063
21064                        DataFlowRelationship relation = analyzeDataFlowRelation(resultColumn, objectNames, EffectType.function, functions);
21065                        if (relation != null) {
21066                                relation.setProcess(process);
21067                        }
21068
21069                        List<TParseTreeNode> constants = visitor.getConstants();
21070                        analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.function, functions);
21071                }
21072
21073                conditionExpressions.addAll(indirectExpressions);
21074                for (int j = 0; j < conditionExpressions.size(); j++) {
21075                        analyzeFilterCondition(resultColumn, conditionExpressions.get(j), null, null, EffectType.function);
21076                }
21077        }
21078
21079        private void analyzeFunctionArgumentsDataFlowRelation(Object resultColumn, TFunctionCall functionCall, int argumentIndex, Process process) {
21080                List<TExpression> directExpressions = new ArrayList<TExpression>();
21081                List<TExpression> indirectExpressions = new ArrayList<TExpression>();
21082                List<TExpression> conditionExpressions = new ArrayList<TExpression>();
21083
21084                getFunctionExpressions(directExpressions, indirectExpressions, functionCall, argumentIndex);
21085
21086                for (int j = 0; j < directExpressions.size(); j++) {
21087                        columnsInExpr visitor = new columnsInExpr();
21088                        directExpressions.get(j).inOrderTraverse(visitor);
21089
21090                        List<TObjectName> objectNames = visitor.getObjectNames();
21091                        List<TParseTreeNode> functions = visitor.getFunctions();
21092
21093                        if (functions != null && !functions.isEmpty()) {
21094                                analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function);
21095                        }
21096
21097                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
21098                        if (subquerys != null && !subquerys.isEmpty()) {
21099                                analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.function);
21100                        }
21101
21102                        DataFlowRelationship relation = analyzeDataFlowRelation(resultColumn, objectNames, EffectType.function, functions);
21103                        if (relation != null) {
21104                                relation.setProcess(process);
21105                        }
21106
21107                        List<TParseTreeNode> constants = visitor.getConstants();
21108                        analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.function, functions);
21109                }
21110
21111                conditionExpressions.addAll(indirectExpressions);
21112                for (int j = 0; j < conditionExpressions.size(); j++) {
21113                        analyzeFilterCondition(resultColumn, conditionExpressions.get(j), null, null, EffectType.function);
21114                }
21115        }
21116
21117        private void analyzeFunctionArgumentsDataFlowRelation(Object resultColumn, TMssqlExecute functionCall, String argumentName, int argumentIndex, Process process) {
21118                List<TExpression> directExpressions = new ArrayList<TExpression>();
21119                List<TExpression> indirectExpressions = new ArrayList<TExpression>();
21120                List<TExpression> conditionExpressions = new ArrayList<TExpression>();
21121
21122                getFunctionExpressions(directExpressions, indirectExpressions, functionCall, argumentName, argumentIndex);
21123
21124                for (int j = 0; j < directExpressions.size(); j++) {
21125                        columnsInExpr visitor = new columnsInExpr();
21126                        directExpressions.get(j).inOrderTraverse(visitor);
21127
21128                        List<TObjectName> objectNames = visitor.getObjectNames();
21129                        List<TParseTreeNode> functions = visitor.getFunctions();
21130
21131                        if (functions != null && !functions.isEmpty()) {
21132                                analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function);
21133                        }
21134
21135                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
21136                        if (subquerys != null && !subquerys.isEmpty()) {
21137                                analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.function);
21138                        }
21139
21140                        DataFlowRelationship relation = analyzeDataFlowRelation(resultColumn, objectNames, EffectType.function, functions);
21141                        if (relation != null) {
21142                                relation.setProcess(process);
21143                        }
21144
21145                        List<TParseTreeNode> constants = visitor.getConstants();
21146                        analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.function, functions);
21147                }
21148
21149                conditionExpressions.addAll(indirectExpressions);
21150                for (int j = 0; j < conditionExpressions.size(); j++) {
21151                        analyzeFilterCondition(resultColumn, conditionExpressions.get(j), null, null, EffectType.function);
21152                }
21153        }
21154
21155
21156        private static void addResultColumnListExpressions(
21157                        TResultColumnList list, List<TExpression> target) {
21158                if (list == null) return;
21159                for (int k = 0; k < list.size(); k++) {
21160                        TExpression e = list.getResultColumn(k).getExpr();
21161                        if (e != null) target.add(e);
21162                }
21163        }
21164
21165        private static void addPassingClauseExpressions(
21166                        TFunctionCall fc, List<TExpression> target) {
21167                TResultColumnList list = null;
21168                if (fc.getXmlPassingClause() != null
21169                                && fc.getXmlPassingClause().getPassingList() != null) {
21170                        list = fc.getXmlPassingClause().getPassingList();
21171                } else if (fc.getPassingClause() != null
21172                                && fc.getPassingClause().getPassingList() != null) {
21173                        list = fc.getPassingClause().getPassingList();
21174                }
21175                addResultColumnListExpressions(list, target);
21176        }
21177
21178        private void collectOracleXmlFunctionExpressions(
21179                        TFunctionCall fc,
21180                        List<TExpression> direct,
21181                        List<TExpression> indirect) {
21182
21183                if (option.getVendor() != EDbVendor.dbvoracle) return;
21184
21185                // XMLELEMENT: value exprs + XMLATTRIBUTES exprs (direct)
21186                addResultColumnListExpressions(fc.getXMLElementValueExprList(), direct);
21187                if (fc.getXMLAttributesClause() != null) {
21188                        addResultColumnListExpressions(
21189                                        fc.getXMLAttributesClause().getValueExprList(), direct);
21190                }
21191
21192                // XMLFOREST (direct)
21193                addResultColumnListExpressions(fc.getXMLForestValueList(), direct);
21194
21195                // EXTRACT(XML) — AST-shape disambiguation (V3)
21196                // XML form populates getXMLType_Instance(); scalar EXTRACT(YEAR FROM d) does not.
21197                if (fc.getXMLType_Instance() != null) {
21198                        direct.add(fc.getXMLType_Instance());
21199                }
21200
21201                String name = fc.getFunctionName() == null
21202                                ? "" : fc.getFunctionName().toString().toUpperCase();
21203
21204                // XMLEXISTS indirect (filter) from PASSING expressions
21205                if ("XMLEXISTS".equals(name)) {
21206                        addPassingClauseExpressions(fc, indirect);
21207                }
21208
21209                // XMLAGG / SYS_XMLAGG ORDER BY → indirect
21210                if (("XMLAGG".equals(name) || "SYS_XMLAGG".equals(name))
21211                                && fc.getSortClause() != null) {
21212                        TOrderByItemList orderByList = fc.getSortClause().getItems();
21213                        if (orderByList != null) {
21214                                for (int k = 0; k < orderByList.size(); k++) {
21215                                        TExpression e = orderByList.getOrderByItem(k).getSortKey();
21216                                        if (e != null) indirect.add(e);
21217                                }
21218                        }
21219                }
21220
21221                // DELIBERATELY NOT HARVESTED (metadata, not data):
21222                //   fc.getTypeExpression()       — target type for XMLCAST
21223                //   fc.getXMLElementNameExpr()   — element tag identifier
21224                //   XPath / XQuery literal strings that appear as function args
21225                //   datatype / format / style tokens
21226        }
21227
21228        private void collectArrayAggOrderByExpressions(
21229                        TFunctionCall fc,
21230                        List<TExpression> direct,
21231                        List<TExpression> indirect) {
21232                if (fc.getFunctionType() != EFunctionType.array_agg_t) return;
21233
21234                TOrderByItemList items = null;
21235                if (fc.getSortClause() != null) items = fc.getSortClause().getItems();
21236                if (items == null || items.size() == 0) {
21237                        items = fc.getOrderByList();
21238                }
21239                if (items == null) return;
21240
21241                // Resolve source tables for ORDER BY columns that the parser
21242                // doesn't link (they're inside a function call, outside the
21243                // resolver's normal scope)
21244                TTable fallbackTable = null;
21245                if (!stmtStack.isEmpty() && stmtStack.peek() instanceof TSelectSqlStatement) {
21246                        TTableList tables = ((TSelectSqlStatement) stmtStack.peek()).tables;
21247                        if (tables != null && tables.size() > 0) {
21248                                fallbackTable = tables.getTable(0);
21249                        }
21250                }
21251
21252                for (int k = 0; k < items.size(); k++) {
21253                        TExpression e = items.getOrderByItem(k).getSortKey();
21254                        if (e != null) {
21255                                if (fallbackTable != null
21256                                                && e.getExpressionType() == EExpressionType.simple_object_name_t
21257                                                && e.getObjectOperand() != null
21258                                                && e.getObjectOperand().getSourceTable() == null) {
21259                                        e.getObjectOperand().setSourceTable(fallbackTable);
21260                                }
21261                                direct.add(e);
21262                        }
21263                }
21264        }
21265
21266        private boolean isArrayAggRowReference(TObjectName on) {
21267                if (on == null) return false;
21268                TTable src = on.getSourceTable();
21269                if (src == null) return false;
21270
21271                String nm = DlineageUtil.getColumnName(on);
21272                if (nm == null || nm.isEmpty()) return false;
21273
21274                // Check if identifier text matches the table alias or table name
21275                boolean matchesAlias =
21276                        (src.getAliasName() != null
21277                                && src.getAliasName().toString().equalsIgnoreCase(nm))
21278                        || (src.getTableName() != null
21279                                && src.getTableName().toString().equalsIgnoreCase(nm));
21280                return matchesAlias;
21281        }
21282
21283        /**
21284         * Collects column names inferred from a row-reference array_agg:
21285         *  - ORDER BY keys inside the function call
21286         *  - GROUP BY keys in the enclosing SELECT
21287         * Excludes the table alias itself (so `original` in `array_agg(original)`
21288         * does not leak into the column list).
21289         */
21290        private List<TObjectName> collectRowReferenceInferredColumns(
21291                        TFunctionCall functionCall, String tableAlias) {
21292                List<TObjectName> columns = new ArrayList<TObjectName>();
21293
21294                TOrderByItemList orderItems = null;
21295                if (functionCall.getSortClause() != null) {
21296                        orderItems = functionCall.getSortClause().getItems();
21297                }
21298                if (orderItems == null || orderItems.size() == 0) {
21299                        orderItems = functionCall.getOrderByList();
21300                }
21301                if (orderItems != null) {
21302                        for (int k = 0; k < orderItems.size(); k++) {
21303                                TExpression sortKey = orderItems.getOrderByItem(k).getSortKey();
21304                                collectInferredColumn(sortKey, tableAlias, columns);
21305                        }
21306                }
21307
21308                if (!stmtStack.isEmpty() && stmtStack.peek() instanceof TSelectSqlStatement) {
21309                        TSelectSqlStatement select = (TSelectSqlStatement) stmtStack.peek();
21310                        if (select.getGroupByClause() != null) {
21311                                TGroupByItemList groupByList = select.getGroupByClause().getItems();
21312                                for (int k = 0; k < groupByList.size(); k++) {
21313                                        TExpression expr = groupByList.getGroupByItem(k).getExpr();
21314                                        collectInferredColumn(expr, tableAlias, columns);
21315                                }
21316                        }
21317                }
21318                return columns;
21319        }
21320
21321        private void collectInferredColumn(TExpression expr, String tableAlias,
21322                        List<TObjectName> columns) {
21323                if (expr == null) return;
21324                if (expr.getExpressionType() != EExpressionType.simple_object_name_t) return;
21325                if (expr.getObjectOperand() == null) return;
21326                String colName = DlineageUtil.getColumnName(expr.getObjectOperand());
21327                if (colName == null || colName.isEmpty()) return;
21328                if (tableAlias != null && tableAlias.equalsIgnoreCase(colName)) return;
21329                if(!columns.contains(expr.getObjectOperand())) {
21330                        columns.add(expr.getObjectOperand());
21331                }
21332        }
21333
21334        /**
21335         * Creates a `*` FunctionResultColumn plus one FunctionResultColumn per
21336         * inferred column. Each column gets its own source-edge back to
21337         * the source table (source.* -> function.*, source.id -> function.id, etc).
21338         *
21339         * Downstream, the `*` column collects all three edges into the inner
21340         * `unique` ResultColumn, and post-processing (rewriteArrayAggRowReferenceTargets)
21341         * splits the collapsed CTAS relation into one per column.
21342         */
21343        private void createRowReferenceStarColumn(Function function, TTable src,
21344                        List<TObjectName> inferredColumns) {
21345                Object tableModel = modelManager.getModel(src);
21346
21347                // Create function.* and bind source.* -> function.*
21348                TObjectName starName = new TObjectName();
21349                starName.setString("*");
21350                ResultColumn starColumn = modelFactory.createFunctionResultColumn(function, starName);
21351                starColumn.setStruct(true);
21352                starColumn.setShowStar(true);
21353
21354                if (tableModel instanceof Table) {
21355                        Table sourceTable = (Table) tableModel;
21356                        TableColumn sourceCol = findOrCreateTableColumn(sourceTable, "*");
21357                        
21358                        for (TObjectName colName : inferredColumns) {
21359                                TableColumn tableColumn = modelFactory.createTableColumn(sourceTable, colName, false);
21360                                if (tableColumn != null) {
21361                                        //order by and group by fdr relation
21362                                        AbstractRelationship relation = modelFactory.createImpactRelation();
21363                                        relation.setEffectType(EffectType.function);
21364                                        relation.setTarget(new ResultColumnRelationshipElement(starColumn));
21365                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
21366                                        
21367                                        // star column fdd relation
21368                                        relation = modelFactory.createDataFlowRelation();
21369                                        relation.setEffectType(EffectType.function);
21370                                        relation.setTarget(new ResultColumnRelationshipElement(starColumn));
21371                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
21372                                }
21373                        }
21374
21375                        
21376                        if (sourceCol != null) {
21377                                DataFlowRelationship rel = modelFactory.createDataFlowRelation();
21378                                rel.setEffectType(EffectType.function);
21379                                rel.setTarget(new ResultColumnRelationshipElement(starColumn));
21380                                rel.addSource(new TableColumnRelationshipElement(sourceCol));
21381                        }
21382                        
21383                } else if (tableModel instanceof ResultSet) {
21384                        ResultSet sourceRS = (ResultSet) tableModel;
21385                        ResultColumn sourceCol = findOrCreateResultColumn(sourceRS, "*");
21386                        for (TObjectName colName : inferredColumns) {
21387                                sourceCol.bindStarLinkColumn(colName);
21388                        }
21389                        
21390                        if (sourceCol != null) {
21391                                DataFlowRelationship rel = modelFactory.createDataFlowRelation();
21392                                rel.setEffectType(EffectType.function);
21393                                rel.setTarget(new ResultColumnRelationshipElement(starColumn));
21394                                rel.addSource(new ResultColumnRelationshipElement(sourceCol));
21395                        }
21396                }
21397        }
21398
21399        /**
21400         * Builds the Function lineage model for a structured-dataflow descriptor.
21401         * Creates per-field FunctionResultColumns and links each to an exact
21402         * structured source TableColumn (e.g. {@code nodes[*].key}). Returns the
21403         * Function on success; returns {@code null} when the source column cannot
21404         * be resolved so the caller falls back to existing function lineage.
21405         */
21406        private Function createStructuredDataflowFunction(StructuredDataflowDescriptor desc) {
21407                TParseTreeNode syntaxNode = desc.getSyntaxNode();
21408                if (!(syntaxNode instanceof TFunctionCall)) {
21409                        return null;
21410                }
21411                TFunctionCall fnCall = (TFunctionCall) syntaxNode;
21412
21413                Function function = null;
21414                Object existing = modelManager.getModel(fnCall);
21415                if (existing instanceof Function) {
21416                        function = (Function) existing;
21417                }
21418                boolean alreadyBuilt = function != null
21419                                && function.getColumns() != null
21420                                && !function.getColumns().isEmpty();
21421                if (alreadyBuilt) {
21422                        return function;
21423                }
21424                if (function == null) {
21425                        function = modelFactory.createFunction(fnCall);
21426                }
21427
21428                StructuredValueSource src = desc.getSource();
21429                TObjectName sourceColumn = src.getSourceColumn();
21430                TTable srcTable = sourceColumn != null ? sourceColumn.getSourceTable() : null;
21431                Object srcTableModel = srcTable != null ? modelManager.getModel(srcTable) : null;
21432                if (!(srcTableModel instanceof Table)) {
21433                        return null;
21434                }
21435                Table sourceTable = (Table) srcTableModel;
21436
21437                for (StructuredFieldBinding fb : desc.getFieldBindings()) {
21438                        String pathDisplay = fb.getSourcePath().toDisplayString();
21439                        TableColumn pathColumn = findOrCreateStructuredPathColumn(sourceTable, pathDisplay);
21440                        if (pathColumn == null) {
21441                                continue;
21442                        }
21443
21444                        TObjectName fieldName = new TObjectName();
21445                        fieldName.setString(fb.getOutputFieldName());
21446                        ResultColumn fnResult = modelFactory.createFunctionResultColumn(function, fieldName);
21447
21448                        DataFlowRelationship rel = modelFactory.createDataFlowRelation();
21449                        rel.setEffectType(EffectType.function);
21450                        rel.setTarget(new ResultColumnRelationshipElement(fnResult));
21451                        rel.addSource(new TableColumnRelationshipElement(pathColumn));
21452                }
21453                return function;
21454        }
21455
21456        /**
21457         * Resolve a struct-field-style outer reference ({@code structAlias.field}
21458         * or {@code relAlias.structAlias.field}) against in-scope subquery
21459         * QueryTables that hold a struct-bearing result column. When found,
21460         * append a relation source pointing at the function's field result
21461         * column so the exact source path (e.g. {@code nodes[*].key}) is
21462         * preserved through the outer projection. Returns {@code true} when
21463         * the reference was bound and the caller should skip orphan-column
21464         * fallback.
21465         */
21466        private boolean tryAppendStructuredFieldRelation(DataFlowRelationship relation,
21467                        TObjectName columnName, TTableList tableList) {
21468                if (relation == null || columnName == null || tableList == null) {
21469                        return false;
21470                }
21471                String fullText = columnName.toString();
21472                if (fullText == null) return false;
21473                int firstDot = fullText.indexOf('.');
21474                if (firstDot < 0) return false;
21475
21476                String structAlias;
21477                String fieldName;
21478                int lastDot = fullText.lastIndexOf('.');
21479                if (firstDot == lastDot) {
21480                        structAlias = fullText.substring(0, firstDot);
21481                        fieldName = fullText.substring(firstDot + 1);
21482                } else {
21483                        structAlias = fullText.substring(firstDot + 1, lastDot);
21484                        fieldName = fullText.substring(lastDot + 1);
21485                }
21486                if (structAlias == null || structAlias.isEmpty()) return false;
21487                if (fieldName == null || fieldName.isEmpty()) return false;
21488
21489                for (int i = 0; i < tableList.size(); i++) {
21490                        TTable tt = tableList.getTable(i);
21491                        if (tt == null) continue;
21492                        Object tm = modelManager.getModel(tt);
21493                        if (!(tm instanceof QueryTable)) continue;
21494                        QueryTable qt = (QueryTable) tm;
21495                        if (qt.getColumns() == null) continue;
21496                        for (ResultColumn rc : qt.getColumns()) {
21497                                if (rc == null || rc.getName() == null) continue;
21498                                if (!structAlias.equalsIgnoreCase(rc.getName())) continue;
21499                                ResultColumn fieldColumn = findStructuredFieldColumn(rc, fieldName);
21500                                if (fieldColumn != null) {
21501                                        relation.addSource(new ResultColumnRelationshipElement(fieldColumn));
21502                                        return true;
21503                                }
21504                        }
21505                }
21506                return false;
21507        }
21508
21509        /**
21510         * Given a QueryTable result column that aliases a structured generator
21511         * (e.g. {@code nodes1} backed by Spark {@code explode(from_json(...))}),
21512         * locate the FunctionResultColumn named {@code fieldName}. The Function
21513         * model is discovered by walking the existing data-flow relations whose
21514         * target is the alias column.
21515         */
21516        private ResultColumn findStructuredFieldColumn(ResultColumn aliasColumn, String fieldName) {
21517                if (aliasColumn == null || fieldName == null) return null;
21518                Relationship[] rels = modelManager.getRelations();
21519                if (rels == null) return null;
21520                String normalField = fieldName.trim();
21521                for (Relationship r : rels) {
21522                        if (!(r instanceof DataFlowRelationship)) continue;
21523                        DataFlowRelationship dfr = (DataFlowRelationship) r;
21524                        if (dfr.getTarget() == null) continue;
21525                        if (!(dfr.getTarget().getElement() == aliasColumn)) continue;
21526                        if (dfr.getSources() == null) continue;
21527                        for (RelationshipElement<?> se : dfr.getSources()) {
21528                                Object srcElement = se.getElement();
21529                                if (!(srcElement instanceof ResultColumn)) continue;
21530                                ResultColumn src = (ResultColumn) srcElement;
21531                                if (normalField.equalsIgnoreCase(src.getName())) {
21532                                        return src;
21533                                }
21534                        }
21535                }
21536                return null;
21537        }
21538
21539        /**
21540         * Find-or-create a TableColumn carrying a structured display path
21541         * (e.g. {@code nodes[*].key}) on the given source table. Unlike the
21542         * standard {@link #findOrCreateTableColumn(Table, String)}, this does
21543         * not treat the dotted/bracketed text as a schema-qualified name;
21544         * the entire path is used as the column display, so existing flat
21545         * columns named after the leaf field (e.g. {@code key}) do not
21546         * collide with structured-path columns.
21547         */
21548        private TableColumn findOrCreateStructuredPathColumn(Table table, String pathDisplay) {
21549                if (table == null || pathDisplay == null || pathDisplay.isEmpty()) return null;
21550                for (TableColumn tc : table.getColumns()) {
21551                        if (pathDisplay.equalsIgnoreCase(tc.getName())) {
21552                                return tc;
21553                        }
21554                }
21555                return new TableColumn(table, pathDisplay);
21556        }
21557
21558        private TableColumn findOrCreateTableColumn(Table table, String colName) {
21559                String normalName = DlineageUtil.getIdentifierNormalColumnName(colName);
21560                for (TableColumn tc : table.getColumns()) {
21561                        if ("*".equals(colName) && "*".equals(tc.getName())) return tc;
21562                        if (DlineageUtil.compareColumnIdentifier(tc.getName(), normalName)) return tc;
21563                }
21564                TObjectName obj = new TObjectName();
21565                obj.setString(colName);
21566                return modelFactory.createTableColumn(table, obj, "*".equals(colName));
21567        }
21568
21569        private ResultColumn findOrCreateResultColumn(ResultSet resultSet, String colName) {
21570                String normalName = DlineageUtil.getIdentifierNormalColumnName(colName);
21571                for (ResultColumn rc : resultSet.getColumns()) {
21572                        if ("*".equals(colName) && "*".equals(rc.getName())) return rc;
21573                        if (DlineageUtil.compareColumnIdentifier(rc.getName(), normalName)) return rc;
21574                }
21575                TObjectName obj = new TObjectName();
21576                obj.setString(colName);
21577                return modelFactory.createResultColumn(resultSet, obj);
21578        }
21579
21580        private void getFunctionExpressions(List<TExpression> directExpressions, List<TExpression> indirectExpressions,
21581                        TFunctionCall functionCall) {
21582                if (functionCall.getArgs() != null) {
21583                        for (int k = 0; k < functionCall.getArgs().size(); k++) {
21584                                TExpression expr = functionCall.getArgs().getExpression(k);
21585                                if(FunctionUtility.isDirectRelation(option.getVendor(), functionCall.getFunctionName().toString(),  functionCall.getArgs().size(), k)) {
21586                                        directExpressions.add(expr);
21587                                }
21588                                if(FunctionUtility.isIndirectRelation(option.getVendor(), functionCall.getFunctionName().toString(),  functionCall.getArgs().size(), k)) {
21589                                        if("DECODE".equalsIgnoreCase(functionCall.getFunctionName().toString()) && option.getVendor() == EDbVendor.dbvoracle) {
21590                                                if(option.isShowCaseWhenAsDirect()) {
21591                                                        directExpressions.add(expr);
21592                                                        continue;
21593                                                }
21594                                        }
21595                                        indirectExpressions.add(expr);
21596                                }
21597                        }
21598                }
21599                collectOracleXmlFunctionExpressions(functionCall, directExpressions, indirectExpressions);
21600                collectArrayAggOrderByExpressions(functionCall, directExpressions, indirectExpressions);
21601                if (functionCall.getTrimArgument() != null) {
21602                        TTrimArgument args = functionCall.getTrimArgument();
21603                        TExpression expr = args.getStringExpression();
21604                        if (expr != null) {
21605                                directExpressions.add(expr);
21606                        }
21607                        expr = args.getTrimCharacter();
21608                        if (expr != null) {
21609                                directExpressions.add(expr);
21610                        }
21611                }
21612
21613                if (functionCall.getAgainstExpr() != null) {
21614                        directExpressions.add(functionCall.getAgainstExpr());
21615                }
21616//              if (functionCall.getBetweenExpr() != null) {
21617//                      directExpressions.add(functionCall.getBetweenExpr());
21618//              }
21619                if (functionCall.getExpr1() != null) {
21620                        directExpressions.add(functionCall.getExpr1());
21621                }
21622                if (functionCall.getExpr2() != null) {
21623                        directExpressions.add(functionCall.getExpr2());
21624                }
21625                if (functionCall.getExpr3() != null) {
21626                        directExpressions.add(functionCall.getExpr3());
21627                }
21628                if (functionCall.getParameter() != null) {
21629                        directExpressions.add(functionCall.getParameter());
21630                }
21631                if (functionCall.getWindowDef() != null && functionCall.getWindowDef().getPartitionClause() != null) {
21632                        TExpressionList args = functionCall.getWindowDef().getPartitionClause().getExpressionList();
21633                        if (args != null) {
21634                                for (int k = 0; k < args.size(); k++) {
21635                                        TExpression expr = args.getExpression(k);
21636                                        if (expr != null) {
21637                                                indirectExpressions.add(expr);
21638                                        }
21639                                }
21640                        }
21641                }
21642                if (functionCall.getWindowDef() != null && functionCall.getWindowDef().getOrderBy() != null) {
21643                        TOrderByItemList orderByList = functionCall.getWindowDef().getOrderBy().getItems();
21644                        for (int i = 0; i < orderByList.size(); i++) {
21645                                TOrderByItem element = orderByList.getOrderByItem(i);
21646                                TExpression expression = element.getSortKey();
21647                                indirectExpressions.add(expression);
21648                        }
21649                }
21650                if (functionCall.getWithinGroup() != null && functionCall.getWithinGroup().getOrderBy() != null) {
21651                        TOrderByItemList orderByList = functionCall.getWithinGroup().getOrderBy().getItems();
21652                        for (int i = 0; i < orderByList.size(); i++) {
21653                                TOrderByItem element = orderByList.getOrderByItem(i);
21654                                TExpression expression = element.getSortKey();
21655                                indirectExpressions.add(expression);
21656                        }
21657                }
21658                if (functionCall.getCallTarget() != null) {
21659                        directExpressions.add(functionCall.getCallTarget().getExpr());
21660                }
21661                if (functionCall.getFieldValues() != null) {
21662                        for (int k = 0; k < functionCall.getFieldValues().size(); k++) {
21663                                TExpression expr = functionCall.getFieldValues().getResultColumn(k).getExpr();
21664                                directExpressions.add(expr);
21665                        }
21666                }
21667                if (functionCall instanceof TJsonObjectFunction) {
21668                        TJsonObjectFunction jsonObject = (TJsonObjectFunction)functionCall;
21669                        for (int k = 0; k < jsonObject.getKeyValues().size(); k++) {
21670                                TExpression expr = jsonObject.getKeyValues().get(k).getValue();
21671                                directExpressions.add(expr);
21672                        }
21673                }
21674                if (functionCall.getGroupConcatParam() != null) {
21675                        for (int k = 0; k < functionCall.getGroupConcatParam().getExprList().size(); k++) {
21676                                TExpression expr = functionCall.getGroupConcatParam().getExprList().getExpression(k);
21677                                directExpressions.add(expr);
21678                        }
21679                }
21680        }
21681
21682        private void getFunctionExpressions(List<TExpression> directExpressions, List<TExpression> indirectExpressions,
21683                                                                                TFunctionCall functionCall, int argumentIndex) {
21684                if (functionCall.getArgs() != null && argumentIndex < functionCall.getArgs().size()) {
21685                        TExpression expr = functionCall.getArgs().getExpression(argumentIndex);
21686                        if (FunctionUtility.isDirectRelation(option.getVendor(), functionCall.getFunctionName().toString(), functionCall.getArgs().size(), argumentIndex)) {
21687                                directExpressions.add(expr);
21688                        }
21689                        if (FunctionUtility.isIndirectRelation(option.getVendor(), functionCall.getFunctionName().toString(), functionCall.getArgs().size(), argumentIndex)) {
21690                                indirectExpressions.add(expr);
21691                        }
21692                }
21693        }
21694        
21695        private void getFunctionExpressions(List<TExpression> directExpressions, List<TExpression> indirectExpressions,
21696                        TCallStatement functionCall, int argumentIndex) {
21697                if (functionCall.getArgs() != null && argumentIndex < functionCall.getArgs().size()) {
21698                        TExpression expr = functionCall.getArgs().getExpression(argumentIndex);
21699                        if (FunctionUtility.isDirectRelation(option.getVendor(), functionCall.getRoutineName().toString(),
21700                                        functionCall.getArgs().size(), argumentIndex)) {
21701                                directExpressions.add(expr);
21702                        }
21703                        if (FunctionUtility.isIndirectRelation(option.getVendor(), functionCall.getRoutineName().toString(),
21704                                        functionCall.getArgs().size(), argumentIndex)) {
21705                                indirectExpressions.add(expr);
21706                        }
21707                }
21708        }
21709
21710        private void getFunctionExpressions(List<TExpression> directExpressions, List<TExpression> indirectExpressions,
21711                        TDb2CallStmt functionCall, int argumentIndex) {
21712                if (functionCall.getParameters() != null && argumentIndex < functionCall.getParameters().size()) {
21713                        TExpression expr = functionCall.getParameters().getExpression(argumentIndex);
21714                        if (FunctionUtility.isDirectRelation(option.getVendor(), functionCall.getProcedureName().toString(),
21715                                        functionCall.getParameters().size(), argumentIndex)) {
21716                                directExpressions.add(expr);
21717                        }
21718                        if (FunctionUtility.isIndirectRelation(option.getVendor(), functionCall.getProcedureName().toString(),
21719                                        functionCall.getParameters().size(), argumentIndex)) {
21720                                indirectExpressions.add(expr);
21721                        }
21722                }
21723        }
21724
21725        private void getFunctionExpressions(List<TExpression> directExpressions, List<TExpression> indirectExpressions,
21726                                                                                TMssqlExecute functionCall, String argumentName, int argumentIndex) {
21727                if (functionCall.getParameters() != null) {
21728                        for (int i = 0; i < functionCall.getParameters().size(); i++) {
21729                                TExecParameter param = functionCall.getParameters().getExecParameter(i);
21730                                if (param.getParameterName() != null) {
21731                                        if (DlineageUtil.compareColumnIdentifier(param.getParameterName().toString(), argumentName)) {
21732                                                TExpression expr = param.getParameterValue();
21733                                                directExpressions.add(expr);
21734                                        }
21735                                } else if (i == argumentIndex) {
21736                                        TExpression expr = param.getParameterValue();
21737                                        directExpressions.add(expr);
21738                                }
21739                        }
21740                }
21741        }
21742
21743        private void analyzeJoin(TJoin join, EffectType effectType) {
21744                if (join.getJoinItems() != null) {
21745                        for (int j = 0; j < join.getJoinItems().size(); j++) {
21746                                TJoinItem joinItem = join.getJoinItems().getJoinItem(j);
21747                                TExpression expr = joinItem.getOnCondition();
21748                                if (expr != null) {
21749                                        analyzeFilterCondition(null, expr, joinItem.getJoinType(), JoinClauseType.on, effectType);
21750                                }
21751                        }
21752                }
21753
21754                if (join.getJoin() != null) {
21755                        analyzeJoin(join.getJoin(), effectType);
21756                }
21757        }
21758
21759        private TSelectSqlStatement getParentSetSelectStmt(TSelectSqlStatement stmt) {
21760                TCustomSqlStatement parent = stmt.getParentStmt();
21761                if (parent == null)
21762                        return null;
21763                if (parent.getStatements() != null) {
21764                        for (int i = 0; i < parent.getStatements().size(); i++) {
21765                                TCustomSqlStatement temp = parent.getStatements().get(i);
21766                                if (temp instanceof TSelectSqlStatement) {
21767                                        TSelectSqlStatement select = (TSelectSqlStatement) temp;
21768                                        if (select.getLeftStmt() == stmt || select.getRightStmt() == stmt)
21769                                                return select;
21770                                }
21771                        }
21772                }
21773                if (parent instanceof TSelectSqlStatement) {
21774                        TSelectSqlStatement select = (TSelectSqlStatement) parent;
21775                        if (select.getLeftStmt() == stmt || select.getRightStmt() == stmt)
21776                                return select;
21777                }
21778                return null;
21779        }
21780
21781        private void createSelectSetResultColumns(SelectSetResultSet resultSet, TSelectSqlStatement stmt) {
21782                if (stmt.getSetOperatorType() != ESetOperatorType.none) {
21783                        createSelectSetResultColumns(resultSet, stmt.getLeftStmt());
21784                } else {
21785                        TResultColumnList columnList = stmt.getResultColumnList();
21786                        ResultSet subqueryResultSet = (ResultSet) modelManager.getModel(columnList);
21787                        if(subqueryResultSet!=null && subqueryResultSet.isDetermined()) {
21788                                for (int j = 0; j < subqueryResultSet.getColumns().size(); j++) {
21789                                        ResultColumn tableColumn = subqueryResultSet.getColumns().get(j);
21790                                        if (tableColumn.getRefColumnName() != null) {
21791                                                TObjectName columnName = new TObjectName();
21792                                                columnName.setString(tableColumn.getRefColumnName());
21793                                                modelFactory.createDeterminedResultColumn(
21794                                                                resultSet, columnName);
21795                                        } else {
21796                                                TObjectName columnName = new TObjectName();
21797                                                columnName.setString(tableColumn.getName());
21798                                                modelFactory.createDeterminedResultColumn(
21799                                                                resultSet, columnName);
21800                                        }
21801                                }
21802                                resultSet.setDetermined(true);
21803                                return;
21804                        }
21805                        
21806                        boolean isDetermined = true;
21807                        for (int i = 0; i < columnList.size(); i++) {
21808                                TResultColumn column = columnList.getResultColumn(i);
21809                                
21810                                if ("*".equals(column.getColumnNameOnly())) {
21811                                        TObjectName columnObject = column.getFieldAttr();
21812                                        TTable sourceTable = columnObject.getSourceTable();
21813                                        if (sourceTable != null) {
21814                                                Object tableModel = modelManager.getModel(sourceTable);
21815                                                if (tableModel instanceof Table && ((Table) tableModel).isCreateTable()) {
21816                                                        Table table = (Table) tableModel;
21817                                                        for (int j = 0; j < table.getColumns().size(); j++) {
21818                                                                TableColumn tableColumn = table.getColumns().get(j);
21819                                                                if (column.getExceptColumnList() != null) {
21820                                                                        boolean except = false;
21821                                                                        for (TObjectName objectName : column.getExceptColumnList()) {
21822                                                                                if (getColumnName(objectName.toString())
21823                                                                                                .equals(getColumnName(tableColumn.getName()))) {
21824                                                                                        except = true;
21825                                                                                        break;
21826                                                                                }
21827                                                                        }
21828                                                                        if (!except && tableColumn.isStruct()) {
21829                                                                                List<String> names = SQLUtil
21830                                                                                                .parseNames(tableColumn.getName());
21831                                                                                for (String name : names) {
21832                                                                                        for (TObjectName objectName : column
21833                                                                                                        .getExceptColumnList()) {
21834                                                                                                if (getColumnName(objectName.toString())
21835                                                                                                                .equals(getColumnName(name))) {
21836                                                                                                        except = true;
21837                                                                                                        break;
21838                                                                                                }
21839                                                                                        }
21840                                                                                        if (except) {
21841                                                                                                break;
21842                                                                                        }
21843                                                                                }
21844                                                                        }
21845                                                                        if (except) {
21846                                                                                continue;
21847                                                                        }
21848                                                                }
21849                                                                TObjectName columnName = new TObjectName();
21850                                                                columnName.setString(tableColumn.getName());
21851                                                                ResultColumn resultColumn = modelFactory.createResultColumn(
21852                                                                                resultSet, columnName);
21853                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
21854                                                                relation.setEffectType(EffectType.select);
21855                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
21856                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
21857                                                        }
21858                                                        continue;
21859                                                } else if (tableModel instanceof ResultSet
21860                                                                && ((ResultSet) tableModel).isDetermined()) {
21861                                                        ResultSet table = (ResultSet) tableModel;
21862                                                        for (int j = 0; j < table.getColumns().size(); j++) {
21863                                                                ResultColumn tableColumn = table.getColumns().get(j);
21864                                                                if (column.getExceptColumnList() != null) {
21865                                                                        boolean except = false;
21866                                                                        for (TObjectName objectName : column.getExceptColumnList()) {
21867                                                                                if (getColumnName(objectName.toString())
21868                                                                                                .equals(getColumnName(tableColumn.getName()))) {
21869                                                                                        except = true;
21870                                                                                        break;
21871                                                                                }
21872                                                                        }
21873                                                                        if (!except && tableColumn.isStruct()) {
21874                                                                                List<String> names = SQLUtil
21875                                                                                                .parseNames(tableColumn.getName());
21876                                                                                for (String name : names) {
21877                                                                                        for (TObjectName objectName : column
21878                                                                                                        .getExceptColumnList()) {
21879                                                                                                if (getColumnName(objectName.toString())
21880                                                                                                                .equals(getColumnName(name))) {
21881                                                                                                        except = true;
21882                                                                                                        break;
21883                                                                                                }
21884                                                                                        }
21885                                                                                        if (except) {
21886                                                                                                break;
21887                                                                                        }
21888                                                                                }
21889                                                                        }
21890                                                                        if (except) {
21891                                                                                continue;
21892                                                                        }
21893                                                                }
21894                                                                if (tableColumn.getRefColumnName() != null) {
21895                                                                        TObjectName columnName = new TObjectName();
21896                                                                        columnName.setString(tableColumn.getRefColumnName());
21897                                                                        ResultColumn resultColumn = modelFactory.createResultColumn(
21898                                                                                        resultSet, columnName);
21899                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
21900                                                                        relation.setEffectType(EffectType.select);
21901                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
21902                                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn));
21903                                                                } else {
21904                                                                        TObjectName columnName = new TObjectName();
21905                                                                        columnName.setString(tableColumn.getName());
21906                                                                        ResultColumn resultColumn = modelFactory.createResultColumn(
21907                                                                                        resultSet, columnName);
21908                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
21909                                                                        relation.setEffectType(EffectType.select);
21910                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
21911                                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn));
21912                                                                }
21913                                                        }
21914                                                        continue;
21915                                                }
21916                                                else {
21917                                                        isDetermined = false;
21918                                                }
21919                                        }
21920                                }
21921                                        
21922                                ResultColumn resultColumn = modelFactory.createSelectSetResultColumn(resultSet, column, i);
21923
21924                                if (resultColumn.getColumnObject() instanceof TResultColumn) {
21925                                        TResultColumn columnObject = (TResultColumn) resultColumn.getColumnObject();
21926                                        if (columnObject.getFieldAttr() != null) {
21927                                                if ("*".equals(getColumnName(columnObject.getFieldAttr()))) {
21928                                                        TObjectName fieldAttr = columnObject.getFieldAttr();
21929                                                        TTable sourceTable = fieldAttr.getSourceTable();
21930                                                        if (fieldAttr.getTableToken() != null && sourceTable != null) {
21931                                                                TObjectName[] columns = modelManager.getTableColumns(sourceTable);
21932                                                                for (int j = 0; j < columns.length; j++) {
21933                                                                        TObjectName columnName = columns[j];
21934                                                                        if (columnName == null) {
21935                                                                                continue;
21936                                                                        }
21937                                                                        if ("*".equals(getColumnName(columnName))) {
21938                                                                                continue;
21939                                                                        }
21940                                                                        resultColumn.bindStarLinkColumn(columnName);
21941                                                                }
21942
21943                                                                if (modelManager.getModel(sourceTable) instanceof Table) {
21944                                                                        Table tableModel = (Table) modelManager.getModel(sourceTable);
21945                                                                        if (tableModel != null && !tableModel.getColumns().isEmpty()) {
21946                                                                                for (int z = 0; z < tableModel.getColumns().size(); z++) {
21947                                                                                        if ("*".equals(
21948                                                                                                        getColumnName(tableModel.getColumns().get(z).getColumnObject()))) {
21949                                                                                                continue;
21950                                                                                        }
21951                                                                                        resultColumn.bindStarLinkColumn(
21952                                                                                                        tableModel.getColumns().get(z).getColumnObject());
21953                                                                                }
21954                                                                        }
21955                                                                } else if (modelManager.getModel(sourceTable) instanceof QueryTable) {
21956                                                                        QueryTable tableModel = (QueryTable) modelManager.getModel(sourceTable);
21957                                                                        if (tableModel != null && !tableModel.getColumns().isEmpty()) {
21958                                                                                for (ResultColumn item : tableModel.getColumns()) {
21959                                                                                        if (item.hasStarLinkColumn()) {
21960                                                                                                for (TObjectName starLinkColumn : item.getStarLinkColumnList()) {
21961                                                                                                        if ("*".equals(getColumnName(starLinkColumn))) {
21962                                                                                                                continue;
21963                                                                                                        }
21964                                                                                                        resultColumn.bindStarLinkColumn(starLinkColumn);
21965                                                                                                }
21966                                                                                        } else if (item.getColumnObject() instanceof TObjectName) {
21967                                                                                                TObjectName starLinkColumn = (TObjectName) item.getColumnObject();
21968                                                                                                if ("*".equals(getColumnName(starLinkColumn))) {
21969                                                                                                        continue;
21970                                                                                                }
21971                                                                                                resultColumn.bindStarLinkColumn(starLinkColumn);
21972                                                                                        }
21973                                                                                }
21974                                                                        }
21975                                                                }
21976
21977                                                        } else {
21978                                                                TTableList tables = stmt.getTables();
21979                                                                for (int k = 0; k < tables.size(); k++) {
21980                                                                        TTable tableElement = tables.getTable(k);
21981                                                                        TObjectName[] columns = modelManager.getTableColumns(tableElement);
21982                                                                        for (int j = 0; j < columns.length; j++) {
21983                                                                                TObjectName columnName = columns[j];
21984                                                                                if (columnName == null) {
21985                                                                                        continue;
21986                                                                                }
21987                                                                                if ("*".equals(getColumnName(columnName))) {
21988                                                                                        if (modelManager.getModel(tableElement) instanceof Table) {
21989                                                                                                Table tableModel = (Table) modelManager.getModel(tableElement);
21990                                                                                                if (tableModel != null && !tableModel.getColumns().isEmpty()) {
21991                                                                                                        for (int z = 0; z < tableModel.getColumns().size(); z++) {
21992                                                                                                                resultColumn.bindStarLinkColumn(
21993                                                                                                                                tableModel.getColumns().get(z).getColumnObject());
21994                                                                                                        }
21995                                                                                                }
21996                                                                                        } else if (modelManager.getModel(tableElement) instanceof QueryTable) {
21997                                                                                                QueryTable tableModel = (QueryTable) modelManager
21998                                                                                                                .getModel(tableElement);
21999                                                                                                if (tableModel != null && !tableModel.getColumns().isEmpty()) {
22000                                                                                                        for (ResultColumn item : tableModel.getColumns()) {
22001                                                                                                                if (item.hasStarLinkColumn()) {
22002                                                                                                                        for (TObjectName starLinkColumn : item
22003                                                                                                                                        .getStarLinkColumnList()) {
22004                                                                                                                                resultColumn.bindStarLinkColumn(starLinkColumn);
22005                                                                                                                        }
22006                                                                                                                } else if (item.getColumnObject() instanceof TObjectName) {
22007                                                                                                                        resultColumn.bindStarLinkColumn(
22008                                                                                                                                        (TObjectName) item.getColumnObject());
22009                                                                                                                }
22010                                                                                                        }
22011                                                                                                }
22012                                                                                        }
22013                                                                                        continue;
22014                                                                                }
22015                                                                                resultColumn.bindStarLinkColumn(columnName);
22016                                                                        }
22017                                                                }
22018                                                        }
22019                                                }
22020                                        }
22021                                }
22022                                
22023                                resultSet.setDetermined(isDetermined);
22024                        }
22025                }
22026        }
22027
22028        private void analyzeResultColumn(TResultColumn column, EffectType effectType) {
22029                // A SELECT * star column whose source resolves to a determined result set
22030                // is bound to a LinkedHashMap of per-column ResultColumns (see
22031                // ModelFactory.createStarResultColumn), and its data-flow relationships
22032                // are already created inline during star expansion in analyzeSelectStmt().
22033                // Re-analyzing the raw "*" here is redundant; skip it. (analyzeDataFlowRelation
22034                // keeps the same guard as a defensive backstop.)
22035                if (modelManager.getModel(column) instanceof LinkedHashMap) {
22036                        return;
22037                }
22038                TExpression expression = column.getExpr();
22039                if (expression.getExpressionType() == EExpressionType.sqlserver_proprietary_column_alias_t) {
22040                        expression = expression.getRightOperand();
22041                }
22042
22043                if (expression.getExpressionType() == EExpressionType.array_t) {
22044                        if (expression.getExprList() != null) {
22045                                for (TExpression expr : expression.getExprList()) {
22046                                        columnsInExpr visitor = new columnsInExpr();
22047                                        expr.inOrderTraverse(visitor);
22048                                        List<TObjectName> objectNames = visitor.getObjectNames();
22049
22050                                        List<TParseTreeNode> functions = visitor.getFunctions();
22051
22052                                        if (functions != null && !functions.isEmpty()) {
22053                                                analyzeFunctionDataFlowRelation(column, functions, effectType);
22054                                        }
22055
22056                                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
22057                                        if (subquerys != null && !subquerys.isEmpty()) {
22058                                                analyzeSubqueryDataFlowRelation(column, subquerys, effectType);
22059                                        }
22060
22061                                        analyzeDataFlowRelation(column, objectNames, column.getExceptColumnList(), effectType, functions);
22062
22063                                        List<TParseTreeNode> constants = visitor.getConstants();
22064                                        Object columnObject = modelManager.getModel(column);
22065                                        analyzeConstantDataFlowRelation(columnObject, constants, effectType, functions);
22066
22067                                        analyzeRecordSetRelation(functions, effectType);
22068                                        // analyzeResultColumnImpact( column, effectType, functions);
22069                                }
22070                        }
22071                        else {
22072                                List<TParseTreeNode> constants = new ArrayList<TParseTreeNode>();
22073                                TConstant constant = new TConstant();
22074                                constant.setString(expression.toString());
22075                                constants.add(constant);
22076                                Object columnObject = modelManager.getModel(column);
22077                                analyzeConstantDataFlowRelation(columnObject, constants, effectType, null);
22078                        }
22079                } else {
22080                        columnsInExpr visitor = new columnsInExpr();
22081                        expression.inOrderTraverse(visitor);
22082                        List<TObjectName> objectNames = visitor.getObjectNames();
22083
22084                        List<TParseTreeNode> functions = visitor.getFunctions();
22085
22086                        if (functions != null && !functions.isEmpty()) {
22087                                analyzeFunctionDataFlowRelation(column, functions, effectType);
22088                        }
22089
22090                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
22091                        if (subquerys != null && !subquerys.isEmpty()) {
22092                                analyzeSubqueryDataFlowRelation(column, subquerys, effectType);
22093                        }
22094
22095                        analyzeDataFlowRelation(column, objectNames, column.getExceptColumnList(), effectType, functions);
22096
22097                        List<TParseTreeNode> constants = visitor.getConstants();
22098                        Object columnObject = modelManager.getModel(column);
22099                        analyzeConstantDataFlowRelation(columnObject, constants, effectType, functions);
22100
22101                        analyzeRecordSetRelation(functions, effectType);
22102                        // analyzeResultColumnImpact( column, effectType, functions);
22103                }
22104        }
22105        
22106        
22107        private void analyzeValueColumn(Object object, TResultColumn column, EffectType effectType) {
22108                TExpression expression = column.getExpr();
22109                if (expression.getExpressionType() == EExpressionType.sqlserver_proprietary_column_alias_t) {
22110                        expression = expression.getRightOperand();
22111                }
22112
22113                if (expression.getExpressionType() == EExpressionType.array_t) {
22114                        if (expression.getExprList() != null) {
22115                                for (TExpression expr : expression.getExprList()) {
22116                                        columnsInExpr visitor = new columnsInExpr();
22117                                        expr.inOrderTraverse(visitor);
22118                                        List<TObjectName> objectNames = visitor.getObjectNames();
22119                                        analyzeDataFlowRelation(object, objectNames, column.getExceptColumnList(), effectType, null, null);
22120                                        List<TParseTreeNode> constants = visitor.getConstants();
22121                                        analyzeConstantDataFlowRelation(object, constants, effectType, null);
22122                                }
22123                        }
22124                        else {
22125                                List<TParseTreeNode> constants = new ArrayList<TParseTreeNode>();
22126                                TConstant constant = new TConstant();
22127                                constant.setString(expression.toString());
22128                                constants.add(constant);
22129                                Object columnObject = modelManager.getModel(column);
22130                                analyzeConstantDataFlowRelation(object, constants, effectType, null);
22131                        }
22132                } else {
22133                        columnsInExpr visitor = new columnsInExpr();
22134                        expression.inOrderTraverse(visitor);
22135                        List<TObjectName> objectNames = visitor.getObjectNames();
22136                        analyzeDataFlowRelation(object, objectNames, column.getExceptColumnList(), effectType, null, null);
22137                        List<TParseTreeNode> constants = visitor.getConstants();
22138                        analyzeConstantDataFlowRelation(object, constants, effectType, null);           }
22139        }
22140
22141        private void analyzeTableColumn(TableColumn tableColumn, TFunctionCall functionCall, EffectType effectType) {
22142                List<TParseTreeNode> functions = new ArrayList<TParseTreeNode>();
22143                functions.add(functionCall);
22144
22145                if (functions != null && !functions.isEmpty()) {
22146                        analyzeFunctionDataFlowRelation(tableColumn, functions, effectType);
22147                }
22148
22149                analyzeRecordSetRelation(functions, effectType);
22150        }
22151
22152        private void analyzeRecordSetRelation(List<TParseTreeNode> functions, EffectType effectType) {
22153                if (functions == null || functions.size() == 0)
22154                        return;
22155
22156                List<TFunctionCall> aggregateFunctions = new ArrayList<TFunctionCall>();
22157                for (TParseTreeNode function : functions) {
22158                        if (function instanceof TFunctionCall && isAggregateFunction((TFunctionCall) function)) {
22159                                aggregateFunctions.add((TFunctionCall) function);
22160                        }
22161                }
22162
22163                if (aggregateFunctions.size() == 0)
22164                        return;
22165
22166                for (int i = 0; i < aggregateFunctions.size(); i++) {
22167                        TFunctionCall function = aggregateFunctions.get(i);
22168
22169                        TCustomSqlStatement stmt = stmtStack.peek();
22170                        if (stmt instanceof TSelectSqlStatement) {
22171                                TSelectSqlStatement select = (TSelectSqlStatement) stmt;
22172                                if (select.getGroupByClause() != null) {
22173                                        if (select.getGroupByClause().isAllModifier()) {
22174                                                // GROUP BY ALL: implicit grouping columns are the
22175                                                // non-aggregate expressions in the SELECT list.
22176                                                TResultColumnList resultColumns = select.getResultColumnList();
22177                                                if (resultColumns != null) {
22178                                                        for (int j = 0; j < resultColumns.size(); j++) {
22179                                                                TResultColumn column = resultColumns.getResultColumn(j);
22180                                                                TExpression expr = column.getExpr();
22181                                                                if (expr == null)
22182                                                                        continue;
22183                                                                columnsInExpr aggVisitor = new columnsInExpr();
22184                                                                expr.inOrderTraverse(aggVisitor);
22185                                                                boolean containsAggregate = false;
22186                                                                for (TParseTreeNode funcNode : aggVisitor.getFunctions()) {
22187                                                                        if (funcNode instanceof TFunctionCall
22188                                                                                        && isAggregateFunction((TFunctionCall) funcNode)) {
22189                                                                                containsAggregate = true;
22190                                                                                break;
22191                                                                        }
22192                                                                }
22193                                                                if (!containsAggregate) {
22194                                                                        analyzeAggregate(function, expr);
22195                                                                }
22196                                                        }
22197                                                }
22198                                        } else {
22199                                                TGroupByItemList groupByList = select.getGroupByClause().getItems();
22200                                                for (int j = 0; j < groupByList.size(); j++) {
22201                                                        TGroupByItem groupBy = groupByList.getGroupByItem(j);
22202                                                        TExpression expr = groupBy.getExpr();
22203                                                        analyzeAggregate(function, expr);
22204                                                }
22205                                        }
22206
22207                                        if (select.getGroupByClause().getHavingClause() != null) {
22208                                                analyzeAggregate(function, select.getGroupByClause().getHavingClause());
22209                                        }
22210                                        // if ("COUNT".equalsIgnoreCase(function.getFunctionName().toString()))
22211                                        {
22212                                                analyzeAggregate(function, null);
22213                                        }
22214                                } else {
22215                                        analyzeAggregate(function, null);
22216                                }
22217                        }
22218                }
22219        }
22220
22221        private void analyzeDataFlowRelation(TParseTreeNode gspObject, List<TObjectName> objectNames,
22222                        TObjectNameList exceptColumnList, EffectType effectType, List<TParseTreeNode> functions) {
22223                Object columnObject = modelManager.getModel(gspObject);
22224                analyzeDataFlowRelation(columnObject, objectNames, exceptColumnList, effectType, functions, null);
22225        }
22226
22227        private DataFlowRelationship analyzeDataFlowRelation(Object modelObject, List<TObjectName> objectNames, EffectType effectType,
22228                        List<TParseTreeNode> functions) {
22229                return analyzeDataFlowRelation(modelObject, objectNames, null, effectType, functions, null);
22230        }
22231        
22232        private DataFlowRelationship analyzeDataFlowRelation(Object modelObject, List<TObjectName> objectNames, EffectType effectType,
22233                        List<TParseTreeNode> functions, Process process) {
22234                return analyzeDataFlowRelation(modelObject, objectNames, null, effectType, functions, process);
22235        }
22236
22237        private DataFlowRelationship analyzeDataFlowRelation(Object modelObject, List<TObjectName> objectNames,
22238                        TObjectNameList exceptColumnList, EffectType effectType, List<TParseTreeNode> functions, Process process) {
22239                return analyzeDataFlowRelation(modelObject, objectNames, exceptColumnList, effectType, functions, process, null);
22240        }
22241        
22242        private DataFlowRelationship analyzeDataFlowRelation(Object modelObject, List<TObjectName> objectNames,
22243                        TObjectNameList exceptColumnList, EffectType effectType, List<TParseTreeNode> functions, Process process, Integer valueIndex) {
22244                if (objectNames == null || objectNames.size() == 0)
22245                        return null;
22246
22247                // Reject model objects this method cannot turn into a relationship BEFORE
22248                // creating one, since createDataFlowRelation() registers the relation
22249                // globally and an early return afterwards would leak an empty relation.
22250                //
22251                // - LinkedHashMap: a SELECT * star column whose source resolves to a
22252                //   determined result set is expanded into per-column ResultColumns held
22253                //   in a LinkedHashMap by ModelFactory.createStarResultColumn(), with its
22254                //   data-flow relationships created inline during star expansion in
22255                //   analyzeSelectStmt(). Re-analyzing the raw "*" here is a no-op.
22256                // - null: modelManager.getModel() found no binding for the gsp object.
22257                if (modelObject == null || modelObject instanceof LinkedHashMap) {
22258                        return null;
22259                }
22260                // Lineage is best-effort: an unexpected model type should not abort the
22261                // whole statement's lineage (this used to throw UnsupportedOperationException).
22262                // Log it for diagnosis and skip just this column instead.
22263                if (!(modelObject instanceof ResultColumn) && !(modelObject instanceof TableColumn)) {
22264                        logger.warn("analyzeDataFlowRelation: unhandled model type "
22265                                        + modelObject.getClass().getName() + ", effectType=" + effectType);
22266                        return null;
22267                }
22268
22269                boolean isStar = false;
22270                boolean showStar = false;
22271
22272                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
22273                relation.setEffectType(effectType);
22274                relation.setProcess(process);
22275
22276                if (functions != null && !functions.isEmpty()) {
22277                        relation.setFunction(getFunctionName(functions.get(0)));
22278                }
22279
22280                int columnIndex = -1;
22281
22282                boolean isOut = false;
22283
22284                if (modelObject instanceof ResultColumn) {
22285                        relation.setTarget(new ResultColumnRelationshipElement((ResultColumn) modelObject));
22286
22287                        if ("*".equals(((ResultColumn) modelObject).getName())) {
22288                                isStar = true;
22289                                showStar = ((ResultColumn) modelObject).isShowStar();
22290                        }
22291
22292                        if (((ResultColumn) modelObject).getResultSet() != null) {
22293                                columnIndex = ((ResultColumn) modelObject).getResultSet().getColumns().indexOf(modelObject);
22294                        }
22295                } else if (modelObject instanceof TableColumn) {
22296                        Table table  = ((TableColumn) modelObject).getTable();
22297                        if(table.getSubType() == SubType.out && isNotInProcedure(table)){
22298                                isOut = true;
22299                                relation.addSource(new TableColumnRelationshipElement((TableColumn) modelObject));
22300                        }
22301                        else {
22302                                relation.setTarget(new TableColumnRelationshipElement((TableColumn) modelObject));
22303                        }
22304
22305                        if ("*".equals(((TableColumn) modelObject).getName())) {
22306                                isStar = true;
22307                        }
22308
22309                        if (((TableColumn) modelObject).getTable() != null) {
22310                                columnIndex = ((TableColumn) modelObject).getTable().getColumns().indexOf(modelObject);
22311                        }
22312                }
22313                // No trailing else: modelObject is guaranteed to be a ResultColumn or
22314                // TableColumn here (validated and logged above before relation creation).
22315
22316                for (int i = 0; i < objectNames.size(); i++) {
22317                        TObjectName columnName = objectNames.get(i);
22318                        if (columnName.toString().indexOf(".") == -1 && isConstant(columnName)) {
22319                                boolean isConstant = true;
22320                                if (columnName.getSourceTable() != null) {
22321                                        Table tableModel = modelManager.getTableByName(
22322                                                        DlineageUtil.getTableFullName(columnName.getSourceTable().getTableName().toString()));
22323                                        if (tableModel != null && tableModel.getColumns() != null) {
22324                                                for (int j = 0; j < tableModel.getColumns().size(); j++) {
22325                                                        if (DlineageUtil.compareColumnIdentifier(getColumnName(columnName),
22326                                                                        getColumnName(tableModel.getColumns().get(j).getName()))) {
22327                                                                isConstant = false;
22328                                                                break;
22329                                                        }
22330                                                }
22331                                        }
22332                                }
22333
22334                                if (isConstant) {
22335                                        if (option.isShowConstantTable()) {
22336                                                Table constantTable = modelFactory.createConstantsTable(stmtStack.peek());
22337                                                TableColumn tableColumn = modelFactory.createTableColumn(constantTable, columnName, false);
22338                                                if(tableColumn!=null) {
22339                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
22340                                                }
22341                                        }
22342                                        continue;
22343                                }
22344                        }
22345                        if (columnName.getDbObjectType() == EDbObjectType.variable) {
22346                                boolean find = false;
22347                                List<String> segments = SQLUtil.parseNames(columnName.toString());
22348                                if (segments.size() == 1) {
22349                                        Table variable = modelManager.getTableByName(DlineageUtil.getTableFullName(columnName.toString()));
22350                                        if (variable != null) {
22351                                                TableColumn columnModel = variable.getColumns().get(0);
22352                                                if(variable.getColumns().size()>0){
22353                                                        TableColumn matchColumn = matchColumn(variable.getColumns(), columnName);
22354                                                        if(matchColumn!=null){
22355                                                                columnModel = matchColumn;
22356                                                        }
22357                                                }
22358                                                if(isOut){
22359                                                        relation.setTarget(new TableColumnRelationshipElement(columnModel));
22360                                                }
22361                                                else {
22362                                                        relation.addSource(new TableColumnRelationshipElement(columnModel));
22363                                                }
22364                                                find = true;
22365                                        } else {
22366                                                if (columnName.toString().matches("\\$\\d+")) {
22367                                                        TCustomSqlStatement stmt = stmtStack.peek();
22368                                                        Procedure procedure = modelManager
22369                                                                        .getProcedureByName(DlineageUtil.getTableFullName(getProcedureParentName(stmt)));
22370                                                        if(procedure!=null) {
22371                                                                Variable cursorVariable = modelFactory.createVariable(procedure.getArguments().get(Integer.valueOf(columnName.toString().replace("$", "")) - 1).getName());
22372                                                                if (isOut) {
22373                                                                        relation.setTarget(new TableColumnRelationshipElement(cursorVariable.getColumns().get(0)));
22374                                                                } else {
22375                                                                        relation.addSource(new TableColumnRelationshipElement(cursorVariable.getColumns().get(0)));
22376                                                                }
22377                                                        }
22378                                                } else {
22379                                                        Variable cursorVariable = modelFactory.createVariable(columnName);
22380                                                        cursorVariable.setCreateTable(true);
22381                                                        cursorVariable.setSubType(SubType.record);
22382                                                        TableColumn variableProperty = null;
22383                                                        if(cursorVariable.getColumns() == null || cursorVariable.getColumns().isEmpty()) {
22384                                variableProperty = modelFactory.createTableColumn(cursorVariable, columnName,
22385                                        true);
22386                            }
22387                                                        else{
22388                                                                variableProperty = cursorVariable.getColumns().get(0);
22389                                                        }
22390                                                        if (isOut) {
22391                                                                relation.setTarget(new TableColumnRelationshipElement(variableProperty));
22392                                                        } else {
22393                                                                relation.addSource(new TableColumnRelationshipElement(variableProperty));
22394                                                        }
22395                                                }
22396                                                find = true;
22397                                        }
22398                                } else if (option.getVendor() == EDbVendor.dbvoracle && columnName.getTableToken()!=null) {
22399                                        Variable cursorVariable = modelFactory
22400                                                        .createVariable(columnName.getTableToken().toString());
22401
22402                                        TObjectName variableColumnName = new TObjectName();
22403                                        variableColumnName.setString(segments.get(segments.size() - 1));
22404
22405                                        if (cursorVariable.getColumns() != null) {
22406                                                for (int j = 0; j < cursorVariable.getColumns().size(); j++) {
22407                                                        if (getColumnName(variableColumnName)
22408                                                                        .equals(getColumnName(cursorVariable.getColumns().get(j).getColumnObject()))) {
22409                                                                TableColumn columnModel = cursorVariable.getColumns().get(j);
22410                                                                if (isOut) {
22411                                                                        relation.setTarget(new TableColumnRelationshipElement(columnModel));
22412                                                                } else {
22413                                                                        relation.addSource(new TableColumnRelationshipElement(columnModel));
22414                                                                }
22415                                                                find = true;
22416                                                        }
22417                                                }
22418                                        }
22419
22420                                        if (!find) {
22421                                                TableColumn variableColumn = new TableColumn(cursorVariable, variableColumnName);
22422                                                cursorVariable.addColumn(variableColumn);
22423                                                if (isOut) {
22424                                                        relation.setTarget(new TableColumnRelationshipElement(variableColumn));
22425                                                } else {
22426                                                        relation.addSource(new TableColumnRelationshipElement(variableColumn));
22427                                                }
22428                                        }
22429                                } else {
22430                                        Table variable = modelManager
22431                                                        .getTableByName(DlineageUtil.getTableFullName(segments.get(segments.size() - 2)));
22432                                        if (variable != null) {
22433                                                for (int j = 0; j < variable.getColumns().size(); j++) {
22434                                                        if (getColumnName(columnName)
22435                                                                        .equals(getColumnName(variable.getColumns().get(j).getColumnObject()))) {
22436                                                                TableColumn columnModel = variable.getColumns().get(j);
22437                                                                if (isOut) {
22438                                                                        relation.setTarget(new TableColumnRelationshipElement(columnModel));
22439                                                                } else {
22440                                                                        relation.addSource(new TableColumnRelationshipElement(columnModel));
22441                                                                }
22442                                                                find = true;
22443                                                        }
22444                                                }
22445                                        }
22446                                }
22447                                if (!find) {
22448                                        TCustomSqlStatement stmt = stmtStack.peek();
22449                                        if (getProcedureParentName(stmt) != null) {
22450                                                Procedure procedure = modelManager
22451                                                                .getProcedureByName(DlineageUtil.getTableFullName(getProcedureParentName(stmt)));
22452                                                if (procedure != null && procedure.getArguments() != null) {
22453                                                        for (Argument argument : procedure.getArguments()) {
22454                                                                if (DlineageUtil.getTableFullName(argument.getName())
22455                                                                                .equals(DlineageUtil.getTableFullName(columnName.toString()))) {
22456                                                                        relation.addSource(new ArgumentRelationshipElement(argument));
22457                                                                }
22458                                                        }
22459                                                }
22460                                        }
22461                                }
22462                                continue;
22463                        }
22464                        
22465                        // Handle sequence pseudocolumn syntax (sequence.NEXTVAL or sequence.CURRVAL)
22466                        // Used by Oracle, Snowflake, and accepted by other vendors for compatibility
22467                        if(("NEXTVAL".equalsIgnoreCase(columnName.getColumnNameOnly()) || "CURRVAL".equalsIgnoreCase(columnName.getColumnNameOnly()))){
22468                                List<String> segments = SQLUtil.parseNames(columnName.toString());
22469                                if (segments.size() > 1) {
22470                                        segments.remove(segments.size()-1);
22471                                        Table table = modelFactory.createTableByName(SQLUtil.mergeSegments(segments, 0), true);
22472                                        table.setSequence(true);
22473                                        TableColumn seqCursor = modelFactory.createTableColumn(table, columnName, true);
22474                                        relation.addSource(new TableColumnRelationshipElement(seqCursor));
22475                                        continue;
22476                                }
22477                        }
22478
22479                        {
22480                                if (columnName.getSourceTable() != null) {
22481
22482                                }
22483                                else {
22484                                        Table variable = modelManager.getTableByName(DlineageUtil.getTableFullName(columnName.toString()));
22485                                        if (variable == null) {
22486                                                variable = modelManager
22487                                                                .getTableByName(DlineageUtil.getTableFullName(columnName.getTableString()));
22488                                                if (variable != null && variable.isCursor()) {
22489                                                        TableColumn variableColumn = modelFactory.createInsertTableColumn(variable, columnName);
22490                                                        if (variableColumn != null) {
22491                                                                if(isOut){
22492                                                                        relation.setTarget(new TableColumnRelationshipElement(variableColumn));
22493                                                                }
22494                                                                else {
22495                                                                        relation.addSource(new TableColumnRelationshipElement(variableColumn));
22496                                                                }
22497                                                        } else {
22498                                                                if(isOut){
22499                                                                        relation.setTarget(new TableColumnRelationshipElement(variable.getColumns().get(0)));
22500                                                                }
22501                                                                else {
22502                                                                        relation.addSource(new TableColumnRelationshipElement(variable.getColumns().get(0)));
22503                                                                }
22504                                                        }
22505                                                        continue;
22506                                                }
22507                                        } else if (variable.isVariable() || variable.isCursor()) {
22508                                                TableColumn columnModel = variable.getColumns().get(0);
22509                                                if (valueIndex != null) {
22510                                                        if(isOut){
22511                                                                relation.setTarget(new TableColumnRelationshipElement(columnModel, valueIndex));
22512                                                        }
22513                                                        else {
22514                                                                relation.addSource(new TableColumnRelationshipElement(columnModel, valueIndex));
22515                                                        }
22516                                                } else {
22517                                                        if(isOut){
22518                                                                relation.setTarget(new TableColumnRelationshipElement(columnModel));
22519                                                        }
22520                                                        else {
22521                                                                relation.addSource(new TableColumnRelationshipElement(columnModel));
22522                                                        }
22523                                                }
22524                                                continue;
22525                                        }
22526                                }
22527                        }
22528                        
22529                        if (columnName.getColumnNameOnly().startsWith("@")
22530                                        && (option.getVendor() == EDbVendor.dbvmssql || option.getVendor() == EDbVendor.dbvazuresql)) {
22531                                continue;
22532                        }
22533
22534                        if (columnName.getColumnNameOnly().startsWith(":")
22535                                        && (option.getVendor() == EDbVendor.dbvhana || option.getVendor() == EDbVendor.dbvteradata)) {
22536                                Table variable = modelManager
22537                                                .getTableByName(DlineageUtil.getTableFullName(columnName.getColumnNameOnly().replace(":", "")));
22538                                if (variable != null) {
22539                                        for (int j = 0; j < variable.getColumns().size(); j++) {
22540                                                if (getColumnName(columnName).replace(":", "")
22541                                                                .equals(getColumnName(variable.getColumns().get(j).getColumnObject()))) {
22542                                                        TableColumn columnModel = variable.getColumns().get(j);
22543                                                        relation.addSource(new TableColumnRelationshipElement(columnModel));
22544                                                }
22545                                        }
22546                                }
22547                                continue;
22548                        }
22549
22550                        boolean linkedFirstTable = false;
22551                        
22552                        TCustomSqlStatement stmt = stmtStack.peek();
22553                        TTableList tableList = stmt.tables;
22554                        if ((tableList == null || tableList.size() == 0) && (hiveFromTables != null && hiveFromTables.size() > 0)) {
22555                                tableList = hiveFromTables;
22556                        }
22557
22558                        List<TTable> tables = new ArrayList<TTable>();
22559                        {
22560                                TTable table = columnName.getSourceTable();
22561
22562                                // 针对CursorVariable特殊处理
22563                                if (columnName.getTableToken() != null) {
22564
22565                                        Table tableModel = null;
22566                                        if (table != null && modelManager.getModel(table) instanceof Table) {
22567                                                tableModel = (Table) modelManager.getModel(table);
22568                                        }
22569
22570                                        if (tableModel == null) {
22571                                                tableModel = modelManager
22572                                                                .getTableByName(DlineageUtil.getTableFullName(columnName.getTableToken().toString()));
22573                                        }
22574
22575                                        if (tableModel == null) {
22576                                                TCustomSqlStatement currentStmt = ModelBindingManager.getGlobalStmtStack().peek();
22577                                                String procedureName = DlineageUtil.getProcedureParentName(currentStmt);
22578                                                String variableString = columnName.getTableToken().toString();
22579                                                if (variableString.startsWith(":")) {
22580                                                        variableString = variableString.substring(variableString.indexOf(":") + 1);
22581                                                }
22582                                                if (!SQLUtil.isEmpty(procedureName)) {
22583                                                        variableString = procedureName + "." + SQLUtil.getIdentifierNormalTableName(variableString);
22584                                                }
22585
22586                                                if (modelManager
22587                                                                .getTableByName(DlineageUtil.getTableFullName(variableString)) instanceof Variable) {
22588                                                        tableModel = modelManager.getTableByName(DlineageUtil.getTableFullName(variableString));
22589                                                }
22590                                        }
22591
22592                                        if (tableModel != null) {
22593
22594                                                if (table == null) {
22595                                                        table = tableModel.getTableObject();
22596                                                }
22597
22598                                                if (tableModel.isVariable()) {
22599                                                        if (!isStar && "*".equals(getColumnName(columnName))) {
22600                                                                TObjectName[] columns = modelManager.getTableColumns(table);
22601                                                                for (int j = 0; j < columns.length; j++) {
22602                                                                        TObjectName objectName = columns[j];
22603                                                                        if (objectName == null || "*".equals(getColumnName(objectName))) {
22604                                                                                continue;
22605                                                                        }
22606                                                                        TableColumn columnModel = modelFactory.createTableColumn(tableModel, objectName,
22607                                                                                        false);
22608                                                                        relation.addSource(new TableColumnRelationshipElement(columnModel));
22609                                                                }
22610                                                        } else {
22611                                                                if ("*".equals(getColumnName(columnName)) && !tableModel.getColumns().isEmpty()) {
22612
22613                                                                        for (int j = 0; j < tableModel.getColumns().size(); j++) {
22614                                                                                TableColumn columnModel = tableModel.getColumns().get(j);
22615                                                                                if (exceptColumnList != null) {
22616                                                                                        boolean flag = false;
22617                                                                                        for (TObjectName objectName : exceptColumnList) {
22618                                                                                                if (getColumnName(objectName)
22619                                                                                                                .equals(getColumnName(columnModel.getColumnObject()))) {
22620                                                                                                        flag = true;
22621                                                                                                        break;
22622                                                                                                }
22623                                                                                        }
22624                                                                                        if (flag) {
22625                                                                                                continue;
22626                                                                                        }
22627                                                                                }
22628                                                                                relation.addSource(new TableColumnRelationshipElement(columnModel));
22629                                                                        }
22630
22631                                                                        if (isStar && showStar) {
22632                                                                                TableColumn columnModel = modelFactory.createTableColumn(tableModel, columnName,
22633                                                                                                false);
22634                                                                                if (columnModel == null) {
22635                                                                                        if (tableModel.isCreateTable()) {
22636                                                                                                for (TableColumn tableColumn : tableModel.getColumns()) {
22637                                                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
22638                                                                                                        relation.setShowStarRelation(true);
22639                                                                                                }
22640                                                                                        }
22641                                                                                } else {
22642                                                                                        relation.addSource(new TableColumnRelationshipElement(columnModel));
22643                                                                                        relation.setShowStarRelation(true);
22644                                                                                }
22645                                                                        }
22646                                                                } else {
22647                                                                        TableColumn columnModel = modelFactory.createTableColumn(tableModel, columnName,
22648                                                                                        false);
22649                                                                        
22650                                                                        if(columnModel == null && containStarColumn(tableModel.getColumns())){
22651                                                                                columnModel = getStarColumn(tableModel.getColumns());
22652                                                                                if (columnModel != null && tableModel.getSubType() == SubType.record_type) {
22653                                                                                        if(!"*".equals(getColumnName(columnName))){
22654                                                                                                columnModel.bindStarLinkColumn(columnName);
22655                                                                                        }
22656                                                                                }
22657                                                                        }
22658                                                                        
22659                                                                        if (columnModel == null) {
22660                                                                                continue;
22661                                                                        }
22662                                                                        if (columnModel.hasStarLinkColumn()) {
22663                                                                                relation.addSource(new TableColumnRelationshipElement(columnModel,
22664                                                                                                columnModel.getStarLinkColumnNames()
22665                                                                                                                .indexOf(DlineageUtil.getColumnName(columnName))));
22666                                                                        } else {
22667                                                                                if (isOut) {
22668                                                                                        relation.setTarget(new TableColumnRelationshipElement(columnModel));
22669                                                                                } else {
22670                                                                                        relation.addSource(new TableColumnRelationshipElement(columnModel));
22671                                                                                }
22672                                                                        }
22673                                                                        if (columnName.getSourceTable() != null
22674                                                                                        && columnName.getSourceTable().getTableType() == ETableSource.function) {
22675                                                                                analyzeTableColumn(columnModel, columnName.getSourceTable().getFuncCall(),
22676                                                                                                effectType);
22677                                                                        }
22678                                                                }
22679                                                        }
22680                                                        continue;
22681                                                }
22682                                        }
22683                                }
22684
22685                                if (table == null) {
22686                                        table = modelManager.getTable(stmt, columnName);
22687                                }
22688
22689                                if (table == null) {
22690                                        if (columnName.getTableToken() != null || !"*".equals(getColumnName(columnName))) {
22691                                                table = columnName.getSourceTable();
22692                                        }
22693                                        
22694                                        if (table == null && !SQLUtil.isEmpty(columnName.getTableString())) {
22695                                                table = modelManager.getTableFromColumn(columnName);
22696                                        }
22697                                }
22698
22699                                if (table == null) {
22700                                        if (tableList != null) {
22701                                                for (int j = 0; j < tableList.size(); j++) {
22702                                                        if (table != null)
22703                                                                break;
22704
22705                                                        TTable tTable = tableList.getTable(j);
22706                                                        if (tTable.getTableType().name().startsWith("open")) {
22707                                                                continue;
22708                                                        }
22709
22710                                                        if (getTableLinkedColumns(tTable) != null && getTableLinkedColumns(tTable).size() > 0) {
22711                                                                for (int z = 0; z < getTableLinkedColumns(tTable).size(); z++) {
22712                                                                        TObjectName refer = getTableLinkedColumns(tTable).getObjectName(z);
22713                                                                        if ("*".equals(getColumnName(refer)))
22714                                                                                continue;
22715                                                                        // For BigQuery struct field access, match base column name from FieldPath
22716                                                                        String structFullName = getStructFieldFullName(columnName);
22717                                                                        if (structFullName != null) {
22718                                                                                String baseName = getStructFieldBaseName(columnName);
22719                                                                                if (baseName != null && DlineageUtil.getIdentifierNormalColumnName(getColumnName(refer))
22720                                                                                                .equals(DlineageUtil.getIdentifierNormalColumnName(baseName))) {
22721                                                                                        table = tTable;
22722                                                                                        break;
22723                                                                                }
22724                                                                        }
22725                                                                        if (getColumnName(refer).equals(getColumnName(columnName))) {
22726                                                                                table = tTable;
22727                                                                                break;
22728                                                                        }
22729                                                                }
22730                                                        }
22731
22732                                                        if (tTable.getLinkTable() != null) {
22733                                                                tTable = tTable.getLinkTable();
22734                                                                for (int z = 0; z < getTableLinkedColumns(tTable).size(); z++) {
22735                                                                        TObjectName refer = getTableLinkedColumns(tTable).getObjectName(z);
22736                                                                        if ("*".equals(getColumnName(refer)))
22737                                                                                continue;
22738                                                                        if (getColumnName(refer).equals(getColumnName(columnName))) {
22739                                                                                table = tTable;
22740                                                                                break;
22741                                                                        }
22742                                                                }
22743                                                        }
22744
22745                                                        if (table != null)
22746                                                                break;
22747
22748                                                        if (columnName.getTableToken() != null && (columnName.getTableToken().getAstext()
22749                                                                        .equalsIgnoreCase(tTable.getName())
22750                                                                        || columnName.getTableToken().getAstext().equalsIgnoreCase(tTable.getAliasName()))) {
22751                                                                table = tTable;
22752                                                                break;
22753                                                        }
22754                                                }
22755                                        }
22756
22757                                        if (table == null) {
22758                                                for (int j = 0; j < tableList.size(); j++) {
22759                                                        if (table != null)
22760                                                                break;
22761
22762                                                        TTable tTable = tableList.getTable(j);
22763                                                        Object model = ModelBindingManager.get().getModel(tTable);
22764                                                        if (model instanceof Table) {
22765                                                                Table tableModel = (Table) model;
22766                                                                for (int z = 0; tableModel.getColumns() != null
22767                                                                                && z < tableModel.getColumns().size(); z++) {
22768                                                                        TableColumn refer = tableModel.getColumns().get(z);
22769                                                                        if (getColumnName(refer.getName()).equals(getColumnName(columnName))) {
22770                                                                                table = tTable;
22771                                                                                break;
22772                                                                        }
22773                                                                        if (refer.hasStarLinkColumn()) {
22774                                                                                for (TObjectName linkColumn : refer.getStarLinkColumnList()) {
22775                                                                                        if (getColumnName(linkColumn).equals(getColumnName(columnName))) {
22776                                                                                                table = tTable;
22777                                                                                                break;
22778                                                                                        }
22779                                                                                }
22780                                                                        }
22781                                                                }
22782                                                        } else if (model instanceof QueryTable) {
22783                                                                QueryTable tableModel = (QueryTable) model;
22784                                                                for (int z = 0; tableModel.getColumns() != null
22785                                                                                && z < tableModel.getColumns().size(); z++) {
22786                                                                        ResultColumn refer = tableModel.getColumns().get(z);
22787                                                                        // Try FieldPath-based matching first
22788                                                                        String structFullName = getStructFieldFullName(columnName);
22789                                                                        if (structFullName != null) {
22790                                                                                String baseName = getStructFieldBaseName(columnName);
22791                                                                                if (baseName != null && DlineageUtil.getIdentifierNormalColumnName(refer.getName())
22792                                                                                                .equals(DlineageUtil.getIdentifierNormalColumnName(baseName))) {
22793                                                                                        table = tTable;
22794                                                                                        break;
22795                                                                                }
22796                                                                        }
22797                                                                        List<String> splits = SQLUtil.parseNames(columnName.toString());
22798                                                                        if (splits.size() > 1 && EDbVendor.dbvbigquery == getOption().getVendor()) {
22799                                                                                if (DlineageUtil.getIdentifierNormalColumnName(refer.getName())
22800                                                                                                .equals(DlineageUtil
22801                                                                                                                .getIdentifierNormalColumnName(getColumnName(splits.get(0))))) {
22802                                                                                        table = tTable;
22803                                                                                        break;
22804                                                                                }
22805                                                                        }
22806                                                                        else if (DlineageUtil.getIdentifierNormalColumnName(refer.getName()).equals(
22807                                                                                        DlineageUtil.getIdentifierNormalColumnName(getColumnName(columnName)))) {
22808                                                                                table = tTable;
22809                                                                                break;
22810                                                                        }
22811                                                                        if (refer.hasStarLinkColumn()) {
22812                                                                                for (TObjectName linkColumn : refer.getStarLinkColumnList()) {
22813                                                                                        if (getColumnName(linkColumn).equals(getColumnName(columnName))) {
22814                                                                                                table = tTable;
22815                                                                                                break;
22816                                                                                        }
22817                                                                                }
22818                                                                        }
22819                                                                }
22820                                                        }
22821                                                }
22822                                        }
22823                                }
22824
22825                                if (columnName.getTableToken() == null && "*".equals(getColumnName(columnName))) {
22826                                        if (!hasJoin(stmt)) {
22827                                                tables.add(table);
22828                                        } else {
22829                                                for (int j = 0; j < tableList.size(); j++) {
22830                                                        tables.add(tableList.getTable(j));
22831                                                }
22832                                        }
22833                                } else if (table != null) {
22834                                        tables.add(table);
22835                                }
22836
22837                                // 此处特殊处理，多表关联无法找到 column 所属的 Table, tTable.getLinkedColumns
22838                                // 也找不到，退而求其次采用第一个表
22839
22840                                if (stmt.getParentStmt() != null && isApplyJoin(stmt.getParentStmt())
22841                                                && (tableList != null && tableList.size() > 0)) {
22842                                        stmt = stmt.getParentStmt();
22843                                        TTable applyTable = tableList.getTable(0);
22844                                        if (modelManager.getModel(table) == null) {
22845                                                modelFactory.createTable(applyTable);
22846                                        }
22847                                }
22848
22849                                if (columnName.toString().indexOf(".")==-1 && isConstant(columnName)) {
22850                                        boolean isConstant = true;
22851                                        if (columnName.getSourceTable() != null) {
22852                                                Table tableModel = modelManager.getTableByName(
22853                                                                DlineageUtil.getTableFullName(columnName.getSourceTable().getTableName().toString()));
22854                                                if (tableModel != null && tableModel.getColumns() != null) {
22855                                                        for (int j = 0; j < tableModel.getColumns().size(); j++) {
22856                                                                if (DlineageUtil.compareColumnIdentifier(getColumnName(columnName),
22857                                                                                getColumnName(tableModel.getColumns().get(j).getName()))) {
22858                                                                        isConstant = false;
22859                                                                        break;
22860                                                                }
22861                                                        }
22862                                                }
22863                                        }
22864
22865                                        if (isConstant) {
22866                                                if (option.isShowConstantTable()) {
22867                                                        Table constantTable = modelFactory.createConstantsTable(stmtStack.peek());
22868                                                        TableColumn tableColumn = modelFactory.createTableColumn(constantTable, columnName, false);
22869                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
22870                                                }
22871                                                continue;
22872                                        }
22873                                }
22874
22875                                if (tableList != null && tableList.size() != 0 && tables.size() == 0
22876                                                && !(isBuiltInFunctionName(columnName) && isFromFunction(columnName))) {
22877                                        if (modelManager.getModel(stmt) instanceof ResultSet) {
22878                                                ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt);
22879                                                boolean find = false;
22880                                                for (ResultColumn resultColumn : resultSetModel.getColumns()) {
22881                                                        if(resultColumn.equals(modelObject)) {
22882                                                                continue;
22883                                                        }
22884                                                        if (!TSQLEnv.isAliasReferenceForbidden.get(option.getVendor())) {
22885                                                                if (getColumnName(columnName).equals(getColumnName(resultColumn.getName()))) {
22886                                                                        if (resultColumn.getColumnObject() != null) {
22887                                                                                int startToken = resultColumn.getColumnObject().getStartToken().posinlist;
22888                                                                                int endToken = resultColumn.getColumnObject().getEndToken().posinlist;
22889                                                                                if (columnName.getStartToken().posinlist >= startToken
22890                                                                                                && columnName.getEndToken().posinlist <= endToken) {
22891                                                                                        continue;
22892                                                                                }
22893                                                                        }
22894                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
22895                                                                        find = true;
22896                                                                        break;
22897                                                                }
22898                                                        }
22899                                                }
22900                                                if (find) {
22901                                                        continue;
22902                                                }
22903                                        }
22904
22905                                        // Structured-dataflow outer field resolution: an unresolved
22906                                        // reference like nodes1.key (or t.nodes1.key) may be a
22907                                        // struct-field projection produced by a structured generator
22908                                        // (e.g. Spark explode(from_json(...))) in a visible subquery.
22909                                        // Link directly to the matching FunctionResultColumn so the
22910                                        // exact source path (nodes[*].key) flows through.
22911                                        if (tryAppendStructuredFieldRelation(relation, columnName, tableList)) {
22912                                                continue;
22913                                        }
22914
22915                                        TObjectName pseudoTableName = new TObjectName();
22916                                        // Use qualified prefix from column name if available (e.g., sch.pk_constv2 from sch.pk_constv2.c_cdsl)
22917                                        // Otherwise fall back to default pseudo table name
22918                                        String qualifiedPrefix = getQualifiedPrefixFromColumn(columnName);
22919                                        pseudoTableName.setString(qualifiedPrefix != null ? qualifiedPrefix : "pseudo_table_include_orphan_column");
22920                                        Table pseudoTable = modelFactory.createTableByName(pseudoTableName);
22921                                        pseudoTable.setPseudo(true);
22922                                        TableColumn pseudoTableColumn = modelFactory.createTableColumn(pseudoTable, columnName, true);
22923
22924                                        // If not linking to first table and column has qualified prefix (3-part name like sch.pkg.col),
22925                                        // add the pseudo table column as source
22926                                        if (!isLinkOrphanColumnToFirstTable() && pseudoTableColumn != null && qualifiedPrefix != null) {
22927                                                if (isOut) {
22928                                                        relation.setTarget(new TableColumnRelationshipElement(pseudoTableColumn));
22929                                                } else {
22930                                                        relation.addSource(new TableColumnRelationshipElement(pseudoTableColumn));
22931                                                }
22932                                        }
22933
22934                                        if (isLinkOrphanColumnToFirstTable()) {
22935                                                TTable orphanTable = tableList.getTable(0);
22936                                                tables.add(orphanTable);
22937                                                Object tableModel = modelManager.getModel(orphanTable);
22938                                                if (tableModel == null) {
22939                                                        if(orphanTable.getSubquery()!=null) {
22940                                                                QueryTable queryTable = modelFactory.createQueryTable(orphanTable);
22941                                                                TSelectSqlStatement subquery = orphanTable.getSubquery();
22942                                                                analyzeSelectStmt(subquery);
22943                                                        }
22944                                                        else {
22945                                                                tableModel = modelFactory.createTable(orphanTable);
22946                                                        }
22947                                                }
22948                                                if (tableModel instanceof Table) {
22949                                                        TableColumn orphanColum = modelFactory.createTableColumn((Table) tableModel, columnName, false);
22950                                                        if(orphanColum!=null) {
22951                                                                ErrorInfo errorInfo = new ErrorInfo();
22952                                                                errorInfo.setErrorType(ErrorInfo.LINK_ORPHAN_COLUMN);
22953                                                                errorInfo.setErrorMessage("Link orphan column [" + columnName.toString()
22954                                                                                + "] to the first table [" + orphanTable.getFullNameWithAliasString() + "]");
22955                                                                errorInfo.setStartPosition(new Pair3<Long, Long, String>(columnName.getStartToken().lineNo,
22956                                                                                columnName.getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
22957                                                                errorInfo.setEndPosition(new Pair3<Long, Long, String>(columnName.getEndToken().lineNo,
22958                                                                                columnName.getEndToken().columnNo + columnName.getEndToken().getAstext().length(),
22959                                                                                ModelBindingManager.getGlobalHash()));
22960                                                                errorInfo.fillInfo(this);
22961                                                                errorInfos.add(errorInfo);
22962                                                        }
22963                                                        if (orphanTable.getSubquery() != null) {
22964                                                                TSelectSqlStatement subquery = orphanTable.getSubquery();
22965                                                                if (subquery.getResultColumnList().toString().endsWith("*") && subquery.getTables().size() == 1) {
22966                                                                        TTable subqueryTable = subquery.getTables().getTable(0);
22967                                                                        Object sourceTable = modelManager.getModel(subqueryTable);
22968                                                                        if(sourceTable instanceof Table) {
22969                                                                                modelFactory.createTableColumn((Table) sourceTable, columnName, false);
22970                                                                        }
22971                                                                        else if(sourceTable instanceof ResultSet) {
22972                                                                                modelFactory.createResultColumn((ResultSet) sourceTable, columnName, false);
22973                                                                        }
22974                                                                }
22975                                                        }
22976                                                        else if (orphanTable.getCTE()!=null && orphanTable.getCTE().getSubquery() != null) {
22977                                                                TSelectSqlStatement subquery = orphanTable.getCTE().getSubquery();
22978                                                                if (subquery.getResultColumnList().toString().endsWith("*") && subquery.getTables().size() == 1) {
22979                                                                        TTable subqueryTable = subquery.getTables().getTable(0);
22980                                                                        Object sourceTable = modelManager.getModel(subqueryTable);
22981                                                                        if(sourceTable instanceof Table) {
22982                                                                                modelFactory.createTableColumn((Table) sourceTable, columnName, false);
22983                                                                        }
22984                                                                        else if(sourceTable instanceof ResultSet) {
22985                                                                                modelFactory.createResultColumn((ResultSet) sourceTable, columnName, false);
22986                                                                        }
22987                                                                }
22988                                                        }
22989                                                }
22990                                                
22991                                                linkedFirstTable = true;
22992                                        }
22993                                }
22994                        }
22995
22996                        for (int k = 0; k < tables.size(); k++) {
22997                                TTable table = tables.get(k);
22998                                if (table != null) {
22999                                        Object object = modelManager.getModel(table);
23000                                        if(object instanceof PivotedTable) {
23001                                                TPivotClause clause = (TPivotClause)((PivotedTable)object).getGspObject();
23002                                                if(clause.getAliasClause()!=null) {
23003                                                        object = modelManager.getModel(clause.getAliasClause());
23004                                                }
23005                                        }
23006                                        if (object == null && table.getTableType() == ETableSource.objectname) {
23007                                                if (table.getCTE() != null) {
23008                                                        QueryTable queryTable = modelFactory.createQueryTable(table);
23009                                                        TSelectSqlStatement subquery = table.getCTE().getSubquery();
23010                                                        analyzeSelectStmt(subquery);
23011                                                        ResultSet resultSetModel = (ResultSet) modelManager.getModel(subquery);
23012
23013                                                        if (resultSetModel != null && resultSetModel != queryTable
23014                                                                        && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
23015                                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
23016                                                                impactRelation.setEffectType(EffectType.select);
23017                                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
23018                                                                                resultSetModel.getRelationRows()));
23019                                                                impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
23020                                                                                queryTable.getRelationRows()));
23021                                                        }
23022
23023                                                        if (resultSetModel != null && resultSetModel != queryTable) {
23024                                                                for (int j = 0; j < resultSetModel.getColumns().size(); j++) {
23025                                                                        ResultColumn sourceColumn = resultSetModel.getColumns().get(j);
23026                                                                        ResultColumn targetColumn = modelFactory.createSelectSetResultColumn(queryTable,
23027                                                                                        sourceColumn);
23028
23029                                                                        DataFlowRelationship queryRalation = modelFactory.createDataFlowRelation();
23030                                                                        queryRalation.setEffectType(EffectType.select);
23031                                                                        queryRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
23032                                                                        queryRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
23033                                                                }
23034                                                        }
23035                                                        
23036                                                        object = queryTable;
23037                                                        
23038                                                } else {
23039                                                        object = modelFactory.createTable(table);
23040                                                }
23041                                        }
23042                                        if (object instanceof Function) {
23043                                                relation.addSource(new ResultColumnRelationshipElement(((Function)object).getColumns().get(0)));
23044                                                continue;
23045                                        } else if (object instanceof ResultSet && !(object instanceof QueryTable)) {
23046                                                //Object tableModel = modelManager.getModel(columnName.getSourceTable());
23047                                                appendResultColumnRelationSource(modelObject, relation, columnIndex, columnName,
23048                                                                (ResultSet)object);
23049                                                if(table.getTableType() == ETableSource.function && !relation.getSources().isEmpty()) {
23050                                                        relation.getSources().stream().reduce((first, second) -> second).get().addTransform(Transform.FUNCTION, table);
23051                                                }
23052                                                continue;
23053                                        }
23054                                        else if (object instanceof Table) {
23055                                                Table tableModel = (Table) modelManager.getModel(table);
23056                                                if (tableModel != null) {
23057                                                        if (!isStar && "*".equals(getColumnName(columnName))) {
23058                                                                TObjectName[] columns = modelManager.getTableColumns(table);
23059                                                                for (int j = 0; j < columns.length; j++) {
23060                                                                        TObjectName objectName = columns[j];
23061                                                                        if (objectName == null || ("*".equals(getColumnName(objectName)) && tableModel.isDetermined())) {
23062                                                                                continue;
23063                                                                        }
23064                                                                        TableColumn columnModel = modelFactory.createTableColumn(tableModel, objectName,
23065                                                                                        false);
23066                                                                        if(columnModel == null) {
23067                                                                                continue;
23068                                                                        }
23069                                                                        relation.addSource(new TableColumnRelationshipElement(columnModel));
23070                                                                }
23071                                                        } else {
23072                                                                if ("*".equals(getColumnName(columnName)) && !tableModel.getColumns().isEmpty()) {
23073                                                                        Map<String, Pair<String, TExpression>> replaceAsIdentifierMap = new HashMap<String, Pair<String, TExpression>>();
23074                                                                        Map<String, TObjectName> replaceColumnMap = new HashMap<String, TObjectName>();
23075                                                                        if(modelObject instanceof ResultColumn && ((ResultColumn)modelObject).getColumnObject() instanceof TResultColumn) {
23076                                                                                TResultColumn resultColumn =  (TResultColumn )((ResultColumn)modelObject).getColumnObject();
23077                                                                                if(resultColumn.getReplaceExprAsIdentifiers()!=null && resultColumn.getReplaceExprAsIdentifiers().size()>0) {
23078                                                                                        for(TReplaceExprAsIdentifier replace: resultColumn.getReplaceExprAsIdentifiers()) {
23079                                                                                                replaceAsIdentifierMap.put(replace.getIdentifier().toString(), new Pair<String, TExpression>(resultColumn.getExpr().getExceptReplaceClause().toString(), replace.getExpr()));
23080                                                                                                replaceColumnMap.put(replace.getIdentifier().toString(), replace.getIdentifier());
23081                                                                                        }
23082                                                                                        ResultSet resultSet = ((ResultColumn)modelObject).getResultSet();
23083                                                                                        if(tableModel.isDetermined()) {
23084                                                                                                resultSet.getColumns().clear();
23085                                                                                        }
23086                                                                                }
23087                                                                        }
23088                                                                        
23089                                                                        for (int j = 0; j < tableModel.getColumns().size(); j++) {
23090                                                                                TableColumn columnModel = tableModel.getColumns().get(j);
23091                                                                                if (exceptColumnList != null) {
23092                                                                                        boolean flag = false;
23093                                                                                        for (TObjectName objectName : exceptColumnList) {
23094                                                                                                if (getColumnName(objectName)
23095                                                                                                                .equals(getColumnName(columnModel.getColumnObject()))) {
23096                                                                                                        flag = true;
23097                                                                                                        break;
23098                                                                                                }
23099                                                                                        }
23100                                                                                        if (flag) {
23101                                                                                                continue;
23102                                                                                        }
23103                                                                                }
23104                                                                                
23105                                                                                if (replaceAsIdentifierMap.containsKey(tableModel.getColumns().get(j).getName())) {
23106                                                                                        Pair<String, TExpression> expr = replaceAsIdentifierMap.get(tableModel.getColumns().get(j).getName());
23107                                                                                        ResultSet resultSet = ((ResultColumn)modelObject).getResultSet();
23108                                                                                        ResultColumn resultColumn = modelFactory.createResultColumn(resultSet, replaceColumnMap.get(tableModel.getColumns().get(j).getName()));
23109                                                                                        Transform transform = new Transform();
23110                                                                                        transform.setType(Transform.EXPRESSION);
23111                                                                                        TObjectName expression = new TObjectName();
23112                                                                                        expression.setString(expr.first);
23113                                                                                transform.setCode(expression);
23114                                                                                        resultColumn.setTransform(transform);
23115                                                                                        analyzeResultColumnExpressionRelation(resultColumn, expr.second);
23116                                                                                } else {
23117                                                                                        if(!replaceAsIdentifierMap.isEmpty()) {
23118                                                                                                ResultSet resultSet = ((ResultColumn) modelObject).getResultSet();
23119                                                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSet,
23120                                                                                                                columnModel.getColumnObject());
23121                                                                                                DataFlowRelationship relation1 = modelFactory.createDataFlowRelation();
23122                                                                                                relation1.setEffectType(effectType);
23123                                                                                                relation1.setProcess(process);
23124                                                                                                relation1.setTarget(new ResultColumnRelationshipElement(resultColumn));
23125                                                                                                relation1.addSource(new TableColumnRelationshipElement(columnModel));
23126                                                                                        }
23127                                                                                        else {
23128                                                                                                relation.addSource(
23129                                                                                                        new TableColumnRelationshipElement(columnModel));
23130                                                                                        }
23131                                                                                }
23132                                                                        }
23133
23134                                                                        if (isStar && showStar) {
23135                                                                                TableColumn columnModel = modelFactory.createTableColumn(tableModel, columnName,
23136                                                                                                false);
23137                                                                                if (columnModel == null) {
23138                                                                                        if(tableModel.isCreateTable()) {
23139                                                                                                for (TableColumn tableColumn : tableModel.getColumns()) {
23140                                                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
23141                                                                                                        relation.setShowStarRelation(true);
23142                                                                                                }
23143                                                                                        }
23144                                                                                } else {
23145                                                                                        relation.addSource(new TableColumnRelationshipElement(columnModel));
23146                                                                                        relation.setShowStarRelation(true);
23147                                                                                }
23148                                                                        }
23149                                                                } else {
23150                                                                        TableColumn columnModel = modelFactory.createTableColumn(tableModel, columnName,
23151                                                                                        false);
23152                                                                        if(columnModel == null) {
23153                                                                                if(tableModel.isCreateTable()) {
23154                                                                                        boolean flag = false;
23155                                                                                        // Try FieldPath-based matching first for BigQuery/Redshift struct fields
23156                                                                                        String structFullName = getStructFieldFullName(columnName);
23157                                                                                        if (structFullName != null && !flag) {
23158                                                                                                for (TableColumn tableColumn : tableModel.getColumns()) {
23159                                                                                                        if (tableColumn.isStruct()
23160                                                                                                                && DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName())
23161                                                                                                                        .equals(DlineageUtil.getIdentifierNormalColumnName(structFullName))) {
23162                                                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
23163                                                                                                                flag = true;
23164                                                                                                                if (modelObject instanceof ResultColumn) {
23165                                                                                                                        ((ResultColumn) modelObject).setStruct(true);
23166                                                                                                                } else if (modelObject instanceof TableColumn) {
23167                                                                                                                        ((TableColumn) modelObject).setStruct(true);
23168                                                                                                                }
23169                                                                                                                break;
23170                                                                                                        }
23171                                                                                                }
23172                                                                                        }
23173                                                                                        if (ModelBindingManager.getGlobalVendor() == EDbVendor.dbvbigquery || ModelBindingManager.getGlobalVendor() == EDbVendor.dbvredshift) {
23174                                                                                                for (TableColumn tableColumn : tableModel.getColumns()) {
23175                                                                                                        if(tableColumn.isStruct()) {
23176                                                                                                                List<String> names = SQLUtil.parseNames(tableColumn.getName());
23177                                                                                                                if (modelObject instanceof TableColumn) {
23178                                                                                                                        TableColumn targetColumn = (TableColumn) modelObject;
23179                                                                                                                        if (targetColumn.isStruct()) {
23180                                                                                                                                List<String> targetNames = SQLUtil
23181                                                                                                                                                .parseNames(targetColumn.getName());
23182                                                                                                                                if (!getColumnName(targetNames.get(0))
23183                                                                                                                                                .equals(getColumnName(names.get(0)))) {
23184                                                                                                                                        continue;
23185                                                                                                                                }
23186                                                                                                                        }
23187                                                                                                                }
23188                                                                                                                else if (modelObject instanceof ResultColumn) {
23189                                                                                                                        ResultColumn targetColumn = (ResultColumn) modelObject;
23190                                                                                                                        if (targetColumn.isStruct()) {
23191                                                                                                                                List<String> targetNames = SQLUtil
23192                                                                                                                                                .parseNames(targetColumn.getName());
23193                                                                                                                                if (!getColumnName(targetNames.get(0))
23194                                                                                                                                                .equals(getColumnName(names.get(0)))) {
23195                                                                                                                                        continue;
23196                                                                                                                                }
23197                                                                                                                        }
23198                                                                                                                }
23199                                                                                                                for(String name: names) {
23200                                                                                                                        if (getColumnName(name)
23201                                                                                                                                        .equals(getColumnName(modelObject.toString()))) {
23202                                                                                                                                relation.addSource(
23203                                                                                                                                                new TableColumnRelationshipElement(tableColumn));
23204                                                                                                                                flag = true;
23205                                                                                                                                if (modelObject instanceof ResultColumn) {
23206                                                                                                                                        ((ResultColumn)modelObject).setStruct(true);
23207                                                                                                                                }
23208                                                                                                                                else if (modelObject instanceof TableColumn) {
23209                                                                                                                                        ((TableColumn)modelObject).setStruct(true);
23210                                                                                                                                }
23211                                                                                                                                break;
23212                                                                                                                        }
23213                                                                                                                }
23214
23215                                                                                                                if (!flag && tableModel.getColumns().size() == 1 && tableModel
23216                                                                                                                                .getColumns().get(0).getSourceColumn() != null) {
23217                                                                                                                        TableColumn sourceColumn = tableModel
23218                                                                                                                                        .getColumns().get(0).getSourceColumn();
23219                                                                                                                        Table sourceTable = sourceColumn.getTable();
23220                                                                                                                        TObjectName sourceColumnName = new TObjectName();
23221                                                                                                                        sourceColumnName.setString(sourceColumn.getName() + "."
23222                                                                                                                                        + columnName.getColumnNameOnly());
23223                                                                                                                        TableColumn sourceTableColumn = modelFactory.createTableColumn(sourceTable, sourceColumnName, true);
23224                                                                                                                        relation.addSource(
23225                                                                                                                                        new TableColumnRelationshipElement(sourceTableColumn));
23226                                                                                                                        flag = true;
23227                                                                                                                        break;
23228                                                                                                                }
23229                                                                                                        }
23230                                                                                                        else if (getColumnName(tableColumn.getName())
23231                                                                                                                        .equals(getColumnName(modelObject.toString()))) {
23232                                                                                                                relation.addSource(
23233                                                                                                                                new TableColumnRelationshipElement(tableColumn));
23234                                                                                                                flag = true;
23235                                                                                                                break;
23236                                                                                                        }
23237                                                                                                }
23238                                                                                                if (modelObject instanceof TableColumn) {
23239                                                                                                        TableColumn column = (TableColumn) modelObject;
23240                                                                                                        for (TableColumn tableColumn : tableModel.getColumns()) {
23241                                                                                                                if (tableColumn.getColumnIndex() == null) {
23242                                                                                                                        continue;
23243                                                                                                                }
23244                                                                                                                if (tableColumn.getName().toLowerCase()
23245                                                                                                                                .indexOf(column.getName().toLowerCase()) == -1
23246                                                                                                                                && tableColumn.getName().toLowerCase()
23247                                                                                                                                                .indexOf(column.getColumnObject().toString()
23248                                                                                                                                                                .toLowerCase()) == -1) {
23249                                                                                                                        continue;
23250                                                                                                                }
23251                                                                                                                flag = true;
23252                                                                                                                relation.addSource(
23253                                                                                                                                new TableColumnRelationshipElement(tableColumn));
23254                                                                                                                relation.setShowStarRelation(true);
23255                                                                                                        }
23256                                                                                                        if (flag)
23257                                                                                                                break;
23258
23259                                                                                                } else if (modelObject instanceof ResultColumn) {
23260                                                                                                        ResultColumn column = (ResultColumn) modelObject;
23261                                                                                                        for (TableColumn tableColumn : tableModel.getColumns()) {
23262                                                                                                                if (tableColumn.getColumnIndex() == null) {
23263                                                                                                                        continue;
23264                                                                                                                }
23265                                                                                                                if (tableColumn.getName().toLowerCase()
23266                                                                                                                                .indexOf(column.getName().toLowerCase()) == -1
23267                                                                                                                                && tableColumn.getName().toLowerCase()
23268                                                                                                                                                .indexOf(column.getColumnObject().toString()
23269                                                                                                                                                                .toLowerCase()) == -1) {
23270                                                                                                                        continue;
23271                                                                                                                }
23272                                                                                                                flag = true;
23273                                                                                                                relation.addSource(
23274                                                                                                                                new TableColumnRelationshipElement(tableColumn));
23275                                                                                                                relation.setShowStarRelation(true);
23276                                                                                                        }
23277                                                                                                        if (flag)
23278                                                                                                                break;
23279
23280                                                                                                }
23281                                                                                        }
23282                                                                                        if (!flag 
23283                                                                                                        && (isStar 
23284                                                                                                                        || getColumnName(columnName).equals(getColumnName(tableModel.getName()))
23285                                                                                                                        || getColumnName(columnName).equals(getColumnName(tableModel.getAlias())))) {
23286                                                                                                for (TableColumn tableColumn : tableModel.getColumns()) {
23287                                                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
23288                                                                                                        relation.setShowStarRelation(true);
23289                                                                                                }
23290                                                                                        }
23291                                                                                }
23292                                                                        }
23293                                                                        else {
23294                                                                                if (linkedFirstTable || columnModel.getCandidateParents() != null) {
23295                                                                                        if (columnName.getCandidateTables() != null
23296                                                                                                        && columnName.getCandidateTables().size() > 1) {
23297                                                                                                List<Object> candidateParents = new ArrayList<Object>();
23298                                                                                                for(TTable tableItem: columnName.getCandidateTables()) {
23299                                                                                                        Object model = modelManager.getModel(tableItem);
23300                                                                                                        if(model!=null) {
23301                                                                                                                candidateParents.add(model);
23302                                                                                                        }
23303                                                                                                }
23304                                                                                                if (candidateParents.size() > 1) {
23305                                                                                                        columnModel.setCandidateParents(candidateParents);
23306                                                                                                }
23307                                                                                        }
23308                                                                                }
23309                                                                                relation.addSource(new TableColumnRelationshipElement(columnModel));
23310                                                                                relation.setShowStarRelation(true);
23311                                                                                if(modelObject instanceof TableColumn) {
23312                                                                                        TableColumn targetTableColumn = (TableColumn)modelObject;
23313                                                                                        if(targetTableColumn.getTable().getSubType() == SubType.unnest && targetTableColumn.getTable().getColumns().size() == 1) {
23314                                                                                                targetTableColumn.setSourceColumn(columnModel);
23315                                                                                                targetTableColumn.setStruct(true);
23316                                                                                        }
23317                                                                                }
23318                                                                                if (columnName.getSourceTable() != null
23319                                                                                                && columnName.getSourceTable().getTableType() == ETableSource.function) {
23320                                                                                        analyzeTableColumn(columnModel, columnName.getSourceTable().getFuncCall(),
23321                                                                                                        effectType);
23322                                                                                }
23323                                                                        }
23324                                                                }
23325                                                        }
23326                                                }
23327                                        } else if (modelManager.getModel(table) instanceof QueryTable) {
23328                                                QueryTable queryTable = (QueryTable) modelManager.getModel(table);
23329
23330                                                TObjectNameList cteColumns = null;
23331                                                TSelectSqlStatement subquery = null;
23332                                                if (queryTable.getTableObject().getCTE() != null) {
23333                                                        subquery = queryTable.getTableObject().getCTE().getSubquery();
23334                                                        cteColumns = queryTable.getTableObject().getCTE().getColumnList();
23335                                                } else if (queryTable.getTableObject().getAliasClause() != null
23336                                                                && queryTable.getTableObject().getAliasClause().getColumns() != null) {
23337
23338                                                } else if (queryTable.getTableObject().getTableExpr() != null
23339                                                                && queryTable.getTableObject().getTableExpr().getSubQuery() != null) {
23340                                                        subquery = queryTable.getTableObject().getTableExpr().getSubQuery();
23341                                                } else {
23342                                                        subquery = queryTable.getTableObject().getSubquery();
23343                                                }
23344
23345                                                if (cteColumns != null) {
23346                                                        for (int j = 0; j < cteColumns.size(); j++) {
23347                                                                modelFactory.createResultColumn(queryTable, cteColumns.getObjectName(j));
23348                                                        }
23349                                                }
23350
23351                                                if (subquery != null && subquery.isCombinedQuery()) {
23352                                                        SelectSetResultSet selectSetResultSetModel = (SelectSetResultSet) modelManager
23353                                                                        .getModel(subquery);
23354
23355                                                        if (selectSetResultSetModel != null
23356                                                                        && !selectSetResultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
23357                                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
23358                                                                impactRelation.setEffectType(EffectType.select);
23359                                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
23360                                                                                selectSetResultSetModel.getRelationRows()));
23361                                                                impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
23362                                                                                queryTable.getRelationRows()));
23363                                                        }
23364
23365                                                        if (selectSetResultSetModel != null) {
23366                                                                if (getColumnName(columnName).equals("*")) {
23367                                                                        Map<String, Pair<String, TExpression>> replaceAsIdentifierMap = new HashMap<String, Pair<String, TExpression>>();
23368                                                                        Map<String, TObjectName> replaceColumnMap = new HashMap<String, TObjectName>();
23369                                                                        if(modelObject instanceof ResultColumn && ((ResultColumn)modelObject).getColumnObject() instanceof TResultColumn) {
23370                                                                                TResultColumn resultColumn =  (TResultColumn )((ResultColumn)modelObject).getColumnObject();
23371                                                                                if(resultColumn.getReplaceExprAsIdentifiers()!=null && resultColumn.getReplaceExprAsIdentifiers().size()>0) {
23372                                                                                        for(TReplaceExprAsIdentifier replace: resultColumn.getReplaceExprAsIdentifiers()) {
23373                                                                                                replaceAsIdentifierMap.put(replace.getIdentifier().toString(), new Pair<String, TExpression>(resultColumn.getExpr().getExceptReplaceClause().toString(), replace.getExpr()));
23374                                                                                                replaceColumnMap.put(replace.getIdentifier().toString(), replace.getIdentifier());
23375                                                                                        }
23376                                                                                        ResultSet resultSet = ((ResultColumn)modelObject).getResultSet();
23377                                                                                        if(selectSetResultSetModel.isDetermined()) {
23378                                                                                                resultSet.getColumns().clear();
23379                                                                                        }
23380                                                                                }
23381                                                                        }
23382                                                                        
23383                                                                        
23384                                                                        for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) {
23385                                                                                ResultColumn sourceColumn = selectSetResultSetModel.getColumns().get(j);
23386                                                                                if (cteColumns != null) {
23387                                                                                        if (j < cteColumns.size()) {
23388                                                                                                ResultColumn targetColumn = queryTable.getColumns().get(j);
23389
23390                                                                                                if (exceptColumnList != null) {
23391                                                                                                        boolean flag = false;
23392                                                                                                        for (TObjectName objectName : exceptColumnList) {
23393                                                                                                                if (getColumnName(objectName)
23394                                                                                                                                .equals(getColumnName(targetColumn.getName()))) {
23395                                                                                                                        flag = true;
23396                                                                                                                        break;
23397                                                                                                                }
23398                                                                                                        }
23399                                                                                                        if (flag) {
23400                                                                                                                continue;
23401                                                                                                        }
23402                                                                                                }
23403
23404                                                                                                if (replaceAsIdentifierMap.containsKey(targetColumn.getName())) {
23405                                                                                                        Pair<String, TExpression> expr = replaceAsIdentifierMap.get(targetColumn.getName());
23406                                                                                                        ResultSet resultSet = ((ResultColumn)modelObject).getResultSet();
23407                                                                                                        ResultColumn resultColumn = modelFactory.createResultColumn(resultSet, replaceColumnMap.get(targetColumn.getName()));
23408                                                                                                        Transform transform = new Transform();
23409                                                                                                        transform.setType(Transform.EXPRESSION);
23410                                                                                                        TObjectName expression = new TObjectName();
23411                                                                                                        expression.setString(expr.first);
23412                                                                                                transform.setCode(expression);
23413                                                                                                        resultColumn.setTransform(transform);
23414                                                                                                        analyzeResultColumnExpressionRelation(resultColumn, expr.second);
23415                                                                                                } else {
23416                                                                                                        if(!replaceAsIdentifierMap.isEmpty()) {
23417                                                                                                                ResultSet resultSet = ((ResultColumn) modelObject).getResultSet();
23418                                                                                                                TObjectName resultColumnName = new TObjectName();
23419                                                                                                                resultColumnName.setString(targetColumn.getName());
23420                                                                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSet,
23421                                                                                                                                resultColumnName);
23422                                                                                                                DataFlowRelationship relation1 = modelFactory.createDataFlowRelation();
23423                                                                                                                relation1.setEffectType(effectType);
23424                                                                                                                relation1.setProcess(process);
23425                                                                                                                relation1.setTarget(new ResultColumnRelationshipElement(resultColumn));
23426                                                                                                                relation1.addSource(new ResultColumnRelationshipElement(targetColumn));
23427                                                                                                        }
23428                                                                                                        else {
23429                                                                                                                relation.addSource(
23430                                                                                                                        new ResultColumnRelationshipElement(targetColumn));
23431                                                                                                        }
23432                                                                                                }
23433                                                                                                
23434                                                                                        }
23435                                                                                } else {
23436                                                                                        ResultColumn targetColumn = modelFactory
23437                                                                                                        .createSelectSetResultColumn(queryTable, sourceColumn);
23438
23439                                                                                        DataFlowRelationship combinedQueryRelation = modelFactory
23440                                                                                                        .createDataFlowRelation();
23441                                                                                        combinedQueryRelation.setEffectType(effectType);
23442                                                                                        combinedQueryRelation
23443                                                                                                        .setTarget(new ResultColumnRelationshipElement(targetColumn));
23444                                                                                        combinedQueryRelation
23445                                                                                                        .addSource(new ResultColumnRelationshipElement(sourceColumn));
23446
23447                                                                                        relation.addSource(new ResultColumnRelationshipElement(targetColumn));
23448                                                                                }
23449                                                                        }
23450                                                                        break;
23451                                                                } else {
23452                                                                        boolean flag = false;
23453
23454                                                                        for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) {
23455                                                                                ResultColumn sourceColumn = selectSetResultSetModel
23456                                                                                                .getColumns().get(j);
23457                                                                                List<String> splits = SQLUtil.parseNames(columnName.toString());        
23458                                                                                if (splits.size() > 1 && EDbVendor.dbvbigquery == getOption().getVendor()) {
23459                                                                                        if (getColumnName(sourceColumn.getName())
23460                                                                                                        .equalsIgnoreCase(getColumnName(splits.get(0)))
23461                                                                                                        || getColumnName(sourceColumn.getName())
23462                                                                                                                        .equalsIgnoreCase(getColumnName(splits.get(1)))) {
23463                                                                                                ResultColumn targetColumn = modelFactory
23464                                                                                                                .createSelectSetResultColumn(queryTable, sourceColumn);
23465
23466                                                                                                DataFlowRelationship combinedQueryRelation = modelFactory
23467                                                                                                                .createDataFlowRelation();
23468                                                                                                combinedQueryRelation.setEffectType(effectType);
23469                                                                                                combinedQueryRelation.setTarget(
23470                                                                                                                new ResultColumnRelationshipElement(targetColumn));
23471                                                                                                combinedQueryRelation.addSource(
23472                                                                                                                new ResultColumnRelationshipElement(sourceColumn));
23473
23474                                                                                                relation.addSource(
23475                                                                                                                new ResultColumnRelationshipElement(targetColumn));
23476                                                                                                flag = true;
23477                                                                                                break;
23478                                                                                        }
23479                                                                                }
23480                                                                                else if (getColumnName(sourceColumn.getName())
23481                                                                                                .equalsIgnoreCase(getColumnName(columnName))) {
23482                                                                                        ResultColumn targetColumn = modelFactory
23483                                                                                                        .createSelectSetResultColumn(queryTable, sourceColumn);
23484
23485                                                                                        DataFlowRelationship combinedQueryRelation = modelFactory
23486                                                                                                        .createDataFlowRelation();
23487                                                                                        combinedQueryRelation.setEffectType(effectType);
23488                                                                                        combinedQueryRelation
23489                                                                                                        .setTarget(new ResultColumnRelationshipElement(targetColumn));
23490                                                                                        combinedQueryRelation
23491                                                                                                        .addSource(new ResultColumnRelationshipElement(sourceColumn));
23492
23493                                                                                        relation.addSource(new ResultColumnRelationshipElement(targetColumn));
23494                                                                                        flag = true;
23495                                                                                        break;
23496                                                                                }
23497                                                                                else if (sourceColumn instanceof SelectSetResultColumn && ((SelectSetResultColumn)sourceColumn).getAliasSet().size() > 1) {
23498                                                                                        for (String alias : ((SelectSetResultColumn)sourceColumn).getAliasSet()) {
23499                                                                                                if (getColumnName(alias).equalsIgnoreCase(getColumnName(columnName))) {
23500                                                                                                        ResultColumn targetColumn = modelFactory
23501                                                                                                                        .createSelectSetResultColumn(queryTable, sourceColumn);
23502
23503                                                                                                        DataFlowRelationship combinedQueryRelation = modelFactory
23504                                                                                                                        .createDataFlowRelation();
23505                                                                                                        combinedQueryRelation.setEffectType(effectType);
23506                                                                                                        combinedQueryRelation.setTarget(
23507                                                                                                                        new ResultColumnRelationshipElement(targetColumn));
23508                                                                                                        combinedQueryRelation.addSource(
23509                                                                                                                        new ResultColumnRelationshipElement(sourceColumn));
23510
23511                                                                                                        relation.addSource(
23512                                                                                                                        new ResultColumnRelationshipElement(targetColumn));
23513                                                                                                        flag = true;
23514                                                                                                        break;
23515                                                                                                }
23516                                                                                        }
23517                                                                                        if (flag) {
23518                                                                                                break;
23519                                                                                        }
23520                                                                                }
23521                                                                        }
23522
23523                                                                        if (flag) {
23524                                                                                break;
23525                                                                        } else if (columnIndex != -1) {
23526                                                                                for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) {
23527                                                                                        ResultColumn sourceColumn = selectSetResultSetModel.getColumns().get(j);
23528                                                                                        if (!sourceColumn.getStarLinkColumns().isEmpty()) {
23529                                                                                                if (cteColumns != null) {
23530                                                                                                        if (j < cteColumns.size()) {
23531                                                                                                                ResultColumn targetColumn = queryTable.getColumns().get(j);
23532                                                                                                                relation.addSource(
23533                                                                                                                                new ResultColumnRelationshipElement(targetColumn));
23534                                                                                                        }
23535                                                                                                } 
23536                                                                                                else {
23537                                                                                                        if(sourceColumn.hasStarLinkColumn()) {
23538                                                                                                                for (TObjectName linkColumn : sourceColumn.getStarLinkColumnList()) {
23539                                                                                                                        if (getColumnName(linkColumn).equals(getColumnName(columnName))) {
23540                                                                                                                                ResultColumn targetColumn = modelFactory
23541                                                                                                                                                .createSelectSetResultColumn(queryTable, sourceColumn);
23542
23543                                                                                                                                DataFlowRelationship combinedQueryRelation = modelFactory
23544                                                                                                                                                .createDataFlowRelation();
23545                                                                                                                                combinedQueryRelation.setEffectType(effectType);
23546                                                                                                                                combinedQueryRelation.setTarget(
23547                                                                                                                                                new ResultColumnRelationshipElement(targetColumn, linkColumn));
23548                                                                                                                                combinedQueryRelation.addSource(
23549                                                                                                                                                new ResultColumnRelationshipElement(sourceColumn));
23550
23551                                                                                                                                relation.addSource(
23552                                                                                                                                                new ResultColumnRelationshipElement(targetColumn, linkColumn));
23553                                                                                                                                flag = true;
23554                                                                                                                                break;
23555                                                                                                                        }
23556                                                                                                                }
23557                                                                                                        }
23558                                                                                                        
23559                                                                                                        if(!flag) {
23560                                                                                                                ResultColumn targetColumn = modelFactory
23561                                                                                                                                .createSelectSetResultColumn(queryTable, sourceColumn);
23562
23563                                                                                                                DataFlowRelationship combinedQueryRelation = modelFactory
23564                                                                                                                                .createDataFlowRelation();
23565                                                                                                                combinedQueryRelation.setEffectType(effectType);
23566                                                                                                                combinedQueryRelation.setTarget(
23567                                                                                                                                new ResultColumnRelationshipElement(targetColumn));
23568                                                                                                                combinedQueryRelation.addSource(
23569                                                                                                                                new ResultColumnRelationshipElement(sourceColumn));
23570
23571                                                                                                                relation.addSource(
23572                                                                                                                                new ResultColumnRelationshipElement(targetColumn));
23573                                                                                                        }
23574                                                                                                }
23575                                                                                                flag = true;
23576                                                                                                break;
23577                                                                                        }
23578                                                                                }
23579                                                                        }
23580
23581                                                                        if (flag) {
23582                                                                                break;
23583                                                                        } else if (columnIndex < selectSetResultSetModel.getColumns().size()
23584                                                                                        && columnIndex != -1) {
23585                                                                                ResultColumn sourceColumn = selectSetResultSetModel.getColumns()
23586                                                                                                .get(columnIndex);
23587                                                                                if (cteColumns != null) {
23588                                                                                        boolean flag1 = false;
23589                                                                                        for (ResultColumn targetColumn : queryTable.getColumns()) {
23590                                                                                                if (getColumnName(targetColumn.getName())
23591                                                                                                                .equalsIgnoreCase(getColumnName(columnName))) {
23592                                                                                                        relation.addSource(
23593                                                                                                                        new ResultColumnRelationshipElement(targetColumn));
23594                                                                                                        flag1 = true;
23595                                                                                                        break;
23596                                                                                                }
23597                                                                                        }
23598                                                                                        if (!flag1 && columnIndex < cteColumns.size()){
23599                                                                                                ResultColumn targetColumn = queryTable.getColumns()
23600                                                                                                                .get(columnIndex);
23601                                                                                                relation.addSource(
23602                                                                                                                new ResultColumnRelationshipElement(targetColumn));
23603                                                                                        }
23604                                                                                } else {
23605                                                                                        ResultColumn targetColumn = modelFactory
23606                                                                                                        .createSelectSetResultColumn(queryTable, sourceColumn);
23607
23608                                                                                        DataFlowRelationship combinedQueryRelation = modelFactory
23609                                                                                                        .createDataFlowRelation();
23610                                                                                        combinedQueryRelation.setEffectType(effectType);
23611                                                                                        combinedQueryRelation
23612                                                                                                        .setTarget(new ResultColumnRelationshipElement(targetColumn));
23613                                                                                        combinedQueryRelation
23614                                                                                                        .addSource(new ResultColumnRelationshipElement(sourceColumn));
23615
23616                                                                                        relation.addSource(new ResultColumnRelationshipElement(targetColumn));
23617                                                                                }
23618                                                                                flag = true;
23619                                                                                break;
23620                                                                        }
23621
23622                                                                        if (flag) {
23623                                                                                break;
23624                                                                        }
23625                                                                }
23626                                                        } else if (cteColumns != null) {
23627                                                                if (getColumnName(columnName).equals("*")) {
23628                                                                        Map<String, Pair<String, TExpression>> replaceAsIdentifierMap = new HashMap<String, Pair<String, TExpression>>();
23629                                                                        Map<String, TObjectName> replaceColumnMap = new HashMap<String, TObjectName>();
23630                                                                        if(modelObject instanceof ResultColumn && ((ResultColumn)modelObject).getColumnObject() instanceof TResultColumn) {
23631                                                                                TResultColumn resultColumn =  (TResultColumn )((ResultColumn)modelObject).getColumnObject();
23632                                                                                if(resultColumn.getReplaceExprAsIdentifiers()!=null && resultColumn.getReplaceExprAsIdentifiers().size()>0) {
23633                                                                                        for(TReplaceExprAsIdentifier replace: resultColumn.getReplaceExprAsIdentifiers()) {
23634                                                                                                replaceAsIdentifierMap.put(replace.getIdentifier().toString(), new Pair<String, TExpression>(resultColumn.getExpr().getExceptReplaceClause().toString(), replace.getExpr()));
23635                                                                                                replaceColumnMap.put(replace.getIdentifier().toString(), replace.getIdentifier());
23636                                                                                        }
23637                                                                                        ResultSet resultSet = ((ResultColumn)modelObject).getResultSet();
23638                                                                                        
23639                                                                                        if (columnName.getSourceColumn() != null) {
23640                                                                                                Object model = modelManager.getModel(columnName.getSourceColumn());
23641                                                                                                if (model instanceof ResultColumn && ((ResultColumn)model).getResultSet().isDetermined()) {
23642                                                                                                        resultSet.getColumns().clear();
23643                                                                                                }
23644                                                                                        } else if (columnName.getSourceTable() != null) {
23645                                                                                                Object tableModel = modelManager.getModel(columnName.getSourceTable());
23646                                                                                                if (tableModel instanceof Table && ((Table)tableModel).isDetermined()) {
23647                                                                                                        resultSet.getColumns().clear();
23648                                                                                                }
23649                                                                                        }
23650                                                                                }
23651                                                                        }
23652                                                                        
23653                                                                        for (int j = 0; j < cteColumns.size(); j++) {
23654                                                                                ResultColumn targetColumn = queryTable.getColumns().get(j);
23655
23656                                                                                if (exceptColumnList != null) {
23657                                                                                        boolean flag = false;
23658                                                                                        for (TObjectName objectName : exceptColumnList) {
23659                                                                                                if (getColumnName(objectName)
23660                                                                                                                .equals(getColumnName(targetColumn.getName()))) {
23661                                                                                                        flag = true;
23662                                                                                                        break;
23663                                                                                                }
23664                                                                                        }
23665                                                                                        if (flag) {
23666                                                                                                continue;
23667                                                                                        }
23668                                                                                }
23669
23670                                                                                if (replaceAsIdentifierMap.containsKey(targetColumn.getName())) {
23671                                                                                        Pair<String, TExpression> expr = replaceAsIdentifierMap.get(targetColumn.getName());
23672                                                                                        ResultSet resultSet = ((ResultColumn)modelObject).getResultSet();
23673                                                                                        ResultColumn resultColumn = modelFactory.createResultColumn(resultSet, replaceColumnMap.get(targetColumn.getName()));
23674                                                                                        Transform transform = new Transform();
23675                                                                                        transform.setType(Transform.EXPRESSION);
23676                                                                                        TObjectName expression = new TObjectName();
23677                                                                                        expression.setString(expr.first);
23678                                                                                transform.setCode(expression);
23679                                                                                        resultColumn.setTransform(transform);
23680                                                                                        analyzeResultColumnExpressionRelation(resultColumn, expr.second);
23681                                                                                } else {
23682                                                                                        if(!replaceAsIdentifierMap.isEmpty()) {
23683                                                                                                ResultSet resultSet = ((ResultColumn) modelObject).getResultSet();
23684                                                                                                TObjectName resultColumnName = new TObjectName();
23685                                                                                                resultColumnName.setString(targetColumn.getName());
23686                                                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSet,
23687                                                                                                                resultColumnName);
23688                                                                                                DataFlowRelationship relation1 = modelFactory.createDataFlowRelation();
23689                                                                                                relation1.setEffectType(effectType);
23690                                                                                                relation1.setProcess(process);
23691                                                                                                relation1.setTarget(new ResultColumnRelationshipElement(resultColumn));
23692                                                                                                relation1.addSource(new ResultColumnRelationshipElement(targetColumn));
23693                                                                                        }
23694                                                                                        else {
23695                                                                                                relation.addSource(
23696                                                                                                        new ResultColumnRelationshipElement(targetColumn));
23697                                                                                        }
23698                                                                                }
23699                                                                        }
23700                                                                        break;
23701                                                                } else {
23702                                                                        boolean flag = false;
23703
23704                                                                        for (int j = 0; j < cteColumns.size(); j++) {
23705                                                                                TObjectName sourceColumn = cteColumns.getObjectName(j);
23706
23707                                                                                if (getColumnName(sourceColumn).equalsIgnoreCase(getColumnName(columnName))) {
23708                                                                                        ResultColumn targetColumn = queryTable.getColumns().get(j);
23709
23710                                                                                        relation.addSource(new ResultColumnRelationshipElement(targetColumn));
23711                                                                                        flag = true;
23712                                                                                        break;
23713                                                                                }
23714                                                                        }
23715
23716                                                                        if (flag) {
23717                                                                                break;
23718                                                                        }
23719                                                                }
23720                                                        }
23721
23722                                                        if (columnName.getSourceColumn() != null) {
23723                                                                Object model = modelManager.getModel(columnName.getSourceColumn());
23724                                                                if (model instanceof ResultColumn) {
23725                                                                        ResultColumn resultColumn = (ResultColumn) model;
23726                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
23727                                                                }
23728                                                        } else if (columnName.getSourceTable() != null) {
23729                                                                Object tableModel = modelManager.getModel(columnName.getSourceTable());
23730                                                                if (tableModel instanceof Table) {
23731                                                                        Object model = modelManager
23732                                                                                        .getModel(new Pair<Table, TObjectName>((Table) tableModel, columnName));
23733                                                                        if (model instanceof TableColumn) {
23734                                                                                relation.addSource(new TableColumnRelationshipElement((TableColumn) model));
23735                                                                        }
23736                                                                }
23737                                                        }
23738                                                } else {
23739                                                        List<ResultColumn> columns = queryTable.getColumns();
23740                                                        if (getColumnName(columnName).equals("*")) {
23741                                                                Map<String, Pair<String, TExpression>> replaceAsIdentifierMap = new HashMap<String, Pair<String, TExpression>>();
23742                                                                Map<String, TObjectName> replaceColumnMap = new HashMap<String, TObjectName>();
23743                                                                if(modelObject instanceof ResultColumn && ((ResultColumn)modelObject).getColumnObject() instanceof TResultColumn) {
23744                                                                        TResultColumn resultColumn =  (TResultColumn )((ResultColumn)modelObject).getColumnObject();
23745                                                                        if(resultColumn.getReplaceExprAsIdentifiers()!=null && resultColumn.getReplaceExprAsIdentifiers().size()>0) {
23746                                                                                for(TReplaceExprAsIdentifier replace: resultColumn.getReplaceExprAsIdentifiers()) {
23747                                                                                        replaceAsIdentifierMap.put(replace.getIdentifier().toString(), new Pair<String, TExpression>(resultColumn.getExpr().getExceptReplaceClause().toString(), replace.getExpr()));
23748                                                                                        replaceColumnMap.put(replace.getIdentifier().toString(), replace.getIdentifier());
23749                                                                                }
23750                                                                                ResultSet resultSet = ((ResultColumn)modelObject).getResultSet();
23751                                                                                if(queryTable.isDetermined()) {
23752                                                                                        resultSet.getColumns().clear();
23753                                                                                }
23754                                                                        }
23755                                                                }
23756                                                                
23757                                                                int index = 0;
23758                                                                for (int j = 0; j < queryTable.getColumns().size(); j++) {
23759                                                                        ResultColumn targetColumn = queryTable.getColumns().get(j);
23760                                                                        if (exceptColumnList != null) {
23761                                                                                boolean flag = false;
23762                                                                                for (TObjectName objectName : exceptColumnList) {
23763                                                                                        if (getColumnName(objectName)
23764                                                                                                        .equals(getColumnName(targetColumn.getName()))) {
23765                                                                                                flag = true;
23766                                                                                                break;
23767                                                                                        }
23768                                                                                }
23769                                                                                if (flag) {
23770                                                                                        continue;
23771                                                                                }
23772                                                                        }
23773                                                                        
23774                                                                        if (replaceAsIdentifierMap.containsKey(targetColumn.getName())) {
23775                                                                                Pair<String, TExpression> expr = replaceAsIdentifierMap.get(targetColumn.getName());
23776                                                                                ResultSet resultSet = ((ResultColumn)modelObject).getResultSet();
23777                                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSet, replaceColumnMap.get(targetColumn.getName()));
23778                                                                                Transform transform = new Transform();
23779                                                                                transform.setType(Transform.EXPRESSION);
23780                                                                                TObjectName expression = new TObjectName();
23781                                                                                expression.setString(expr.first);
23782                                                                        transform.setCode(expression);
23783                                                                                resultColumn.setTransform(transform);
23784                                                                                analyzeResultColumnExpressionRelation(resultColumn, expr.second);
23785                                                                        } else {
23786                                                                                if(!replaceAsIdentifierMap.isEmpty()) {
23787                                                                                        ResultSet resultSet = ((ResultColumn) modelObject).getResultSet();
23788                                                                                        TObjectName resultColumnName = new TObjectName();
23789                                                                                        resultColumnName.setString(targetColumn.getName());
23790                                                                                        ResultColumn resultColumn = modelFactory.createResultColumn(resultSet,
23791                                                                                                        resultColumnName);
23792                                                                                        DataFlowRelationship relation1 = modelFactory.createDataFlowRelation();
23793                                                                                        relation1.setEffectType(effectType);
23794                                                                                        relation1.setProcess(process);
23795                                                                                        relation1.setTarget(new ResultColumnRelationshipElement(resultColumn));
23796                                                                                        relation1.addSource(new ResultColumnRelationshipElement(targetColumn));
23797                                                                                }
23798                                                                                else {
23799                                                                                        relation.addSource(
23800                                                                                                new ResultColumnRelationshipElement(targetColumn));
23801                                                                                }
23802                                                                        }
23803                                                                        index++;
23804                                                                }
23805                                                        } else {
23806                                                                if (table.getCTE() != null) {
23807                                                                        
23808                                                                        if (modelObject instanceof TableColumn) {
23809                                                                                Table modelTable = ((TableColumn) modelObject).getTable();
23810                                                                                if (modelTable.getSubType() == SubType.unnest) {
23811                                                                                        boolean find = false;
23812                                                                                        for (k = 0; k < columns.size(); k++) {
23813                                                                                                ResultColumn column = columns.get(k);
23814                                                                                                if (column.isStruct()) {
23815                                                                                                        List<String> names = SQLUtil.parseNames(column.getName());
23816                                                                                                        for (String name : names) {
23817                                                                                                                if (getColumnName(name).equals(getColumnName(columnName))) {
23818                                                                                                                        DataFlowRelationship unnestRelation = modelFactory.createDataFlowRelation();
23819                                                                                                                        unnestRelation.setEffectType(effectType);
23820                                                                                                                        unnestRelation.setProcess(process);
23821                                                                                                                        unnestRelation.addSource(new ResultColumnRelationshipElement(
23822                                                                                                                                        column, columnName));
23823                                                                                                                        TObjectName unnestTableColumnName = new TObjectName();
23824                                                                                                                        unnestTableColumnName.setString(names.get(names.size()-1));
23825                                                                                                                        TableColumn unnestTableColumn = modelFactory.createTableColumn(modelTable, unnestTableColumnName, true);
23826                                                                                                                        unnestRelation.setTarget(new TableColumnRelationshipElement(unnestTableColumn));
23827                                                                                                                        find = true;
23828                                                                                                                }
23829                                                                                                        }
23830                                                                                                        List<String> names1 = SQLUtil.parseNames(column.getName());
23831                                                                                                        if (names.size() == 1 && names1.size() >= 1) {
23832                                                                                                                for (String name : names1) {
23833                                                                                                                        if (getColumnName(name)
23834                                                                                                                                        .equals(getColumnName(column.getName()))) {
23835                                                                                                                                DataFlowRelationship unnestRelation = modelFactory.createDataFlowRelation();
23836                                                                                                                                unnestRelation.setEffectType(effectType);
23837                                                                                                                                unnestRelation.setProcess(process);
23838                                                                                                                                unnestRelation.addSource(new ResultColumnRelationshipElement(
23839                                                                                                                                                column, columnName));
23840                                                                                                                                TObjectName unnestTableColumnName = new TObjectName();
23841                                                                                                                                unnestTableColumnName.setString(names1.get(names.size()-1));
23842                                                                                                                                TableColumn unnestTableColumn = modelFactory.createTableColumn(modelTable, unnestTableColumnName, true);
23843                                                                                                                                unnestRelation.setTarget(new TableColumnRelationshipElement(unnestTableColumn));
23844                                                                                                                                find = true;
23845                                                                                                                        }
23846                                                                                                                }
23847                                                                                                        }
23848                                                                                                }
23849                                                                                        }
23850                                                                                        if (find) {
23851                                                                                                modelTable.getColumns().remove(modelObject);
23852                                                                                                break;
23853                                                                                        }
23854                                                                                }
23855                                                                        }
23856                                                                        
23857                                                                        for (k = 0; k < columns.size(); k++) {
23858                                                                                ResultColumn column = columns.get(k);
23859                                                                                if ("*".equals(column.getName())) {
23860                                                                                        if (!containsStarColumn(column, columnName)) {
23861                                                                                                column.bindStarLinkColumn(columnName);
23862                                                                                        }
23863                                                                                        relation.addSource(new ResultColumnRelationshipElement(column, columnName));
23864                                                                                } else if (DlineageUtil.compareColumnIdentifier(getColumnName(columnName),
23865                                                                                                DlineageUtil.getIdentifierNormalColumnName(column.getName()))) {
23866                                                                                        if (!column.equals(modelObject)) {
23867                                                                                                relation.addSource(
23868                                                                                                                new ResultColumnRelationshipElement(column, columnName));
23869                                                                                        }
23870                                                                                        break;
23871                                                                                } else if(column.isStruct()) {
23872                                                                                        List<String> names = SQLUtil.parseNames(column.getName());
23873                                                                                        for(String name: names) {
23874                                                                                                if (getColumnName(name)
23875                                                                                                                .equals(getColumnName(columnName))) {
23876                                                                                                        relation.addSource(
23877                                                                                                                        new ResultColumnRelationshipElement(column, columnName));
23878                                                                                                }
23879                                                                                        }
23880                                                                                        List<String> names1 = SQLUtil.parseNames(column.getName());
23881                                                                                        if (names.size() == 1 && names1.size() >= 1) {
23882                                                                                                for(String name: names1) {
23883                                                                                                        if (getColumnName(name)
23884                                                                                                                        .equals(getColumnName(column.getName()))) {
23885                                                                                                                relation.addSource(
23886                                                                                                                                new ResultColumnRelationshipElement(column, columnName));
23887                                                                                                        }
23888                                                                                                }
23889                                                                                        }
23890                                                                                }
23891                                                                        }
23892                                                                } else if (table.getAliasClause() != null
23893                                                                                && table.getAliasClause().getColumns() != null) {
23894                                                                        for (k = 0; k < columns.size(); k++) {
23895                                                                                ResultColumn column = columns.get(k);
23896                                                                                List<String> splits = SQLUtil.parseNames(columnName.toString());        
23897                                                                                if ("*".equals(column.getName())) {
23898                                                                                        if (!containsStarColumn(column, columnName)) {
23899                                                                                                column.bindStarLinkColumn(columnName);
23900                                                                                        }
23901                                                                                        relation.addSource(new ResultColumnRelationshipElement(column, columnName));
23902                                                                                } else if (splits.size() > 1 && EDbVendor.dbvbigquery == getOption().getVendor()) {
23903                                                                                        if (DlineageUtil.compareColumnIdentifier(getColumnName(splits.get(0)),
23904                                                                                                        DlineageUtil.getIdentifierNormalColumnName(column.getName()))) {
23905                                                                                                if (!column.equals(modelObject)) {
23906                                                                                                        relation.addSource(new ResultColumnRelationshipElement(column,
23907                                                                                                                        columnName));
23908                                                                                                }
23909                                                                                                break;
23910                                                                                        }
23911                                                                                } else if (DlineageUtil.compareColumnIdentifier(getColumnName(columnName),
23912                                                                                                DlineageUtil.getIdentifierNormalColumnName(column.getName()))) {
23913                                                                                        if (!column.equals(modelObject)) {
23914                                                                                                relation.addSource(
23915                                                                                                                new ResultColumnRelationshipElement(column, columnName));
23916                                                                                        }
23917                                                                                        break;
23918                                                                                }
23919                                                                        }
23920                                                                } else if (table.getSubquery() != null || (table.getTableExpr() != null
23921                                                                                && table.getTableExpr().getSubQuery() != null)) {
23922                                                                        TSelectSqlStatement select = table.getSubquery();
23923                                                                        if (select == null) {
23924                                                                                select = table.getTableExpr().getSubQuery();
23925                                                                        }
23926                                                                        if (columnName.getSourceTable() != null) {
23927                                                                                Object tableModel = modelManager.getModel(columnName.getSourceTable());
23928                                                                                appendResultColumnRelationSource(modelObject, relation, columnIndex, columnName,
23929                                                                                                tableModel);
23930                                                                        } else if (columnName.getObjectToken() != null
23931                                                                                        && !SQLUtil.isEmpty(table.getAliasName())) {
23932                                                                                if (DlineageUtil.compareTableIdentifier(columnName.getObjectToken().toString(),
23933                                                                                                table.getAliasName())) {
23934                                                                                        Object tableModel = modelManager.getModel(table);
23935                                                                                        appendResultColumnRelationSource(modelObject, relation, columnIndex,
23936                                                                                                        columnName, tableModel);
23937                                                                                }
23938                                                                        } else if(columns!=null) {
23939                                                                                for (k = 0; k < columns.size(); k++) {
23940                                                                                        ResultColumn column = columns.get(k);
23941                                                                                        List<String> splits = SQLUtil.parseNames(columnName.toString());        
23942                                                                                        if ("*".equals(column.getName())) {
23943                                                                                                if (!containsStarColumn(column, columnName)) {
23944                                                                                                        column.bindStarLinkColumn(columnName);
23945                                                                                                }
23946                                                                                                relation.addSource(new ResultColumnRelationshipElement(column, columnName));
23947                                                                                        } else if (splits.size() > 1 && EDbVendor.dbvbigquery == getOption().getVendor()) {
23948                                                                                                if (DlineageUtil.compareColumnIdentifier(getColumnName(splits.get(0)),
23949                                                                                                                DlineageUtil.getIdentifierNormalColumnName(column.getName()))) {
23950                                                                                                        if (!column.equals(modelObject)) {
23951                                                                                                                relation.addSource(new ResultColumnRelationshipElement(column,
23952                                                                                                                                columnName));
23953                                                                                                        }
23954                                                                                                        break;
23955                                                                                                }
23956                                                                                        } else if (DlineageUtil.compareColumnIdentifier(getColumnName(columnName),
23957                                                                                                        DlineageUtil.getIdentifierNormalColumnName(column.getName()))) {
23958                                                                                                if (!column.equals(modelObject)) {
23959                                                                                                        relation.addSource(
23960                                                                                                                        new ResultColumnRelationshipElement(column, columnName));
23961                                                                                                }
23962                                                                                                break;
23963                                                                                        }
23964                                                                                }
23965                                                                        }
23966                                                                } else if (table.getOutputMerge() != null) {
23967                                                                        if (columnName.getSourceColumn() != null) {
23968                                                                                Object model = modelManager.getModel(columnName.getSourceColumn());
23969                                                                                if (model instanceof ResultColumn) {
23970                                                                                        ResultColumn resultColumn = (ResultColumn) model;
23971                                                                                        if ("*".equals(resultColumn.getName())
23972                                                                                                        && !containsStarColumn(resultColumn, columnName)) {
23973                                                                                                resultColumn.bindStarLinkColumn(columnName);
23974                                                                                        }
23975                                                                                        relation.addSource(
23976                                                                                                        new ResultColumnRelationshipElement(resultColumn, columnName));
23977                                                                                }
23978                                                                        } else if (columnName.getSourceTable() != null) {
23979                                                                                Object tableModel = modelManager.getModel(columnName.getSourceTable());
23980                                                                                appendResultColumnRelationSource(modelObject, relation, columnIndex, columnName,
23981                                                                                                tableModel);
23982                                                                        } else if (columnName.getObjectToken() != null
23983                                                                                        && !SQLUtil.isEmpty(table.getAliasName())) {
23984                                                                                if (DlineageUtil.compareTableIdentifier(columnName.getObjectToken().toString(),
23985                                                                                                table.getAliasName())) {
23986                                                                                        Object tableModel = modelManager.getModel(table);
23987                                                                                        appendResultColumnRelationSource(modelObject, relation, columnIndex,
23988                                                                                                        columnName, tableModel);
23989                                                                                }
23990                                                                        }
23991                                                                }
23992                                                        }
23993                                                }
23994                                        }
23995                                }
23996                        }
23997                        if (relation.getSources().size() == 0 && isKeyword(columnName)) {
23998                                Table constantTable = modelFactory.createConstantsTable(stmtStack.peek());
23999                                TableColumn constantColumn = modelFactory.createTableColumn(constantTable, columnName, true);
24000                                relation.addSource(new ConstantRelationshipElement(constantColumn));
24001                        }
24002
24003                        if (relation.getSources().size() > 0) {
24004                                for (RelationshipElement<?> sourceItem: relation.getSources()) {
24005                                        Object source = sourceItem.getElement();
24006                                        ImpactRelationship impactRelation = null;
24007                                        if (source instanceof ResultColumn
24008                                                        && !((ResultColumn) source).getResultSet().getRelationRows().getHoldRelations().isEmpty()) {
24009                                                impactRelation = modelFactory.createImpactRelation();
24010                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
24011                                                                ((ResultColumn) source).getResultSet().getRelationRows()));
24012                                                impactRelation.setEffectType(effectType);
24013                                        } else if (source instanceof TableColumn
24014                                                        && !((TableColumn) source).getTable().getRelationRows().getHoldRelations().isEmpty()) {
24015                                                impactRelation = modelFactory.createImpactRelation();
24016                                                impactRelation.addSource(new RelationRowsRelationshipElement<TableRelationRows>(
24017                                                                ((TableColumn) source).getTable().getRelationRows()));
24018                                                impactRelation.setEffectType(effectType);;
24019                                        }
24020
24021                                        if (impactRelation == null) {
24022                                                continue;
24023                                        }
24024
24025                                        Object target = relation.getTarget().getElement();
24026                                        if (target instanceof ResultColumn) {
24027                                                impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
24028                                                                ((ResultColumn) target).getResultSet().getRelationRows()));
24029                                        } else if (target instanceof TableColumn) {
24030                                                impactRelation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(
24031                                                                ((TableColumn) target).getTable().getRelationRows()));
24032                                        }
24033
24034                                        if (impactRelation.getSources().iterator().next().getElement() == impactRelation.getTarget().getElement()) {
24035                                                modelManager.removeRelation(impactRelation);
24036                                        }
24037                                }
24038                        }
24039                }
24040                return relation;
24041        }
24042
24043        private boolean isNotInProcedure(Table table) {
24044        TStoredProcedureSqlStatement stmt = getProcedureParent(stmtStack.peek());
24045                Procedure procedure = this.modelFactory.createProcedure(stmt);
24046                if(procedure!=null && procedure.getName().equals(table.getParent())){
24047                        return false;
24048                }
24049                return true;
24050        }
24051
24052        private boolean hasJoin(TCustomSqlStatement stmt) {
24053                if (stmt.getJoins() == null || stmt.getJoins().size() == 0)
24054                        return false;
24055                if (stmt.getJoins().size() > 1) {
24056                        return true;
24057                }
24058                TJoinItemList joinItems = stmt.getJoins().getJoin(0).getJoinItems();
24059                if (joinItems == null || joinItems.size() == 0) {
24060                        return false;
24061                }
24062                return true;
24063        }
24064
24065        private boolean isInQuery(TSelectSqlStatement query, TResultColumn column) {
24066                if(query == null)
24067                        return false;
24068                TResultColumnList columns = query.getResultColumnList();
24069                if (columns != null) {
24070                        for (int i = 0; i < columns.size(); i++) {
24071                                if (columns.getResultColumn(i).equals(column)) {
24072                                        return true;
24073                                }
24074                        }
24075                }
24076                return false;
24077        }
24078
24079        private void appendResultColumnRelationSource(Object modelObject, DataFlowRelationship relation, int columnIndex,
24080                        TObjectName columnName, Object tableModel) {
24081                if (tableModel instanceof Table) {
24082                        Object model = modelManager.getModel(new Pair<Table, TObjectName>((Table) tableModel, columnName));
24083                        if (model instanceof TableColumn) {
24084                                relation.addSource(new TableColumnRelationshipElement((TableColumn) model));
24085                        }
24086                } else if (tableModel instanceof ResultSet) {
24087                        List<ResultColumn> queryColumns = ((ResultSet) tableModel).getColumns();
24088                        boolean flag = false;
24089                        for (int l = 0; l < queryColumns.size(); l++) {
24090                                ResultColumn column = queryColumns.get(l);
24091                                if (DlineageUtil.compareColumnIdentifier(getColumnName(columnName),
24092                                                DlineageUtil.getIdentifierNormalColumnName(column.getName()))) {
24093                                        if (!column.equals(modelObject)) {
24094                                                relation.addSource(new ResultColumnRelationshipElement(column, columnName));
24095                                                flag = true;
24096                                        }
24097                                        break;
24098                                }
24099                        }
24100                        if (!flag) {
24101                                for (int l = 0; l < queryColumns.size(); l++) {
24102                                        ResultColumn column = queryColumns.get(l);
24103                                        if ("*".equals(column.getName())) {
24104                                                if (!containsStarColumn(column, columnName)) {
24105                                                        column.bindStarLinkColumn(columnName);
24106                                                }
24107                                                relation.addSource(new ResultColumnRelationshipElement(column, columnName));
24108                                                flag = true;
24109                                                break;
24110                                        }
24111                                }
24112                        }
24113                        if (!flag && columnIndex < queryColumns.size() && columnIndex != -1) {
24114                                relation.addSource(new ResultColumnRelationshipElement(queryColumns.get(columnIndex), columnName));
24115                        }
24116                }
24117        }
24118
24119        private boolean isApplyJoin(TCustomSqlStatement stmt) {
24120                if (stmt.getJoins() == null || stmt.getJoins().size() == 0)
24121                        return false;
24122                TJoinItemList joinItems = stmt.getJoins().getJoin(0).getJoinItems();
24123                if (joinItems == null || joinItems.size() == 0) {
24124                        return false;
24125                }
24126                if (joinItems.getJoinItem(0).getJoinType() == EJoinType.crossapply
24127                                || joinItems.getJoinItem(0).getJoinType() == EJoinType.outerapply)
24128                        return true;
24129                return false;
24130        }
24131
24132        private boolean containsStarColumn(ResultColumn resultColumn, TObjectName columnName) {
24133                String targetColumnName = getColumnName(columnName);
24134                if (resultColumn.hasStarLinkColumn()) {
24135                        return resultColumn.getStarLinkColumns().containsKey(targetColumnName);
24136                }
24137                return false;
24138        }
24139
24140        private void analyzeAggregate(TFunctionCall function, TExpression expr) {
24141                TCustomSqlStatement stmt = stmtStack.peek();
24142                ResultSet resultSet = (ResultSet) modelManager.getModel(stmt.getResultColumnList());
24143                if (resultSet == null) {
24144                        return;
24145                }
24146
24147                if (expr != null) {
24148                        columnsInExpr visitor = new columnsInExpr();
24149                        expr.inOrderTraverse(visitor);
24150                        List<TObjectName> objectNames = visitor.getObjectNames();
24151                        for (int j = 0; j < objectNames.size(); j++) {
24152                                TObjectName columnName = objectNames.get(j);
24153
24154                                if (columnName.getDbObjectType() == EDbObjectType.variable) {
24155                                        continue;
24156                                }
24157
24158                                if (columnName.getColumnNameOnly().startsWith("@")
24159                                                && (option.getVendor() == EDbVendor.dbvmssql || option.getVendor() == EDbVendor.dbvazuresql)) {
24160                                        continue;
24161                                }
24162
24163                                if (columnName.getColumnNameOnly().startsWith(":")
24164                                                && (option.getVendor() == EDbVendor.dbvhana || option.getVendor() == EDbVendor.dbvteradata)) {
24165                                        continue;
24166                                }
24167
24168                                Object targetModel0 = modelManager.getModel(function.getFunctionName());
24169                                if (targetModel0 == null && function.getFunctionType() == EFunctionType.array_agg_t && modelManager.getModel(function) instanceof Function) {
24170                                        Function functionModel = (Function) modelManager.getModel(function);
24171                                        if (functionModel.getColumns().size() == 1) {
24172                                                targetModel0 = ((Function) functionModel).getColumns().get(0);
24173                                        }
24174                                }
24175                                if (!(targetModel0 instanceof ResultColumn)) {
24176                                        continue;
24177                                }
24178                                ResultColumn targetResultColumn0 = (ResultColumn) targetModel0;
24179                                AbstractRelationship relation = modelFactory.createRecordSetRelation();
24180                                relation.setEffectType(EffectType.function);
24181                                relation.setFunction(function.getFunctionName().toString());
24182                                relation.setTarget(new ResultColumnRelationshipElement(targetResultColumn0));
24183                                TTable table = modelManager.getTable(stmt, columnName);
24184                                if (table != null) {
24185                                        if (modelManager.getModel(table) instanceof Table) {
24186                                                Table tableModel = (Table) modelManager.getModel(table);
24187                                                if (tableModel != null) {
24188                                                        TableColumn columnModel = modelFactory.createTableColumn(tableModel, columnName, false);
24189                                                        if (columnModel != null) {
24190                                                                relation.addSource(
24191                                                                                new TableColumnRelationshipElement(columnModel, columnName.getLocation()));
24192                                                        }
24193                                                }
24194                                        } else if (modelManager.getModel(table) instanceof QueryTable) {
24195                                                Object model = modelManager.getModel(columnName.getSourceColumn());
24196                                                if (model instanceof ResultColumn) {
24197                                                        ResultColumn resultColumn = (ResultColumn) model;
24198                                                        if (resultColumn != null) {
24199                                                                relation.addSource(
24200                                                                                new ResultColumnRelationshipElement(resultColumn, columnName.getLocation()));
24201                                                        }
24202                                                }
24203                                        }
24204                                }
24205                        }
24206
24207                        List<TParseTreeNode> functions = visitor.getFunctions();
24208                        for (int j = 0; j < functions.size(); j++) {
24209                                TParseTreeNode functionObj = functions.get(j);
24210                                Object functionModel = modelManager.getModel(functionObj);
24211                                if (functionModel == null) {
24212                                        functionModel = createFunction(functionObj);
24213                                }
24214                                if (functionModel instanceof Function) {
24215                                        Object targetModel1 = modelManager.getModel(function.getFunctionName());
24216                                        if (targetModel1 == null && function.getFunctionType() == EFunctionType.array_agg_t && modelManager.getModel(function) instanceof Function) {
24217                                                Function functionModel1 = (Function) modelManager.getModel(function);
24218                                                if (functionModel1.getColumns().size() == 1) {
24219                                                        targetModel1 = ((Function) functionModel1).getColumns().get(0);
24220                                                }
24221                                        }
24222                                        if (!(targetModel1 instanceof ResultColumn)) {
24223                                                continue;
24224                                        }
24225                                        ResultColumn targetRC_func = (ResultColumn) targetModel1;
24226                                        AbstractRelationship relation;
24227                                        if ("COUNT".equalsIgnoreCase(function.getFunctionName().toString())) {
24228                                                // relation = modelFactory.createDataFlowRelation();
24229                                                relation = modelFactory.createRecordSetRelation();
24230                                        } else {
24231                                                relation = modelFactory.createRecordSetRelation();
24232                                        }
24233                                        relation.setEffectType(EffectType.function);
24234                                        relation.setFunction(function.getFunctionName().toString());
24235                                        relation.setTarget(new ResultColumnRelationshipElement(targetRC_func));
24236
24237                                        if (functionObj instanceof TFunctionCall) {
24238                                                ResultColumn resultColumn = (ResultColumn) modelManager
24239                                                                .getModel(((TFunctionCall) functionObj).getFunctionName());
24240                                                if (resultColumn != null) {
24241                                                        ResultColumnRelationshipElement element = new ResultColumnRelationshipElement(resultColumn,
24242                                                                        ((TFunctionCall) functionObj).getFunctionName().getLocation());
24243                                                        relation.addSource(element);
24244                                                }
24245                                        }
24246                                        if (functionObj instanceof TCaseExpression) {
24247                                                ResultColumn resultColumn = (ResultColumn) modelManager
24248                                                                .getModel(((TCaseExpression) functionObj).getWhenClauseItemList());
24249                                                if (resultColumn != null) {
24250                                                        ResultColumnRelationshipElement element = new ResultColumnRelationshipElement(resultColumn);
24251                                                        relation.addSource(element);
24252                                                }
24253                                        }
24254                                } else if (functionModel instanceof Table) {
24255                                        Object targetModel2 = modelManager.getModel(function.getFunctionName());
24256                                        if (targetModel2 == null && function.getFunctionType() == EFunctionType.array_agg_t && modelManager.getModel(function) instanceof Function) {
24257                                                Function functionModel1 = (Function) modelManager.getModel(function);
24258                                                if (functionModel1.getColumns().size() == 1) {
24259                                                        targetModel2 = ((Function) functionModel1).getColumns().get(0);
24260                                                }
24261                                        }
24262                                        if (!(targetModel2 instanceof ResultColumn)) {
24263                                                continue;
24264                                        }
24265                                        ResultColumn targetRC_table = (ResultColumn) targetModel2;
24266                                        TableColumn tableColumn = modelFactory.createTableColumn((Table) functionModel,
24267                                                        function.getFunctionName(), false);
24268                                        AbstractRelationship relation;
24269                                        if ("COUNT".equalsIgnoreCase(function.getFunctionName().toString())) {
24270                                                // relation = modelFactory.createDataFlowRelation();
24271                                                relation = modelFactory.createRecordSetRelation();
24272                                        } else {
24273                                                relation = modelFactory.createRecordSetRelation();
24274                                        }
24275                                        relation.setEffectType(EffectType.function);
24276                                        relation.setFunction(function.getFunctionName().toString());
24277                                        relation.setTarget(new ResultColumnRelationshipElement(targetRC_table));
24278                                        TableColumnRelationshipElement element = new TableColumnRelationshipElement(tableColumn);
24279                                        relation.addSource(element);
24280                                }
24281                        }
24282                }
24283
24284                if (expr == null || "COUNT".equalsIgnoreCase(function.getFunctionName().toString())) {
24285                        if ("COUNT".equalsIgnoreCase(function.getFunctionName().toString())
24286                                        // https://e.gitee.com/gudusoft/issues/list?issue=I4L5EO
24287                                        // 对于非 count() 函数，当有group by clause时， RelationRows 不参与indirect dataflow,
24288                                        // 让位与group by clause中的column.
24289                                        || ((TSelectSqlStatement) stmt).getGroupByClause() == null) {
24290                                TTableList tables = stmt.getTables();
24291                                if (tables != null) {
24292                                        for (int i = 0; i < tables.size(); i++) {
24293                                                TTable table = tables.getTable(i);
24294                                                if (modelManager.getModel(table) == null && table.getSubquery() == null) {
24295                                                        modelFactory.createTable(table);
24296                                                }
24297                                                if (modelManager.getModel(table) instanceof Table) {
24298                                                        Object targetModel3 = modelManager.getModel(function.getFunctionName());
24299                                                        if (targetModel3 == null && function.getFunctionType() == EFunctionType.array_agg_t && modelManager.getModel(function) instanceof Function) {
24300                                                                Function functionModel = (Function) modelManager.getModel(function);
24301                                                                if (functionModel.getColumns().size() == 1) {
24302                                                                        targetModel3 = functionModel.getColumns().get(0);
24303                                                                }
24304                                                        }
24305                                                        if (!(targetModel3 instanceof ResultColumn)) {
24306                                                                continue;
24307                                                        }
24308                                                        ResultColumn targetRC_tbl = (ResultColumn) targetModel3;
24309                                                        Table tableModel = (Table) modelManager.getModel(table);
24310                                                        AbstractRelationship relation;
24311                                                        if ("COUNT".equalsIgnoreCase(function.getFunctionName().toString())) {
24312                                                                // relation = modelFactory.createDataFlowRelation();
24313                                                                relation = modelFactory.createRecordSetRelation();
24314                                                        } else {
24315                                                                relation = modelFactory.createRecordSetRelation();
24316                                                        }
24317                                                        relation.setEffectType(EffectType.function);
24318                                                        relation.setFunction(function.getFunctionName().toString());
24319                                                        relation.setTarget(new ResultColumnRelationshipElement(targetRC_tbl));
24320
24321                                                        if (relation instanceof DataFlowRelationship && option.isShowCountTableColumn()) {
24322                                                                List<TExpression> expressions = new ArrayList<TExpression>();
24323                                                                getFunctionExpressions(expressions, new ArrayList<TExpression>(), function);
24324                                                                for (int j = 0; j < expressions.size(); j++) {
24325                                                                        columnsInExpr visitor = new columnsInExpr();
24326                                                                        expressions.get(j).inOrderTraverse(visitor);
24327                                                                        List<TObjectName> objectNames = visitor.getObjectNames();
24328                                                                        if (objectNames != null) {
24329                                                                                for (TObjectName columnName : objectNames) {
24330                                                                                        TTable tempTable = modelManager.getTable(stmt, columnName);
24331                                                                                        if (table.equals(tempTable)) {
24332                                                                                                TableColumn tableColumn = modelFactory.createTableColumn(tableModel,
24333                                                                                                                columnName, false);
24334                                                                                                TableColumnRelationshipElement element = new TableColumnRelationshipElement(
24335                                                                                                                tableColumn);
24336                                                                                                relation.addSource(element);
24337                                                                                        }
24338                                                                                }
24339                                                                        }
24340                                                                }
24341                                                        }
24342                                                        if (relation.getSources().size() == 0) {
24343                                                                RelationRowsRelationshipElement element = new RelationRowsRelationshipElement<TableRelationRows>(
24344                                                                                tableModel.getRelationRows());
24345                                                                relation.addSource(element);
24346                                                        }
24347                                                } else if (modelManager.getModel(table) instanceof QueryTable) {
24348                                                        Object targetModel4 = modelManager.getModel(function.getFunctionName());
24349                                                        if (targetModel4 == null && function.getFunctionType() == EFunctionType.array_agg_t && modelManager.getModel(function) instanceof Function) {
24350                                                                Function functionModel = (Function) modelManager.getModel(function);
24351                                                                targetModel4 = functionModel.getColumns().get(0);
24352                                                        }
24353                                                        if (!(targetModel4 instanceof ResultColumn)) {
24354                                                                continue;
24355                                                        }
24356                                                        ResultColumn targetRC_qt = (ResultColumn) targetModel4;
24357                                                        QueryTable tableModel = (QueryTable) modelManager.getModel(table);
24358                                                        AbstractRelationship relation;
24359                                                        if ("COUNT".equalsIgnoreCase(function.getFunctionName().toString())) {
24360                                                                // relation = modelFactory.createDataFlowRelation();
24361                                                                relation = modelFactory.createRecordSetRelation();
24362                                                        } else {
24363                                                                relation = modelFactory.createRecordSetRelation();
24364                                                        }
24365                                                        relation.setEffectType(EffectType.function);
24366                                                        relation.setFunction(function.getFunctionName().toString());
24367                                                        relation.setTarget(new ResultColumnRelationshipElement(targetRC_qt));
24368                                                        RelationRowsRelationshipElement element = new RelationRowsRelationshipElement<ResultSetRelationRows>(
24369                                                                        tableModel.getRelationRows());
24370                                                        relation.addSource(element);
24371                                                }
24372                                        }
24373                                }
24374                        }
24375
24376                        if (stmt.getWhereClause() == null || stmt.getWhereClause().getCondition() == null) {
24377                                return;
24378                        }
24379
24380                        columnsInExpr visitor = new columnsInExpr();
24381                        stmt.getWhereClause().getCondition().inOrderTraverse(visitor);
24382                        List<TObjectName> objectNames = visitor.getObjectNames();
24383                        for (int j = 0; j < objectNames.size(); j++) {
24384                                TObjectName columnName = objectNames.get(j);
24385                                if (columnName.getDbObjectType() == EDbObjectType.variable) {
24386                                        Variable tableModel;
24387                                        if (columnName.toString().indexOf(".") != -1) {
24388                                                List<String> splits = SQLUtil.parseNames(columnName.toString());
24389                                                tableModel = modelFactory.createVariable(splits.get(splits.size() - 2));
24390                                        } else {
24391                                                tableModel = modelFactory.createVariable(columnName);
24392                                        }
24393                                        tableModel.setCreateTable(true);
24394                                        tableModel.setSubType(SubType.record);
24395                                        TObjectName variableProperties = new TObjectName();
24396                                        variableProperties.setString("*");
24397                                        modelFactory.createTableColumn(tableModel, variableProperties, true);
24398                                }
24399
24400//                              if (columnName.getColumnNameOnly().startsWith("@")
24401//                                              && (option.getVendor() == EDbVendor.dbvmssql || option.getVendor() == EDbVendor.dbvazuresql)) {
24402//                                      continue;
24403//                              }
24404//
24405//                              if (columnName.getColumnNameOnly().startsWith(":") && (option.getVendor() == EDbVendor.dbvhana || option.getVendor() == EDbVendor.dbvteradata)) {
24406//                                      continue;
24407//                              }
24408
24409                                Object targetModel5 = modelManager.getModel(function.getFunctionName());
24410                                if (targetModel5 == null && function.getFunctionType() == EFunctionType.array_agg_t && modelManager.getModel(function) instanceof Function) {
24411                                        Function functionModel = (Function) modelManager.getModel(function);
24412                                        if (functionModel.getColumns().size() == 1) {
24413                                                targetModel5 = functionModel.getColumns().get(0);
24414                                        }
24415                                }
24416                                if (!(targetModel5 instanceof ResultColumn)) {
24417                                        continue;
24418                                }
24419                                AbstractRelationship relation = modelFactory.createRecordSetRelation();
24420                                relation.setEffectType(EffectType.function);
24421                                relation.setFunction(function.getFunctionName().toString());
24422                                relation.setTarget(new ResultColumnRelationshipElement((ResultColumn) targetModel5));
24423
24424                                TTable table = modelManager.getTable(stmt, columnName);
24425                                if (table != null) {
24426                                        if (modelManager.getModel(table) instanceof Table) {
24427                                                Table tableModel = (Table) modelManager.getModel(table);
24428                                                if (tableModel != null) {
24429                                                        TableColumn columnModel = modelFactory.createTableColumn(tableModel, columnName, false);
24430                                                        if(columnModel == null) {
24431                                                                continue;
24432                                                        }
24433                                                        relation.addSource(
24434                                                                        new TableColumnRelationshipElement(columnModel, columnName.getLocation()));
24435                                                }
24436                                        } else if (modelManager.getModel(table) instanceof QueryTable) {
24437                                                Object model = modelManager.getModel(columnName.getSourceColumn());
24438                                                if (model instanceof ResultColumn) {
24439                                                        ResultColumn resultColumn = (ResultColumn) model;
24440                                                        if (resultColumn != null) {
24441                                                                relation.addSource(
24442                                                                                new ResultColumnRelationshipElement(resultColumn, columnName.getLocation()));
24443                                                        }
24444                                                }
24445                                        }
24446                                }
24447                        }
24448                }
24449        }
24450
24451        private void analyzeFilterCondition(Object modelObject, TExpression expr, EJoinType joinType,
24452                        JoinClauseType joinClauseType, EffectType effectType) {
24453                if (expr == null) {
24454                        return;
24455                }
24456
24457                TCustomSqlStatement stmt = stmtStack.peek();
24458
24459                columnsInExpr visitor = new columnsInExpr();
24460                expr.inOrderTraverse(visitor);
24461
24462                List<TObjectName> objectNames = visitor.getObjectNames();
24463                List<TParseTreeNode> functions = visitor.getFunctions();
24464                List<TResultColumn> resultColumns = visitor.getResultColumns();
24465                List<TParseTreeNode> constants = visitor.getConstants();
24466
24467                ImpactRelationship relation = modelFactory.createImpactRelation();
24468                relation.setEffectType(effectType);
24469                relation.setJoinClauseType(joinClauseType);
24470                if (modelObject instanceof ResultColumn) {
24471                        relation.setTarget(new ResultColumnRelationshipElement((ResultColumn) modelObject));
24472                } else {
24473                        ResultSet resultSet = (ResultSet) modelManager.getModel(stmt.getResultColumnList());
24474                        if (resultSet == null && stmt instanceof TUpdateSqlStatement) {
24475                                resultSet = (ResultSet) modelManager.getModel(stmt);
24476                        }
24477                        if (resultSet == null && stmt instanceof TMergeSqlStatement) {
24478                                TSelectSqlStatement subquery = ((TMergeSqlStatement) stmt).getUsingTable().getSubquery();
24479                                if (subquery != null) {
24480                                        resultSet = (ResultSet) modelManager.getModel(((TMergeSqlStatement) stmt).getUsingTable());
24481                                }
24482                                else {
24483                                        resultSet = modelFactory.createQueryTable(((TMergeSqlStatement) stmt).getUsingTable());
24484                                }
24485                        }
24486                        if (resultSet != null) {
24487                                relation.setTarget(
24488                                                new RelationRowsRelationshipElement<ResultSetRelationRows>(resultSet.getRelationRows()));
24489                        }
24490                        if (stmt instanceof TDeleteSqlStatement) {
24491                                Table table = (Table) modelManager.getModel(((TDeleteSqlStatement) stmt).getTargetTable());
24492                                if (table != null) {
24493                                        relation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(table.getRelationRows()));
24494                                }
24495                        }
24496                }
24497                if (relation.getTarget() != null) {
24498
24499                        if (constants != null && constants.size() > 0) {
24500                                if (option.isShowConstantTable()) {
24501                                        Table constantTable = modelFactory.createConstantsTable(stmtStack.peek());
24502                                        for (int i = 0; i < constants.size(); i++) {
24503                                                TParseTreeNode constant = constants.get(i);
24504                                                if (constant instanceof TConstant) {
24505                                                        TableColumn constantColumn = modelFactory.createTableColumn(constantTable,
24506                                                                        (TConstant) constant);
24507                                                        relation.addSource(new ConstantRelationshipElement(constantColumn));
24508                                                } else if (constant instanceof TObjectName) {
24509                                                        TableColumn constantColumn = modelFactory.createTableColumn(constantTable,
24510                                                                        (TObjectName) constant, false);
24511                                                        if(constantColumn == null) {
24512                                                                continue;
24513                                                        }
24514                                                        relation.addSource(new ConstantRelationshipElement(constantColumn));
24515                                                }
24516                                        }
24517                                }
24518                        }
24519
24520                        for (int j = 0; j < objectNames.size(); j++) {
24521                                TObjectName columnName = objectNames.get(j);
24522                                if (columnName.getDbObjectType() == EDbObjectType.variable) {
24523                                        Variable variable = modelFactory.createVariable(columnName);
24524                                        variable.setSubType(SubType.record);
24525                                        if (variable.getColumns().isEmpty()) {
24526                                                TObjectName variableProperties = new TObjectName();
24527                                                variableProperties.setString("*");
24528                                                modelFactory.createTableColumn(variable, variableProperties, true);
24529                                        }
24530                                        relation.addSource(new TableColumnRelationshipElement(variable.getColumns().get(0), columnName.getLocation()));
24531                                        continue;
24532                                }
24533
24534                                if (columnName.getColumnNameOnly().startsWith("@")
24535                                                && (option.getVendor() == EDbVendor.dbvmssql || option.getVendor() == EDbVendor.dbvazuresql)) {
24536                                        continue;
24537                                }
24538
24539                                if (columnName.getColumnNameOnly().startsWith(":")
24540                                                && (option.getVendor() == EDbVendor.dbvhana || option.getVendor() == EDbVendor.dbvteradata)) {
24541                                        continue;
24542                                }
24543
24544                                TTable table = modelManager.getTable(stmt, columnName);
24545
24546                                if (table == null) {
24547                                        table = columnName.getSourceTable();
24548                                }
24549
24550                                if (table == null && stmt.tables != null) {
24551                                        for (int k = 0; k < stmt.tables.size(); k++) {
24552                                                if (table != null)
24553                                                        break;
24554
24555                                                TTable tTable = stmt.tables.getTable(k);
24556                                                if (tTable.getTableType().name().startsWith("open")) {
24557                                                        continue;
24558                                                } else if (getTableLinkedColumns(tTable) != null && getTableLinkedColumns(tTable).size() > 0) {
24559                                                        for (int z = 0; z < getTableLinkedColumns(tTable).size(); z++) {
24560                                                                TObjectName refer = getTableLinkedColumns(tTable).getObjectName(z);
24561                                                                if ("*".equals(getColumnName(refer)))
24562                                                                        continue;
24563                                                                if (getColumnName(refer).equals(getColumnName(columnName))) {
24564                                                                        table = tTable;
24565                                                                        break;
24566                                                                }
24567                                                        }
24568                                                } else if (columnName.getTableToken() != null
24569                                                                && (columnName.getTableToken().getAstext().equalsIgnoreCase(tTable.getName())
24570                                                                                || columnName.getTableToken().getAstext().equalsIgnoreCase(tTable.getAliasName()))) {
24571                                                        table = tTable;
24572                                                        break;
24573                                                }
24574                                        }
24575
24576                                        if (table == null) {
24577                                                for (int k = 0; k < stmt.tables.size(); k++) {
24578                                                        if (table != null)
24579                                                                break;
24580
24581                                                        TTable tTable = stmt.tables.getTable(k);
24582                                                        Object model = ModelBindingManager.get().getModel(tTable);
24583                                                        if (model instanceof Table) {
24584                                                                Table tableModel = (Table) model;
24585                                                                for (int z = 0; tableModel.getColumns() != null
24586                                                                                && z < tableModel.getColumns().size(); z++) {
24587                                                                        TableColumn refer = tableModel.getColumns().get(z);
24588                                                                        if (getColumnName(refer.getName()).equals(getColumnName(columnName))) {
24589                                                                                table = tTable;
24590                                                                                break;
24591                                                                        }
24592                                                                        if (refer.hasStarLinkColumn()) {
24593                                                                                for (TObjectName linkColumn : refer.getStarLinkColumnList()) {
24594                                                                                        if (getColumnName(linkColumn).equals(getColumnName(columnName))) {
24595                                                                                                table = tTable;
24596                                                                                                break;
24597                                                                                        }
24598                                                                                }
24599                                                                        }
24600                                                                }
24601                                                        } else if (model instanceof QueryTable) {
24602                                                                QueryTable tableModel = (QueryTable) model;
24603                                                                for (int z = 0; tableModel.getColumns() != null
24604                                                                                && z < tableModel.getColumns().size(); z++) {
24605                                                                        ResultColumn refer = tableModel.getColumns().get(z);
24606                                                                        if (DlineageUtil.getIdentifierNormalColumnName(refer.getName()).equals(
24607                                                                                        DlineageUtil.getIdentifierNormalColumnName(getColumnName(columnName)))) {
24608                                                                                table = tTable;
24609                                                                                break;
24610                                                                        }
24611                                                                        if (refer.hasStarLinkColumn()) {
24612                                                                                for (TObjectName linkColumn : refer.getStarLinkColumnList()) {
24613                                                                                        if (getColumnName(linkColumn).equals(getColumnName(columnName))) {
24614                                                                                                table = tTable;
24615                                                                                                break;
24616                                                                                        }
24617                                                                                }
24618                                                                        }
24619                                                                }
24620                                                        }
24621                                                }
24622                                        }
24623                                }
24624
24625                                if (table == null && stmt.tables != null && stmt.tables.size() != 0
24626                                                && !(isBuiltInFunctionName(columnName) && isFromFunction(columnName))) {
24627                                        
24628                                        if (modelManager.getModel(stmt) instanceof ResultSet) {
24629                                                ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt);
24630                                                boolean find = false;
24631                                                for (ResultColumn resultColumn : resultSetModel.getColumns()) {
24632                                                        if(resultColumn.equals(modelObject)) {
24633                                                                continue;
24634                                                        }
24635                                                        if (!TSQLEnv.isAliasReferenceForbidden.get(option.getVendor())) {
24636                                                                if (getColumnName(columnName).equals(getColumnName(resultColumn.getName()))) {
24637                                                                        if (resultColumn.getColumnObject() != null) {
24638                                                                                int startToken = resultColumn.getColumnObject().getStartToken().posinlist;
24639                                                                                int endToken = resultColumn.getColumnObject().getEndToken().posinlist;
24640                                                                                if (columnName.getStartToken().posinlist >= startToken
24641                                                                                                && columnName.getEndToken().posinlist <= endToken) {
24642                                                                                        continue;
24643                                                                                }
24644                                                                        }
24645                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
24646                                                                        find = true;
24647                                                                        break;
24648                                                                }
24649                                                        }
24650                                                }
24651                                                if (find) {
24652                                                        continue;
24653                                                }
24654                                        }
24655
24656                                        TObjectName pseudoTableName = new TObjectName();
24657                                        // Use qualified prefix from column name if available (e.g., sch.pk_constv2 from sch.pk_constv2.c_cdsl)
24658                                        // Otherwise fall back to default pseudo table name
24659                                        String qualifiedPrefix = getQualifiedPrefixFromColumn(columnName);
24660                                        pseudoTableName.setString(qualifiedPrefix != null ? qualifiedPrefix : "pseudo_table_include_orphan_column");
24661                                        Table pseudoTable = modelFactory.createTableByName(pseudoTableName);
24662                                        pseudoTable.setPseudo(true);
24663                                        TableColumn pseudoTableColumn = modelFactory.createTableColumn(pseudoTable, columnName, true);
24664
24665                                        // If not linking to first table and column has qualified prefix (3-part name like sch.pkg.col),
24666                                        // add the pseudo table column as source
24667                                        if (!isLinkOrphanColumnToFirstTable() && pseudoTableColumn != null && qualifiedPrefix != null) {
24668                                                relation.addSource(new TableColumnRelationshipElement(pseudoTableColumn));
24669                                        }
24670
24671                                        if (isLinkOrphanColumnToFirstTable()) {
24672                                                TTable orphanTable = stmt.tables.getTable(0);
24673                                                table = stmt.tables.getTable(0);
24674                                                Object tableModel = modelManager.getModel(table);
24675                                                if (tableModel == null) {
24676                                                        tableModel = modelFactory.createTable(orphanTable);
24677                                                }
24678                                                if (tableModel instanceof Table) {
24679                                                        modelFactory.createTableColumn((Table) tableModel, columnName, false);
24680                                                        ErrorInfo errorInfo = new ErrorInfo();
24681                                                        errorInfo.setErrorType(ErrorInfo.LINK_ORPHAN_COLUMN);
24682                                                        errorInfo.setErrorMessage("Link orphan column [" + columnName.toString()
24683                                                                        + "] to the first table [" + orphanTable.getFullNameWithAliasString() + "]");
24684                                                        errorInfo.setStartPosition(new Pair3<Long, Long, String>(columnName.getStartToken().lineNo,
24685                                                                        columnName.getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
24686                                                        errorInfo.setEndPosition(new Pair3<Long, Long, String>(columnName.getEndToken().lineNo,
24687                                                                        columnName.getEndToken().columnNo + columnName.getEndToken().getAstext().length(),
24688                                                                        ModelBindingManager.getGlobalHash()));
24689                                                        errorInfo.fillInfo(this);
24690                                                        errorInfos.add(errorInfo);
24691                                                }
24692                                        }
24693                                }
24694
24695                                if (table != null) {
24696                                        if (modelManager.getModel(table) instanceof Table) {
24697                                                Table tableModel = (Table) modelManager.getModel(table);
24698                                                if (tableModel != null) {
24699                                                        TableColumn columnModel = modelFactory.createTableColumn(tableModel, columnName, false);
24700                                                        if(columnModel!=null) {
24701                                                                TableColumnRelationshipElement element = new TableColumnRelationshipElement(columnModel,
24702                                                                                columnName.getLocation());
24703                                                                relation.addSource(element);
24704                                                        }
24705                                                }
24706                                        } else if (modelManager.getModel(table) instanceof QueryTable) {
24707                                                QueryTable tableModel = (QueryTable)modelManager.getModel(table);
24708                                                if (table.getSubquery() != null && table.getSubquery().isCombinedQuery()) {
24709                                                        TSelectSqlStatement subquery = table.getSubquery();
24710                                                        List<ResultSet> resultSets = new ArrayList<ResultSet>();
24711                                                        if (!subquery.getLeftStmt().isCombinedQuery()) {
24712                                                                ResultSet sourceResultSet = (ResultSet) modelManager
24713                                                                                .getModel(subquery.getLeftStmt().getResultColumnList());
24714                                                                resultSets.add(sourceResultSet);
24715                                                        } else {
24716                                                                ResultSet sourceResultSet = (ResultSet) modelManager.getModel(subquery.getLeftStmt());
24717                                                                resultSets.add(sourceResultSet);
24718                                                        }
24719
24720                                                        if (!subquery.getRightStmt().isCombinedQuery()) {
24721                                                                ResultSet sourceResultSet = (ResultSet) modelManager
24722                                                                                .getModel(subquery.getRightStmt().getResultColumnList());
24723                                                                resultSets.add(sourceResultSet);
24724                                                        } else {
24725                                                                ResultSet sourceResultSet = (ResultSet) modelManager.getModel(subquery.getRightStmt());
24726                                                                resultSets.add(sourceResultSet);
24727                                                        }
24728
24729                                                        for (ResultSet sourceResultSet : resultSets) {
24730                                                                if (sourceResultSet != null && columnName.getSourceColumn() != null) {
24731                                                                        for (int k = 0; k < sourceResultSet.getColumns().size(); k++) {
24732                                                                                if (getColumnName(sourceResultSet.getColumns().get(k).getName()).equals(
24733                                                                                                getColumnName(columnName.getSourceColumn().getColumnNameOnly()))) {
24734                                                                                        Set<TObjectName> starLinkColumnSet = sourceResultSet.getColumns().get(k)
24735                                                                                                        .getStarLinkColumns().get(getColumnName(columnName));
24736                                                                                        if (starLinkColumnSet != null && !starLinkColumnSet.isEmpty()) {
24737                                                                                                ResultColumn column = modelFactory.createResultColumn(sourceResultSet,
24738                                                                                                                starLinkColumnSet.iterator().next(), true);
24739                                                                                                relation.addSource(new ResultColumnRelationshipElement(column));
24740                                                                                        } else {
24741                                                                                                relation.addSource(new ResultColumnRelationshipElement(
24742                                                                                                                sourceResultSet.getColumns().get(k)));
24743                                                                                        }
24744                                                                                }
24745                                                                        }
24746                                                                }
24747                                                        }
24748                                                } else {
24749                                                        Object model = modelManager.getModel(columnName.getSourceColumn());
24750                                                        if (model instanceof ResultColumn) {
24751                                                                ResultColumn resultColumn = (ResultColumn) model;
24752                                                                if (resultColumn != null) {
24753                                                                        if (resultColumn.hasStarLinkColumn()) {
24754                                                                                Set<TObjectName> starLinkColumnSet = resultColumn.getStarLinkColumns()
24755                                                                                                .get(getColumnName(columnName));
24756                                                                                if (starLinkColumnSet != null && !starLinkColumnSet.isEmpty()) {
24757                                                                                        ResultColumn column = modelFactory.createResultColumn(
24758                                                                                                        resultColumn.getResultSet(), starLinkColumnSet.iterator().next(), true);
24759                                                                                        relation.addSource(new ResultColumnRelationshipElement(column));
24760                                                                                } else {
24761                                                                                        resultColumn.bindStarLinkColumn(columnName);
24762                                                                                        ResultColumn column = modelFactory
24763                                                                                                        .createResultColumn(resultColumn.getResultSet(), columnName, true);
24764                                                                                        relation.addSource(new ResultColumnRelationshipElement(column));
24765                                                                                }
24766                                                                        } else {
24767                                                                                ResultColumnRelationshipElement element = new ResultColumnRelationshipElement(
24768                                                                                                resultColumn, columnName.getLocation());
24769                                                                                relation.addSource(element);
24770                                                                        }
24771                                                                }
24772                                                        }
24773                                                        else{
24774                                                                boolean find = false;
24775                                                                for (int i = 0; i < tableModel.getColumns().size(); i++) {
24776                                                                        ResultColumn resultColumn = tableModel.getColumns().get(i);
24777                                                                        if (DlineageUtil.getIdentifierNormalColumnName(resultColumn.getName()).equals(
24778                                                                                        DlineageUtil.getIdentifierNormalColumnName(getColumnName(columnName)))) {
24779                                                                                ResultColumnRelationshipElement element = new ResultColumnRelationshipElement(
24780                                                                                                resultColumn, columnName.getLocation());
24781                                                                                relation.addSource(element);
24782                                                                                find = true;
24783                                                                                break;
24784                                                                        }
24785                                                                        else if (resultColumn.getName().endsWith("*")) {
24786                                                                                resultColumn.bindStarLinkColumn(columnName);
24787                                                                        }
24788                                                                }
24789                                                                if(!find){
24790                                                                        ResultColumn resultColumn = new ResultColumn(tableModel, columnName);
24791                                                                        ResultColumnRelationshipElement element = new ResultColumnRelationshipElement(
24792                                                                                                resultColumn, columnName.getLocation());
24793                                                                                relation.addSource(element);
24794                                                                }
24795                                                        }
24796                                                }
24797                                        }
24798                                }
24799                        }
24800
24801                        for (int j = 0; j < functions.size(); j++) {
24802                                TParseTreeNode functionObj = functions.get(j);
24803                                Object functionModel = modelManager.getModel(functionObj);
24804                                if (functionModel == null) {
24805                                        functionModel = createFunction(functionObj);
24806                                }
24807                                if (functionModel instanceof Function) {
24808                                        if (functionObj instanceof TFunctionCall) {
24809                                                ResultColumn resultColumn = (ResultColumn) modelManager
24810                                                                .getModel(((TFunctionCall) functionObj).getFunctionName());
24811                                                if (resultColumn != null) {
24812                                                        ResultColumnRelationshipElement element = new ResultColumnRelationshipElement(resultColumn,
24813                                                                        ((TFunctionCall) functionObj).getFunctionName().getLocation());
24814                                                        relation.addSource(element);
24815                                                }
24816                                        }
24817                                        if (functionObj instanceof TCaseExpression) {
24818                                                ResultColumn resultColumn = (ResultColumn) modelManager
24819                                                                .getModel(((TCaseExpression) functionObj).getWhenClauseItemList());
24820                                                if (resultColumn != null) {
24821                                                        ResultColumnRelationshipElement element = new ResultColumnRelationshipElement(resultColumn);
24822                                                        relation.addSource(element);
24823                                                }
24824                                        }
24825                                } else if (functionModel instanceof Table) {
24826                                        TableColumn tableColumn = modelFactory.createTableColumn((Table) functionModel,
24827                                                        ((TFunctionCall) functionObj).getFunctionName(), false);
24828                                        TableColumnRelationshipElement element = new TableColumnRelationshipElement(tableColumn);
24829                                        relation.addSource(element);
24830                                }
24831                        }
24832
24833                        for (int j = 0; j < resultColumns.size(); j++) {
24834                                TResultColumn resultColumn = resultColumns.get(j);
24835                                if (modelManager.getModel(resultColumn) instanceof ResultColumn) {
24836                                        ResultColumn resultColumnModel = (ResultColumn) modelManager.getModel(resultColumn);
24837                                        relation.addSource(new ResultColumnRelationshipElement(resultColumnModel, ESqlClause.selectList));
24838                                }
24839                        }
24840                }
24841
24842                if (isShowJoin() && joinClauseType != null) {
24843                        joinInExpr joinVisitor = new joinInExpr(joinType, joinClauseType, effectType);
24844                        expr.inOrderTraverse(joinVisitor);
24845                }
24846        }
24847
24848        public void dispose() {
24849                accessedSubqueries.clear();
24850                accessedStatements.clear();
24851                stmtStack.clear();
24852                viewDDLMap.clear();
24853                procedureDDLMap.clear();
24854                structObjectMap.clear();
24855                appendResultSets.clear();
24856                appendStarColumns.clear();
24857                appendTableStarColumns.clear();
24858                modelManager.DISPLAY_ID.clear();
24859                modelManager.DISPLAY_NAME.clear();
24860                tableIds.clear();
24861                ModelBindingManager.remove();
24862        }
24863
24864        class joinTreatColumnsInExpr implements IExpressionVisitor {
24865
24866                private List<TObjectName> objectNames = new ArrayList<TObjectName>();
24867                
24868                private TTable table;
24869
24870                public joinTreatColumnsInExpr(TTable table) {
24871                        this.table = table;
24872                }
24873
24874                public List<TObjectName> getObjectNames() {
24875                        return objectNames;
24876                }
24877
24878                boolean is_compare_condition(EExpressionType t) {
24879                        return t == EExpressionType.simple_comparison_t;
24880                }
24881
24882                @Override
24883                public boolean exprVisit(TParseTreeNode pNode, boolean isLeafNode) {
24884                        TExpression expr = (TExpression) pNode;
24885                        if (is_compare_condition(expr.getExpressionType())) {
24886                                TExpression leftExpr = expr.getLeftOperand();
24887                                columnsInExpr leftVisitor = new columnsInExpr();
24888                                leftExpr.inOrderTraverse(leftVisitor);
24889                                List<TObjectName> leftObjectNames = leftVisitor.getObjectNames();
24890
24891                                TExpression rightExpr = expr.getRightOperand();
24892                                columnsInExpr rightVisitor = new columnsInExpr();
24893                                rightExpr.inOrderTraverse(rightVisitor);
24894                                List<TObjectName> rightObjectNames = rightVisitor.getObjectNames();
24895
24896                                if (!leftObjectNames.isEmpty() && !rightObjectNames.isEmpty()) {
24897                                        for (TObjectName column : leftObjectNames) {
24898                                                if (column.getSourceTable() != null && column.getSourceTable().equals(table)) {
24899                                                        objectNames.add(column);
24900                                                        return false;
24901                                                }
24902                                        }
24903                                        for (TObjectName column : rightObjectNames) {
24904                                                if (column.getSourceTable() != null && column.getSourceTable().equals(table)) {
24905                                                        objectNames.add(column);
24906                                                        return false;
24907                                                }
24908                                        }
24909                                }
24910                                return false;
24911                        }
24912                        return true;
24913                }
24914        }
24915        
24916        class columnsInExpr implements IExpressionVisitor {
24917
24918                private List<TParseTreeNode> constants = new ArrayList<TParseTreeNode>();
24919                private List<TObjectName> objectNames = new ArrayList<TObjectName>();
24920                private List<TParseTreeNode> functions = new ArrayList<TParseTreeNode>();
24921                private List<TResultColumn> resultColumns = new ArrayList<TResultColumn>();
24922                private List<TSelectSqlStatement> subquerys = new ArrayList<TSelectSqlStatement>();
24923                private boolean skipFunction = false;
24924
24925                public void setSkipFunction(boolean skipFunction) {
24926                        this.skipFunction = skipFunction;
24927                }
24928
24929                public List<TParseTreeNode> getFunctions() {
24930                        return functions;
24931                }
24932
24933                public List<TSelectSqlStatement> getSubquerys() {
24934                        return subquerys;
24935                }
24936
24937                public List<TParseTreeNode> getConstants() {
24938                        return constants;
24939                }
24940
24941                public List<TObjectName> getObjectNames() {
24942                        return objectNames;
24943                }
24944
24945                public List<TResultColumn> getResultColumns() {
24946                        return resultColumns;
24947                }
24948
24949                @Override
24950                public boolean exprVisit(TParseTreeNode pNode, boolean isLeafNode) {
24951                        TExpression lcexpr = (TExpression) pNode;
24952                        // Handle named argument expressions (e.g., "INPUT => value" in Snowflake FLATTEN)
24953                        // The left operand is the parameter name, NOT a column reference.
24954                        // Only traverse the right operand (the value).
24955                        if (lcexpr.getExpressionType() == EExpressionType.assignment_t) {
24956                                // Skip left operand (parameter name) - only traverse right operand (value)
24957                                if (lcexpr.getRightOperand() != null) {
24958                                        lcexpr.getRightOperand().inOrderTraverse(this);
24959                                }
24960                                return false; // Don't continue default traversal
24961                        }
24962                        if (lcexpr.getExpressionType() == EExpressionType.simple_constant_t) {
24963                                if (lcexpr.getConstantOperand() != null) {
24964                                        if(lcexpr.getConstantOperand().getInt64_expression()!=null 
24965                                                        && lcexpr.getConstantOperand().getInt64_expression().getExpressionType() == EExpressionType.function_t) {
24966                                                lcexpr.getConstantOperand().getInt64_expression().inOrderTraverse(this);
24967                                        }
24968                                        else {
24969                                                constants.add(lcexpr.getConstantOperand());
24970                                        }
24971                                }
24972                        } else if (lcexpr.getExpressionType() == EExpressionType.array_t) {
24973                                if(lcexpr.getObjectOperand()!=null) {
24974                                        TObjectName object = lcexpr.getObjectOperand();
24975                                        objectNames.add(object);
24976                                } else if (lcexpr.getExprList() != null) {
24977                                        for (int j = 0; j < lcexpr.getExprList().size(); j++) {
24978                                                TExpression expr = lcexpr.getExprList().getExpression(j);
24979                                                if (expr != null)
24980                                                        expr.inOrderTraverse(this);
24981                                        }
24982                                }
24983                        } else if (lcexpr.getExpressionType() == EExpressionType.simple_object_name_t) {
24984                                if (lcexpr.getObjectOperand() != null && !(isBuiltInFunctionName(lcexpr.getObjectOperand())
24985                                                && isFromFunction(lcexpr.getObjectOperand()))) {
24986                                        TObjectName object = lcexpr.getObjectOperand();
24987                                        // Skip named argument parameter names (e.g., INPUT in "INPUT => value")
24988                                        // These are function parameter names, NOT column references
24989                                        if (object.getObjectType() == TObjectName.ttobjNamedArgParameter) {
24990                                                // Skip - this is a named argument parameter name
24991                                        } else if (object.getDbObjectType() == EDbObjectType.column
24992                                                        || object.getDbObjectType() == EDbObjectType.column_alias
24993                                                        || object.getDbObjectType() == EDbObjectType.alias
24994                                                        || object.getDbObjectType() == EDbObjectType.unknown
24995                                                        || object.getDbObjectType() == EDbObjectType.variable) {
24996                                                objectNames.add(object);
24997                                        } else if (object.getDbObjectType() == EDbObjectType.notAColumn
24998                                                        || object.getDbObjectType() == EDbObjectType.date_time_part ) {
24999                                                constants.add(object);
25000                                        }
25001                                }
25002                        } else if (lcexpr.getExpressionType() == EExpressionType.between_t) {
25003                                if (lcexpr.getBetweenOperand() != null && lcexpr.getBetweenOperand().getObjectOperand() != null) {
25004                                        TObjectName object = lcexpr.getBetweenOperand().getObjectOperand();
25005                                        if (object.getDbObjectType() == EDbObjectType.column
25006                                                        || object.getDbObjectType() == EDbObjectType.column_alias
25007                                                        || object.getDbObjectType() == EDbObjectType.alias
25008                                                        || object.getDbObjectType() == EDbObjectType.unknown
25009                                                        || object.getDbObjectType() == EDbObjectType.variable) {
25010                                                objectNames.add(object);
25011                                        }
25012                                }
25013                        } else if (lcexpr.getExpressionType() == EExpressionType.object_access_t) {
25014                                if (lcexpr.getObjectAccess() != null) {
25015                                        TObjectNameList objects = lcexpr.getObjectAccess().getAttributes();
25016                                        TFunctionCall function = lcexpr.getObjectAccess().getObjectExpr().getFunctionCall();
25017                                        if (objects != null && function != null) {
25018                                                for (TObjectName object : objects) {
25019                                                        TGSqlParser sqlparser = new TGSqlParser(option.getVendor());
25020                                                        sqlparser.sqltext = "select " + function.getFunctionName().toString() + "."
25021                                                                        + object.getColumnNameOnly() + " from " + function.getFunctionName().toString();
25022                                                        if (sqlparser.parse() == 0) {
25023                                                                TObjectName objectName = sqlparser.sqlstatements.get(0).getResultColumnList()
25024                                                                                .getResultColumn(0).getFieldAttr();
25025                                                                objectNames.add(objectName);
25026                                                        }
25027                                                }
25028                                        }
25029                                }
25030                        } else if (lcexpr.getExpressionType() == EExpressionType.function_t || lcexpr.getExpressionType() == EExpressionType.fieldselection_t) {
25031                                TFunctionCall func = lcexpr.getFunctionCall();
25032                                if (func == null) {
25033                                        return true;
25034                                }
25035                                if (skipFunction) {
25036                                        if (func.getArgs() != null) {
25037                                                for (int k = 0; k < func.getArgs().size(); k++) {
25038                                                        TExpression expr = func.getArgs().getExpression(k);
25039                                                        if (expr != null)
25040                                                                expr.inOrderTraverse(this);
25041                                                }
25042                                        }
25043
25044                                        if (func.getTrimArgument() != null) {
25045                                                TTrimArgument args = func.getTrimArgument();
25046                                                TExpression expr = args.getStringExpression();
25047                                                if (expr != null) {
25048                                                        expr.inOrderTraverse(this);
25049                                                }
25050                                                expr = args.getTrimCharacter();
25051                                                if (expr != null) {
25052                                                        expr.inOrderTraverse(this);
25053                                                }
25054                                        }
25055
25056                                        if (func.getAgainstExpr() != null) {
25057                                                func.getAgainstExpr().inOrderTraverse(this);
25058                                        }
25059//                                      if (func.getBetweenExpr() != null) {
25060//                                              func.getBetweenExpr().inOrderTraverse(this);
25061//                                      }
25062                                        if (func.getExpr1() != null) {
25063                                                func.getExpr1().inOrderTraverse(this);
25064                                        }
25065                                        if (func.getExpr2() != null) {
25066                                                func.getExpr2().inOrderTraverse(this);
25067                                        }
25068                                        if (func.getExpr3() != null) {
25069                                                func.getExpr3().inOrderTraverse(this);
25070                                        }
25071                                        if (func.getParameter() != null) {
25072                                                func.getParameter().inOrderTraverse(this);
25073                                        }
25074                                } else {
25075                                        functions.add(func);
25076                                }
25077
25078                        } else if (lcexpr.getExpressionType() == EExpressionType.case_t) {
25079                                TCaseExpression expr = lcexpr.getCaseExpression();
25080                                if (skipFunction) {
25081                                        TExpression defaultExpr = expr.getElse_expr();
25082                                        if (defaultExpr != null) {
25083                                                defaultExpr.inOrderTraverse(this);
25084                                        }
25085                                        TWhenClauseItemList list = expr.getWhenClauseItemList();
25086                                        for (int i = 0; i < list.size(); i++) {
25087                                                TWhenClauseItem element = (TWhenClauseItem) list.getElement(i);
25088                                                (((TWhenClauseItem) element).getReturn_expr()).inOrderTraverse(this);
25089
25090                                        }
25091                                } else {
25092                                        functions.add(expr);
25093                                }
25094                        } else if (lcexpr.getSubQuery() != null) {
25095                                TSelectSqlStatement select = lcexpr.getSubQuery();
25096                                analyzeSelectStmt(select);
25097                                subquerys.add(select);
25098                                if (select.getResultColumnList() != null && select.getResultColumnList().size() > 0) {
25099                                        for (TResultColumn column : select.getResultColumnList()) {
25100                                                resultColumns.add(column);
25101                                        }
25102                                }
25103                        }
25104                        return true;
25105                }
25106        }
25107
25108        class joinInExpr implements IExpressionVisitor {
25109
25110                private EJoinType joinType;
25111                private JoinClauseType joinClauseType;
25112                private EffectType effectType;
25113
25114                public joinInExpr(EJoinType joinType, JoinClauseType joinClauseType, EffectType effectType) {
25115                        this.joinType = joinType;
25116                        this.joinClauseType = joinClauseType;
25117                        this.effectType = effectType;
25118                }
25119
25120                boolean is_compare_condition(EExpressionType t) {
25121                        return ((t == EExpressionType.simple_comparison_t) || (t == EExpressionType.group_comparison_t)
25122                                        || (t == EExpressionType.in_t) || (t == EExpressionType.pattern_matching_t)
25123                                        || (t == EExpressionType.left_join_t) || (t == EExpressionType.right_join_t));
25124                }
25125
25126                @Override
25127                public boolean exprVisit(TParseTreeNode pNode, boolean isLeafNode) {
25128                        TExpression expr = (TExpression) pNode;
25129                        if (is_compare_condition(expr.getExpressionType())) {
25130                                TExpression leftExpr = expr.getLeftOperand();
25131                                columnsInExpr leftVisitor = new columnsInExpr();
25132                                leftExpr.inOrderTraverse(leftVisitor);
25133                                List<TObjectName> leftObjectNames = leftVisitor.getObjectNames();
25134                                List<TParseTreeNode> leftObjects = leftVisitor.getFunctions();
25135                                leftObjects.addAll(leftObjectNames);
25136                                
25137                                TExpression rightExpr = expr.getRightOperand();
25138                                columnsInExpr rightVisitor = new columnsInExpr();
25139                                rightExpr.inOrderTraverse(rightVisitor);
25140                                List<TObjectName> rightObjectNames = rightVisitor.getObjectNames();
25141                                List<TParseTreeNode> rightObjects = rightVisitor.getFunctions();
25142                                rightObjects.addAll(rightObjectNames);
25143
25144                                if (!leftObjects.isEmpty() && !rightObjects.isEmpty()) {
25145                                        TCustomSqlStatement stmt = stmtStack.peek();
25146
25147                                        for (int i = 0; i < leftObjects.size(); i++) {
25148                                                TParseTreeNode leftObject = leftObjects.get(i);
25149                                                TTable leftTable = null;
25150                                                TFunctionCall leftFunction = null;
25151                                                TObjectName leftObjectName = null;
25152                                                if (leftObject instanceof TObjectName) {
25153                                                        leftObjectName = (TObjectName)leftObject;
25154
25155                                                        if (leftObjectName.getDbObjectType() == EDbObjectType.variable) {
25156                                                                continue;
25157                                                        }
25158
25159                                                        if (leftObjectName.getColumnNameOnly().startsWith("@")
25160                                                                        && (option.getVendor() == EDbVendor.dbvmssql
25161                                                                                        || option.getVendor() == EDbVendor.dbvazuresql)) {
25162                                                                continue;
25163                                                        }
25164
25165                                                        if (leftObjectName.getColumnNameOnly().startsWith(":")
25166                                                                        && (option.getVendor() == EDbVendor.dbvhana
25167                                                                                        || option.getVendor() == EDbVendor.dbvteradata)) {
25168                                                                continue;
25169                                                        }
25170
25171                                                        leftTable = modelManager.getTable(stmt, leftObjectName);
25172
25173                                                        if (leftTable == null) {
25174                                                                leftTable = leftObjectName.getSourceTable();
25175                                                        }
25176
25177                                                        if (leftTable == null) {
25178                                                                leftTable = modelManager.guessTable(stmt, leftObjectName);
25179                                                        }
25180                                                }
25181                                                else if(leftObject instanceof TFunctionCall){
25182                                                        leftFunction = (TFunctionCall)leftObject;
25183                                                }
25184
25185                                                if (leftTable != null || leftFunction != null) {
25186                                                        for (int j = 0; j < rightObjects.size(); j++) {
25187                                                                JoinRelationship joinRelation = modelFactory.createJoinRelation();
25188                                                                joinRelation.setEffectType(effectType);
25189                                                                if (joinType != null) {
25190                                                                        joinRelation.setJoinType(joinType);
25191                                                                } else {
25192                                                                        if (expr.getLeftOperand().isOracleOuterJoin()) {
25193                                                                                joinRelation.setJoinType(right);
25194                                                                        } else if (expr.getRightOperand().isOracleOuterJoin()) {
25195                                                                                joinRelation.setJoinType(EJoinType.left);
25196                                                                        } else if (expr.getExpressionType() == EExpressionType.left_join_t) {
25197                                                                                joinRelation.setJoinType(EJoinType.left);
25198                                                                        } else if (expr.getExpressionType() == EExpressionType.right_join_t) {
25199                                                                                joinRelation.setJoinType(right);
25200                                                                        } else {
25201                                                                                joinRelation.setJoinType(EJoinType.inner);
25202                                                                        }
25203                                                                }
25204
25205                                                                joinRelation.setJoinClauseType(joinClauseType);
25206                                                                joinRelation.setJoinCondition(expr.toString());
25207
25208
25209                                                                if (leftTable != null) {
25210                                                                        if (modelManager.getModel(leftTable) instanceof Table) {
25211                                                                                Table tableModel = (Table) modelManager.getModel(leftTable);
25212                                                                                if (tableModel != null) {
25213                                                                                        TableColumn columnModel = modelFactory.createTableColumn(tableModel,
25214                                                                                                        leftObjectName, false);
25215                                                                                        if (columnModel != null) {
25216                                                                                                joinRelation.addSource(new TableColumnRelationshipElement(columnModel));
25217                                                                                        }
25218                                                                                }
25219                                                                        } else if (modelManager.getModel(leftTable) instanceof QueryTable) {
25220                                                                                QueryTable table = (QueryTable) modelManager.getModel(leftTable);
25221                                                                                TSelectSqlStatement subquery = table.getTableObject().getSubquery();
25222                                                                                if (subquery != null && subquery.isCombinedQuery()) {
25223                                                                                        ResultColumn resultColumn = matchResultColumn(table.getColumns(),
25224                                                                                                        leftObjectName);
25225                                                                                        if (resultColumn != null) {
25226                                                                                                joinRelation
25227                                                                                                                .addSource(new ResultColumnRelationshipElement(resultColumn));
25228                                                                                        }
25229                                                                                } else if (leftObjectName.getSourceColumn() != null) {
25230                                                                                        Object model = modelManager.getModel(leftObjectName);
25231                                                                                        if (model == null) {
25232                                                                                                model = modelFactory.createResultColumn(table, leftObjectName);
25233                                                                                        }
25234                                                                                        if (model instanceof ResultColumn) {
25235                                                                                                ResultColumn resultColumn = (ResultColumn) model;
25236                                                                                                if (resultColumn != null) {
25237                                                                                                        joinRelation.addSource(
25238                                                                                                                        new ResultColumnRelationshipElement(resultColumn));
25239                                                                                                }
25240                                                                                        } else if (model instanceof LinkedHashMap) {
25241                                                                                                String columnName = getColumnNameOnly(leftObjectName.toString());
25242                                                                                                LinkedHashMap<String, ResultColumn> resultColumns = (LinkedHashMap<String, ResultColumn>) model;
25243                                                                                                if (resultColumns.containsKey(columnName)) {
25244                                                                                                        ResultColumn resultColumn = resultColumns.get(columnName);
25245                                                                                                        joinRelation.addSource(
25246                                                                                                                        new ResultColumnRelationshipElement(resultColumn));
25247                                                                                                }
25248                                                                                        }
25249                                                                                } else {
25250                                                                                        ResultColumn resultColumn = matchResultColumn(table.getColumns(),
25251                                                                                                        leftObjectName);
25252                                                                                        if (resultColumn != null) {
25253                                                                                                joinRelation
25254                                                                                                                .addSource(new ResultColumnRelationshipElement(resultColumn));
25255                                                                                        }
25256                                                                                }
25257                                                                        }
25258                                                                }
25259                                                                else if(leftFunction!=null) {
25260                                                                        Object functionObj = createFunction(leftFunction);
25261                                                                        if(functionObj instanceof Function) {
25262                                                                                Function function = (Function)functionObj;
25263                                                                                joinRelation.addSource(new ResultColumnRelationshipElement(function.getColumns().get(0)));
25264                                                                        }
25265                                                                }
25266
25267                                                                TParseTreeNode rightObject = rightObjects.get(j);
25268                                                                if(rightObject instanceof TObjectName) {
25269                                                                        TObjectName rightObjectName = (TObjectName)rightObject;
25270
25271                                                                        if (rightObjectName.getDbObjectType() == EDbObjectType.variable) {
25272                                                                                continue;
25273                                                                        }
25274
25275                                                                        if (rightObjectName.getColumnNameOnly().startsWith("@")
25276                                                                                        && (option.getVendor() == EDbVendor.dbvmssql
25277                                                                                                        || option.getVendor() == EDbVendor.dbvazuresql)) {
25278                                                                                continue;
25279                                                                        }
25280
25281                                                                        if (rightObjectName.getColumnNameOnly().startsWith(":")
25282                                                                                        && (option.getVendor() == EDbVendor.dbvhana
25283                                                                                                        || option.getVendor() == EDbVendor.dbvteradata)) {
25284                                                                                continue;
25285                                                                        }
25286
25287                                                                        TTable rightTable = modelManager.getTable(stmt, rightObjectName);
25288                                                                        if (rightTable == null) {
25289                                                                                rightTable = rightObjectName.getSourceTable();
25290                                                                        }
25291
25292                                                                        if (rightTable == null) {
25293                                                                                rightTable = modelManager.guessTable(stmt, rightObjectName);
25294                                                                        }
25295
25296                                                                        if (modelManager.getModel(rightTable) instanceof Table) {
25297                                                                                Table tableModel = (Table) modelManager.getModel(rightTable);
25298                                                                                if (tableModel != null) {
25299                                                                                        TableColumn columnModel = modelFactory.createTableColumn(tableModel,
25300                                                                                                        rightObjectName, false);
25301                                                                                        if(columnModel != null) {
25302                                                                                                joinRelation.setTarget(new TableColumnRelationshipElement(columnModel));
25303                                                                                        }
25304                                                                                }
25305                                                                        } else if (modelManager.getModel(rightTable) instanceof QueryTable) {
25306                                                                                QueryTable table = (QueryTable) modelManager.getModel(rightTable);
25307                                                                                TSelectSqlStatement subquery = table.getTableObject().getSubquery();
25308                                                                                if (subquery != null && subquery.isCombinedQuery()) {
25309                                                                                        ResultColumn resultColumn = matchResultColumn(table.getColumns(),
25310                                                                                                        rightObjectName);
25311                                                                                        if (resultColumn != null) {
25312                                                                                                joinRelation.setTarget(new ResultColumnRelationshipElement(resultColumn));
25313                                                                                        }
25314                                                                                } else if (rightObjectName.getSourceColumn() != null) {
25315                                                                                        Object model = modelManager.getModel(rightObjectName);
25316                                                                                        if (model == null) {
25317                                                                                                model = modelManager
25318                                                                                                                .getModel(rightObjectName.getSourceColumn());
25319                                                                                        }
25320                                                                                        if (model instanceof ResultColumn) {
25321                                                                                                joinRelation.setTarget(new ResultColumnRelationshipElement((ResultColumn)model));
25322                                                                                        }
25323                                                                                        else if (model instanceof LinkedHashMap) {
25324                                                                                                String columnName = getColumnNameOnly(rightObjectName.toString());
25325                                                                                                LinkedHashMap<String, ResultColumn> resultColumns = (LinkedHashMap<String, ResultColumn>)model;
25326                                                                                                if (resultColumns.containsKey(columnName)) {
25327                                                                                                        ResultColumn resultColumn = resultColumns.get(columnName);
25328                                                                                                        joinRelation.setTarget(new ResultColumnRelationshipElement(resultColumn));
25329                                                                                                }
25330                                                                                        }
25331                                                                                } else {
25332                                                                                        ResultColumn resultColumn = matchResultColumn(table.getColumns(),
25333                                                                                                        rightObjectName);
25334                                                                                        if (resultColumn != null) {
25335                                                                                                joinRelation.setTarget(new ResultColumnRelationshipElement(resultColumn));
25336                                                                                        }
25337                                                                                }
25338                                                                        }
25339                                                                }
25340                                                                else if(rightObject instanceof TFunctionCall) {
25341                                                                        Object functionObj = createFunction(rightObject);
25342                                                                        if(functionObj instanceof Function) {
25343                                                                                Function function = (Function)functionObj;
25344                                                                                joinRelation.setTarget(new ResultColumnRelationshipElement(function.getColumns().get(0)));
25345                                                                        }
25346                                                                }
25347                                                        }
25348                                                }
25349                                        }
25350                                }
25351                        }
25352                        return true;
25353                }
25354        }
25355
25356        @Deprecated
25357        public static Dataflow getSqlflowJSONModel(dataflow dataflow) {
25358                EDbVendor vendor = ModelBindingManager.getGlobalVendor();
25359                if(vendor == null) {
25360                        throw new IllegalArgumentException("getSqlflowJSONModel(dataflow dataflow) is deprecated, please call method getSqlflowJSONModel(dataflow dataflow, EDbVendor vendor).");
25361                }
25362                return getSqlflowJSONModel(vendor, dataflow, false);
25363        }
25364
25365        public static Dataflow getSqlflowJSONModel(dataflow dataflow, EDbVendor vendor) {
25366                return getSqlflowJSONModel(vendor, dataflow, false);
25367        }
25368
25369        public static Dataflow getSqlflowJSONModel(EDbVendor vendor, dataflow dataflow, boolean normalizeIdentifier) {
25370                Dataflow model = new Dataflow();
25371                
25372                if (dataflow.getErrors() != null && !dataflow.getErrors().isEmpty()) {
25373                        List<Error> errorList = new ArrayList<Error>();
25374                        for (error error : dataflow.getErrors()) {
25375                                Error err = new Error();
25376                                err.setErrorMessage(error.getErrorMessage());
25377                                err.setErrorType(error.getErrorType());
25378                                err.setCoordinates(Coordinate.parse(error.getCoordinate()));
25379                                err.setFile(err.getFile());
25380                                err.setOriginCoordinates(Coordinate.parse(error.getOriginCoordinate()));
25381                                errorList.add(err);
25382                        }
25383                        model.setErrors(errorList.toArray(new Error[0]));
25384                }
25385
25386                Sqlflow sqlflow = MetadataUtil.convertDataflowToMetadata(vendor, dataflow);
25387                sqlflow.setErrorMessages(null);
25388                model.setDbobjs(sqlflow);
25389                model.setOrientation(dataflow.getOrientation());
25390
25391
25392                List<gudusoft.gsqlparser.dlineage.dataflow.model.json.Process> processes = new ArrayList<gudusoft.gsqlparser.dlineage.dataflow.model.json.Process>();
25393                if(dataflow.getProcesses()!=null){
25394                        for(process process: dataflow.getProcesses()){
25395                                gudusoft.gsqlparser.dlineage.dataflow.model.json.Process processModel = new gudusoft.gsqlparser.dlineage.dataflow.model.json.Process();
25396                                processModel.setId(process.getId());
25397                                processModel.setName(process.getName());
25398                                processModel.setProcedureId(process.getProcedureId());
25399                                processModel.setProcedureName(process.getProcedureName());
25400                                processModel.setType(process.getType());
25401                                processModel.setCoordinate(process.getCoordinate());
25402                                processModel.setDatabase(process.getDatabase());
25403                                processModel.setSchema(process.getSchema());
25404                                processModel.setServer(process.getServer());
25405                                processModel.setQueryHashId(process.getQueryHashId());
25406                                if (process.getTransforms() != null && !process.getTransforms().isEmpty()) {
25407                                        List<gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform> transforms = new ArrayList<gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform>();
25408                                        for (transform transform : process.getTransforms()) {
25409                                                gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform item = new gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform();
25410                                                item.setCode(transform.getCode());
25411                                                item.setType(transform.getType());
25412                                                item.setCoordinate(transform.getCoordinate(true));
25413                                                transforms.add(item);
25414                                        }
25415                                        processModel.setTransforms(transforms
25416                                                        .toArray(new gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform[0]));
25417                                }
25418                                processes.add(processModel);
25419                        }
25420                }
25421                model.setProcesses(processes.toArray(new gudusoft.gsqlparser.dlineage.dataflow.model.json.Process[0]));
25422
25423                List<gudusoft.gsqlparser.dlineage.dataflow.model.json.Relationship> relations = new ArrayList<gudusoft.gsqlparser.dlineage.dataflow.model.json.Relationship>();
25424                if (dataflow.getRelationships() != null) {
25425                        for (relationship relation : dataflow.getRelationships()) {
25426                                gudusoft.gsqlparser.dlineage.dataflow.model.json.Relationship relationModel;
25427                                if (relation.getType().equals("join")) {
25428                                        gudusoft.gsqlparser.dlineage.dataflow.model.json.JoinRelationship joinRelationModel = new gudusoft.gsqlparser.dlineage.dataflow.model.json.JoinRelationship();
25429                                        joinRelationModel.setCondition(relation.getCondition());
25430                                        joinRelationModel.setJoinType(relation.getJoinType());
25431                                        joinRelationModel.setClause(relation.getClause());
25432                                        relationModel = joinRelationModel;
25433                                } else {
25434                                        relationModel = new gudusoft.gsqlparser.dlineage.dataflow.model.json.Relationship();
25435                                }
25436
25437                                relationModel.setId(relation.getId());
25438                                relationModel.setProcessId(relation.getProcessId());
25439                                relationModel.setProcessType(relation.getProcessType());
25440                                relationModel.setType(relation.getType());
25441                                relationModel.setEffectType(relation.getEffectType());
25442                                relationModel.setPartition(relation.getPartition());
25443                                relationModel.setFunction(relation.getFunction());
25444                                relationModel.setProcedureId(relation.getProcedureId());
25445                                relationModel.setSqlHash(relation.getSqlHash());
25446                                relationModel.setCondition(relation.getCondition());
25447                                relationModel.setSqlComment(relation.getSqlComment());
25448                                relationModel.setTimestampMax(relation.getTimestampMax());
25449                                relationModel.setTimestampMin(relation.getTimestampMin());
25450                                if (Boolean.TRUE.equals(relation.getBuiltIn())) {
25451                                        relationModel.setBuiltIn(relation.getBuiltIn());
25452                                }
25453                                relationModel.setCallStmt(relation.getCallStmt());
25454                                relationModel.setCallCoordinate(relation.getCallCoordinate());
25455                                
25456                                if (relation.getTarget() != null && relation.getSources() != null && !relation.getSources().isEmpty()) {
25457                                        {
25458                                                gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement targetModel = new gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement();
25459                                                targetColumn target = relation.getTarget();
25460                                                if (normalizeIdentifier) {
25461                                                        targetModel.setColumn(SQLUtil.getIdentifierNormalColumnName(vendor, target.getColumn()));
25462                                                        targetModel.setParentName(
25463                                                                        SQLUtil.getIdentifierNormalTableName(vendor, target.getParent_name()));
25464                                                        targetModel.setTargetName(
25465                                                                        SQLUtil.getIdentifierNormalColumnName(vendor, target.getTarget_name()));
25466                                                } else {
25467                                                        targetModel.setColumn(target.getColumn());
25468                                                        targetModel.setParentName(target.getParent_name());
25469                                                        targetModel.setTargetName(target.getTarget_name());
25470                                                }
25471                                                targetModel.setId(target.getId());
25472                                                targetModel.setTargetId(target.getTarget_id());
25473                                                targetModel.setParentId(target.getParent_id());
25474                                                targetModel.setCoordinates(Coordinate.parse(target.getCoordinate()));
25475                                                targetModel.setFunction(target.getFunction());
25476                                                targetModel.setType(target.getType());
25477                                                relationModel.setTarget(targetModel);
25478                                        }
25479
25480                                        List<gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement> sourceModels = new ArrayList<gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement>();
25481                                        for (sourceColumn source : relation.getSources()) {
25482                                                gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement sourceModel = new gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement();
25483                                                if (normalizeIdentifier) {
25484                                                        sourceModel.setColumn(SQLUtil.getIdentifierNormalColumnName(vendor, source.getColumn()));
25485                                                        sourceModel.setParentName(
25486                                                                        SQLUtil.getIdentifierNormalTableName(vendor, source.getParent_name()));
25487                                                        sourceModel.setSourceName(
25488                                                                        SQLUtil.getIdentifierNormalColumnName(vendor, source.getSource_name()));
25489                                                } else {
25490                                                        sourceModel.setColumn(source.getColumn());
25491                                                        sourceModel.setParentName(source.getParent_name());
25492                                                        sourceModel.setSourceName(source.getSource_name());
25493                                                }
25494                                                sourceModel.setColumnType(source.getColumn_type());
25495                                                sourceModel.setId(source.getId());
25496                                                sourceModel.setParentId(source.getParent_id());
25497                                                sourceModel.setSourceId(source.getSource_id());
25498                                                sourceModel.setCoordinates(Coordinate.parse(source.getCoordinate()));
25499                                                sourceModel.setClauseType(source.getClauseType());
25500                                                sourceModel.setType(source.getType());
25501                                                sourceModels.add(sourceModel);
25502                                                if (source.getTransforms() != null && !source.getTransforms().isEmpty()) {
25503                                                        List<gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform> transforms = new ArrayList<gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform>();
25504                                                        for (transform transform : source.getTransforms()) {
25505                                                                gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform item = new gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform();
25506                                                                item.setCode(transform.getCode());
25507                                                                item.setType(transform.getType());
25508                                                                item.setCoordinate(transform.getCoordinate(true));
25509                                                                transforms.add(item);
25510                                                        }
25511                                                        sourceModel.setTransforms(transforms
25512                                                                        .toArray(new gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform[0]));
25513                                                }
25514                                                
25515                                                if (source.getCandidateParents() != null && !source.getCandidateParents().isEmpty()) {
25516                                                        List<gudusoft.gsqlparser.dlineage.dataflow.model.json.CandidateTable> candidateParents = new ArrayList<gudusoft.gsqlparser.dlineage.dataflow.model.json.CandidateTable>();
25517                                                        for (candidateTable candidateTable : source.getCandidateParents()) {
25518                                                                gudusoft.gsqlparser.dlineage.dataflow.model.json.CandidateTable item = new gudusoft.gsqlparser.dlineage.dataflow.model.json.CandidateTable();
25519                                                                item.setId(candidateTable.getId());
25520                                                                if (normalizeIdentifier) {
25521                                                                        item.setName(
25522                                                                                        SQLUtil.getIdentifierNormalTableName(vendor, candidateTable.getName()));
25523                                                                } else {
25524                                                                        item.setName(candidateTable.getName());
25525                                                                }
25526                                                                candidateParents.add(item);
25527                                                        }
25528                                                        sourceModel.setCandidateParents(candidateParents
25529                                                                        .toArray(new gudusoft.gsqlparser.dlineage.dataflow.model.json.CandidateTable[0]));
25530                                                }
25531                                        }
25532                                        relationModel.setSources(sourceModels
25533                                                        .toArray(new gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement[0]));
25534                                        relations.add(relationModel);
25535                                } else if (relation.getCaller() != null && relation.getCallees() != null
25536                                                && !relation.getCallees().isEmpty()) {
25537                                        {
25538                                                gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement targetModel = new gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement();
25539                                                targetColumn target = relation.getCaller();
25540                                                if (normalizeIdentifier) {
25541                                                        targetModel.setName(SQLUtil.getIdentifierNormalColumnName(vendor, target.getName()));
25542                                                } else {
25543                                                        targetModel.setName(target.getName());
25544                                                }
25545                                                targetModel.setId(target.getId());
25546                                                targetModel.setCoordinates(Coordinate.parse(target.getCoordinate()));
25547                                                targetModel.setType(target.getType());
25548                                                relationModel.setCaller(targetModel);
25549                                        }
25550
25551                                        List<gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement> sourceModels = new ArrayList<gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement>();
25552                                        for (sourceColumn source : relation.getCallees()) {
25553                                                gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement sourceModel = new gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement();
25554                                                if (normalizeIdentifier) {
25555                                                        sourceModel.setName(SQLUtil.getIdentifierNormalColumnName(vendor, source.getName()));
25556                                                } else {
25557                                                        sourceModel.setName(source.getName());
25558                                                }
25559                                                sourceModel.setId(source.getId());
25560                                                sourceModel.setCoordinates(Coordinate.parse(source.getCoordinate()));
25561                                                sourceModel.setType(source.getType());
25562                                                sourceModels.add(sourceModel);
25563                                        }
25564                                        relationModel.setCallees(sourceModels
25565                                                        .toArray(new gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement[0]));
25566                                        relations.add(relationModel);
25567                                }
25568                        }
25569                }
25570                model.setRelationships(relations.toArray(new gudusoft.gsqlparser.dlineage.dataflow.model.json.Relationship[0]));
25571                return model;
25572        }
25573
25574        public static String getVersion() {
25575                return "3.1.4";
25576        }
25577
25578        public static String getReleaseDate() {
25579                return "2023-03-04";
25580        }
25581
25582        public static void main(String[] args) {
25583                if (args.length < 1) {
25584                        System.out.println(
25585                                        "Usage: java DataFlowAnalyzer [/f <path_to_sql_file>] [/d <path_to_directory_includes_sql_files>] [/s [/text]] [/json] [/traceView] [/t <database type>] [/o <output file path>][/version]");
25586                        System.out.println("/f: Option, specify the sql file path to analyze fdd relation.");
25587                        System.out.println("/d: Option, specify the sql directory path to analyze fdd relation.");
25588                        System.out.println("/j: Option, analyze the join relation.");
25589                        System.out.println("/s: Option, simple output, ignore the intermediate results.");
25590                        System.out.println("/i: Option, ignore all result sets.");
25591                        System.out.println("/traceView: Option, analyze the source tables of views.");
25592                        System.out.println("/text: Option, print the plain text format output.");
25593                        System.out.println("/json: Option, print the json format output.");
25594                        System.out.println(
25595                                        "/t: Option, set the database type. Support oracle, mysql, mssql, db2, netezza, teradata, informix, sybase, postgresql, hive, greenplum and redshift, the default type is oracle");
25596                        System.out.println("/o: Option, write the output stream to the specified file.");
25597                        System.out.println("/log: Option, generate a dataflow.log file to log information.");
25598                        return;
25599                }
25600
25601                File sqlFiles = null;
25602
25603                List<String> argList = Arrays.asList(args);
25604
25605                if (argList.indexOf("/version") != -1) {
25606                        System.out.println("Version: " + DataFlowAnalyzer.getVersion());
25607                        System.out.println("Release Date: " + DataFlowAnalyzer.getReleaseDate());
25608                        return;
25609                }
25610
25611                if (argList.indexOf("/f") != -1 && argList.size() > argList.indexOf("/f") + 1) {
25612                        sqlFiles = new File(args[argList.indexOf("/f") + 1]);
25613                        if (!sqlFiles.exists() || !sqlFiles.isFile()) {
25614                                System.out.println(sqlFiles + " is not a valid file.");
25615                                return;
25616                        }
25617                } else if (argList.indexOf("/d") != -1 && argList.size() > argList.indexOf("/d") + 1) {
25618                        sqlFiles = new File(args[argList.indexOf("/d") + 1]);
25619                        if (!sqlFiles.exists() || !sqlFiles.isDirectory()) {
25620                                System.out.println(sqlFiles + " is not a valid directory.");
25621                                return;
25622                        }
25623                } else {
25624                        System.out.println("Please specify a sql file path or directory path to analyze dlineage.");
25625                        return;
25626                }
25627
25628                EDbVendor vendor = EDbVendor.dbvoracle;
25629
25630                int index = argList.indexOf("/t");
25631
25632                if (index != -1 && args.length > index + 1) {
25633                        vendor = TGSqlParser.getDBVendorByName(args[index + 1]);
25634                }
25635
25636                String outputFile = null;
25637
25638                index = argList.indexOf("/o");
25639
25640                if (index != -1 && args.length > index + 1) {
25641                        outputFile = args[index + 1];
25642                }
25643
25644                FileOutputStream writer = null;
25645                if (outputFile != null) {
25646                        try {
25647                                writer = new FileOutputStream(outputFile);
25648                                System.setOut(new PrintStream(writer));
25649                        } catch (FileNotFoundException e) {
25650                                logger.error("output file is not found.", e);
25651                        }
25652                }
25653
25654                boolean simple = argList.indexOf("/s") != -1;
25655                boolean ignoreResultSets = argList.indexOf("/i") != -1;
25656                boolean showJoin = argList.indexOf("/j") != -1;
25657                boolean textFormat = false;
25658                boolean jsonFormat = false;
25659                if (simple) {
25660                        textFormat = argList.indexOf("/text") != -1;
25661                }
25662
25663                boolean traceView = argList.indexOf("/traceView") != -1;
25664                if (traceView) {
25665                        simple = true;
25666                }
25667
25668                jsonFormat = argList.indexOf("/json") != -1;
25669
25670                DataFlowAnalyzer dlineage = new DataFlowAnalyzer(sqlFiles, vendor, simple);
25671
25672                dlineage.setShowJoin(showJoin);
25673                dlineage.setIgnoreRecordSet(ignoreResultSets);
25674                // dlineage.setShowImplicitSchema(true);
25675
25676                if (simple && !jsonFormat) {
25677                        dlineage.setTextFormat(textFormat);
25678                }
25679
25680                String result = dlineage.generateDataFlow();
25681
25682//              dataflow dataflow = ProcessUtility.generateTableLevelLineage(dlineage, dlineage.getDataFlow());
25683//              System.out.println(result);
25684
25685                if (jsonFormat) {
25686                        // Map jsonResult = new LinkedHashMap();
25687                        Dataflow model = getSqlflowJSONModel(vendor, dlineage.getDataFlow(), true);
25688                        // jsonResult.put("data", BeanUtils.bean2Map(model));
25689                        result = JSON.toJSONString(model);
25690                } else if (traceView) {
25691                        result = dlineage.traceView();
25692                }
25693
25694                if (result != null) {
25695                        System.out.println(result);
25696
25697                        if (writer != null && result.length() < 1024 * 1024) {
25698                                System.err.println(result);
25699                        }
25700                }
25701
25702                try {
25703                        if (writer != null) {
25704                                writer.close();
25705                        }
25706                } catch (IOException e) {
25707                        logger.error("close writer failed.", e);
25708                }
25709
25710                boolean log = argList.indexOf("/log") != -1;
25711
25712                PrintStream systemSteam = System.err;
25713                ByteArrayOutputStream sw = new ByteArrayOutputStream();
25714                PrintStream pw = new PrintStream(sw);
25715                System.setErr(pw);
25716
25717
25718                List<ErrorInfo> errors = dlineage.getErrorMessages();
25719                if (!errors.isEmpty()) {
25720                        System.err.println("Error log:\n");
25721                        for (int i = 0; i < errors.size(); i++) {
25722                                System.err.println(errors.get(i).getErrorMessage());
25723                        }
25724                }
25725
25726                if (sw != null) {
25727                        String errorMessage = sw.toString().trim();
25728                        if (errorMessage.length() > 0) {
25729                                if (log) {
25730                                        try {
25731                                                pw = new PrintStream(new File(".", "dataflow.log"));
25732                                                pw.print(errorMessage);
25733                                        } catch (FileNotFoundException e) {
25734                                                logger.error("error log file is not found.", e);
25735                                        }
25736                                }
25737
25738                                System.setErr(systemSteam);
25739                                System.err.println(errorMessage);
25740                        }
25741                }
25742        }
25743
25744        public List<ErrorInfo> getErrorMessages() {
25745                return errorInfos;
25746        }
25747
25748        /**
25749         * Every dynamic-SQL execution site (T-SQL {@code EXEC(...)} / {@code sp_executesql}) the analyzer
25750         * encountered, with whether it was statically resolved. Diagnostic only; not consumed by lineage.
25751         * Read-only snapshot; see {@link DynamicSqlSite}.
25752         */
25753        public List<DynamicSqlSite> getDynamicSqlSites() {
25754                return java.util.Collections.unmodifiableList(new ArrayList<DynamicSqlSite>(dynamicSqlSites));
25755        }
25756
25757        /**
25758         * Resolve dynamic-SQL object names by abstractly evaluating a stored procedure
25759         * with a set of parameter bindings.
25760         *
25761         * <p>Given a T-SQL {@code CREATE PROC} parse tree and the values its parameters
25762         * actually had at a call site (e.g. {@code @SourceDB='QSP'},
25763         * {@code @STG_AEG='STG_AEG'}), this evaluates the procedure's dynamic-SQL string
25764         * building (assignment + concatenation + name-building functions), materializes
25765         * the concrete SQL each {@code EXEC} / {@code sp_executesql} site runs, analyzes
25766         * it with ordinary dlineage, and returns the lineage edges tagged with dynamic
25767         * provenance ({@code origin=DYNAMIC_RESOLVED}, source proc, stable dynamic-site
25768         * id, binding hash). Sites that cannot be reduced are reported as honest
25769         * {@code UNRESOLVED} diagnostics — never guessed.
25770         *
25771         * <p>This is abstract string evaluation, NOT execution: no table reads, no
25772         * cursors, no data-dependent control flow. It is purely additive — it builds
25773         * throwaway analyzers for the materialized strings and does not change the
25774         * default dlineage output of this analyzer.
25775         *
25776         * @param procAst         the {@code CREATE PROC} parse tree (currently T-SQL)
25777         * @param vendor          the SQL dialect ({@code dbvmssql})
25778         * @param sqlEnv          catalog env for the resolved analysis (may be null)
25779         * @param currentDatabase the proc's database (resolves unqualified names)
25780         * @param defaultSchema   the proc's default schema (e.g. {@code dbo})
25781         * @param bindings        parameter name → concrete value
25782         * @param options         resource bounds (may be null for defaults)
25783         */
25784        public gudusoft.gsqlparser.dlineage.dynamicsql.DynamicLineageResult resolveDynamicSqlLineage(
25785                        TCustomSqlStatement procAst, EDbVendor vendor, gudusoft.gsqlparser.sqlenv.TSQLEnv sqlEnv,
25786                        String currentDatabase, String defaultSchema,
25787                        Map<String, gudusoft.gsqlparser.dlineage.dynamicsql.SqlValue> bindings,
25788                        gudusoft.gsqlparser.dlineage.dynamicsql.DynamicLineageOptions options) {
25789                return gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.resolve(
25790                                procAst, vendor, sqlEnv, currentDatabase, defaultSchema, bindings, options);
25791        }
25792
25793        public String traceView() {
25794                StringBuilder buffer = new StringBuilder();
25795                dataflow dataflow = this.getDataFlow();
25796                Map<table, Set<table>> traceViewMap = new LinkedHashMap<table, Set<table>>();
25797                if (dataflow != null && dataflow.getViews() != null) {
25798                        List<relationship> relations = dataflow.getRelationships();
25799                        Map<String, table> viewMap = new HashMap<String, table>();
25800                        Map<String, table> tableMap = new HashMap<String, table>();
25801                        for (table view : dataflow.getViews()) {
25802                                viewMap.put(view.getId(), view);
25803                                tableMap.put(view.getId(), view);
25804                        }
25805                        for (table table : dataflow.getTables()) {
25806                                tableMap.put(table.getId(), table);
25807                        }
25808                        for (relationship relation : relations) {
25809                                if (!RelationshipType.fdd.name().equals(relation.getType())) {
25810                                        continue;
25811                                }
25812                                String parentId = relation.getTarget().getParent_id();
25813                                if (viewMap.containsKey(parentId)) {
25814                                        if (!traceViewMap.containsKey(viewMap.get(parentId))) {
25815                                                traceViewMap.put(viewMap.get(parentId), new LinkedHashSet<table>());
25816                                        }
25817
25818                                        for (sourceColumn sourceColumn : relation.getSources()) {
25819                                                traceViewMap.get(viewMap.get(parentId)).add(tableMap.get(sourceColumn.getParent_id()));
25820                                        }
25821                                }
25822                        }
25823
25824                        Map<table, Set<table>> viewTableMap = new LinkedHashMap<table, Set<table>>();
25825                        for (table view : traceViewMap.keySet()) {
25826                                Set<table> tables = new LinkedHashSet<table>();
25827                                traverseViewSourceTables(tables, view, traceViewMap);
25828                                viewTableMap.put(view, tables);
25829                        }
25830
25831                        for (table view : viewTableMap.keySet()) {
25832                                buffer.append(view.getFullName());
25833                                for (table table : viewTableMap.get(view)) {
25834                                        buffer.append(",").append(table.getFullName());
25835                                }
25836                                buffer.append(System.getProperty("line.separator"));
25837                        }
25838                }
25839                return buffer.toString().trim();
25840        }
25841
25842        private void traverseViewSourceTables(Set<table> tables, table view, Map<table, Set<table>> traceViewMap) {
25843                Set<table> sourceTables = traceViewMap.get(view);
25844                for (table sourceTable : sourceTables) {
25845                        if (sourceTable.isTable()) {
25846                                tables.add(sourceTable);
25847                        } else if (sourceTable.isView()) {
25848                                traverseViewSourceTables(tables, sourceTable, traceViewMap);
25849                        }
25850                }
25851        }
25852
25853        protected List<SqlInfo> convertSQL(EDbVendor vendor, String json) {
25854                List<SqlInfo> sqlInfos = new ArrayList<SqlInfo>();
25855                List sqlContents = (List) JSON.parseObject(json);
25856                for (int j = 0; j < sqlContents.size(); j++) {
25857                        Map sqlContent = (Map) sqlContents.get(j);
25858                        String sql = (String) sqlContent.get("sql");
25859                        String fileName = (String) sqlContent.get("fileName");
25860                        String filePath = (String) sqlContent.get("filePath");
25861                        if (sql != null && sql.trim().startsWith("{")) {
25862                                if (sql.indexOf("createdBy") != -1) {
25863                                        if (this.sqlenv == null) {
25864                                                TSQLEnv[] sqlenvs = new TJSONSQLEnvParser(option.getDefaultServer(),
25865                                                                option.getDefaultDatabase(), option.getDefaultSchema()).parseSQLEnv(vendor, sql);
25866                                                if (sqlenvs != null && sqlenvs.length > 0) {
25867                                                        this.sqlenv = sqlenvs[0];
25868                                                }
25869                                        }
25870                                        if (sql.toLowerCase().indexOf("sqldep") != -1 || sql.toLowerCase().indexOf("grabit") != -1) {
25871                                                Map queryObject = (Map) JSON.parseObject(sql);
25872                                                List querys = (List) queryObject.get("queries");
25873                                                if (querys != null) {
25874                                                        for (int i = 0; i < querys.size(); i++) {
25875                                                                Map object = (Map) querys.get(i);
25876                                                                SqlInfo info = new SqlInfo();
25877                                                                info.setSql(JSON.toJSONString(object));
25878                                                                info.setFileName(fileName);
25879                                                                info.setFilePath(filePath);
25880                                                                info.setOriginIndex(i);
25881                                                                sqlInfos.add(info);
25882                                                        }
25883                                                        queryObject.remove("queries");
25884                                                        SqlInfo info = new SqlInfo();
25885                                                        info.setSql(JSON.toJSONString(queryObject));
25886                                                        info.setFileName(fileName);
25887                                                        info.setFilePath(filePath);
25888                                                        info.setOriginIndex(querys.size());
25889                                                        sqlInfos.add(info);
25890                                                } else {
25891                                                        SqlInfo info = new SqlInfo();
25892                                                        info.setSql(JSON.toJSONString(queryObject));
25893                                                        info.setFileName(fileName);
25894                                                        info.setFilePath(filePath);
25895                                                        info.setOriginIndex(0);
25896                                                        sqlInfos.add(info);
25897                                                }
25898                                        } else if (sql.toLowerCase().indexOf("sqlflow") != -1) {
25899                                                Map sqlflow = (Map) JSON.parseObject(sql);
25900                                                List<Map> servers = (List<Map>) sqlflow.get("servers");
25901                                                if (servers != null) {
25902                                                        for (Map queryObject : servers) {
25903                                                                String name = (String) queryObject.get("name");
25904                                                                String dbVendor = (String) queryObject.get("dbVendor");
25905                                                                List querys = (List) queryObject.get("queries");
25906                                                                if (querys != null) {
25907                                                                        for (int i = 0; i < querys.size(); i++) {
25908                                                                                Map object = (Map) querys.get(i);
25909                                                                                SqlInfo info = new SqlInfo();
25910                                                                                info.setSql(JSON.toJSONString(object));
25911                                                                                info.setFileName(fileName);
25912                                                                                info.setFilePath(filePath);
25913                                                                                info.setOriginIndex(i);
25914                                                                                info.setDbVendor(dbVendor);
25915                                                                                info.setServer(name);
25916                                                                                sqlInfos.add(info);
25917                                                                        }
25918                                                                        queryObject.remove("queries");
25919                                                                        Map serverObject = new IndexedLinkedHashMap();
25920                                                                        serverObject.put("createdBy", sqlflow.get("createdBy"));
25921                                                                        serverObject.put("servers", Arrays.asList(queryObject));
25922                                                                        SqlInfo info = new SqlInfo();
25923                                                                        info.setSql(JSON.toJSONString(serverObject));
25924                                                                        info.setFileName(fileName);
25925                                                                        info.setFilePath(filePath);
25926                                                                        info.setOriginIndex(querys.size());
25927                                                                        info.setDbVendor(dbVendor);
25928                                                                        info.setServer(filePath);
25929                                                                        sqlInfos.add(info);
25930                                                                } else {
25931                                                                        SqlInfo info = new SqlInfo();
25932                                                                        info.setSql(JSON.toJSONString(queryObject));
25933                                                                        info.setFileName(fileName);
25934                                                                        info.setFilePath(filePath);
25935                                                                        info.setOriginIndex(0);
25936                                                                        sqlInfos.add(info);
25937                                                                }
25938                                                        }
25939                                                }
25940                                                
25941                                                List<Map> errorMessages =  (List<Map>) sqlflow.get("errorMessages");
25942                                                if(errorMessages!=null && !errorMessages.isEmpty()) {
25943                                                        for(Map error: errorMessages){
25944                                                                ErrorInfo errorInfo = new ErrorInfo();
25945                                                                errorInfo.setErrorType(ErrorInfo.METADATA_ERROR);
25946                                                                errorInfo.setErrorMessage((String)error.get("errorMessage"));
25947                                                                errorInfo.setFileName(fileName);
25948                                                                errorInfo.setFilePath(filePath);
25949                                                                errorInfo.setStartPosition(new Pair3<Long, Long, String>(-1L, -1L,
25950                                                                                ModelBindingManager.getGlobalHash()));
25951                                                                errorInfo.setEndPosition(new Pair3<Long, Long, String>(-1L, -1L,
25952                                                                                ModelBindingManager.getGlobalHash()));
25953                                                                errorInfo.setOriginStartPosition(new Pair<Long, Long>(-1L, -1L));
25954                                                                errorInfo.setOriginEndPosition(new Pair<Long, Long>(-1L, -1L));
25955                                                                metadataErrors.add(errorInfo);
25956                                                        }
25957                                                }
25958                                        }
25959                                }
25960                        } else if (sql != null) {
25961                                SqlInfo info = new SqlInfo();
25962                                info.setSql(sql);
25963                                info.setFileName(fileName);
25964                                info.setFilePath(filePath);
25965                                info.setOriginIndex(0);
25966                                sqlInfos.add(info);
25967                        }
25968                }
25969                return sqlInfos;
25970        }
25971
25972        public void setTextFormat(boolean textFormat) {
25973                option.setTextFormat(textFormat);
25974        }
25975
25976        public boolean isBuiltInFunctionName(TObjectName object) {
25977                if (object == null || object.getGsqlparser() == null)
25978                        return false;
25979                try {
25980                        EDbVendor vendor = object.getGsqlparser().getDbVendor();
25981                        if (vendor == EDbVendor.dbvteradata) {
25982                                boolean result = TERADATA_BUILTIN_FUNCTIONS.contains(object.toString().toUpperCase());
25983                                if (result) {
25984                                        return true;
25985                                }
25986                        }
25987
25988                        List<String> versions = functionChecker.getAvailableDbVersions(vendor);
25989                        if (versions != null && versions.size() > 0) {
25990                                for (int i = 0; i < versions.size(); i++) {
25991                                        boolean result = functionChecker.isBuiltInFunction(object.toString(),
25992                                                        object.getGsqlparser().getDbVendor(), versions.get(i));
25993                                        if (result) {
25994                                                return result;
25995                                        }
25996                                }
25997
25998                                // boolean result =
25999                                // TERADATA_BUILTIN_FUNCTIONS.contains(object.toString());
26000                                // if (result) {
26001                                // return true;
26002                                // }
26003                        }
26004                } catch (Exception e) {
26005                }
26006
26007                return false;
26008        }
26009        
26010        public boolean isBuiltInFunctionName(String functionName) {
26011                if (functionName == null)
26012                        return false;
26013                try {
26014                        EDbVendor vendor = getOption().getVendor();
26015                        if (vendor == EDbVendor.dbvteradata) {
26016                                boolean result = TERADATA_BUILTIN_FUNCTIONS.contains(functionName.toUpperCase());
26017                                if (result) {
26018                                        return true;
26019                                }
26020                        }
26021
26022                        List<String> versions = functionChecker.getAvailableDbVersions(vendor);
26023                        if (versions != null && versions.size() > 0) {
26024                                for (int i = 0; i < versions.size(); i++) {
26025                                        boolean result = functionChecker.isBuiltInFunction(functionName.toUpperCase(),
26026                                                        vendor, versions.get(i));
26027                                        if (result) {
26028                                                return result;
26029                                        }
26030                                }
26031
26032                                // boolean result =
26033                                // TERADATA_BUILTIN_FUNCTIONS.contains(object.toString());
26034                                // if (result) {
26035                                // return true;
26036                                // }
26037                        }
26038                } catch (Exception e) {
26039                }
26040
26041                return false;
26042        }
26043
26044        public boolean isKeyword(TObjectName object) {
26045                if (object == null || object.getGsqlparser() == null)
26046                        return false;
26047                try {
26048                        EDbVendor vendor = object.getGsqlparser().getDbVendor();
26049
26050                        List<String> versions = keywordChecker.getAvailableDbVersions(vendor);
26051                        if (versions != null && versions.size() > 0) {
26052                                for (int i = 0; i < versions.size(); i++) {
26053                                        List<String> segments = SQLUtil.parseNames(object.toString());
26054                                        boolean result = keywordChecker.isKeyword(segments.get(segments.size() - 1),
26055                                                        object.getGsqlparser().getDbVendor(), versions.get(i), true);
26056                                        if (result) {
26057                                                return result;
26058                                        }
26059                                }
26060                        }
26061                } catch (Exception e) {
26062                }
26063
26064                return false;
26065        }
26066        
26067        public boolean isKeyword(String objectName) {
26068                if (objectName == null)
26069                        return false;
26070                try {
26071                        EDbVendor vendor = getOption().getVendor();
26072
26073                        List<String> versions = keywordChecker.getAvailableDbVersions(vendor);
26074                        if (versions != null && versions.size() > 0) {
26075                                for (int i = 0; i < versions.size(); i++) {
26076                                        List<String> segments = SQLUtil.parseNames(objectName);
26077                                        boolean result = keywordChecker.isKeyword(segments.get(segments.size() - 1),
26078                                                        vendor, versions.get(i), false);
26079                                        if (result) {
26080                                                return result;
26081                                        }
26082                                }
26083                        }
26084                } catch (Exception e) {
26085                }
26086
26087                return false;
26088        }
26089
26090        public boolean isAggregateFunction(TFunctionCall func) {
26091                if (func == null)
26092                        return false;
26093                return Arrays
26094                                .asList(new String[] { "AVG", "COUNT", "MAX", "MIN", "SUM", "COLLECT", "CORR", "COVAR_POP",
26095                                                "COVAR_SAMP", "CUME_DIST", "DENSE_RANK", "FIRST", "GROUP_ID", "GROUPING", "GROUPING_ID", "LAST",
26096                                                "LISTAGG", "MEDIAN", "PERCENT_RANK", "PERCENTILE_CONT", "PERCENTILE_DISC", "RANK",
26097                                                "STATS_BINOMIAL_TEST", "STATS_CROSSTAB", "STATS_F_TEST", "STATS_KS_TEST", "STATS_MODE",
26098                                                "STATS_MW_TEST", "STATS_ONE_WAY_ANOVA", "STATS_WSR_TEST", "STDDEV", "STDDEV_POP", "STDDEV_SAMP",
26099                                                "SYS_XMLAGG", "VAR_ POP", "VAR_ SAMP", "VARI ANCE", "XMLAGG", "ARRAY_AGG" })
26100                                .contains(func.getFunctionName().toString().toUpperCase());
26101        }
26102
26103        public boolean isConstant(TObjectName object) {
26104                if (object == null || object.getGsqlparser() == null)
26105                        return false;
26106                List<String> constants = Arrays.asList(new String[] { "NEXTVAL", "CURRVAL", "SYSDATE", "CENTURY", "YEAR",
26107                                "MONTH", "DAY", "HOUR", "MINUTE", "SECOND" });
26108                List<String> segments = SQLUtil.parseNames(object.toString());
26109                // sequence.NEXTVAL or sequence.CURRVAL is a sequence reference, not a constant
26110                // This syntax is used by Oracle, Snowflake, and accepted by other vendors for compatibility
26111                String columnNameOnly = object.getColumnNameOnly();
26112                if (segments.size() > 1 && ("NEXTVAL".equalsIgnoreCase(columnNameOnly) || "CURRVAL".equalsIgnoreCase(columnNameOnly))) {
26113                        return false;
26114                }
26115                boolean result = constants.indexOf(segments.get(segments.size() - 1).toUpperCase()) != -1;
26116                if (result) {
26117                        return result;
26118                }
26119                if (isKeyword(object)) {
26120                        return true;
26121                }
26122                return false;
26123        }
26124
26125        private Pair3<Long, Long, Integer> convertCoordinate(Pair3<Long, Long, String> position) {
26126//              if (ModelBindingManager.getGlobalOption()!=null && ModelBindingManager.getGlobalOption().isIgnoreCoordinate()) {
26127//                      return new Pair3<>(-1L, -1L, -1);
26128//              }
26129                return new Pair3<Long, Long, Integer>(position.first, position.second,
26130                                ModelBindingManager.getGlobalSqlInfo().getIndexOf(position.third));
26131        }
26132
26133        /**
26134         * Analyze MDX SELECT statement to generate data lineage.
26135         * Maps MDX cube as source table, measures/dimensions as columns,
26136         * and creates dataflow relationships to the result set.
26137         */
26138        private void analyzeMdxSelectStmt(gudusoft.gsqlparser.stmt.mdx.TMdxSelect stmt) {
26139                // Get cube name from FROM clause
26140                gudusoft.gsqlparser.nodes.mdx.TMdxIdentifierNode cube = stmt.getCube();
26141                if (cube == null) {
26142                        return;
26143                }
26144
26145                String cubeName = getMdxIdentifierName(cube);
26146                Table cubeTable = modelFactory.createTableByName(cubeName, false);
26147
26148                // Collect all MDX identifier references from axes and WHERE clause
26149                List<String> measureNames = new ArrayList<String>();
26150                List<String> dimensionNames = new ArrayList<String>();
26151
26152                // Process axes (COLUMNS, ROWS, etc.)
26153                if (stmt.getAxes() != null) {
26154                        for (int i = 0; i < stmt.getAxes().size(); i++) {
26155                                gudusoft.gsqlparser.nodes.mdx.TMdxAxisNode axis = stmt.getAxes().getElement(i);
26156                                if (axis.getExpNode() != null) {
26157                                        collectMdxReferences(axis.getExpNode(), measureNames, dimensionNames);
26158                                }
26159                        }
26160                }
26161
26162                // Process WHERE clause (slicer dimension)
26163                if (stmt.getWhere() != null && stmt.getWhere().getFilter() != null) {
26164                        collectMdxReferences(stmt.getWhere().getFilter(), measureNames, dimensionNames);
26165                }
26166
26167                // Process WITH MEMBER definitions
26168                if (stmt.getWiths() != null) {
26169                        for (int i = 0; i < stmt.getWiths().size(); i++) {
26170                                gudusoft.gsqlparser.nodes.mdx.TMdxWithNode withNode = stmt.getWiths().getElement(i);
26171                                if (withNode.getNameNode() != null) {
26172                                        String withName = getMdxIdentifierName(withNode.getNameNode());
26173                                        // Calculated members are treated as derived measures
26174                                        if (withName.toLowerCase().startsWith("[measures].")
26175                                                        || withName.toLowerCase().startsWith("measures.")) {
26176                                                measureNames.add(withName);
26177                                        }
26178                                }
26179                                // Also collect references used in the WITH expression
26180                                if (withNode.getExprNode() != null) {
26181                                        collectMdxReferences(withNode.getExprNode(), measureNames, dimensionNames);
26182                                }
26183                        }
26184                }
26185
26186                // Create columns on the cube table for all referenced measures and dimensions
26187                Set<String> addedColumns = new LinkedHashSet<String>();
26188                for (String measure : measureNames) {
26189                        if (addedColumns.add(measure)) {
26190                                modelFactory.createTableColumn(cubeTable, measure);
26191                        }
26192                }
26193                for (String dimension : dimensionNames) {
26194                        if (addedColumns.add(dimension)) {
26195                                modelFactory.createTableColumn(cubeTable, dimension);
26196                        }
26197                }
26198
26199                // Create result set with all referenced columns
26200                ResultSet resultSet = modelFactory.createResultSet(stmt, false);
26201                if (resultSet != null) {
26202                        for (String colName : addedColumns) {
26203                                TableColumn sourceCol = findTableColumnByName(cubeTable, colName);
26204                                if (sourceCol != null) {
26205                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
26206                                        relation.setEffectType(EffectType.select);
26207                                        relation.addSource(new TableColumnRelationshipElement(sourceCol));
26208                                        relation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
26209                                                        resultSet.getRelationRows()));
26210                                }
26211                        }
26212                }
26213        }
26214
26215        private TableColumn findTableColumnByName(Table table, String name) {
26216                for (TableColumn col : table.getColumns()) {
26217                        if (col.getName().equals(name)) {
26218                                return col;
26219                        }
26220                }
26221                return null;
26222        }
26223
26224        /**
26225         * Extract the display name from an MDX identifier node.
26226         * E.g., [Measures].[Departures NEAT] -> [Measures].[Departures NEAT]
26227         */
26228        private String getMdxIdentifierName(gudusoft.gsqlparser.nodes.mdx.TMdxIdentifierNode idNode) {
26229                StringBuilder sb = new StringBuilder();
26230                for (int i = 0; i < idNode.getSegmentList().size(); i++) {
26231                        if (i > 0) sb.append(".");
26232                        gudusoft.gsqlparser.nodes.mdx.IMdxIdentifierSegment seg = idNode.getSegmentList().getElement(i);
26233                        if (seg.getQuoting() == gudusoft.gsqlparser.nodes.mdx.EMdxQuoting.QUOTED) {
26234                                sb.append("[").append(seg.getName()).append("]");
26235                        } else {
26236                                sb.append(seg.getName());
26237                        }
26238                }
26239                return sb.toString();
26240        }
26241
26242        /**
26243         * Recursively collect measure and dimension references from MDX expression tree.
26244         * Uses iterative DFS to avoid StackOverflow on deeply nested expressions.
26245         */
26246        private void collectMdxReferences(gudusoft.gsqlparser.nodes.mdx.TMdxExpNode expr,
26247                        List<String> measures, List<String> dimensions) {
26248                Deque<gudusoft.gsqlparser.nodes.mdx.TMdxExpNode> stack = new ArrayDeque<gudusoft.gsqlparser.nodes.mdx.TMdxExpNode>();
26249                stack.push(expr);
26250
26251                while (!stack.isEmpty()) {
26252                        gudusoft.gsqlparser.nodes.mdx.TMdxExpNode current = stack.pop();
26253                        if (current == null) continue;
26254
26255                        if (current instanceof gudusoft.gsqlparser.nodes.mdx.TMdxIdentifierNode) {
26256                                gudusoft.gsqlparser.nodes.mdx.TMdxIdentifierNode idNode =
26257                                                (gudusoft.gsqlparser.nodes.mdx.TMdxIdentifierNode) current;
26258                                String name = getMdxIdentifierName(idNode);
26259                                if (name.toLowerCase().startsWith("[measures].")
26260                                                || name.toLowerCase().startsWith("measures.")) {
26261                                        measures.add(name);
26262                                } else if (idNode.getSegmentList().size() > 1) {
26263                                        // Multi-segment identifiers that aren't measures are dimensions
26264                                        dimensions.add(name);
26265                                }
26266                        } else if (current instanceof gudusoft.gsqlparser.nodes.mdx.TMdxSetNode) {
26267                                gudusoft.gsqlparser.nodes.mdx.TMdxSetNode setNode =
26268                                                (gudusoft.gsqlparser.nodes.mdx.TMdxSetNode) current;
26269                                if (setNode.getTupleList() != null) {
26270                                        for (int i = 0; i < setNode.getTupleList().size(); i++) {
26271                                                stack.push(setNode.getTupleList().getElement(i));
26272                                        }
26273                                }
26274                        } else if (current instanceof gudusoft.gsqlparser.nodes.mdx.TMdxTupleNode) {
26275                                gudusoft.gsqlparser.nodes.mdx.TMdxTupleNode tupleNode =
26276                                                (gudusoft.gsqlparser.nodes.mdx.TMdxTupleNode) current;
26277                                if (tupleNode.getExprList() != null) {
26278                                        for (int i = 0; i < tupleNode.getExprList().size(); i++) {
26279                                                stack.push(tupleNode.getExprList().getElement(i));
26280                                        }
26281                                }
26282                        } else if (current instanceof gudusoft.gsqlparser.nodes.mdx.TMdxBinOpNode) {
26283                                gudusoft.gsqlparser.nodes.mdx.TMdxBinOpNode binOp =
26284                                                (gudusoft.gsqlparser.nodes.mdx.TMdxBinOpNode) current;
26285                                if (binOp.getRightExprNode() != null) stack.push(binOp.getRightExprNode());
26286                                if (binOp.getLeftExprNode() != null) stack.push(binOp.getLeftExprNode());
26287                        } else if (current instanceof gudusoft.gsqlparser.nodes.mdx.TMdxFunctionNode) {
26288                                gudusoft.gsqlparser.nodes.mdx.TMdxFunctionNode funcNode =
26289                                                (gudusoft.gsqlparser.nodes.mdx.TMdxFunctionNode) current;
26290                                if (funcNode.getArguments() != null) {
26291                                        for (int i = 0; i < funcNode.getArguments().size(); i++) {
26292                                                stack.push(funcNode.getArguments().getElement(i));
26293                                        }
26294                                }
26295                        }
26296                }
26297        }
26298
26299        /**
26300         * Analyze a Power Query M-language document for data lineage.
26301         *
26302         * Delegates to TPowerQueryAnalyzer to extract navigation chains and
26303         * NativeQuery embedded SQL, then feeds the results back through the
26304         * standard SQL analysis pipeline so the dataflow model contains
26305         * regular table/column lineage.
26306         */
26307        private void analyzePowerQueryDocumentStmt(TPowerQueryDocumentStmt pqStmt) {
26308                TPowerQueryAnalyzer pqAnalyzer = new TPowerQueryAnalyzer(pqStmt);
26309
26310                if (option.getPowerQueryInnerVendor() != null) {
26311                        pqAnalyzer.withExplicitInnerVendor(option.getPowerQueryInnerVendor());
26312                }
26313
26314                PowerQueryLineageResult result = pqAnalyzer.analyze();
26315
26316                if (result.isEmpty()) {
26317                        for (String w : result.getWarnings()) {
26318                                logger.warn("Power Query: " + w);
26319                        }
26320                        return;
26321                }
26322
26323                for (PowerQueryLineageResult.NativeQueryRef nq : result.getNativeQueryReferences()) {
26324                        if (nq.innerParser != null && nq.innerParseReturnCode == 0) {
26325                                for (int i = 0; i < nq.innerParser.sqlstatements.size(); i++) {
26326                                        analyzeCustomSqlStmt(nq.innerParser.sqlstatements.get(i));
26327                                }
26328                        }
26329                }
26330
26331                for (PowerQueryLineageResult.NavigationRef nav : result.getNavigationReferences()) {
26332                        if (nav.syntheticSelect != null && nav.resolvedVendor != null) {
26333                                TGSqlParser synParser = new TGSqlParser(nav.resolvedVendor);
26334                                synParser.sqltext = nav.syntheticSelect;
26335                                int rc = synParser.parse();
26336                                if (rc == 0) {
26337                                        for (int i = 0; i < synParser.sqlstatements.size(); i++) {
26338                                                analyzeCustomSqlStmt(synParser.sqlstatements.get(i));
26339                                        }
26340                                }
26341                        }
26342                }
26343
26344                for (String w : result.getWarnings()) {
26345                        logger.warn("Power Query: " + w);
26346                }
26347        }
26348
26349}