001
002package gudusoft.gsqlparser.dlineage;
003
004import gudusoft.gsqlparser.*;
005import gudusoft.gsqlparser.common.structured.StructuredAdapterContext;
006import gudusoft.gsqlparser.common.structured.StructuredDataflowDescriptor;
007import gudusoft.gsqlparser.common.structured.StructuredDataflowRegistry;
008import gudusoft.gsqlparser.common.structured.StructuredFieldBinding;
009import gudusoft.gsqlparser.common.structured.StructuredValueSource;
010import gudusoft.gsqlparser.dlineage.dataflow.listener.DataFlowHandleListener;
011import gudusoft.gsqlparser.dlineage.dataflow.metadata.MetadataReader;
012import gudusoft.gsqlparser.dlineage.dataflow.metadata.grabit.GrabitMetadataAnalyzer;
013import gudusoft.gsqlparser.dlineage.dataflow.metadata.sqldep.SQLDepMetadataAnalyzer;
014import gudusoft.gsqlparser.dlineage.dataflow.metadata.sqlflow.SqlflowMetadataAnalyzer;
015import gudusoft.gsqlparser.dlineage.dataflow.metadata.sqlflow.sharded.SqlflowShardedMetadataAnalyzer;
016import gudusoft.gsqlparser.dlineage.impl.powerquery.PowerQueryLineageResult;
017import gudusoft.gsqlparser.dlineage.impl.powerquery.TPowerQueryAnalyzer;
018import gudusoft.gsqlparser.dlineage.dataflow.model.*;
019import gudusoft.gsqlparser.dlineage.dataflow.model.Process;
020import gudusoft.gsqlparser.dlineage.dataflow.model.JoinRelationship.JoinClauseType;
021import gudusoft.gsqlparser.dlineage.dataflow.model.json.Coordinate;
022import gudusoft.gsqlparser.dlineage.dataflow.model.json.Dataflow;
023import gudusoft.gsqlparser.dlineage.dataflow.model.json.Error;
024import gudusoft.gsqlparser.dlineage.dataflow.model.xml.*;
025import gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlTrustMode;
026import gudusoft.gsqlparser.dlineage.dataflow.sqlenv.SQLEnvParser;
027import gudusoft.gsqlparser.dlineage.metadata.MetadataUtil;
028import gudusoft.gsqlparser.dlineage.metadata.Sqlflow;
029import gudusoft.gsqlparser.dlineage.util.*;
030import gudusoft.gsqlparser.nodes.TTable;
031import gudusoft.gsqlparser.nodes.*;
032import gudusoft.gsqlparser.nodes.couchbase.TObjectConstruct;
033import gudusoft.gsqlparser.nodes.couchbase.TPair;
034import gudusoft.gsqlparser.nodes.functions.TJsonObjectFunction;
035import gudusoft.gsqlparser.nodes.hive.THiveTransformClause;
036import gudusoft.gsqlparser.nodes.hive.THiveTransformClause.ETransformType;
037import gudusoft.gsqlparser.resolver2.ScopeBuildResult;
038import gudusoft.gsqlparser.resolver2.TSQLResolver2;
039import gudusoft.gsqlparser.sqlenv.*;
040import gudusoft.gsqlparser.sqlenv.parser.TJSONSQLEnvParser;
041import gudusoft.gsqlparser.stmt.*;
042import gudusoft.gsqlparser.stmt.db2.TDb2CallStmt;
043import gudusoft.gsqlparser.stmt.db2.TDb2ReturnStmt;
044import gudusoft.gsqlparser.stmt.db2.TDb2SqlVariableDeclaration;
045import gudusoft.gsqlparser.stmt.hive.THiveLoad;
046import gudusoft.gsqlparser.stmt.mssql.*;
047import gudusoft.gsqlparser.stmt.mysql.TLoadDataStmt;
048import gudusoft.gsqlparser.stmt.oracle.*;
049import gudusoft.gsqlparser.stmt.powerquery.TPowerQueryDocumentStmt;
050import gudusoft.gsqlparser.stmt.redshift.TRedshiftCopy;
051import gudusoft.gsqlparser.stmt.redshift.TRedshiftDeclare;
052import gudusoft.gsqlparser.stmt.snowflake.*;
053import gudusoft.gsqlparser.stmt.teradata.TTeradataCreateProcedure;
054import gudusoft.gsqlparser.util.*;
055import gudusoft.gsqlparser.util.json.JSON;
056
057import javax.script.ScriptEngine;
058import javax.script.ScriptEngineManager;
059import javax.script.ScriptException;
060import java.io.*;
061import java.util.zip.GZIPInputStream;
062import java.util.*;
063import java.util.concurrent.atomic.AtomicInteger;
064import java.util.regex.Matcher;
065import java.util.regex.Pattern;
066import java.util.stream.Collectors;
067
068import static gudusoft.gsqlparser.EJoinType.right;
069
070@SuppressWarnings("rawtypes")
071/**
072 * Single-threaded data flow analyzer that performs lineage analysis on SQL scripts.
073 *
074 * <h2>Avoiding OOM with Large Files</h2>
075 * <p>
076 * Analyzing very large SQL files or manifests (e.g. sqlflow-sharded exports with many
077 * shards) in a single JVM can easily exhaust heap memory and cause {@code OutOfMemoryError}.
078 * To avoid OOM, enable automatic large-file detection and delegation:
079 *
080 * <pre>{@code
081 * Option option = new Option();
082 * option.setAutoDetectLargeFile(true);                        // enable auto-detection
083 * option.setLargeSqlInfoCountThreshold(1000);                 // max SqlInfo entries before delegating
084 * option.setLargeSqlTotalSizeThreshold(25 * 1024 * 1024);     // max total SQL size (25 MB) before delegating
085 * option.setLargeQueryCountThreshold(1000);                   // max manifest queries before delegating
086 * option.setLargeShardCountThreshold(10);                     // max sharded source files before delegating
087 * option.setLargeFileSplitSizeMB(5);                          // split files larger than 5 MB
088 * option.setEstimatedMemoryPerTaskMB(2560);                   // estimated memory per parallel task (2.5 GB)
089 * option.setParallel(0);                                      // 0 = auto-calculate from CPU cores
090 *
091 * DataFlowAnalyzer analyzer = new DataFlowAnalyzer(sqlFiles, option);
092 * analyzer.generateDataFlow();
093 * }</pre>
094 *
095 * <p>
096 * When {@code autoDetectLargeFile} is enabled, {@link #generateDataFlow(boolean)} invokes
097 * {@link gudusoft.gsqlparser.dlineage.util.LargeFileDetector} to check the input scale against
098 * the configured thresholds. If any threshold is exceeded, the analyzer automatically delegates
099 * to {@link gudusoft.gsqlparser.dlineage.ParallelDataFlowAnalyzer}, which:
100 * <ul>
101 *   <li>Splits large files into smaller chunks via {@link gudusoft.gsqlparser.util.FileSplitter}</li>
102 *   <li>Processes each chunk in a separate {@code DataFlowAnalyzer} instance within a bounded
103 *       thread pool (size capped by {@code estimatedMemoryPerTaskMB})</li>
104 *   <li>For sharded manifests, distributes individual shard source files across parallel tasks</li>
105 *   <li>Merges results from all parallel tasks into a single dataflow output</li>
106 * </ul>
107 *
108 * <p>
109 * If delegation fails for any reason, the analyzer falls back to single-threaded processing
110 * with a warning log, so the analysis still completes (though it may OOM on very large inputs).
111 *
112 * @see gudusoft.gsqlparser.dlineage.util.LargeFileDetector
113 * @see gudusoft.gsqlparser.dlineage.ParallelDataFlowAnalyzer
114 * @see gudusoft.gsqlparser.dlineage.dataflow.model.Option#setAutoDetectLargeFile(boolean)
115 */
116public class DataFlowAnalyzer implements IDataFlowAnalyzer {
117
118        private static final Logger logger = LoggerFactory.getLogger(DataFlowAnalyzer.class);
119
120        private static final List<String> TERADATA_BUILTIN_FUNCTIONS = Arrays
121                        .asList(new String[] { "ACCOUNT", "CURRENT_DATE", "CURRENT_ROLE", "CURRENT_TIME", "CURRENT_TIMESTAMP",
122                                        "CURRENT_USER", "DATABASE", "DATE", "PROFILE", "ROLE", "SESSION", "TIME", "USER", "SYSDATE", });
123
124        private static final List<String> CONSTANT_BUILTIN_FUNCTIONS = Arrays.asList(new String[] { "ACCOUNT",
125                        "CURRENT_DATE", "CURRENT_ROLE", "CURRENT_TIME", "CURRENT_TIMESTAMP", "CURRENT_USER", "DATABASE", "DATE",
126                        "PROFILE", "ROLE", "SESSION", "TIME", "USER", "SYSDATE", "GETDATE" });
127
128        private Stack<TCustomSqlStatement> stmtStack = new Stack<TCustomSqlStatement>();
129        private List<ResultSet> appendResultSets = new ArrayList<ResultSet>();
130        private Set<TCustomSqlStatement> accessedStatements = new HashSet<TCustomSqlStatement>();
131        private Set<TSelectSqlStatement> accessedSubqueries = new HashSet<TSelectSqlStatement>();
132        private Map<String, TCustomSqlStatement> viewDDLMap = new HashMap<String, TCustomSqlStatement>();
133        private Map<String, TCustomSqlStatement> procedureDDLMap = new HashMap<String, TCustomSqlStatement>();
134        private Map<TTable, TObjectNameList> structObjectMap = new HashMap<TTable, TObjectNameList>();
135        private final Map<String, String> normalizedColumnNameCache = new HashMap<String, String>();
136        private final Set<TParseTreeNode> processingFunctions = new HashSet<TParseTreeNode>();
137
138        // perf (10x, perf/column-lineage-10x): normalized-column-name -> first ResultColumn index over a
139        // resultset's column list. analyzeDataFlowRelation otherwise LINEAR-scans queryTable.getColumns()
140        // once per target column => O(W^2) for wide resultsets (now the dominant dlineage hotspot after the
141        // CTENamespace memo). Keyed by the getColumns() list identity, which is the stable ResultSet.columns
142        // field. Only built for "simple" column lists (no star, no struct) because the linear scan has ordered
143        // side effects on star/struct columns; other lists fall back to the scan. Rebuilt when size changes.
144        // The fast path is used only when it is byte-identical to the scan (verified over the full corpus).
145        private final java.util.IdentityHashMap<List<ResultColumn>, ColumnNameIndex> columnNameIndexCache =
146                        new java.util.IdentityHashMap<List<ResultColumn>, ColumnNameIndex>();
147
148        /** Cached normalized-name -> first ResultColumn map for a simple resultset column list. */
149        private static final class ColumnNameIndex {
150                final int size;
151                /** null when the column list is NOT simple (has a star or struct column) — use the linear scan. */
152                final Map<String, ResultColumn> byNormalizedName;
153                ColumnNameIndex(int size, Map<String, ResultColumn> byNormalizedName) {
154                        this.size = size;
155                        this.byNormalizedName = byNormalizedName;
156                }
157        }
158
159        /**
160         * Return the normalized-name index for {@code columns}, building (and caching) it on first use and
161         * rebuilding when the list size changes. Returns an index whose {@code byNormalizedName} is null when
162         * the list contains any star ("*") or struct column — signalling the caller to use the linear scan.
163         */
164        private ColumnNameIndex getColumnNameIndex(List<ResultColumn> columns) {
165                ColumnNameIndex idx = columnNameIndexCache.get(columns);
166                if (idx != null && idx.size == columns.size()) {
167                        return idx;
168                }
169                Map<String, ResultColumn> map = new HashMap<String, ResultColumn>();
170                boolean simple = true;
171                for (int i = 0; i < columns.size(); i++) {
172                        ResultColumn c = columns.get(i);
173                        String nm = c.getName();
174                        if ("*".equals(nm) || c.isStruct()) {
175                                simple = false;
176                                break;
177                        }
178                        // putIfAbsent semantics: keep the FIRST column for a given normalized name, matching the
179                        // linear scan's first-match-wins + break.
180                        String key = DlineageUtil.getIdentifierNormalColumnName(nm);
181                        if (!map.containsKey(key)) {
182                                map.put(key, c);
183                        }
184                }
185                idx = new ColumnNameIndex(columns.size(), simple ? map : null);
186                columnNameIndexCache.put(columns, idx);
187                return idx;
188        }
189
190        // perf: normalized-column-name -> ResultColumn map for a resultset's column list, used by
191        // appendStarRelation to avoid O(S*C) linear scans per star column. Unlike ColumnNameIndex,
192        // this cache does NOT require "simple" column lists — it works for any list including those
193        // with star/struct columns, because appendStarRelation only needs O(1) lookup, not the
194        // ordered side effects that the linear scan in analyzeDataFlowRelation provides.
195        private final java.util.IdentityHashMap<List<ResultColumn>, Map<String, ResultColumn>> resultSetColumnLookupCache =
196                        new java.util.IdentityHashMap<List<ResultColumn>, Map<String, ResultColumn>>();
197
198        private Map<String, ResultColumn> getResultSetColumnLookup(List<ResultColumn> columns) {
199                Map<String, ResultColumn> map = resultSetColumnLookupCache.get(columns);
200                if (map != null) {
201                        return map;
202                }
203                map = new HashMap<String, ResultColumn>(columns.size() * 2);
204                for (ResultColumn c : columns) {
205                        String key = DlineageUtil.getIdentifierNormalColumnName(c.getName());
206                        if (!map.containsKey(key)) {
207                                map.put(key, c);
208                        }
209                }
210                resultSetColumnLookupCache.put(columns, map);
211                return map;
212        }
213
214        // perf: normalized-column-name -> TableColumn map for a table's column list, used by
215        // appendStarRelation to avoid O(C) linear scans via searchTableColumn.
216        private final java.util.IdentityHashMap<List<TableColumn>, Map<String, TableColumn>> tableColumnLookupCache =
217                        new java.util.IdentityHashMap<List<TableColumn>, Map<String, TableColumn>>();
218
219        private Map<String, TableColumn> getTableColumnLookup(List<TableColumn> columns) {
220                Map<String, TableColumn> map = tableColumnLookupCache.get(columns);
221                if (map != null) {
222                        return map;
223                }
224                map = new HashMap<String, TableColumn>(columns.size() * 2);
225                for (TableColumn c : columns) {
226                        String key = DlineageUtil.getIdentifierNormalColumnName(c.getName());
227                        if (!map.containsKey(key)) {
228                                map.put(key, c);
229                        }
230                }
231                tableColumnLookupCache.put(columns, map);
232                return map;
233        }
234
235        private SqlInfo[] sqlInfos;
236        private File[] originalFiles;
237        private List<ErrorInfo> errorInfos = new ArrayList<ErrorInfo>();
238        private final List<DynamicSqlSite> dynamicSqlSites = new ArrayList<DynamicSqlSite>();
239        private final Set<CteSelectRelationKey> cteSelectRelationKeys = new HashSet<CteSelectRelationKey>();
240        /**
241         * Per-procedure evaluation variants for the default dynamic-SQL path (identity keys).
242         * Index 0 is the no-bindings evaluation; further entries are one per literal call-site
243         * binding set discovered in the same file (capped).
244         */
245        private final Map<TCustomSqlStatement, List<Map<TCustomSqlStatement, gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite>>> dynamicEvalCache = new IdentityHashMap<TCustomSqlStatement, List<Map<TCustomSqlStatement, gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite>>>();
246        /**
247         * Multi-file mssql analyses only (plan Phase 5): the raw text of every analysis
248         * unit, so parameter-dependent dynamic SQL in one file can pick up literal EXEC
249         * bindings from another. Null in single-file analyses — the cross-file discovery
250         * path is then never entered and per-file output is byte-identical to before.
251         */
252        private final Map<String, TStatementList> crossFileParseCache = new HashMap<String, TStatementList>();
253        /** Shared routine/call-site index over all units (plan §5.6); null in single-file analyses. */
254        private gudusoft.gsqlparser.dlineage.dynamicsql.RoutineCatalog routineCatalog;
255        /** Oracle files (by unit text) already run through the PL/SQL interpreter this analysis. */
256        private final Set<String> plsqlEvaluatedTexts = new HashSet<String>();
257        private IndexedLinkedHashMap<String, List<SqlInfo>> sqlInfoMap = new IndexedLinkedHashMap<String, List<SqlInfo>>();
258        private TSQLEnv sqlenv = null;
259        private ModelBindingManager modelManager = new ModelBindingManager();
260        private ModelFactory modelFactory = new ModelFactory(modelManager);
261        private PipelinedFunctionAnalyzer pipelinedAnalyzer;
262        private List<Long> tableIds = new ArrayList<Long>();
263        private TTableList hiveFromTables;
264        private dataflow dataflow;
265        private String dataflowString;
266        private final AuthoritativeLineageEvidenceCollector authoritativeEvidenceCollector =
267                        new AuthoritativeLineageEvidenceCollector();
268
269        /**
270         * Whether this analyzer is still holding authoritative-evidence DRAFT state.
271         *
272         * <p>Drafts are per-analysis working state; the lifecycle invariant is that
273         * none are retained once a generation completes or the analyzer is disposed.
274         * Always false in normal use — a true here means evidence accumulated across
275         * analyses, which would leak one analysis's findings into the next.</p>
276         *
277         * <p>This deliberately exposes the INVARIANT, not the collector: the
278         * collector class and its fields stay package-private, so they remain free to
279         * change, and nothing internal is frozen into the public API. It is public so
280         * the assertion can be made against the shipped (ProGuard-obfuscated) jar,
281         * where package-private members are renamed and unreachable.</p>
282         *
283         * @return true if any evidence draft state is still held
284         * @since 4.2.6
285         */
286        public boolean hasRetainedEvidenceDrafts() {
287                return authoritativeEvidenceCollector.hasRetainedDrafts();
288        }
289        private AuthoritativeLineageEvidence authoritativeLineageEvidence =
290                        AuthoritativeLineageEvidence.empty();
291        private Option option = new Option();
292
293        {
294                modelManager.TABLE_COLUMN_ID = option.getStartId();
295                modelManager.RELATION_ID = option.getStartId();
296                ModelBindingManager.set(modelManager);
297                ModelBindingManager.setGlobalStmtStack(stmtStack);
298                ModelBindingManager.setGlobalOption(option);
299                ModelBindingManager.setGlobalSqlInfo(sqlInfoMap);
300        }
301
302    private void initPipelinedFunctionAnalyzer() {
303        // Only initialize pipelinedAnalyzer for Oracle
304        if (option.getVendor() == EDbVendor.dbvoracle)
305        {
306            pipelinedAnalyzer = new PipelinedFunctionAnalyzer(modelManager, modelFactory, option);
307        }
308        else{
309            pipelinedAnalyzer = null;
310        }
311    }
312
313    public DataFlowAnalyzer(Sqlflow sqlflow, Option option) {
314        this.option = option;
315        ModelBindingManager.setGlobalOption(this.option);
316
317        SqlInfo info = new SqlInfo();
318        info.setOriginIndex(0);
319
320        File tempFile = LargeFileDetector.writeSqlflowToTempFile(sqlflow);
321
322        if (option.isAutoDetectLargeFile() && tempFile.length() > option.getLargeSqlThresholdBytes()) {
323            info.setFilePath(tempFile.getAbsolutePath());
324            info.setFileName(tempFile.getName());
325            this.sqlInfos = new SqlInfo[] { info };
326            return;
327        }
328
329        String sql = SQLUtil.getFileContent(tempFile);
330        tempFile.delete();
331        info.setSql(sql);
332
333        this.sqlInfos = convertSQL(option.getVendor(), JSON.toJSONString(new SqlInfo[] { info })).toArray(new SqlInfo[0]);
334    }
335
336    public DataFlowAnalyzer(String sqlContent, Option option) {
337                SqlInfo[] sqlInfos = new SqlInfo[1];
338                SqlInfo info = new SqlInfo();
339                info.setSql(sqlContent);
340                info.setOriginIndex(0);
341                sqlInfos[0] = info;
342                this.option = option;
343                ModelBindingManager.setGlobalOption(this.option);
344                if (option.isLargeSql(sqlContent)) {
345                        LargeFileDetector.persistSqlToTempFile(info, sqlContent);
346            this.sqlInfos = sqlInfos;
347                }
348        else {
349            this.sqlInfos = convertSQL(option.getVendor(), JSON.toJSONString(sqlInfos)).toArray(new SqlInfo[0]);
350        }
351    }
352
353    public DataFlowAnalyzer(String sqlContent, EDbVendor dbVendor, boolean simpleOutput, String defaultServer,
354                        String defaultDatabase, String defaltSchema) {
355                SqlInfo[] sqlInfos = new SqlInfo[1];
356                SqlInfo info = new SqlInfo();
357                info.setSql(sqlContent);
358                info.setOriginIndex(0);
359                sqlInfos[0] = info;
360                option.setVendor(dbVendor);
361                option.setSimpleOutput(simpleOutput);
362                option.setDefaultServer(defaultServer);
363                option.setDefaultDatabase(defaultDatabase);
364                option.setDefaultSchema(defaltSchema);
365                this.sqlInfos = convertSQL(dbVendor, JSON.toJSONString(sqlInfos)).toArray(new SqlInfo[0]);
366        }
367
368        public DataFlowAnalyzer(String sqlContent, EDbVendor dbVendor, boolean simpleOutput) {
369                SqlInfo[] sqlInfos = new SqlInfo[1];
370                SqlInfo info = new SqlInfo();
371                info.setSql(sqlContent);
372                info.setOriginIndex(0);
373                sqlInfos[0] = info;
374                option.setVendor(dbVendor);
375                option.setSimpleOutput(simpleOutput);
376                this.sqlInfos = convertSQL(dbVendor, JSON.toJSONString(sqlInfos)).toArray(new SqlInfo[0]);
377    }
378
379        public DataFlowAnalyzer(String[] sqlContents, Option option) {
380                SqlInfo[] sqlInfos = new SqlInfo[sqlContents.length];
381                for (int i = 0; i < sqlContents.length; i++) {
382                        SqlInfo info = new SqlInfo();
383                        info.setSql(sqlContents[i]);
384                        info.setOriginIndex(0);
385                        sqlInfos[i] = info;
386                }
387                this.option = option;
388                ModelBindingManager.setGlobalOption(this.option);
389                this.sqlInfos = convertSQL(option.getVendor(), JSON.toJSONString(sqlInfos)).toArray(new SqlInfo[0]);
390        }
391
392        public DataFlowAnalyzer(String[] sqlContents, EDbVendor dbVendor, boolean simpleOutput, String defaultServer,
393                        String defaultDatabase, String defaltSchema) {
394                SqlInfo[] sqlInfos = new SqlInfo[sqlContents.length];
395                for (int i = 0; i < sqlContents.length; i++) {
396                        SqlInfo info = new SqlInfo();
397                        info.setSql(sqlContents[i]);
398                        info.setOriginIndex(0);
399                        sqlInfos[i] = info;
400                }
401                option.setVendor(dbVendor);
402                option.setSimpleOutput(simpleOutput);
403                option.setDefaultServer(defaultServer);
404                option.setDefaultDatabase(defaultDatabase);
405                option.setDefaultSchema(defaltSchema);
406                this.sqlInfos = convertSQL(dbVendor, JSON.toJSONString(sqlInfos)).toArray(new SqlInfo[0]);
407        }
408
409        public DataFlowAnalyzer(String[] sqlContents, EDbVendor dbVendor, boolean simpleOutput) {
410                SqlInfo[] sqlInfos = new SqlInfo[sqlContents.length];
411                for (int i = 0; i < sqlContents.length; i++) {
412                        SqlInfo info = new SqlInfo();
413                        info.setSql(sqlContents[i]);
414                        info.setOriginIndex(0);
415                        sqlInfos[i] = info;
416                }
417                option.setVendor(dbVendor);
418                option.setSimpleOutput(simpleOutput);
419                this.sqlInfos = convertSQL(dbVendor, JSON.toJSONString(sqlInfos)).toArray(new SqlInfo[0]);
420        }
421
422        public DataFlowAnalyzer(SqlInfo[] sqlInfos, Option option) {
423                this.sqlInfos = sqlInfos;
424                this.option = option;
425                ModelBindingManager.setGlobalOption(this.option);
426        }
427
428        public DataFlowAnalyzer(SqlInfo[] sqlInfos, EDbVendor dbVendor, boolean simpleOutput) {
429                this.sqlInfos = sqlInfos;
430                option.setVendor(dbVendor);
431                option.setSimpleOutput(simpleOutput);
432        }
433
434        public DataFlowAnalyzer(File[] sqlFiles, Option option) {
435                this.originalFiles = sqlFiles;
436                SqlInfo[] sqlInfos = new SqlInfo[sqlFiles.length];
437                for (int i = 0; i < sqlFiles.length; i++) {
438                        SqlInfo info = new SqlInfo();
439                        info.setFileName(sqlFiles[i].getName());
440                        info.setFilePath(sqlFiles[i].getAbsolutePath());
441                        info.setOriginIndex(0);
442                        sqlInfos[i] = info;
443                }
444                this.sqlInfos = sqlInfos;
445                this.option = option;
446                ModelBindingManager.setGlobalOption(this.option);
447        }
448
449        public DataFlowAnalyzer(File[] sqlFiles, EDbVendor dbVendor, boolean simpleOutput) {
450                this.originalFiles = sqlFiles;
451                SqlInfo[] sqlInfos = new SqlInfo[sqlFiles.length];
452                for (int i = 0; i < sqlFiles.length; i++) {
453                        SqlInfo info = new SqlInfo();
454                        info.setFileName(sqlFiles[i].getName());
455                        info.setFilePath(sqlFiles[i].getAbsolutePath());
456                        info.setOriginIndex(0);
457                        sqlInfos[i] = info;
458                }
459                this.sqlInfos = sqlInfos;
460                option.setVendor(dbVendor);
461                option.setSimpleOutput(simpleOutput);
462        }
463
464        public DataFlowAnalyzer(File sqlFile, Option option) {
465                File[] children = SQLUtil.listFiles(sqlFile);
466                this.originalFiles = children;
467                SqlInfo[] sqlInfos = new SqlInfo[children.length];
468                for (int i = 0; i < children.length; i++) {
469                        SqlInfo info = new SqlInfo();
470                        info.setFileName(children[i].getName());
471                        info.setFilePath(children[i].getAbsolutePath());
472                        info.setOriginIndex(0);
473                        sqlInfos[i] = info;
474                }
475                this.sqlInfos = sqlInfos;
476                this.option = option;
477                ModelBindingManager.setGlobalOption(this.option);
478        }
479
480        public DataFlowAnalyzer(File sqlFile, EDbVendor dbVendor, boolean simpleOutput) {
481                File[] children = SQLUtil.listFiles(sqlFile);
482                this.originalFiles = children;
483                List<SqlInfo> sqlInfos = new ArrayList<>();
484                for (int i = 0; i < children.length; i++) {
485                        SqlInfo info = new SqlInfo();
486                        info.setSql(SQLUtil.getFileContent(children[i]));
487            if(children[i].getName().toLowerCase().endsWith(".csv")){
488                if(!MetadataReader.isMetadata(info.getSql())){
489                    continue;
490                }
491            }
492            else if(children[i].getName().toLowerCase().endsWith(".json")){
493                if (!(MetadataReader.isGrabit(info.getSql()) || MetadataReader.isSqlflow(info.getSql()) || MetadataReader.isSqlflowSharded(info.getSql()))){
494                    continue;
495                }
496            }
497                        info.setFileName(children[i].getName());
498                        info.setFilePath(children[i].getAbsolutePath());
499                        info.setOriginIndex(0);
500                        sqlInfos.add(info);
501                }
502                this.sqlInfos = sqlInfos.toArray(new SqlInfo[0]);
503                option.setVendor(dbVendor);
504                option.setSimpleOutput(simpleOutput);
505        }
506
507        public boolean isIgnoreRecordSet() {
508                return option.isIgnoreRecordSet();
509        }
510
511        public void setIgnoreRecordSet(boolean ignoreRecordSet) {
512                option.setIgnoreRecordSet(ignoreRecordSet);
513        }
514
515        public boolean isSimpleShowTopSelectResultSet() {
516                return option.isSimpleShowTopSelectResultSet();
517        }
518
519        public void setSimpleShowTopSelectResultSet(boolean simpleShowTopSelectResultSet) {
520                option.setSimpleShowTopSelectResultSet(simpleShowTopSelectResultSet);
521        }
522
523        public boolean isSimpleShowFunction() {
524                return option.isSimpleShowFunction();
525        }
526
527        public void setSimpleShowFunction(boolean simpleShowFunction) {
528                option.setSimpleShowFunction(simpleShowFunction);
529        }
530
531        public boolean isShowJoin() {
532                return option.isShowJoin();
533        }
534
535        public void setShowJoin(boolean showJoin) {
536                option.setShowJoin(showJoin);
537        }
538
539        public void setShowCallRelation(boolean showCallRelation) {
540                option.setShowCallRelation(showCallRelation);
541        }
542
543        public boolean isShowCallRelation() {
544                return option.isShowCallRelation();
545        }
546
547        public boolean isShowImplicitSchema() {
548                return option.isShowImplicitSchema();
549        }
550
551        public void setShowImplicitSchema(boolean showImplicitSchema) {
552                option.setShowImplicitSchema(showImplicitSchema);
553        }
554
555        public boolean isShowConstantTable() {
556                return option.isShowConstantTable();
557        }
558
559        public void setShowConstantTable(boolean showConstantTable) {
560                option.setShowConstantTable(showConstantTable);
561        }
562
563        public boolean isShowCountTableColumn() {
564                return option.isShowCountTableColumn();
565        }
566
567        public void setShowCountTableColumn(boolean showCountTableColumn) {
568                option.setShowCountTableColumn(showCountTableColumn);
569        }
570
571        public boolean isTransform() {
572                return option.isTransform();
573        }
574
575        public void setTransform(boolean transform) {
576                option.setTransform(transform);
577                if (option.isTransformCoordinate()) {
578                        option.setTransform(true);
579                }
580        }
581
582        public boolean isTransformCoordinate() {
583                return option.isTransformCoordinate();
584        }
585
586        public void setTransformCoordinate(boolean transformCoordinate) {
587                option.setTransformCoordinate(transformCoordinate);
588                if (transformCoordinate) {
589                        option.setTransform(true);
590                }
591        }
592
593        public boolean isLinkOrphanColumnToFirstTable() {
594                return option.isLinkOrphanColumnToFirstTable();
595        }
596
597        public void setLinkOrphanColumnToFirstTable(boolean linkOrphanColumnToFirstTable) {
598                option.setLinkOrphanColumnToFirstTable(linkOrphanColumnToFirstTable);
599        }
600
601        public boolean isIgnoreTemporaryTable() {
602                return option.isIgnoreTemporaryTable();
603        }
604
605        public void setIgnoreTemporaryTable(boolean ignoreTemporaryTable) {
606                option.setIgnoreTemporaryTable(ignoreTemporaryTable);
607        }
608        
609        public boolean isIgnoreCoordinate() {
610                return option.isIgnoreCoordinate();
611        }
612
613        public void setIgnoreCoordinate(boolean ignoreCoordinate) {
614                option.setIgnoreCoordinate(ignoreCoordinate);
615        }
616
617        public void setHandleListener(DataFlowHandleListener listener) {
618                option.setHandleListener(listener);
619        }
620
621        public void setSqlEnv(TSQLEnv sqlenv) {
622                this.sqlenv = sqlenv;
623        }
624
625        public void setOption(Option option) {
626                this.option = option;
627                ModelBindingManager.setGlobalOption(this.option);
628        }
629
630        public Option getOption() {
631                return option;
632        }
633
634        /**
635         * Create a parser carrying the syntax compatibility options owned by this
636         * analyzer. All primary and nested SQL parsing paths must use this method.
637         */
638        private TGSqlParser createSqlParser(EDbVendor vendor) {
639                TGSqlParser sqlparser = new TGSqlParser(vendor);
640                sqlparser.setEnableMssqlColonBindVariables(
641                                option != null && option.isMssqlColonBindVariablesEnabled());
642                return sqlparser;
643        }
644
645        public synchronized String chechSyntax() {
646                StringBuilder builder = new StringBuilder();
647                if (sqlInfos != null) {
648                        for (SqlInfo sqlInfo : sqlInfos) {
649                                String content = sqlInfo.getSql();
650                                if (content != null && content.indexOf("<dlineage") != -1) {
651                                        try {
652                                                XML2Model.loadXML(dataflow.class, content);
653                                                continue;
654                                        } catch (Exception e) {
655                                                builder.append("Parsing dataflow ").append("occurs errors.\n").append(e.getMessage())
656                                                                .append("\n");
657                                        }
658                                }
659                                if (content != null && content.trim().startsWith("{")) {
660                                        Map queryObject = (Map) JSON.parseObject(content);
661                                        if (isSourceUnavailable(queryObject)) {
662                                                continue; // DDL body not retrieved; its text is not the definition
663                                        }
664                                        content = (String) queryObject.get("sourceCode");
665                                }
666                                if (MetadataReader.isMetadata(content)) {
667                                        continue;
668                                }
669                                TGSqlParser sqlparser = createSqlParser(option.getVendor());
670                                sqlparser.sqltext = content;
671                                int result = sqlparser.parse();
672                                if (result != 0) {
673                                        builder.append("Parsing sql ").append("occurs errors.\n").append(sqlparser.getErrormessage())
674                                                        .append("\n");
675                                }
676                        }
677                }
678                return builder.toString();
679        }
680
681        public synchronized String generateDataFlow(boolean withExtraInfo) {
682                // Gated here as well as in TGSqlParser.parse(), which this reaches
683                // transitively: refusing at the API boundary gives the caller a clear
684                // failure instead of one surfacing from deep inside the analysis.
685                gudusoft.gsqlparser.runtime.GspRuntime.requireActivated();
686                authoritativeLineageEvidence = AuthoritativeLineageEvidence.empty();
687                if (option.isAutoDetectLargeFile()) {
688                        LargeFileDetector.LargeFileDetectionResult result = LargeFileDetector.detect(sqlInfos, originalFiles, option);
689                        if (result.shouldDelegate && delegateToParallel(result)) {
690                                return finishGenerateDataFlow(withExtraInfo);
691                        }
692                }
693
694                if (ModelBindingManager.get() == null) {
695                        ModelBindingManager.set(modelManager);
696                }
697
698        initPipelinedFunctionAnalyzer();
699
700                dataflow = analyzeSqlScript();
701
702                return finishGenerateDataFlow(withExtraInfo);
703        }
704
705        private boolean delegateToParallel(LargeFileDetector.LargeFileDetectionResult result) {
706                ParallelDataFlowAnalyzer parallelAnalyzer = null;
707                //autogenerate 委托路径产生的拆分目录、gsp-input 临时文件及拆分产物统一在此清理
708                File splitDir = new File(System.getProperty("java.io.tmpdir"), "gsp-split-" + UUID.randomUUID().toString());
709                if (!splitDir.exists() && !splitDir.mkdirs()) {
710                        logger.error("Failed to create split directory: " + splitDir.getAbsolutePath());
711                        return false;
712                }
713                try {
714                        logger.info("Delegating to ParallelDataFlowAnalyzer: " + result.reason);
715
716                        parallelAnalyzer = buildParallelAnalyzer(splitDir);
717                        parallelAnalyzer.setSqlEnv(sqlenv);
718
719                        parallelAnalyzer.generateDataFlow(false, true);
720
721                        this.dataflow = parallelAnalyzer.getDataFlow();
722                        this.dataflowString = null;
723                        this.errorInfos.clear();
724                        this.errorInfos.addAll(parallelAnalyzer.getErrorMessages());
725                        this.sqlInfoMap = (IndexedLinkedHashMap<String, List<SqlInfo>>) parallelAnalyzer.getSqlInfos();
726
727                        return true;
728                } catch (Exception e) {
729                        logger.error("Delegation to ParallelDataFlowAnalyzer failed, falling back to single-threaded", e);
730                        return false;
731                } finally {
732                        //autogenerate dispose 异常不能阻塞 splitDir 清理,避免临时产物残留
733                        if (parallelAnalyzer != null) {
734                                try {
735                                        parallelAnalyzer.dispose();
736                                } catch (Exception ex) {
737                                        logger.error("Failed to dispose parallel analyzer", ex);
738                                }
739                        }
740                        //autogenerate 清理委托路径产生的拆分目录、gsp-input 临时文件及拆分产物,成功或失败均必须执行
741                        SQLUtil.deltree(splitDir);
742                }
743        }
744
745        private ParallelDataFlowAnalyzer buildParallelAnalyzer(File splitDir) {
746                if (originalFiles != null) {
747                        return new ParallelDataFlowAnalyzer(originalFiles, option,
748                                        option.getLargeFileSplitSizeMB(), splitDir);
749                }
750                // SqlInfo[] constructor path: collect files from all entries. Entries
751                // without a filePath (e.g., String/String[] constructors, or large
752                // raw SQL strings) are written to temp files so they can be split.
753                // All paths must go through split, otherwise ParallelDataFlowAnalyzer
754                // would create DataFlowAnalyzer instances internally which would
755                // detect the same large input and delegate again → infinite recursion.
756                if (sqlInfos != null && sqlInfos.length > 0) {
757                        List<File> files = new ArrayList<>();
758                        for (SqlInfo info : sqlInfos) {
759                                if (info == null) continue;
760                                File f = null;
761                                if (info.getFilePath() != null) {
762                                        f = new File(info.getFilePath());
763                                }
764                                if (f != null && f.exists()) {
765                                        if (!files.contains(f)) {
766                                                files.add(f);
767                                        }
768                                } else if (info.getSql() != null) {
769                                        try {
770                                                f = File.createTempFile("gsp-input-", ".tmp", splitDir);
771                                                try (FileWriter fw = new FileWriter(f)) {
772                                                        fw.write(info.getSql());
773                                                }
774                                                files.add(f);
775                                        } catch (Exception e) {
776                                                logger.error("Failed to create temp file for split", e);
777                                        }
778                                }
779                        }
780                        if (!files.isEmpty()) {
781                                return new ParallelDataFlowAnalyzer(files.toArray(new File[0]), option,
782                                                option.getLargeFileSplitSizeMB(), splitDir);
783                        }
784                }
785                return new ParallelDataFlowAnalyzer(sqlInfos, option);
786        }
787
788        private String finishGenerateDataFlow(boolean withExtraInfo) {
789                if (dataflow != null && !withExtraInfo && dataflow.getResultsets() != null) {
790                        for (table t : dataflow.getResultsets()) {
791                                t.setIsTarget(null);
792                t.setProcedureId(null);
793                                if (t.getColumns() != null) {
794                                        for (column t1 : t.getColumns()) {
795                                                t1.setIsFunction(null);
796                                        }
797                                }
798                        }
799                }
800
801        if (dataflow != null && !withExtraInfo && dataflow.getVariables() != null) {
802            for (table t : dataflow.getVariables()) {
803                t.setIsTarget(null);
804                t.setProcedureId(null);
805            }
806        }
807
808                if(dataflow!=null && dataflow.getRelationships()!=null && option.getFilterRelationTypes()!=null && !option.getFilterRelationTypes().isEmpty()) {
809                        List<relationship> relationships = new ArrayList<relationship>();
810                        for(relationship relationship: dataflow.getRelationships()) {
811                                if(option.getFilterRelationTypes().contains(relationship.getType())) {
812                                        relationships.add(relationship);
813                                }
814                        }
815                        dataflow.setRelationships(relationships);
816                }
817
818                if (option.getHandleListener() != null) {
819                        option.getHandleListener().endAnalyze(dataflow);
820                }
821
822                if (option.isOutput()) {
823                        if (option.getHandleListener() != null) {
824                                option.getHandleListener().startOutputDataFlowXML();
825                        }
826                        if (dataflow != null) {
827                                if (option.isTextFormat()) {
828                                        dataflowString = getTextOutput(dataflow);
829                                } else {
830                                        try {
831                                                dataflowString = XML2Model.saveXML(dataflow);
832                                        }catch (Exception e){
833                                                logger.error("Output dataflow to xml failed.", e);
834                                                dataflowString = null;
835                                        }
836                                }
837                        }
838                        if (option.getHandleListener() != null) {
839                                option.getHandleListener().endOutputDataFlowXML(dataflowString == null ? 0 : dataflowString.length());
840                        }
841                }
842
843                return dataflowString;
844        }
845
846        private dataflow removeDuplicateColumns(dataflow dataflow) {
847                List<table> tables = new ArrayList<table>();
848                if (dataflow.getTables() != null) {
849                        tables.addAll(dataflow.getTables());
850                }
851                if (dataflow.getViews() != null) {
852                        tables.addAll(dataflow.getViews());
853                }
854                if (dataflow.getStages() != null) {
855                        tables.addAll(dataflow.getStages());
856                }
857                if (dataflow.getDatasources() != null) {
858                        tables.addAll(dataflow.getDatasources());
859                }
860                if (dataflow.getStreams() != null) {
861                        tables.addAll(dataflow.getStreams());
862                }
863                if (dataflow.getPaths() != null) {
864                        tables.addAll(dataflow.getPaths());
865                }
866                if (dataflow.getVariables() != null) {
867                        tables.addAll(dataflow.getVariables());
868                }
869                if (dataflow.getResultsets() != null) {
870                        tables.addAll(dataflow.getResultsets());
871                }
872                for (table table : tables) {
873                        if (table.getColumns() == null) {
874                                continue;
875                        }
876                        Set<String> columnIds = new HashSet<String>();
877                        Iterator<column> iter = table.getColumns().iterator();
878                        while(iter.hasNext()) {
879                                column column = iter.next();
880                                String id = column.getId();
881                                if (columnIds.contains(id)) {
882                                        iter.remove();
883                                } else {
884                                        columnIds.add(id);
885                                }
886                        }
887                }
888                return dataflow;
889        }
890
891        public synchronized String generateDataFlow() {
892                return generateDataFlow(false);
893        }
894
895        public synchronized String generateSqlInfos() {
896                return JSON.toJSONString(sqlInfoMap);
897        }
898
899        public Map<String, List<SqlInfo>> getSqlInfos() {
900                return sqlInfoMap;
901        }
902
903        public Map getHashSQLMap() {
904                return modelManager.getHashSQLMap();
905        }
906        
907        public Map getDynamicSQLMap() {
908                return modelManager.getDynamicSQLMap();
909        }
910
911        /**
912         * @deprecated please use SqlInfoHelper.getSelectedDbObjectInfo
913         */
914        public DbObjectPosition getSelectedDbObjectInfo(Coordinate start, Coordinate end) {
915                if (start == null || end == null) {
916                        throw new IllegalArgumentException("Coordinate can't be null.");
917                }
918
919                String hashCode = start.getHashCode();
920
921                if (hashCode == null) {
922                        throw new IllegalArgumentException("Coordinate hashcode can't be null.");
923                }
924
925                int dbObjectStartLine = (int) start.getX() - 1;
926                int dbObjectStarColumn = (int) start.getY() - 1;
927                int dbObjectEndLine = (int) end.getX() - 1;
928                int dbObjectEndColumn = (int) end.getY() - 1;
929                List<SqlInfo> sqlInfoList;
930                if (hashCode.matches("\\d+")) {
931                        sqlInfoList = sqlInfoMap.getValueAtIndex(Integer.valueOf(hashCode));
932                } else {
933                        sqlInfoList = sqlInfoMap.get(hashCode);
934                }
935                for (int j = 0; j < sqlInfoList.size(); j++) {
936                        SqlInfo sqlInfo = sqlInfoList.get(j);
937                        int startLine = sqlInfo.getLineStart();
938                        int endLine = sqlInfo.getLineEnd();
939                        if (dbObjectStartLine >= startLine && dbObjectStartLine <= endLine) {
940                                DbObjectPosition position = new DbObjectPosition();
941                                position.setFile(sqlInfo.getFileName());
942                                position.setFilePath(sqlInfo.getFilePath());
943                                position.setSql(sqlInfo.getSql());
944                                position.setIndex(sqlInfo.getOriginIndex());
945                                List<Pair<Integer, Integer>> positions = position.getPositions();
946                                positions.add(new Pair<Integer, Integer>(
947                                                dbObjectStartLine - startLine + sqlInfo.getOriginLineStart() + 1, dbObjectStarColumn + 1));
948                                positions.add(new Pair<Integer, Integer>(dbObjectEndLine - startLine + sqlInfo.getOriginLineStart() + 1,
949                                                dbObjectEndColumn + 1));
950                                return position;
951                        }
952                }
953                return null;
954        }
955
956        /**
957         * Whether a serialized coordinate ("[line,col,offset],[line,col,offset]...")
958         * points at a real reference site. A position synthesized for a column that
959         * has no source token renders as "[1,1,0],[1,&lt;n&gt;,0]" (start-of-module,
960         * width = name length) or carries -1 markers; a genuine column reference can
961         * never start at line 1, column 1, because the statement head keyword
962         * occupies that position.
963         */
964        private static boolean hasUsableCoordinate(String coordinate) {
965                return DlineageUtil.hasUsableCoordinate(coordinate);
966        }
967
968        public static dataflow mergeTables(dataflow dataflow, Long startId) {
969                return mergeTables(dataflow, startId, new Option());
970        }
971
972        public static dataflow mergeTables(dataflow dataflow, Long startId, Option option) {
973                List<table> tableCopy = new ArrayList<table>();
974                List<table> viewCopy = new ArrayList<table>();
975                List<table> databaseCopy = new ArrayList<table>();
976                List<table> schemaCopy = new ArrayList<table>();
977                List<table> stageCopy = new ArrayList<table>();
978                List<table> dataSourceCopy = new ArrayList<table>();
979                List<table> streamCopy = new ArrayList<table>();
980                List<table> fileCopy = new ArrayList<table>();
981                List<table> variableCopy = new ArrayList<table>();
982                List<table> cursorCopy = new ArrayList<table>();
983                List<table> resultSetCopy = new ArrayList<table>();
984                if (dataflow.getTables() != null) {
985                        tableCopy.addAll(dataflow.getTables());
986                }
987                dataflow.setTables(tableCopy);
988                if (dataflow.getViews() != null) {
989                        viewCopy.addAll(dataflow.getViews());
990                }
991                dataflow.setViews(viewCopy);
992                if (dataflow.getDatabases() != null) {
993                        databaseCopy.addAll(dataflow.getDatabases());
994                }
995                dataflow.setDatabases(databaseCopy);
996                if (dataflow.getSchemas() != null) {
997                        schemaCopy.addAll(dataflow.getSchemas());
998                }
999                dataflow.setSchemas(schemaCopy);
1000                if (dataflow.getStages() != null) {
1001                        stageCopy.addAll(dataflow.getStages());
1002                }
1003                dataflow.setStages(stageCopy);
1004                if (dataflow.getDatasources() != null) {
1005                        dataSourceCopy.addAll(dataflow.getDatasources());
1006                }
1007                dataflow.setDatasources(dataSourceCopy);
1008                if (dataflow.getStreams() != null) {
1009                        streamCopy.addAll(dataflow.getStreams());
1010                }
1011                dataflow.setStreams(streamCopy);
1012                if (dataflow.getPaths() != null) {
1013                        fileCopy.addAll(dataflow.getPaths());
1014                }
1015                dataflow.setPaths(fileCopy);
1016                if (dataflow.getVariables() != null) {
1017                        variableCopy.addAll(dataflow.getVariables());
1018                }
1019                dataflow.setVariables(variableCopy);
1020                if (dataflow.getResultsets() != null) {
1021                        resultSetCopy.addAll(dataflow.getResultsets());
1022                }
1023                dataflow.setResultsets(resultSetCopy);
1024
1025                Map<String, List<table>> tableMap = new HashMap<String, List<table>>();
1026                Map<String, String> tableTypeMap = new HashMap<String, String>();
1027                Map<String, TMssqlCreateType> mssqlTypeMap = new HashMap<String, TMssqlCreateType>();
1028                Map<String, String> tableIdMap = new HashMap<String, String>();
1029
1030                Map<String, List<column>> columnMap = new HashMap<String, List<column>>();
1031                Map<String, Set<String>> tableColumnMap = new HashMap<String, Set<String>>();
1032                Map<String, String> columnIdMap = new HashMap<String, String>();
1033                Map<String, column> columnMergeIdMap = new HashMap<String, column>();
1034
1035                List<table> tables = new ArrayList<table>();
1036                tables.addAll(dataflow.getTables());
1037                tables.addAll(dataflow.getViews());
1038                tables.addAll(dataflow.getDatabases());
1039                tables.addAll(dataflow.getSchemas());
1040                tables.addAll(dataflow.getStages());
1041                tables.addAll(dataflow.getDatasources());
1042                tables.addAll(dataflow.getStreams());
1043                tables.addAll(dataflow.getPaths());
1044                tables.addAll(dataflow.getResultsets());
1045                tables.addAll(dataflow.getVariables());
1046                
1047                Set<String> columnIds = new HashSet<String>();
1048
1049                for (table table : tables) {
1050                        String qualifiedTableName = DlineageUtil.getQualifiedTableName(table);
1051                        String tableFullName = DlineageUtil.getIdentifierNormalTableName(qualifiedTableName);
1052                        if ("variable".endsWith(table.getType()) && !SQLUtil.isEmpty(table.getParent())) {
1053                                tableFullName = table.getParent() + "." + tableFullName;
1054                        }
1055                        
1056                        if (!tableMap.containsKey(tableFullName)) {
1057                                tableMap.put(tableFullName, new ArrayList<table>());
1058                        }
1059
1060                        tableMap.get(tableFullName).add(table);
1061
1062                        if (!tableTypeMap.containsKey(tableFullName)) {
1063                                tableTypeMap.put(tableFullName, table.getType());
1064                        } else if ("view".equals(table.getSubType())) {
1065                                tableTypeMap.put(tableFullName, table.getType());
1066                        } else if ("database".equals(table.getSubType())) {
1067                                tableTypeMap.put(tableFullName, table.getType());
1068                        } else if ("schema".equals(table.getSubType())) {
1069                                tableTypeMap.put(tableFullName, table.getType());
1070                        } else if ("stage".equals(table.getSubType())) {
1071                                tableTypeMap.put(tableFullName, table.getType());
1072                        } else if ("sequence".equals(table.getSubType())) {
1073                                tableTypeMap.put(tableFullName, table.getType());
1074                        } else if ("datasource".equals(table.getSubType())) {
1075                                tableTypeMap.put(tableFullName, table.getType());
1076                        } else if ("stream".equals(table.getSubType())) {
1077                                tableTypeMap.put(tableFullName, table.getType());
1078                        } else if ("file".equals(table.getSubType())) {
1079                                tableTypeMap.put(tableFullName, table.getType());
1080                        } else if ("table".equals(tableTypeMap.get(tableFullName))) {
1081                                tableTypeMap.put(tableFullName, table.getType());
1082                        } else if ("variable".equals(tableTypeMap.get(tableFullName))) {
1083                                tableTypeMap.put(tableFullName, table.getType());
1084                        }
1085
1086                        if (table.getColumns() != null) {
1087                                if (!tableColumnMap.containsKey(tableFullName)) {
1088                                        tableColumnMap.put(tableFullName, new LinkedHashSet<String>());
1089                                }
1090                                for (column column : table.getColumns()) {
1091                                        String columnFullName = tableFullName + "."
1092                                                        + (column.getQualifiedTable() != null
1093                                                                        ? (DlineageUtil.getIdentifierNormalTableName(column.getQualifiedTable()) + ".")
1094                                                                        : "")
1095                                                        + ("false".equals(table.getIsTarget()) ? DlineageUtil.normalizeColumnName(column.getName())
1096                                                                        : DlineageUtil.getIdentifierNormalColumnName(column.getName()));
1097
1098                                        if (!columnMap.containsKey(columnFullName)) {
1099                                                columnMap.put(columnFullName, new ArrayList<column>());
1100                                                tableColumnMap.get(tableFullName).add(columnFullName);
1101                                        }
1102
1103                                        columnMap.get(columnFullName).add(column);
1104                                        columnIds.add(column.getId());
1105                                }
1106                        }
1107                }
1108
1109                Set<String> relationParentIds = new HashSet<String>();
1110                if (dataflow.getRelationships() != null) {
1111                        for (relationship rel : dataflow.getRelationships()) {
1112                                if (rel.getSources() != null) {
1113                                        for (sourceColumn src : rel.getSources()) {
1114                                                if (src.getParent_id() != null) {
1115                                                        relationParentIds.add(src.getParent_id());
1116                                                }
1117                                        }
1118                                }
1119                                if (rel.getTarget() != null) {
1120                                        if (rel.getTarget().getParent_id() != null) {
1121                                                relationParentIds.add(rel.getTarget().getParent_id());
1122                                        }
1123                                }
1124                        }
1125                }
1126
1127                Iterator<String> tableNameIter = tableMap.keySet().iterator();
1128                while (tableNameIter.hasNext()) {
1129                        String tableName = tableNameIter.next();
1130                        List<table> tableList = tableMap.get(tableName);
1131                        table table;
1132                        if (tableList.size() > 1) {
1133                                table standardTable = tableList.get(0);
1134                                // Function允许重名,不做合并处理
1135                                if (standardTable.isFunction()) {
1136                                        continue;
1137                                }
1138                                
1139                                // Variable允许重名,不做合并处理
1140                                if (standardTable.isVariable()) {
1141                                        continue;
1142                                }
1143                                
1144                                // 临时表不做合并处理
1145                                if(SQLUtil.isTempTable(standardTable)) {
1146                                        continue;
1147                                }
1148
1149                                String type = tableTypeMap.get(tableName);
1150                                table = new table();
1151                                table.setId(String.valueOf(++startId));
1152                                table.setServer(standardTable.getServer());
1153                                table.setDatabase(standardTable.getDatabase());
1154                                table.setSchema(standardTable.getSchema());
1155                                table.setName(standardTable.getName());
1156                                table.setDisplayName(standardTable.getDisplayName());
1157                                table.setParent(standardTable.getParent());
1158                                table.setMore(standardTable.getMore());
1159                                if (standardTable.getCandidateTables() != null && !standardTable.getCandidateTables().isEmpty()) {
1160                                        table.setCandidateTables(new ArrayList<String>(standardTable.getCandidateTables()));
1161                                }
1162                                table.setColumns(new ArrayList<column>());
1163                                String subType = null;
1164                                for(table item: tableList){
1165                                        if (item.getSubType() != null) {
1166                                                subType = item.getSubType();
1167                                                break;
1168                                        }
1169                                }
1170                                if (subType != null) {
1171                                        table.setSubType(subType);
1172                                } else {
1173                                        table.setSubType(standardTable.getSubType());
1174                                }
1175                                Set<String> processIds = new LinkedHashSet<String>();
1176                                for (int k = 0; k < tableList.size(); k++) {
1177                                        if (tableList.get(k).getProcessIds() != null) {
1178                                                processIds.addAll(tableList.get(k).getProcessIds());
1179                                        }
1180                                }
1181                                if (!processIds.isEmpty()) {
1182                                        table.setProcessIds(new ArrayList<String>(processIds));
1183                                }
1184                                Set<String> alias = new LinkedHashSet<String>();
1185                                for (int k = 0; k < tableList.size(); k++) {
1186                                        if (tableList.get(k).getAlias() != null) {
1187                                                alias.addAll(Arrays.asList(tableList.get(k).getAlias().split("\\s*,\\s*")));
1188                                        }
1189                                }
1190                                if (!alias.isEmpty()) {
1191                                        String aliasString = Arrays.toString(alias.toArray(new String[0]));
1192                                        table.setAlias(aliasString.substring(1, aliasString.length() - 1));
1193                                }
1194                                table.setType(type);
1195                                for (table item : tableList) {
1196                                        if (!SQLUtil.isEmpty(table.getCoordinate()) && !SQLUtil.isEmpty(item.getCoordinate())) {
1197                                                if (table.getCoordinate().indexOf(item.getCoordinate()) == -1) {
1198                                                        table.appendCoordinate(item.getCoordinate());
1199                                                }
1200                                        } else if (!SQLUtil.isEmpty(item.getCoordinate())) {
1201                                                table.setCoordinate(item.getCoordinate());
1202                                        }
1203
1204                                        if (item.getStarStmt() != null) {
1205                                                table.setStarStmt(item.getStarStmt());
1206                                        }
1207
1208                                        tableIdMap.put(item.getId(), table.getId());
1209
1210                                        if (item.isView()) {
1211                                                dataflow.getViews().remove(item);
1212                                        } else if (item.isDatabaseType()) {
1213                                                dataflow.getDatabases().remove(item);
1214                                        } else if (item.isSchemaType()) {
1215                                                dataflow.getSchemas().remove(item);
1216                                        } else if (item.isStage()) {
1217                                                dataflow.getStages().remove(item);
1218                                        } else if (item.isDataSource()) {
1219                                                dataflow.getDatasources().remove(item);
1220                                        } else if (item.isStream()) {
1221                                                dataflow.getStreams().remove(item);
1222                                        } else if (item.isFile()) {
1223                                                dataflow.getPaths().remove(item);
1224                                        } else if (item.isVariable()) {
1225                                                dataflow.getVariables().remove(item);
1226                                        } else if (item.isTable()) {
1227                                                dataflow.getTables().remove(item);
1228                                        } else if (item.isResultSet()) {
1229                                                dataflow.getResultsets().remove(item);
1230                                        }
1231                                }
1232
1233                                if (table.isView()) {
1234                                        dataflow.getViews().add(table);
1235                                } else if (table.isDatabaseType()) {
1236                                        dataflow.getDatabases().add(table);
1237                                } else if (table.isSchemaType()) {
1238                                        dataflow.getSchemas().add(table);
1239                                } else if (table.isStage()) {
1240                                        dataflow.getStages().add(table);
1241                                } else if (table.isDataSource()) {
1242                                        dataflow.getDatasources().add(table);
1243                                } else if (table.isStream()) {
1244                                        dataflow.getStreams().add(table);
1245                                } else if (table.isFile()) {
1246                                        dataflow.getPaths().add(table);
1247                                } else if (table.isVariable()) {
1248                                        dataflow.getVariables().add(table);
1249                                } else if (table.isResultSet()) {
1250                                        dataflow.getResultsets().add(table);
1251                                } else {
1252                                        dataflow.getTables().add(table);
1253                                }
1254                        } else {
1255                                table = tableList.get(0);
1256                                if(Boolean.TRUE.toString().equals(table.getIsDetermined())){
1257                                        continue;
1258                                }
1259                                
1260                                if (option.isIgnoreUnusedSynonym() && SubType.synonym.name().equals(table.getSubType())) {
1261                                        boolean hasSourceRelation = relationParentIds.contains(table.getId());
1262                                        if (!hasSourceRelation) {
1263                                                dataflow.getTables().remove(table);
1264                                                tableColumnMap.get(tableName).clear();
1265                                                continue;
1266                                        }
1267                                }
1268                        }
1269
1270                        Set<String> columns = tableColumnMap.get(tableName);
1271                        Iterator<String> columnIter = columns.iterator();
1272                        List<column> mergeColumns = new ArrayList<column>();
1273                        while (columnIter.hasNext()) {
1274                                String columnName = columnIter.next();
1275                                List<column> columnList = columnMap.get(columnName);
1276                                List<column> functions = new ArrayList<column>();
1277                                for (column t : columnList) {
1278                                        if (Boolean.TRUE.toString().equals(t.getIsFunction())) {
1279                                                functions.add(t);
1280                                        }
1281                                }
1282                                if (functions != null && !functions.isEmpty()) {
1283                                        for (column function : functions) {
1284                                                mergeColumns.add(function);
1285                                                columnIdMap.put(function.getId(), function.getId());
1286                                                columnMergeIdMap.put(function.getId(), function);
1287                                        }
1288
1289                                        columnList.removeAll(functions);
1290                                }
1291                                if (!columnList.isEmpty()) {
1292                                        column firstColumn = columnList.iterator().next();
1293                                        if (columnList.size() > 1) {
1294                                                column mergeColumn = new column();
1295                                                mergeColumn.setId(String.valueOf(++startId));
1296                                                mergeColumn.setName(firstColumn.getName());
1297                                                mergeColumn.setDisplayName(firstColumn.getDisplayName());
1298                                                mergeColumn.setSource(firstColumn.getSource());
1299                                                mergeColumn.setQualifiedTable(firstColumn.getQualifiedTable());
1300                                                mergeColumn.setDataType(firstColumn.getDataType());
1301                                                mergeColumn.setForeignKey(firstColumn.isForeignKey());
1302                                                mergeColumn.setPrimaryKey(firstColumn.isPrimaryKey());
1303                                                mergeColumn.setUnqiueKey(firstColumn.isUnqiueKey());
1304                                                mergeColumn.setIndexKey(firstColumn.isIndexKey());
1305                                                mergeColumns.add(mergeColumn);
1306                                                for (column item : columnList) {
1307                                                        mergeColumn.appendCoordinate(item.getCoordinate());
1308                                                        columnIdMap.put(item.getId(), mergeColumn.getId());
1309                                                }
1310                                                columnMergeIdMap.put(mergeColumn.getId(), mergeColumn);
1311                                                columnIds.add(mergeColumn.getId());
1312                                        } else {
1313                                                mergeColumns.add(firstColumn);
1314                                                columnIdMap.put(firstColumn.getId(), firstColumn.getId());
1315                                                columnMergeIdMap.put(firstColumn.getId(), firstColumn);
1316                                        }
1317                                }
1318                        }
1319                        table.setColumns(mergeColumns);
1320                }
1321
1322                if (dataflow.getRelationships() != null) {
1323                        Map<Long, relationship> mergeRelations = new LinkedHashMap<Long, relationship>();
1324                        for (int i = 0; i < dataflow.getRelationships().size(); i++) {
1325                                relationship relation = dataflow.getRelationships().get(i);
1326                                
1327                                if("crud".equals(relation.getType()) && option.getAnalyzeMode() == AnalyzeMode.crud) {
1328                                        long key = relation.toDedupHash();
1329                                        if (!mergeRelations.containsKey(key)) {
1330                                                mergeRelations.put(key, relation);
1331                                        }
1332                                        continue;
1333                                }
1334                                
1335                                targetColumn target = relation.getTarget();
1336                                if ("call".equals(relation.getType())) {
1337                                        target = relation.getCaller();
1338                                }
1339                                if (target != null && tableIdMap.containsKey(target.getParent_id())) {
1340                                        target.setParent_id(tableIdMap.get(target.getParent_id()));
1341                                }
1342
1343                                if (columnIdMap.containsKey(target.getId())) {
1344                                        target.setId(columnIdMap.get(target.getId()));
1345                                        target.setCoordinate(columnMergeIdMap.get(target.getId()).getCoordinate());
1346                                }
1347                                else if(option.isIgnoreUnusedSynonym() && EffectType.synonym.name().equals(relation.getEffectType())){
1348                                        continue;
1349                                }
1350                                
1351                                if (!"call".equals(relation.getType()) && !columnIds.contains(target.getId())) {
1352                                        continue;
1353                                }
1354
1355                                List<sourceColumn> sources = relation.getSources();
1356                                if ("call".equals(relation.getType())) {
1357                                        sources = relation.getCallees();
1358                                }
1359                                // Sources of merged columns keep their own reference-site
1360                                // coordinate (the ON/WHERE operand span) instead of the merged
1361                                // column's stamped definition coordinate. Source and relation
1362                                // dedup, however, must still collapse EXACTLY as they did under
1363                                // the stamped coordinates — sourceColumn.equals and
1364                                // relationship.toDedupHash both include the coordinate — so the
1365                                // stamped coordinate is applied for the dedup steps and the
1366                                // reference-site coordinate is restored on the survivors below.
1367                                Map<sourceColumn, String> referenceCoordinates = new IdentityHashMap<sourceColumn, String>();
1368                                Set<sourceColumn> sourceSet = new LinkedHashSet<sourceColumn>();
1369                                if (sources != null) {
1370                                        for (sourceColumn source : sources) {
1371                                                // Reference-published source of an UNMERGED column: stamp
1372                                                // the definition span it carried before reference-site
1373                                                // publishing, so dedup keeps its legacy behavior.
1374                                                String dedupCoordinate = source.getDedupCoordinate();
1375                                                if (dedupCoordinate != null && !dedupCoordinate.equals(source.getCoordinate())) {
1376                                                        referenceCoordinates.put(source, source.getCoordinate());
1377                                                        source.setCoordinate(dedupCoordinate);
1378                                                }
1379                                                if (!"call".equals(relation.getType()) && !columnIds.contains(source.getId())) {
1380                                                        continue;
1381                                                }
1382                                                if (tableIdMap.containsKey(source.getParent_id())) {
1383                                                        source.setParent_id(tableIdMap.get(source.getParent_id()));
1384                                                }
1385                                                if (tableIdMap.containsKey(source.getSource_id())) {
1386                                                        source.setSource_id(tableIdMap.get(source.getSource_id()));
1387                                                }
1388                                                if (columnIdMap.containsKey(source.getId())) {
1389                                                        String ownCoordinate = referenceCoordinates.containsKey(source)
1390                                                                        ? referenceCoordinates.get(source)
1391                                                                        : source.getCoordinate();
1392                                                        source.setId(columnIdMap.get(source.getId()));
1393                                                        source.setCoordinate(columnMergeIdMap.get(source.getId()).getCoordinate());
1394                                                        if (hasUsableCoordinate(ownCoordinate)) {
1395                                                                // The operand's own coordinate survives the merge;
1396                                                                // only a synthetic/absent one is replaced by the
1397                                                                // merged column's definition coordinate for good.
1398                                                                referenceCoordinates.put(source, ownCoordinate);
1399                                                        } else {
1400                                                                referenceCoordinates.remove(source);
1401                                                        }
1402                                                }
1403                                        }
1404
1405                                        sourceSet.addAll(sources);
1406                                        if ("call".equals(relation.getType())) {
1407                                                relation.setCallees(new ArrayList<sourceColumn>(sourceSet));
1408                                        } else {
1409                                                relation.setSources(new ArrayList<sourceColumn>(sourceSet));
1410                                        }
1411                                }
1412
1413                                long key = relation.toDedupHash();
1414                                if (!mergeRelations.containsKey(key)) {
1415                                        mergeRelations.put(key, relation);
1416                                        // Restore the reference-site coordinates on the surviving
1417                                        // relation's sources, now that dedup ran under the stamped
1418                                        // legacy coordinates.
1419                                        for (Map.Entry<sourceColumn, String> entry : referenceCoordinates.entrySet()) {
1420                                                entry.getKey().setCoordinate(entry.getValue());
1421                                        }
1422                                }
1423                        }
1424
1425                        dataflow.setRelationships(new ArrayList<relationship>(mergeRelations.values()));
1426                }
1427
1428                tableMap.clear();
1429                tableTypeMap.clear();
1430                tableIdMap.clear();
1431                columnMap.clear();
1432                tableColumnMap.clear();
1433                columnIdMap.clear();
1434                columnMergeIdMap.clear();
1435                tables.clear();
1436                return dataflow;
1437        }
1438
1439        public synchronized dataflow getDataFlow() {
1440                if (dataflow != null) {
1441                        return dataflow;
1442                } else if (dataflowString != null) {
1443                        return XML2Model.loadXML(dataflow.class, dataflowString);
1444                }
1445                return null;
1446        }
1447
1448        /**
1449         * Returns immutable AST/catalog evidence captured during this analyzer run.
1450         * The evidence is correlated to {@link #getDataFlow()} by document-local IDs
1451         * and is intentionally absent from the legacy serialized contract.
1452         */
1453        @Override
1454        public synchronized AuthoritativeLineageEvidence getAuthoritativeLineageEvidence() {
1455                return authoritativeLineageEvidence;
1456        }
1457
1458        List<ErrorInfo> metadataErrors = new ArrayList<>();
1459        
1460        private synchronized dataflow analyzeSqlScript() {
1461                init();
1462
1463                try {
1464                        dataflow dataflow = new dataflow();
1465                        
1466                        if (sqlInfos != null) {
1467                                if (option.getHandleListener() != null) {
1468                                        if (sqlInfos.length == 1) {
1469                                                int sqlLength = sqlInfos[0].getSql() != null ? sqlInfos[0].getSql().length() : 0;
1470                                                option.getHandleListener().startAnalyze(null, sqlLength, false);
1471                                        } else {
1472                                                option.getHandleListener().startAnalyze(null, sqlInfos.length, true);
1473                                        }
1474                                }
1475
1476                                if (sqlenv == null) {
1477                                        if (option.getHandleListener() != null) {
1478                                                option.getHandleListener().startParseSQLEnv();
1479                                        }
1480                                        TSQLEnv[] sqlenvs = new SQLEnvParser(option.getDefaultServer(), option.getDefaultDatabase(),
1481                                                        option.getDefaultSchema()).parseSQLEnv(option.getVendor(), sqlInfos);
1482                                        if (sqlenvs != null && sqlenvs.length > 0) {
1483                                                sqlenv = sqlenvs[0];
1484                                        }
1485                                        if (option.getHandleListener() != null) {
1486                                                option.getHandleListener().endParseSQLEnv();
1487                                        }
1488                                }
1489                                TGSqlParser sqlparser = createSqlParser(option.getVendor());
1490                                Map<String, Pair3<StringBuilder, AtomicInteger, String>> databaseMap = new LinkedHashMap<String, Pair3<StringBuilder, AtomicInteger, String>>();
1491                                for (int i = 0; i < sqlInfos.length; i++) {
1492                                        SqlInfo sqlInfo = sqlInfos[i];
1493                                        if (sqlInfo == null) {
1494                                                sqlInfoMap.put(String.valueOf(i), new ArrayList<SqlInfo>());
1495                                                continue;
1496                                        }
1497                                        String sql = sqlInfo.getSql();
1498                                        if (SQLUtil.isEmpty(sql) && sqlInfo.getFileName() != null
1499                                                        && new File(sqlInfo.getFileName()).exists()) {
1500                                                sql = SQLUtil.getFileContent(sqlInfo.getFileName());
1501                                        }
1502                                        if (SQLUtil.isEmpty(sql) && sqlInfo.getFilePath() != null
1503                                                        && new File(sqlInfo.getFilePath()).exists()) {
1504                                                sql = SQLUtil.getFileContent(sqlInfo.getFilePath());
1505                                        }
1506                                        String sqlTrim = null;
1507                                        if (sql != null) {
1508                                                sqlTrim = sql.substring(0, Math.min(sql.length(), 512)).trim();
1509                                        }
1510                                        if(sql!=null && sqlTrim.startsWith("<") && sqlTrim.indexOf("<dlineage")!=-1){
1511                                                dataflow temp = XML2Model.loadXML(dataflow.class, sql);
1512                                                if(sqlInfos.length == 1){
1513                                                        dataflow = temp;
1514                                                }
1515                                                else {
1516                                                        if (temp.getTables() != null) {
1517                                                                dataflow.getTables().addAll(temp.getTables());
1518                                                        }
1519                                                        if (temp.getViews() != null) {
1520                                                                dataflow.getViews().addAll(temp.getViews());
1521                                                        }
1522                                                        if (temp.getResultsets() != null) {
1523                                                                dataflow.getResultsets().addAll(temp.getResultsets());
1524                                                        }
1525                                                        if (temp.getRelationships() != null) {
1526                                                                dataflow.getRelationships().addAll(temp.getRelationships());
1527                                                        }
1528                                                        if (temp.getErrors() != null) {
1529                                                                dataflow.getErrors().addAll(temp.getErrors());
1530                                                        }
1531                                                }
1532                                        }
1533                                        else if (sql != null && sqlTrim.startsWith("{")) {
1534                                                EDbVendor vendor = SQLUtil.isEmpty(sqlInfo.getDbVendor()) ? option.getVendor()
1535                                                                : EDbVendor.valueOf(sqlInfo.getDbVendor());
1536                                                // Pass the manifest's directory so a sharded manifest supplied
1537                                                // as inline content can still resolve its relative catalog
1538                                                // paths (otherwise baseDir is null and the catalog is not read).
1539                                                String envBaseDir = sqlInfo.getFilePath() != null
1540                                                                ? new File(sqlInfo.getFilePath()).getParent() : null;
1541                                                TSQLEnv[] sqlenvs = new TJSONSQLEnvParser(option.getDefaultServer(),
1542                                                                option.getDefaultDatabase(), option.getDefaultSchema()).parseSQLEnv(vendor, sql, envBaseDir);
1543                                                if (sqlenvs != null && sqlenvs.length > 0) {
1544                                                        if (sqlenv == null) {
1545                                                                sqlenv = sqlenvs[0];
1546                                                        } else {
1547                                                                sqlenv = SQLEnvParser.mergeSQLEnv(Arrays.asList(sqlenv, sqlenvs[0]));
1548                                                        }
1549                                                }
1550                                                if (sqlenv != null) {
1551                                                        if (MetadataReader.isGrabit(sql) || MetadataReader.isSqlflow(sql) || MetadataReader.isSqlflowSharded(sql)) {
1552                                                                String hash = SHA256.getMd5(sql);
1553                                                                String fileHash = SHA256.getMd5(hash);
1554                                                                if (!sqlInfoMap.containsKey(fileHash)) {
1555                                                                        sqlInfoMap.put(fileHash, new ArrayList<SqlInfo>());
1556                                                                        sqlInfoMap.get(fileHash).add(sqlInfo);
1557                                                                }
1558                                                                ModelBindingManager.setGlobalHash(fileHash);
1559                                                                dataflow temp = null;
1560
1561                                                                // Check the authoritative sharded `format` before the
1562                                                                // grabit brand: a grabit-branded sharded manifest must go
1563                                                                // to the sharded reader, not GrabitMetadataAnalyzer.
1564                                                                if (MetadataReader.isSqlflowSharded(sql)) {
1565                                                                        String baseDir = null;
1566                                                                        if (sqlInfo.getFilePath() != null) {
1567                                                                                baseDir = new File(sqlInfo.getFilePath()).getParent();
1568                                                                        }
1569                                                                        temp = new SqlflowShardedMetadataAnalyzer(baseDir).analyzeMetadata(option.getVendor(), sql);
1570                                                                } else if (MetadataReader.isGrabit(sql)) {
1571                                                                        temp = new GrabitMetadataAnalyzer().analyzeMetadata(option.getVendor(), sql);
1572                                                                } else {
1573                                                                        temp = new SqlflowMetadataAnalyzer(sqlenv).analyzeMetadata(option.getVendor(), sql);
1574                                                                }
1575//                                                              if (temp.getPackages() != null) {
1576//                                                                      dataflow.getPackages().addAll(temp.getPackages());
1577//                                                              }
1578//                                                              if (temp.getProcedures() != null) {
1579//                                                                      dataflow.getProcedures().addAll(temp.getProcedures());
1580//                                                              }
1581                                                                if (temp.getTables() != null) {
1582                                                                        dataflow.getTables().addAll(temp.getTables());
1583                                                                }
1584                                                                if (temp.getViews() != null) {
1585                                                                        dataflow.getViews().addAll(temp.getViews());
1586                                                                }
1587                                                                if (temp.getResultsets() != null) {
1588                                                                        dataflow.getResultsets().addAll(temp.getResultsets());
1589                                                                }
1590                                                                if (temp.getRelationships() != null) {
1591                                                                        dataflow.getRelationships().addAll(temp.getRelationships());
1592                                                                }
1593                                                                if (temp.getErrors() != null) {
1594                                                                        dataflow.getErrors().addAll(temp.getErrors());
1595                                                                }
1596                                                                if (sql.indexOf("createdBy") != -1) {
1597                                                                        // Route on the authoritative `format` field, not the
1598                                                                        // `createdBy` product brand. A sqlflow-sharded manifest
1599                                                                        // carries the exporter brand (SQLdep/sqlflow/grabit) in
1600                                                                        // createdBy; a plain substring check on the brand hijacked
1601                                                                        // SQLdep-branded sharded exports into the single-file
1602                                                                        // `queries` path, which finds no `queries` array and read
1603                                                                        // nothing. Exclude sharded manifests from the brand branch
1604                                                                        // so they fall through to the sharded reader below.
1605                                                                        if ((sql.toLowerCase().indexOf("sqldep") != -1
1606                                                                                        || sql.toLowerCase().indexOf("grabit") != -1)
1607                                                                                        && !MetadataReader.isSqlflowSharded(sql)) {
1608                                                                                Map jsonObject = (Map) JSON.parseObject(sql);
1609                                                                                List<Map> queries = (List<Map>) jsonObject.get("queries");
1610                                                                                if (queries != null) {
1611                                                                                        for (int j = 0; j < queries.size(); j++) {
1612                                                                                                Map queryObject = queries.get(j);
1613                                                                                                appendSqlInfo(databaseMap, j, sqlInfo, queryObject);
1614                                                                                        }
1615                                                                                }
1616                                                                        } else if (sql.toLowerCase().indexOf("sqlflow") != -1) {
1617                                                                                Map sqlflow = (Map) JSON.parseObject(sql);
1618                                                                                if ("sqlflow-sharded".equals(sqlflow.get("format"))
1619                                                                                                && !MetadataReader.isSupportedSqlflowSharded(sql)) {
1620                                                                                        ErrorInfo errorInfo = new ErrorInfo();
1621                                                                                        errorInfo.setErrorType(ErrorInfo.METADATA_ERROR);
1622                                                                                        errorInfo.setErrorMessage("Unsupported sqlflow-sharded formatVersion "
1623                                                                                                        + MetadataReader.shardedFormatVersion(sql)
1624                                                                                                        + "; this build supports up to "
1625                                                                                                        + MetadataReader.SUPPORTED_SHARDED_FORMAT_VERSION
1626                                                                                                        + ". Source SQL not loaded.");
1627                                                                                        errorInfo.setFileName(sqlInfo.getFileName());
1628                                                                                        errorInfo.setFilePath(sqlInfo.getFilePath());
1629                                                                                        errorInfo.setStartPosition(new Pair3<Long, Long, String>(-1L, -1L,
1630                                                                                                        ModelBindingManager.getGlobalHash()));
1631                                                                                        errorInfo.setEndPosition(new Pair3<Long, Long, String>(-1L, -1L,
1632                                                                                                        ModelBindingManager.getGlobalHash()));
1633                                                                                        errorInfo.setOriginStartPosition(new Pair<Long, Long>(-1L, -1L));
1634                                                                                        errorInfo.setOriginEndPosition(new Pair<Long, Long>(-1L, -1L));
1635                                                                                        metadataErrors.add(errorInfo);
1636                                                                                } else if ("sqlflow-sharded".equals(sqlflow.get("format"))) {
1637                                                                                        String baseDir = null;
1638                                                                                        if (sqlInfo.getFilePath() != null) {
1639                                                                                                baseDir = new File(sqlInfo.getFilePath()).getParent();
1640                                                                                        }
1641                                                                                        String sourceCompression = (String) sqlflow.get("sourceCompression");
1642                                                                                        List<Map> servers = (List<Map>) sqlflow.get("servers");
1643                                                                                        if (servers != null) {
1644                                                                                                for (Map serverObject : servers) {
1645                                                                                                        List<Map> databases = (List<Map>) serverObject.get("databases");
1646                                                                                                        // Source shards live under databases[] (catalog topology) or
1647                                                                                                        // schemas[] (schema topology, the natural Oracle shape). Both
1648                                                                                                        // carry a per-shard `source`; iterate whichever is present.
1649                                                                                                        // Reading only databases[] silently dropped every source record
1650                                                                                                        // of a schema-topology manifest -> empty lineage.
1651                                                                                                        List<Map> schemaShards = (List<Map>) serverObject.get("schemas");
1652                                                                                                        if (schemaShards != null && !schemaShards.isEmpty()) {
1653                                                                                                                databases = schemaShards;
1654                                                                                                        }
1655                                                                                                        if (databases != null) {
1656                                                                                                                for (Map database : databases) {
1657                                                                                                                        Map source = (Map) database.get("source");
1658                                                                                                                        if (source != null) {
1659                                                                                                                                String sourcePath = (String) source.get("path");
1660                                                                                                                                if (sourcePath != null && baseDir != null) {
1661                                                                    File sourceFile = new File(baseDir, sourcePath);
1662                                                                                                                                        String fullPath = sourceFile.getAbsolutePath();
1663                                                                                                                                        if ("block".equals(sourceCompression)) {
1664                                                                                                                                                readGzipBlockSource(fullPath, databaseMap, sqlInfo);
1665                                                                                                                                        } else {
1666                                                                                                                                                String sourceContent = SQLUtil.getFileContent(fullPath);
1667                                                                                                                                                if (sourceContent != null) {
1668                                                                                                                                                        String[] lines = sourceContent.split("\\r?\\n");
1669                                                                                                                                                        for (int j = 0; j < lines.length; j++) {
1670                                                                                                                                                                String line = lines[j].trim();
1671                                                                                                                                                                if (line.isEmpty()) {
1672                                                                                                                                                                        continue;
1673                                                                                                                                                                }
1674                                                                                                                                                                try {
1675                                                                                                                                                                        Map sourceObject = (Map) JSON.parseObject(line);
1676                                                                                                                                                                        String sourceCode = (String) sourceObject.get("sourceCode");
1677                                                                                                                                                                        if (sourceCode != null && !sourceCode.isEmpty()) {
1678                                                                                                                                                                                SqlInfo sourceSqlInfo = new SqlInfo();
1679                                                                                                                                                                                sourceSqlInfo.setFileName(sourceFile.getName());
1680                                                                                                                                                                                sourceSqlInfo.setFilePath(sourceFile.getAbsolutePath());
1681                                                                                                                                                                                sourceSqlInfo.setSql(sourceCode);
1682                                                                                                                                                                                sourceSqlInfo.setOriginIndex(j);
1683                                                                                                                                                                                appendSqlInfo(databaseMap, j, sourceSqlInfo, sourceObject);
1684                                                                                                                                                                        }
1685                                                                                                                                                                } catch (Exception e) {
1686                                                                                                                                                                        logger.warn("Parse source jsonl line failed.", e);
1687                                                                                                                                                                }
1688                                                                                                                                                        }
1689                                                                                                                                                }
1690                                                                                                                                        }
1691                                                                                                                                }
1692                                                                                                                        }
1693                                                                                                                }
1694                                                                                                        }
1695                                                                                                }
1696                                                                                        }
1697                                                                                } else {
1698                                                                                        List<Map> servers = (List<Map>) sqlflow.get("servers");
1699                                                                                        if (servers != null) {
1700                                                                                                for (Map serverObject : servers) {
1701                                                                                                        List<Map> queries = (List<Map>) serverObject.get("queries");
1702                                                                                                        if (queries != null) {
1703                                                                                                                for (int j = 0; j < queries.size(); j++) {
1704                                                                                                                        Map queryObject = queries.get(j);
1705                                                                                                                        appendSqlInfo(databaseMap, j, sqlInfo, queryObject);
1706                                                                                                                }
1707                                                                                                        }
1708                                                                                                }
1709                                                                                        }
1710                                                                                }
1711                                                                                List<Map> errorMessages =  (List<Map>) sqlflow.get("errorMessages");
1712                                                                                if(errorMessages!=null && !errorMessages.isEmpty()) {
1713                                                                                        for(Map error: errorMessages){
1714                                                                                                ErrorInfo errorInfo = new ErrorInfo();
1715                                                                                                errorInfo.setErrorType(ErrorInfo.METADATA_ERROR);
1716                                                                                                errorInfo.setErrorMessage((String)error.get("errorMessage"));
1717                                                                                                errorInfo.setFileName(sqlInfo.getFileName());
1718                                                                                                errorInfo.setFilePath(sqlInfo.getFilePath());
1719                                                                                                errorInfo.setStartPosition(new Pair3<Long, Long, String>(-1L, -1L,
1720                                                                                                                ModelBindingManager.getGlobalHash()));
1721                                                                                                errorInfo.setEndPosition(new Pair3<Long, Long, String>(-1L, -1L,
1722                                                                                                                ModelBindingManager.getGlobalHash()));
1723                                                                                                errorInfo.setOriginStartPosition(new Pair<Long, Long>(-1L, -1L));
1724                                                                                                errorInfo.setOriginEndPosition(new Pair<Long, Long>(-1L, -1L));
1725                                                                                                metadataErrors.add(errorInfo);
1726                                                                                        }
1727                                                                                }
1728                                                                        }
1729                                                                }
1730                                                        } else {
1731                                                                Map queryObject = (Map) JSON.parseObject(sql);
1732                                                                appendSqlInfo(databaseMap, i, sqlInfo, queryObject);
1733                                                        }
1734                                                } else {
1735                                                        Map queryObject = (Map) JSON.parseObject(sql);
1736                                                        appendSqlInfo(databaseMap, i, sqlInfo, queryObject);
1737                                                }
1738                                        } else {
1739                                                ModelBindingManager.removeGlobalDatabase();
1740                                                ModelBindingManager.removeGlobalSchema();
1741                                                ModelBindingManager.removeGlobalHash();
1742
1743                                                String content = sql;
1744
1745                                                if (content == null) {
1746                                                        continue;
1747                                                }
1748
1749                                                String delimiterChar = String.valueOf(sqlparser.getDelimiterChar());
1750
1751                                                if (sqlInfos.length > 1) {
1752                                                        String endTrim = SQLUtil.endTrim(content);
1753                                                        if (endTrim.endsWith(delimiterChar) || endTrim.endsWith(";")) {
1754                                                                content += "\n";
1755                                                        } else if (option.getVendor() == EDbVendor.dbvredshift
1756                                                                        || option.getVendor() == EDbVendor.dbvgaussdb
1757                                                                        || option.getVendor() == EDbVendor.dbvedb
1758                                                                        || option.getVendor() == EDbVendor.dbvpostgresql
1759                                                                        || option.getVendor() == EDbVendor.dbvmysql
1760                                                                        || option.getVendor() == EDbVendor.dbvteradata) {
1761                                                                content += ("\n\n-- " + TBaseType.sqlflow_stmt_delimiter_str + "\n\n");
1762                                                        } else {
1763                                                                content = endTrim + ";" + "\n";
1764                                                        }
1765                                                }
1766
1767                                                sqlInfo.setSql(content);
1768
1769                                                if (MetadataReader.isMetadata(content)) {
1770                                                        String hash = SHA256.getMd5(content);
1771                                                        ModelBindingManager.setGlobalHash(hash);
1772                                                        dataflow temp = new SQLDepMetadataAnalyzer().analyzeMetadata(option.getVendor(), content);
1773                                                        if (temp.getProcedures() != null) {
1774                                                                dataflow.getProcedures().addAll(temp.getProcedures());
1775                                                        }
1776                                                        if (temp.getTables() != null) {
1777                                                                dataflow.getTables().addAll(temp.getTables());
1778                                                        }
1779                                                        if (temp.getViews() != null) {
1780                                                                dataflow.getViews().addAll(temp.getViews());
1781                                                        }
1782                                                        if (temp.getResultsets() != null) {
1783                                                                dataflow.getResultsets().addAll(temp.getResultsets());
1784                                                        }
1785                                                        if (temp.getRelationships() != null) {
1786                                                                dataflow.getRelationships().addAll(temp.getRelationships());
1787                                                        }
1788                                                        if (temp.getErrors() != null) {
1789                                                                dataflow.getErrors().addAll(temp.getErrors());
1790                                                        }
1791                                                        String fileHash = SHA256.getMd5(hash);
1792                                                        if (!sqlInfoMap.containsKey(fileHash)) {
1793                                                                sqlInfoMap.put(fileHash, new ArrayList<SqlInfo>());
1794                                                                sqlInfoMap.get(fileHash).add(sqlInfo);
1795                                                        }
1796                                                } else {
1797                                                        String sqlHash = SHA256.getMd5(content);
1798                                                        String fileHash = SHA256.getMd5(sqlHash);
1799                                                        if (!sqlInfoMap.containsKey(fileHash)) {
1800                                                                sqlInfoMap.put(fileHash, new ArrayList<SqlInfo>());
1801                                                        }
1802
1803                                                        String database = TSQLEnv.DEFAULT_DB_NAME;
1804                                                        String schema = TSQLEnv.DEFAULT_SCHEMA_NAME;
1805                                                        if (sqlenv != null) {
1806                                                                // Prefer the per-analysis Option default database over the attached
1807                                                                // env's default catalog, so a shared multi-catalog TSQLEnv can be
1808                                                                // reused across analyses with different default databases without
1809                                                                // the env's default catalog deciding name resolution.
1810                                                                if (!SQLUtil.isEmpty(option.getDefaultDatabase())
1811                                                                                && !TSQLEnv.DEFAULT_DB_NAME.equals(option.getDefaultDatabase())) {
1812                                                                        database = option.getDefaultDatabase();
1813                                                                } else {
1814                                                                        database = sqlenv.getDefaultCatalogName();
1815                                                                }
1816                                                                if (database == null) {
1817                                                                        database = TSQLEnv.DEFAULT_DB_NAME;
1818                                                                }
1819                                                                schema = sqlenv.getDefaultSchemaName();
1820                                                                if (schema == null) {
1821                                                                        schema = TSQLEnv.DEFAULT_SCHEMA_NAME;
1822                                                                }
1823                                                        }
1824
1825                                                        boolean supportCatalog = TSQLEnv.supportCatalog(option.getVendor());
1826                                                        boolean supportSchema = TSQLEnv.supportSchema(option.getVendor());
1827                                                        String group;
1828                                                        if (sqlInfo.getGroup() != null && !sqlInfo.getGroup().isEmpty()) {
1829                                                                group = sqlInfo.getGroup();
1830                                                        } else {
1831                                                                StringBuilder builder = new StringBuilder();
1832                                                                if (supportCatalog) {
1833                                                                        builder.append(database);
1834                                                                }
1835                                                                if (supportSchema) {
1836                                                                        if (builder.length() > 0) {
1837                                                                                builder.append(".");
1838                                                                        }
1839                                                                        builder.append(schema);
1840                                                                }
1841                                                                group = builder.toString();
1842                                                        }
1843                                                        SqlInfo sqlInfoItem = new SqlInfo();
1844                                                        sqlInfoItem.setFileName(sqlInfo.getFileName());
1845                                                        sqlInfoItem.setFilePath(sqlInfo.getFilePath());
1846                                                        sqlInfoItem.setSql(sqlInfo.getSql());
1847                                                        sqlInfoItem.setOriginIndex(0);
1848                                                        sqlInfoItem.setOriginLineStart(0);
1849                                                        int lineEnd = sqlInfo.getSql().split("\n").length - 1;
1850                                                        sqlInfoItem.setOriginLineEnd(lineEnd);
1851                                                        sqlInfoItem.setIndex(0);
1852                                                        sqlInfoItem.setLineStart(0);
1853                                                        sqlInfoItem.setLineEnd(lineEnd);
1854                                                        sqlInfoItem.setHash(SHA256.getMd5(sqlHash));
1855                                                        sqlInfoItem.setGroup(group);
1856                                                        sqlInfoMap.get(fileHash).add(sqlInfoItem);
1857                                                        if (!databaseMap.containsKey(sqlHash)) {
1858                                                                databaseMap.put(sqlHash, new Pair3<StringBuilder, AtomicInteger, String>(
1859                                                                                new StringBuilder(), new AtomicInteger(), group));
1860                                                        }
1861                                                        databaseMap.get(sqlHash).first.append(sqlInfoItem.getSql());
1862                                                        databaseMap.get(sqlHash).second.incrementAndGet();
1863                                                }
1864                                        }
1865                                }
1866
1867                                boolean supportCatalog = TSQLEnv.supportCatalog(option.getVendor());
1868                                boolean supportSchema = TSQLEnv.supportSchema(option.getVendor());
1869
1870                                // Cross-file call-site binding discovery (plan Phase 5) needs every
1871                                // unit's text before the first unit is analyzed: units are parsed and
1872                                // analyzed strictly one at a time, so a caller in a later file would
1873                                // otherwise be invisible while an earlier file's procedure is analyzed.
1874                                if ((option.getVendor() == EDbVendor.dbvmssql || option.getVendor() == EDbVendor.dbvoracle)
1875                                                && databaseMap.size() > 1) {
1876                                        List<String> unitTexts = new ArrayList<String>();
1877                                        for (Pair3<StringBuilder, AtomicInteger, String> unit : databaseMap.values()) {
1878                                                unitTexts.add(unit.first.toString());
1879                                        }
1880                                        routineCatalog = new gudusoft.gsqlparser.dlineage.dynamicsql.RoutineCatalog(
1881                                                        option.getVendor(), unitTexts, crossFileParseCache);
1882                                }
1883
1884                                Iterator<String> schemaIter = databaseMap.keySet().iterator();
1885                                while (schemaIter.hasNext()) {
1886                                        if (option.getHandleListener() != null && option.getHandleListener().isCanceled()) {
1887                                                break;
1888                                        }
1889                                        String key = schemaIter.next();
1890                                        String group = databaseMap.get(key).third;
1891                                        String[] split = SQLUtil.parseNames(group).toArray(new String[0]);
1892
1893                                        ModelBindingManager.removeGlobalDatabase();
1894                                        ModelBindingManager.removeGlobalSchema();
1895                                        ModelBindingManager.removeGlobalSQLEnv();
1896                                        ModelBindingManager.removeGlobalHash();
1897
1898                                        if (sqlenv == null) {
1899                                                sqlenv = new TSQLEnv(option.getVendor()) {
1900
1901                                                        @Override
1902                                                        public void initSQLEnv() {
1903                                                                // TODO Auto-generated method stub
1904
1905                                                        }
1906                                                };
1907                                        }
1908                                        
1909                                        String defaultDatabase = null;
1910                                        String defaultSchema = null;
1911                                        if(sqlenv!=null){
1912                                                defaultDatabase = sqlenv.getDefaultCatalogName();
1913                                                defaultSchema = sqlenv.getDefaultSchemaName();
1914                                        }
1915                                        if (supportCatalog && supportSchema) {
1916                                                if (split.length >= 2) {
1917                                                        if (!TSQLEnv.DEFAULT_DB_NAME.equals(split[split.length - 2])) {
1918                                                                ModelBindingManager.setGlobalDatabase(split[split.length - 2]);
1919                                                                sqlenv.setDefaultCatalogName(ModelBindingManager.getGlobalDatabase());
1920                                                        }
1921                                                }
1922                                                if (split.length >= 1) {
1923                                                        if (!TSQLEnv.DEFAULT_SCHEMA_NAME.equals(split[split.length - 1])) {
1924                                                                ModelBindingManager.setGlobalSchema(split[split.length - 1]);
1925                                                                sqlenv.setDefaultSchemaName(ModelBindingManager.getGlobalSchema());
1926                                                        }
1927                                                }
1928                                        } else if (supportCatalog) {
1929                                                if (!TSQLEnv.DEFAULT_DB_NAME.equals(split[split.length - 1])) {
1930                                                        ModelBindingManager.setGlobalDatabase(split[split.length - 1]);
1931                                                        sqlenv.setDefaultCatalogName(ModelBindingManager.getGlobalDatabase());
1932                                                }
1933                                        } else if (supportSchema) {
1934                                                if (!TSQLEnv.DEFAULT_SCHEMA_NAME.equals(split[split.length - 1])) {
1935                                                        ModelBindingManager.setGlobalSchema(split[split.length - 1]);
1936                                                        sqlenv.setDefaultSchemaName(ModelBindingManager.getGlobalSchema());
1937                                                }
1938                                        }
1939                                        if (option.getHandleListener() != null) {
1940                                                option.getHandleListener().startParse(null, databaseMap.get(key).first.toString());
1941                                        }
1942
1943                                        ModelBindingManager.setGlobalSQLEnv(sqlenv);
1944                                        sqlparser.sqltext = databaseMap.get(key).first.toString();
1945                                        ModelBindingManager.setGlobalHash(SHA256.getMd5(key));
1946                                        analyzeAndOutputResult(sqlparser);
1947                                        if(sqlenv!=null){
1948                                                sqlenv.setDefaultCatalogName(defaultDatabase);
1949                                                sqlenv.setDefaultSchemaName(defaultSchema);
1950                                        }
1951                                }
1952
1953                                materializeSqlEnvSynonyms();
1954
1955                                appendProcesses(dataflow);
1956                                appendOraclePackages(dataflow);
1957                                appendProcedures(dataflow);
1958                                appendTables(dataflow);
1959                                appendViews(dataflow);
1960                                appendResultSets(dataflow);
1961                                appendRelations(dataflow);
1962                                appendErrors(dataflow);
1963                        }
1964
1965                        dataflow = handleDataflowExecProcedure(dataflow);
1966
1967                        if (dataflow != null && option.getAnalyzeMode() != AnalyzeMode.crud) {
1968                                if (!isShowJoin()) {
1969                                        dataflow = mergeTables(dataflow, modelManager.TABLE_COLUMN_ID, option);
1970                                } else {
1971                                        dataflow = removeDuplicateColumns(dataflow);
1972                                }
1973                        }
1974
1975                        if (dataflow != null && option.isNormalizeOutput()) {
1976                                dataflow = getNormalizeDataflow(dataflow);
1977                        }
1978
1979                        if (dataflow != null && option.isIgnoreCoordinate()) {
1980                                dataflow = filterDataflowCoordinate(dataflow);
1981                        }
1982                        
1983                        if (option.isSimpleOutput() || option.isIgnoreRecordSet()) {
1984                                List<String> showTypes = new ArrayList<>();
1985                                if(option.getSimpleShowRelationTypes()!=null) {
1986                                        showTypes.addAll(option.getSimpleShowRelationTypes());
1987                                }
1988                                if(showTypes.isEmpty()) {
1989                                        showTypes.add("fdd");
1990                                }
1991                                if(option.isShowCallRelation()) {
1992                                        showTypes.add("call");
1993                                }
1994                                if(option.isShowERDiagram()) {
1995                                        showTypes.add("er");
1996                                }
1997                                dataflow simpleDataflow = getSimpleDataflow(dataflow, option.isSimpleOutput(), showTypes);
1998                                if (simpleDataflow.getResultsets() != null) {
1999                                        for (table t : simpleDataflow.getResultsets()) {
2000                                                t.setIsTarget(null);
2001                                        }
2002                                }
2003                                return finalizeAuthoritativeLineageEvidence(simpleDataflow);
2004                        } else {
2005                                return finalizeAuthoritativeLineageEvidence(dataflow);
2006                        }
2007                        
2008                } catch (Exception e) {
2009                        logger.error("analyze sql failed.", e);
2010                        ErrorInfo errorInfo = new ErrorInfo();
2011                        errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
2012                        if (e.getMessage() == null) {
2013                                if (e.getStackTrace() != null && e.getStackTrace().length > 0) {
2014                                        errorInfo.setErrorMessage(e.getClass().getSimpleName() + ": " + e.getStackTrace()[0].toString());
2015                                } else {
2016                                        errorInfo.setErrorMessage(e.getClass().getSimpleName());
2017                                }
2018                        } else {
2019                                errorInfo.setErrorMessage(e.getClass().getSimpleName() + ": " + e.getMessage());
2020                        }
2021                        errorInfo.fillInfo(this);
2022                        errorInfos.add(errorInfo);
2023                }
2024
2025                modelManager.reset();
2026                return null;
2027        }
2028
2029        private dataflow finalizeAuthoritativeLineageEvidence(dataflow completed) {
2030                if (!shouldCollectAuthoritativeLineageEvidence()) {
2031                        authoritativeEvidenceCollector.reset();
2032                        authoritativeLineageEvidence = AuthoritativeLineageEvidence.empty();
2033                        return completed;
2034                }
2035                try {
2036                        authoritativeLineageEvidence = authoritativeEvidenceCollector.build(modelManager, completed);
2037                } catch (ThreadDeath fatal) {
2038                        throw fatal;
2039                } catch (VirtualMachineError fatal) {
2040                        throw fatal;
2041                } catch (Throwable evidenceFailure) {
2042                        // Additive evidence must never make the established legacy analyzer fail.
2043                        logger.error("Capture authoritative lineage evidence failed; legacy output is preserved.",
2044                                        evidenceFailure);
2045                        authoritativeLineageEvidence = AuthoritativeLineageEvidence.empty();
2046                } finally {
2047                        // Drafts retain AST/model objects. The published bundle contains only
2048                        // immutable scalar values, so release all per-run references immediately.
2049                        authoritativeEvidenceCollector.reset();
2050                }
2051                return completed;
2052        }
2053
2054        private boolean shouldCollectAuthoritativeLineageEvidence() {
2055                return option != null && option.isCollectAuthoritativeLineageEvidence();
2056        }
2057
2058        private dataflow handleDataflowExecProcedure(dataflow dataflow) {
2059                Set<String> procedures = new HashSet<String>();
2060                for (procedure procedure : dataflow.getProcedures()) {
2061                        procedures.add(DlineageUtil.getIdentifierNormalTableName(procedure.getName()));
2062                }
2063                List<table> removeResultSets = new ArrayList<table>();
2064                
2065                Map<String, table> allMap = new HashMap<String, table>();
2066                
2067                if (dataflow.getResultsets() != null) {
2068                        for (table t : dataflow.getResultsets()) {
2069                                allMap.put(t.getId().toLowerCase(), t);
2070                        }
2071                }
2072
2073                if (dataflow.getTables() != null) {
2074                        for (table t : dataflow.getTables()) {
2075                                allMap.put(t.getId().toLowerCase(), t);
2076                        }
2077                }
2078
2079                if (dataflow.getViews() != null) {
2080                        for (table t : dataflow.getViews()) {
2081                                allMap.put(t.getId().toLowerCase(), t);
2082                        }
2083                }
2084
2085                if (dataflow.getVariables() != null) {
2086                        for (table t : dataflow.getVariables()) {
2087                                allMap.put(t.getId().toLowerCase(), t);
2088                        }
2089                }
2090
2091                
2092                for (table resultSet : dataflow.getResultsets()) {
2093                        String resultSetName = DlineageUtil.getIdentifierNormalTableName(resultSet.getName());
2094                        if (!(ResultSetType.function.name().equals(resultSet.getType())
2095                                        && modelManager.getFunctionTable(resultSetName) != null && procedures.contains(resultSetName))) {
2096                                continue;
2097                        }
2098
2099                        Set<Object> functionTables = modelManager
2100                                        .getFunctionTable(DlineageUtil.getIdentifierNormalTableName(resultSet.getName()));
2101                        List<ResultSet> functionResults = functionTables.stream().filter(t -> t instanceof ResultSet)
2102                                        .map(t -> (ResultSet) t).collect(Collectors.toList());
2103                        if (!(resultSet.getColumns().size() == 1
2104                                        && DlineageUtil.getIdentifierNormalTableName(resultSet.getColumns().get(0).getName()).equals(resultSetName))
2105                                        || functionResults.isEmpty()) {
2106                                continue;
2107                        }
2108
2109                        column column = resultSet.getColumns().get(0);
2110                        String sourceColumnId = column.getId();
2111                        boolean reserveResultSet = false;
2112                        List<relationship> appendRelations = new ArrayList<relationship>();
2113                        Set<relationship> removeRelations = new HashSet<relationship>();
2114                        
2115                        for (relationship relationship : dataflow.getRelationships()) {
2116                                targetColumn targetColumn = relationship.getTarget();
2117                                List<sourceColumn> sourceColumns = relationship.getSources();
2118                                List<sourceColumn> newSourceColumns = new ArrayList<sourceColumn>();
2119                                for (sourceColumn sourceColumn : sourceColumns) {
2120                                        if (!sourceColumn.getId().equals(sourceColumnId)) {
2121                                                continue;
2122                                        }
2123                                        for (ResultSet sourceResultSet : functionResults) {
2124                                                for (ResultColumn resultColumn : sourceResultSet.getColumns()) {
2125                                                        if (!DlineageUtil.compareColumnIdentifier(resultColumn.getName(),
2126                                                                        targetColumn.getColumn())) {
2127                                                                if (targetColumn.getColumn().endsWith("*")) {
2128                                                                        table parent = allMap.get(targetColumn.getParent_id());
2129                                                                        if (parent != null && parent.getColumns().size() > 1) {
2130                                                                                for (column item : parent.getColumns()) {
2131                                                                                        if (DlineageUtil.compareColumnIdentifier(resultColumn.getName(),
2132                                                                                                        item.getName())) {
2133                                                                                                relationship newRelation = appendStarRelation(dataflow, relationship, item, resultColumn);
2134                                                                                                appendRelations.add(newRelation);
2135                                                                                                removeRelations.add(relationship);
2136                                                                                                newSourceColumns.add(newRelation.getSources().get(0));
2137                                                                                                if (resultColumn.getResultSet() != null && resultColumn.getResultSet().isTarget()) {
2138                                                                                                        dataflow.getResultsets().stream().filter(
2139                                                                                                                        t -> t.getId().equals(String.valueOf(resultColumn.getResultSet().getId())))
2140                                                                                                                        .forEach(t -> t.setIsTarget(Boolean.FALSE.toString()));
2141                                                                                                }
2142                                                                                        }
2143                                                                                }
2144                                                                        }
2145                                                                }
2146                                                                continue;
2147                                                        }
2148                                                        
2149                                                        sourceColumn sourceColumn1 = new sourceColumn();
2150                                                        sourceColumn1.setId(String.valueOf(resultColumn.getId()));
2151                                                        sourceColumn1.setColumn(resultColumn.getName());
2152                                                        sourceColumn1.setParent_id(String.valueOf(resultColumn.getResultSet().getId()));
2153                                                        sourceColumn1.setParent_name(getResultSetName(resultColumn.getResultSet()));
2154                                                        if (resultColumn.getStartPosition() != null
2155                                                                        && resultColumn.getEndPosition() != null) {
2156                                                                sourceColumn1.setCoordinate(convertCoordinate(resultColumn.getStartPosition())
2157                                                                                + "," + convertCoordinate(resultColumn.getEndPosition()));
2158                                                        }
2159                                                        newSourceColumns.add(sourceColumn1);
2160                                                        
2161                                                        if (resultColumn.getResultSet() != null && resultColumn.getResultSet().isTarget()) {
2162                                                                dataflow.getResultsets().stream().filter(
2163                                                                                t -> t.getId().equals(String.valueOf(resultColumn.getResultSet().getId())))
2164                                                                                .forEach(t -> t.setIsTarget(Boolean.FALSE.toString()));
2165                                                        }
2166                                                }
2167                                        }
2168
2169                                        if (!newSourceColumns.isEmpty()) {
2170                                                if (removeRelations.isEmpty()) {
2171                                                        relationship.setSources(newSourceColumns);
2172                                                }
2173                                        }
2174                                        else {
2175                                                reserveResultSet = true;
2176                                        }
2177                                }
2178                        }
2179                        
2180                        dataflow.getRelationships().addAll(appendRelations);
2181                        dataflow.getRelationships().removeAll(removeRelations);
2182                        
2183                        if(!reserveResultSet) {
2184                                removeResultSets.add(resultSet);
2185                        }
2186                }
2187                dataflow.getResultsets().removeAll(removeResultSets);
2188                return dataflow;
2189        }
2190
2191        private relationship appendStarRelation(dataflow dataflow,
2192                        relationship relationship, column item, ResultColumn resultColumn) {
2193                relationship relationElement = new relationship();
2194                relationElement.setType(relationship.getType());
2195                relationElement.setEffectType(relationship.getEffectType());
2196                relationElement.setSqlHash(relationship.getSqlHash());
2197                relationElement.setSqlComment(relationship.getSqlComment());
2198                relationElement.setProcedureId(relationship.getProcedureId());
2199                relationElement.setId(String.valueOf(++ModelBindingManager.get().RELATION_ID));
2200                relationElement.setProcessId(relationship.getProcessId());
2201                relationElement.setProcessType(relationship.getProcessType());
2202                
2203                targetColumn targetColumn1 = new targetColumn();
2204                targetColumn1.setId(String.valueOf(item.getId()));
2205                targetColumn1.setColumn(item.getName());
2206                targetColumn1.setParent_id(relationship.getTarget().getParent_id());
2207                targetColumn1.setParent_name(relationship.getTarget().getParent_name());
2208                targetColumn1.setParent_alias(relationship.getTarget().getParent_alias());
2209                if (relationship.getTarget().getCoordinate() != null) {
2210                        targetColumn1.setCoordinate(item.getCoordinate());
2211                }
2212                relationElement.setTarget(targetColumn1);       
2213                
2214                sourceColumn sourceColumn1 = new sourceColumn();
2215                sourceColumn1.setId(String.valueOf(resultColumn.getId()));
2216                sourceColumn1.setColumn(resultColumn.getName());
2217                sourceColumn1.setParent_id(String.valueOf(resultColumn.getResultSet().getId()));
2218                sourceColumn1.setParent_name(getResultSetName(resultColumn.getResultSet()));
2219                if (resultColumn.getStartPosition() != null
2220                                && resultColumn.getEndPosition() != null) {
2221                        sourceColumn1.setCoordinate(convertCoordinate(resultColumn.getStartPosition())
2222                                        + "," + convertCoordinate(resultColumn.getEndPosition()));
2223                }
2224                relationElement.addSource(sourceColumn1);
2225                return relationElement;
2226        }
2227
2228        private dataflow getNormalizeDataflow(dataflow instance) {
2229                List<table> tables = new ArrayList<>();
2230                if (instance.getResultsets() != null) {
2231                        for (table t : instance.getResultsets()) {
2232                                tables.add(t);
2233                        }
2234                }
2235
2236                if (instance.getTables() != null) {
2237                        for (table t : instance.getTables()) {
2238                                tables.add(t);
2239                        }
2240                }
2241
2242                if (instance.getViews() != null) {
2243                        for (table t : instance.getViews()) {
2244                                tables.add(t);
2245                        }
2246                }
2247
2248                if (instance.getPaths() != null) {
2249                        for (table t : instance.getPaths()) {
2250                                tables.add(t);
2251                        }
2252                }
2253
2254                if (instance.getStages() != null) {
2255                        for (table t : instance.getStages()) {
2256                                tables.add(t);
2257                        }
2258                }
2259
2260                if (instance.getSequences() != null) {
2261                        for (table t : instance.getSequences()) {
2262                                tables.add(t);
2263                        }
2264                }
2265
2266                if (instance.getDatasources() != null) {
2267                        for (table t : instance.getDatasources()) {
2268                                tables.add(t);
2269                        }
2270                }
2271
2272                if (instance.getDatabases() != null) {
2273                        for (table t : instance.getDatabases()) {
2274                                tables.add(t);
2275                        }
2276                }
2277
2278                if (instance.getSchemas() != null) {
2279                        for (table t : instance.getSchemas()) {
2280                                tables.add(t);
2281                        }
2282                }
2283
2284                if (instance.getStreams() != null) {
2285                        for (table t : instance.getStreams()) {
2286                                tables.add(t);
2287                        }
2288                }
2289
2290                if (instance.getVariables() != null) {
2291                        for (table t : instance.getVariables()) {
2292                                tables.add(t);
2293                        }
2294                }
2295
2296                Map<String, String> idNameMap = new HashMap();
2297
2298                for(table table: tables){
2299                        if(table.getDatabase()!=null) {
2300                                table.setDatabase(DlineageUtil.getIdentifierNormalName(table.getDatabase(), ESQLDataObjectType.dotCatalog));
2301                        }
2302                        if(table.getSchema()!=null) {
2303                                table.setSchema(DlineageUtil.getIdentifierNormalName(table.getSchema(), ESQLDataObjectType.dotSchema));
2304                        }
2305                        if(table.getName()!=null) {
2306                                table.setName(DlineageUtil.getIdentifierNormalName(table.getName(), ESQLDataObjectType.dotTable));
2307                                idNameMap.put(table.getId(), table.getName());
2308                        }
2309                        if(table.getColumns()!=null){
2310                                for(column column: table.getColumns()){
2311                                        column.setName(DlineageUtil.getIdentifierNormalName(column.getName(), ESQLDataObjectType.dotColumn));
2312                                        idNameMap.put(column.getId(), column.getName());
2313                                }
2314                        }
2315                }
2316                if(instance.getPackages()!=null){
2317                        for(oraclePackage oraclePackage: instance.getPackages()){
2318                                if(oraclePackage.getDatabase()!=null) {
2319                                        oraclePackage.setDatabase(DlineageUtil.getIdentifierNormalName(oraclePackage.getDatabase(), ESQLDataObjectType.dotCatalog));
2320                                }
2321                                if(oraclePackage.getSchema()!=null) {
2322                                        oraclePackage.setSchema(DlineageUtil.getIdentifierNormalName(oraclePackage.getSchema(), ESQLDataObjectType.dotSchema));
2323                                }
2324                                if(oraclePackage.getName()!=null) {
2325                                        oraclePackage.setName(DlineageUtil.getIdentifierNormalName(oraclePackage.getName(), ESQLDataObjectType.dotTable));
2326                                        idNameMap.put(oraclePackage.getId(), oraclePackage.getName());
2327                                }
2328                                if(oraclePackage.getArguments()!=null){
2329                                        for(argument argument: oraclePackage.getArguments()){
2330                                                argument.setName(DlineageUtil.getIdentifierNormalName(argument.getName(), ESQLDataObjectType.dotColumn));
2331                                                idNameMap.put(argument.getId(), argument.getName());
2332                                        }
2333                                }
2334                                if(oraclePackage.getProcedures()!=null){
2335                                        for(procedure procedure: oraclePackage.getProcedures()){
2336                                                if(procedure.getDatabase()!=null) {
2337                                                        procedure.setDatabase(DlineageUtil.getIdentifierNormalName(procedure.getDatabase(), ESQLDataObjectType.dotCatalog));
2338                                                }
2339                                                if(procedure.getSchema()!=null) {
2340                                                        procedure.setSchema(DlineageUtil.getIdentifierNormalName(procedure.getSchema(), ESQLDataObjectType.dotSchema));
2341                                                }
2342                                                if(procedure.getName()!=null) {
2343                                                        procedure.setName(DlineageUtil.getIdentifierNormalName(procedure.getName(), ESQLDataObjectType.dotTable));
2344                                                        idNameMap.put(procedure.getId(), procedure.getName());
2345                                                }
2346                                                for(argument argument: procedure.getArguments()){
2347                                                        argument.setName(DlineageUtil.getIdentifierNormalName(argument.getName(), ESQLDataObjectType.dotColumn));
2348                                                        idNameMap.put(argument.getId(), argument.getName());
2349                                                }
2350                                        }
2351                                }
2352                        }
2353                }
2354                if(instance.getProcedures()!=null){
2355                        for(procedure procedure: instance.getProcedures()){
2356                                if(procedure.getDatabase()!=null) {
2357                                        procedure.setDatabase(DlineageUtil.getIdentifierNormalName(procedure.getDatabase(), ESQLDataObjectType.dotCatalog));
2358                                }
2359                                if(procedure.getSchema()!=null) {
2360                                        procedure.setSchema(DlineageUtil.getIdentifierNormalName(procedure.getSchema(), ESQLDataObjectType.dotSchema));
2361                                }
2362                                if(procedure.getName()!=null) {
2363                                        procedure.setName(DlineageUtil.getIdentifierNormalName(procedure.getName(), ESQLDataObjectType.dotTable));
2364                                        idNameMap.put(procedure.getId(), procedure.getName());
2365                                }
2366                                for(argument argument: procedure.getArguments()){
2367                                        argument.setName(DlineageUtil.getIdentifierNormalName(argument.getName(), ESQLDataObjectType.dotColumn));
2368                                        idNameMap.put(argument.getId(), argument.getName());
2369                                }
2370                        }
2371                }
2372                if (instance.getProcesses() != null) {
2373                        for (process process : instance.getProcesses()) {
2374                                if(process.getDatabase()!=null) {
2375                                        process.setDatabase(DlineageUtil.getIdentifierNormalName(process.getDatabase(), ESQLDataObjectType.dotCatalog));
2376                                }
2377                                if(process.getSchema()!=null) {
2378                                        process.setSchema(DlineageUtil.getIdentifierNormalName(process.getSchema(), ESQLDataObjectType.dotSchema));
2379                                }
2380                                if (process.getProcedureId() != null) {
2381                                        process.setProcedureName(DlineageUtil.getIdentifierNormalName(process.getProcedureName(), ESQLDataObjectType.dotTable));
2382                                        idNameMap.put(process.getId(), process.getName());
2383                                }
2384                        }
2385                }
2386
2387                if(instance.getRelationships()!=null){
2388                        for(relationship relation: instance.getRelationships()){
2389                                targetColumn targetColumn = relation.getTarget();
2390                                if(targetColumn != null) {
2391                                        targetColumn.setColumn(idNameMap.get(targetColumn.getId()));
2392                                        targetColumn.setTarget_name(idNameMap.get(targetColumn.getTarget_id()));
2393                                        targetColumn.setParent_name(idNameMap.get(targetColumn.getParent_id()));
2394                                }
2395                                List<sourceColumn> sourceColumns = relation.getSources();
2396                                if(sourceColumns!=null){
2397                                        for(sourceColumn sourceColumn: sourceColumns){
2398                                                sourceColumn.setColumn(idNameMap.get(sourceColumn.getId()));
2399                                                sourceColumn.setSource_name(idNameMap.get(sourceColumn.getSource_id()));
2400                                                sourceColumn.setParent_name(idNameMap.get(sourceColumn.getParent_id()));
2401                                        }
2402                                }
2403                                targetColumn = relation.getCaller();
2404                                if(targetColumn != null) {
2405                                        targetColumn.setName(idNameMap.get(targetColumn.getId()));
2406                                }
2407                                sourceColumns = relation.getCallees();
2408                                if(sourceColumns!=null){
2409                                        for(sourceColumn sourceColumn: sourceColumns){
2410                                                sourceColumn.setName(idNameMap.get(sourceColumn.getId()));
2411                                        }
2412                                }
2413                        }
2414                }
2415
2416
2417                return instance;
2418        }
2419
2420        private dataflow filterDataflowCoordinate(dataflow instance) {
2421                List<table> tables = new ArrayList<>();
2422                if (instance.getResultsets() != null) {
2423                        for (table t : instance.getResultsets()) {
2424                                tables.add(t);
2425                        }
2426                }
2427
2428                if (instance.getTables() != null) {
2429                        for (table t : instance.getTables()) {
2430                                tables.add(t);
2431                        }
2432                }
2433
2434                if (instance.getViews() != null) {
2435                        for (table t : instance.getViews()) {
2436                                tables.add(t);
2437                        }
2438                }
2439
2440                if (instance.getPaths() != null) {
2441                        for (table t : instance.getPaths()) {
2442                                tables.add(t);
2443                        }
2444                }
2445
2446                if (instance.getStages() != null) {
2447                        for (table t : instance.getStages()) {
2448                                tables.add(t);
2449                        }
2450                }
2451
2452                if (instance.getSequences() != null) {
2453                        for (table t : instance.getSequences()) {
2454                                tables.add(t);
2455                        }
2456                }
2457
2458                if (instance.getDatasources() != null) {
2459                        for (table t : instance.getDatasources()) {
2460                                tables.add(t);
2461                        }
2462                }
2463
2464                if (instance.getDatabases() != null) {
2465                        for (table t : instance.getDatabases()) {
2466                                tables.add(t);
2467                        }
2468                }
2469
2470                if (instance.getSchemas() != null) {
2471                        for (table t : instance.getSchemas()) {
2472                                tables.add(t);
2473                        }
2474                }
2475
2476                if (instance.getStreams() != null) {
2477                        for (table t : instance.getStreams()) {
2478                                tables.add(t);
2479                        }
2480                }
2481
2482                if (instance.getVariables() != null) {
2483                        for (table t : instance.getVariables()) {
2484                                tables.add(t);
2485                        }
2486                }
2487
2488                for(table table: tables){
2489                        table.clearCoordinate();
2490                        if(table.getColumns()!=null){
2491                                for(column column: table.getColumns()){
2492                                        column.clearCoordinate();
2493                                }
2494                        }
2495                }
2496                if(instance.getPackages()!=null){
2497                        for(oraclePackage oraclePackage: instance.getPackages()){
2498                                oraclePackage.setCoordinate(null);
2499                                if(oraclePackage.getProcedures()!=null){
2500                                        for(procedure procedure: oraclePackage.getProcedures()){
2501                                                procedure.setCoordinate(null);
2502                                                for(argument argument: procedure.getArguments()){
2503                                                        argument.setCoordinate(null);
2504                                                }
2505                                        }
2506                                }
2507                        }
2508                }
2509                if(instance.getProcedures()!=null){
2510                        for(procedure procedure: instance.getProcedures()){
2511                                procedure.setCoordinate(null);
2512                                for(argument argument: procedure.getArguments()){
2513                                        argument.setCoordinate(null);
2514                                }
2515                        }
2516                }
2517                if (instance.getProcesses() != null) {
2518                        for (process process : instance.getProcesses()) {
2519                                process.setCoordinate(null);
2520                        }
2521                }
2522
2523                if(instance.getRelationships()!=null){
2524                        for(relationship relation: instance.getRelationships()){
2525                                targetColumn targetColumn = relation.getTarget();
2526                                if(targetColumn != null) {
2527                                        targetColumn.setCoordinate(null);
2528                                }
2529                                List<sourceColumn> sourceColumns = relation.getSources();
2530                                if(sourceColumns!=null){
2531                                        for(sourceColumn sourceColumn: sourceColumns){
2532                                                sourceColumn.setCoordinate(null);
2533                                        }
2534                                }
2535                                targetColumn = relation.getCaller();
2536                                if(targetColumn != null) {
2537                                        targetColumn.setCoordinate(null);
2538                                }
2539                                sourceColumns = relation.getCallees();
2540                                if(sourceColumns!=null){
2541                                        for(sourceColumn sourceColumn: sourceColumns){
2542                                                sourceColumn.setCoordinate(null);
2543                                        }
2544                                }
2545                        }
2546                }
2547
2548                return instance;
2549        }
2550
2551        /**
2552         * Read a sqlflow-sharded manifest's source records into {@link SqlInfo}s (for the
2553         * convertSQL / text-constructor path). Relative source paths resolve against
2554         * {@code baseDir}; without a base directory the shard files cannot be located, so
2555         * nothing is added (a manifest passed as inline text with no file location simply
2556         * cannot be followed to its shards). Unavailable records are skipped. Block-compressed
2557         * sources are handled on the generateDataFlow path, not here.
2558         */
2559        private void appendShardedSourceSqlInfos(String manifestSql, String baseDir, String fileName,
2560                        List<SqlInfo> sqlInfos) {
2561                if (baseDir == null) {
2562                        return;
2563                }
2564                Map sqlflow = (Map) JSON.parseObject(manifestSql);
2565                boolean block = "block".equals(sqlflow.get("sourceCompression"));
2566                List<Map> servers = (List<Map>) sqlflow.get("servers");
2567                if (servers == null) {
2568                        return;
2569                }
2570                for (Map server : servers) {
2571                        String dbVendor = (String) server.get("dbVendor");
2572                        String serverName = (String) server.get("name");
2573                        // A sharded export is organized either as catalog shards
2574                        // (servers[].databases[]) or as schema shards
2575                        // (servers[].schemas[], the natural shape for schema-topology
2576                        // vendors like Oracle). Both carry a per-shard `source` pointer;
2577                        // prefer schema shards when present, treating an empty list as
2578                        // absent so a manifest with "databases":[] alongside schemas[]
2579                        // still loads. Handling only `databases` silently dropped every
2580                        // source record of a schema-topology manifest -> empty lineage.
2581                        List<Map> shards = (List<Map>) server.get("schemas");
2582                        if (shards == null || shards.isEmpty()) {
2583                                shards = (List<Map>) server.get("databases");
2584                        }
2585                        if (shards == null || shards.isEmpty()) {
2586                                continue;
2587                        }
2588                        for (Map shard : shards) {
2589                                appendShardedSourceRecords((Map) shard.get("source"), baseDir, block,
2590                                                dbVendor, serverName, sqlInfos);
2591                        }
2592                }
2593        }
2594
2595        /**
2596         * Read one shard's source jsonl (catalog- or schema-topology) into SqlInfos.
2597         * Shared by the {@code databases[]} and {@code schemas[]} shard shapes so a
2598         * schema-topology manifest loads its source SQL identically to a
2599         * catalog-topology one.
2600         */
2601        private void appendShardedSourceRecords(Map source, String baseDir, boolean block,
2602                        String dbVendor, String serverName, List<SqlInfo> sqlInfos) {
2603                if (source == null) {
2604                        return;
2605                }
2606                String sourcePath = (String) source.get("path");
2607                if (sourcePath == null) {
2608                        return;
2609                }
2610                File sourceFile = new File(baseDir, sourcePath);
2611                String content = block ? readGzipContent(sourceFile.getAbsolutePath())
2612                                : SQLUtil.getFileContent(sourceFile.getAbsolutePath());
2613                if (content == null) {
2614                        return;
2615                }
2616                String[] lines = content.split("\\r?\\n");
2617                EDbVendor vendor = EDbVendor.valueOf(dbVendor);
2618                boolean supportCatalog = TSQLEnv.supportCatalog(vendor);
2619                boolean supportSchema = TSQLEnv.supportSchema(vendor);
2620
2621                for (int j = 0; j < lines.length; j++) {
2622                        String line = lines[j].trim();
2623                        if (line.isEmpty()) {
2624                                continue;
2625                        }
2626                        try {
2627                                Map rec = (Map) JSON.parseObject(line);
2628                                String sc = (String) rec.get("sourceCode");
2629                                if (sc != null && !sc.isEmpty() && !isSourceUnavailable(rec)) {
2630                                        SqlInfo info = new SqlInfo();
2631                                        info.setSql(sc);
2632                                        info.setFileName(sourceFile.getName());
2633                                        info.setFilePath(sourceFile.getAbsolutePath());
2634                                        info.setOriginIndex(j);
2635                                        info.setDbVendor(dbVendor);
2636                                        info.setServer(serverName);
2637
2638                                        String database = (String) rec.get("database");
2639                                        String schema = (String) rec.get("schema");
2640                                        StringBuilder groupBuilder = new StringBuilder();
2641                                        if (supportCatalog) {
2642                                                if (database.indexOf(".") != -1) {
2643                                                        database = SQLUtil.quoteDottedName(vendor, ESQLDataObjectType.dotCatalog, database);
2644                                                }
2645                                                groupBuilder.append(database);
2646                                        }
2647                                        if (supportSchema) {
2648                                                if (schema.indexOf(".") != -1) {
2649                                                        schema = SQLUtil.quoteDottedName(vendor, ESQLDataObjectType.dotSchema, schema);
2650                                                }
2651                                                if (groupBuilder.length() > 0) {
2652                                                        groupBuilder.append(".");
2653                                                }
2654                                                groupBuilder.append(schema);
2655                                        }
2656                                        info.setGroup(groupBuilder.toString());
2657
2658                                        sqlInfos.add(info);
2659                                }
2660                        } catch (Exception e) {
2661                                logger.warn("Parse sharded source jsonl line failed.", e);
2662                        }
2663                }
2664        }
2665
2666        /** Decompress a gzip/block-compressed source shard to its text, or null on failure. */
2667        private static String readGzipContent(String path) {
2668                File f = new File(path);
2669                if (!f.exists()) {
2670                        return null;
2671                }
2672                StringBuilder sb = new StringBuilder();
2673                try (java.io.BufferedReader reader = new java.io.BufferedReader(new java.io.InputStreamReader(
2674                                new java.util.zip.GZIPInputStream(new java.io.FileInputStream(f)),
2675                                java.nio.charset.StandardCharsets.UTF_8))) {
2676                        String line;
2677                        while ((line = reader.readLine()) != null) {
2678                                sb.append(line).append('\n');
2679                        }
2680                } catch (IOException e) {
2681                        logger.warn("Failed to read gzip source shard: " + path, e);
2682                        return null;
2683                }
2684                return sb.toString();
2685        }
2686
2687        /**
2688         * True when a sqlflow source record declares its DDL body unavailable
2689         * ({@code sourceUnavailable:true}). Robust to the JSON value arriving as a
2690         * boolean or a string. A record with no such field is available.
2691         */
2692        static boolean isSourceUnavailable(Map record) {
2693                if (record == null) {
2694                        return false;
2695                }
2696                Object v = record.get("sourceUnavailable");
2697                if (v instanceof Boolean) {
2698                        return (Boolean) v;
2699                }
2700                return v != null && "true".equalsIgnoreCase(v.toString().trim()); // non-identifier-compare: JSON boolean literal, not a DB name
2701        }
2702
2703        private void appendSqlInfo(Map<String, Pair3<StringBuilder, AtomicInteger, String>> databaseMap, int index,
2704                        SqlInfo sqlInfo, Map queryObject) {
2705                if (queryObject == null) {
2706                        return;
2707                }
2708                EDbVendor vendor = option.getVendor();
2709                if (!SQLUtil.isEmpty(sqlInfo.getDbVendor())) {
2710                        vendor = EDbVendor.valueOf(sqlInfo.getDbVendor());
2711                }
2712
2713                boolean supportCatalog = TSQLEnv.supportCatalog(vendor);
2714                boolean supportSchema = TSQLEnv.supportSchema(vendor);
2715
2716                String groupName = (String) queryObject.get("groupName");
2717                if (DlineageUtil.isProcedureExcluded(groupName)) {
2718                        return;
2719                }
2720
2721                // sourceUnavailable: the DDL body was not retrieved (encrypted proc,
2722                // permission denied). Any sourceCode present is a placeholder or a
2723                // reason string, not the real definition, so it must NOT be parsed as
2724                // live SQL — doing so fabricates lineage from text that is not the
2725                // object's body.
2726                if (isSourceUnavailable(queryObject)) {
2727                        return;
2728                }
2729
2730                String content = (String) queryObject.get("sourceCode");
2731                if (SQLUtil.isEmpty(content)) {
2732                        return;
2733                }
2734                StringBuilder builder = new StringBuilder();
2735                if (supportCatalog) {
2736                        String database = (String) queryObject.get("database");
2737                        if (database.indexOf(".") != -1) {
2738                                database = SQLUtil.quoteDottedName(vendor, ESQLDataObjectType.dotCatalog, database);
2739                        }
2740                        builder.append(database);
2741                }
2742                if (supportSchema) {
2743                        String schema = (String) queryObject.get("schema");
2744                        if (schema.indexOf(".") != -1) {
2745                                schema = SQLUtil.quoteDottedName(vendor, ESQLDataObjectType.dotSchema, schema);
2746                        }
2747                        if (builder.length() > 0) {
2748                                builder.append(".");
2749                        }
2750                        builder.append(schema);
2751                }
2752                String group = builder.toString();
2753                String sqlHash = SHA256.getMd5(content);
2754                String hash = SHA256.getMd5(sqlHash);
2755                if (!databaseMap.containsKey(sqlHash)) {
2756                        databaseMap.put(sqlHash,
2757                                        new Pair3<StringBuilder, AtomicInteger, String>(new StringBuilder(), new AtomicInteger(), group));
2758                }
2759                String delimiterChar = String.valueOf(TGSqlParser.getDelimiterChar(option.getVendor()));
2760                StringBuilder buffer = new StringBuilder(content);
2761                if (content.trim().endsWith(delimiterChar) || content.trim().endsWith(";")) {
2762                        buffer.append("\n");
2763                } else if(vendor == EDbVendor.dbvredshift
2764                                || vendor == EDbVendor.dbvgaussdb
2765                                || vendor == EDbVendor.dbvedb
2766                                || vendor == EDbVendor.dbvpostgresql
2767                                || vendor == EDbVendor.dbvmysql
2768                                || vendor == EDbVendor.dbvteradata){
2769                        buffer.append("\n\n-- " + TBaseType.sqlflow_stmt_delimiter_str + "\n\n");
2770                } else{
2771                        SQLUtil.endTrim(buffer);
2772                        buffer.append(";").append("\n");
2773                }
2774
2775                int lineStart = databaseMap.get(sqlHash).first.toString().split("\n", -1).length - 1;
2776                if (databaseMap.get(sqlHash).first.toString().length() == 0) {
2777                        lineStart = 0;
2778                }
2779                databaseMap.get(sqlHash).first.append(buffer.toString());
2780                SqlInfo sqlInfoItem = new SqlInfo();
2781                sqlInfoItem.setServer(sqlInfo.getServer());
2782                sqlInfoItem.setDbVendor(sqlInfo.getDbVendor());
2783                sqlInfoItem.setFileName(sqlInfo.getFileName());
2784                sqlInfoItem.setFilePath(sqlInfo.getFilePath());
2785                sqlInfoItem.setSql(buffer.toString());
2786                sqlInfoItem.setOriginIndex(index);
2787                sqlInfoItem.setOriginLineStart(0);
2788                sqlInfoItem.setOriginLineEnd(buffer.toString().split("\n", -1).length - 1);
2789                sqlInfoItem.setIndex(databaseMap.get(sqlHash).second.getAndIncrement());
2790                sqlInfoItem.setLineStart(lineStart);
2791                sqlInfoItem.setLineEnd(databaseMap.get(sqlHash).first.toString().split("\n", -1).length - 1);
2792                sqlInfoItem.setGroup(group);
2793                sqlInfoItem.setHash(hash);
2794
2795                if (!sqlInfoMap.containsKey(hash)) {
2796                        sqlInfoMap.put(hash, new ArrayList<SqlInfo>());
2797                }
2798                sqlInfoMap.get(hash).add(sqlInfoItem);
2799        }
2800
2801        private void readGzipBlockSource(String fullPath, Map<String, Pair3<StringBuilder, AtomicInteger, String>> databaseMap, SqlInfo sqlInfo) {
2802                try (FileInputStream fis = new FileInputStream(fullPath);
2803                         GZIPInputStream gzis = new GZIPInputStream(fis);
2804                         BufferedReader reader = new BufferedReader(new InputStreamReader(gzis, "UTF-8"))) {
2805                        String line;
2806                        int j = 0;
2807                        while ((line = reader.readLine()) != null) {
2808                                line = line.trim();
2809                                if (line.isEmpty()) {
2810                                        continue;
2811                                }
2812                                try {
2813                                        Map sourceObject = (Map) JSON.parseObject(line);
2814                                        String sourceCode = (String) sourceObject.get("sourceCode");
2815                                        if (sourceCode != null && !sourceCode.isEmpty()
2816                                                        && !isSourceUnavailable(sourceObject)) {
2817                                                SqlInfo sourceSqlInfo = new SqlInfo();
2818                                                sourceSqlInfo.setFileName(sqlInfo.getFileName());
2819                                                sourceSqlInfo.setFilePath(sqlInfo.getFilePath());
2820                                                sourceSqlInfo.setSql(sourceCode);
2821                                                sourceSqlInfo.setOriginIndex(j);
2822                                                // Pass the record (not null): appendSqlInfo dereferences it
2823                                                // for groupName/database/schema; null NPE'd and the catch
2824                                                // silently discarded every source record on this path.
2825                                                appendSqlInfo(databaseMap, j, sourceSqlInfo, sourceObject);
2826                                        }
2827                                } catch (Exception e) {
2828                                        logger.warn("Parse gzip source jsonl line failed.", e);
2829                                }
2830                                j++;
2831                        }
2832                } catch (Exception e) {
2833                        logger.warn("Read gzip source file failed: " + fullPath, e);
2834                }
2835        }
2836
2837        static String getTextOutput(dataflow dataflow) {
2838                StringBuffer buffer = new StringBuffer();
2839                List<relationship> relations = dataflow.getRelationships();
2840                if (relations != null) {
2841                        for (int i = 0; i < relations.size(); i++) {
2842                                relationship relation = relations.get(i);
2843                                targetColumn target = relation.getTarget();
2844                                List<sourceColumn> sources = relation.getSources();
2845                                if (target != null && sources != null && sources.size() > 0) {
2846                                        buffer.append(target.getColumn()).append(" depends on: ");
2847                                        Set<String> columnSet = new LinkedHashSet<String>();
2848                                        for (int j = 0; j < sources.size(); j++) {
2849                                                sourceColumn sourceColumn = sources.get(j);
2850                                                String columnName = sourceColumn.getColumn();
2851                                                if (sourceColumn.getParent_name() != null && sourceColumn.getParent_name().length() > 0) {
2852                                                        columnName = sourceColumn.getParent_name() + "." + columnName;
2853                                                }
2854                                                columnSet.add(columnName);
2855                                        }
2856                                        String[] columns = columnSet.toArray(new String[0]);
2857                                        for (int j = 0; j < columns.length; j++) {
2858                                                buffer.append(columns[j]);
2859                                                if (j == columns.length - 1) {
2860                                                        buffer.append("\n");
2861                                                } else
2862                                                        buffer.append(", ");
2863                                        }
2864                                }
2865                        }
2866                }
2867                return buffer.toString();
2868        }
2869
2870        private String mergeRelationType(List<Pair<sourceColumn, List<String>>> typePaths) {
2871                RelationshipType relationType = RelationshipType.join;
2872                for (int i = 0; i < typePaths.size(); i++) {
2873                        List<String> path = typePaths.get(i).second;
2874                        RelationshipType type = RelationshipType.valueOf(getRelationType(path));
2875                        if (type.ordinal() < relationType.ordinal()) {
2876                                relationType = type;
2877                        }
2878                }
2879                return relationType.name();
2880        }
2881
2882        private String getRelationType(List<String> typePaths) {
2883                if (typePaths.contains("join"))
2884                        return "join";
2885                if (typePaths.contains("fdr"))
2886                        return "fdr";
2887                if (typePaths.contains("frd"))
2888                        return "frd";
2889                if (typePaths.contains("fddi"))
2890                        return "fddi";
2891                return "fdd";
2892        }
2893
2894        public dataflow getSimpleDataflow(dataflow instance, boolean simpleOutput) throws Exception {
2895                return getSimpleDataflow(instance, simpleOutput, Arrays.asList("fdd"));
2896        }
2897
2898        public dataflow getSimpleDataflow(dataflow instance, boolean simpleOutput, List<String> types) throws Exception {
2899                ModelBindingManager.setGlobalVendor(option.getVendor());
2900                allMap.clear();
2901                targetTables.clear();
2902                resultSetMap.clear();
2903                tableMap.clear();
2904                viewMap.clear();
2905                cursorMap.clear();
2906                variableMap.clear();
2907                fileMap.clear();
2908                stageMap.clear();
2909                sequenceMap.clear();
2910                dataSourceMap.clear();
2911                databaseMap.clear();
2912                schemaMap.clear();
2913                streamMap.clear();
2914                dataflow simple = new dataflow();
2915                List<relationship> simpleRelations = new ArrayList<relationship>();
2916                List<relationship> relations = instance.getRelationships();
2917                if (instance.getResultsets() != null) {
2918                        for (table t : instance.getResultsets()) {
2919                                resultSetMap.put(t.getId().toLowerCase(), t);
2920                                allMap.put(t.getId().toLowerCase(), t);
2921                        }
2922                        if (option.isSimpleRetainIntermediate()) {
2923                                for (table t : instance.getResultsets()) {
2924                                        if (t.isFunction() || t.getProcedureId() != null) {
2925                                                t.setIsTarget("true");
2926                                        }
2927                                }
2928                        }
2929                }
2930
2931                if (instance.getTables() != null) {
2932                        for (table t : instance.getTables()) {
2933                                tableMap.put(t.getId().toLowerCase(), t);
2934                                allMap.put(t.getId().toLowerCase(), t);
2935                        }
2936                }
2937
2938                if (instance.getViews() != null) {
2939                        for (table t : instance.getViews()) {
2940                                viewMap.put(t.getId().toLowerCase(), t);
2941                                allMap.put(t.getId().toLowerCase(), t);
2942                        }
2943                }
2944
2945                if (instance.getPaths() != null) {
2946                        for (table t : instance.getPaths()) {
2947                                fileMap.put(t.getId().toLowerCase(), t);
2948                                allMap.put(t.getId().toLowerCase(), t);
2949                        }
2950                }
2951
2952                if (instance.getStages() != null) {
2953                        for (table t : instance.getStages()) {
2954                                stageMap.put(t.getId().toLowerCase(), t);
2955                                allMap.put(t.getId().toLowerCase(), t);
2956                        }
2957                }
2958                
2959                if (instance.getSequences() != null) {
2960                        for (table t : instance.getSequences()) {
2961                                sequenceMap.put(t.getId().toLowerCase(), t);
2962                                allMap.put(t.getId().toLowerCase(), t);
2963                        }
2964                }
2965
2966                if (instance.getDatasources() != null) {
2967                        for (table t : instance.getDatasources()) {
2968                                dataSourceMap.put(t.getId().toLowerCase(), t);
2969                                allMap.put(t.getId().toLowerCase(), t);
2970                        }
2971                }
2972
2973                if (instance.getDatabases() != null) {
2974                        for (table t : instance.getDatabases()) {
2975                                databaseMap.put(t.getId().toLowerCase(), t);
2976                                allMap.put(t.getId().toLowerCase(), t);
2977                        }
2978                }
2979
2980                if (instance.getSchemas() != null) {
2981                        for (table t : instance.getSchemas()) {
2982                                schemaMap.put(t.getId().toLowerCase(), t);
2983                                allMap.put(t.getId().toLowerCase(), t);
2984                        }
2985                }
2986
2987                if (instance.getStreams() != null) {
2988                        for (table t : instance.getStreams()) {
2989                                streamMap.put(t.getId().toLowerCase(), t);
2990                                allMap.put(t.getId().toLowerCase(), t);
2991                        }
2992                }
2993
2994                if (instance.getVariables() != null) {
2995                        for (table t : instance.getVariables()) {
2996                                if(SubType.cursor.name().equals(t.getSubType())){
2997                                        cursorMap.put(t.getId().toLowerCase(), t);
2998                                }
2999                                else {
3000                                        variableMap.put(t.getId().toLowerCase(), t);
3001                                }
3002                                allMap.put(t.getId().toLowerCase(), t);
3003                        }
3004                }
3005
3006                if (relations != null) {
3007                        
3008                        List<relationship> filterRelations = new ArrayList<>();
3009                        for (relationship relationElem : relations) {
3010                                if (!types.contains(relationElem.getType())) {
3011                                        if (option.isSimpleRetainIntermediate()
3012                                                        && RelationshipType.call.name().equals(relationElem.getType())) {
3013                                                filterRelations.add(relationElem);
3014                                        }
3015                                        continue;
3016                                }
3017                                else {
3018                                        filterRelations.add(relationElem);
3019                                }
3020                        }
3021                        
3022                        relations = filterRelations;
3023                        
3024                        Map<String, Set<relationship>> targetIdRelationMap = new LinkedHashMap<String, Set<relationship>>();
3025                        for (relationship relation : relations) {
3026                                if (relation.getTarget() != null) {
3027                                        String key = relation.getTarget().getParent_id() + "." + relation.getTarget().getId();
3028                                        if (!targetIdRelationMap.containsKey(key)) {
3029                                                targetIdRelationMap.put(key, new TreeSet<relationship>(new Comparator<relationship>() {
3030                                                        @Override
3031                                                        public int compare(relationship o1, relationship o2) {
3032                                                                return o1.getId().compareTo(o2.getId());
3033                                                        }
3034                                                }));
3035                                        }
3036                                        targetIdRelationMap.get(key).add(relation);
3037                                }
3038                        }
3039
3040                        Iterator<String> keys = targetIdRelationMap.keySet().iterator();
3041                        while (keys.hasNext()) {
3042                                String key = keys.next();
3043                                if (targetIdRelationMap.get(key).size() > 500) {
3044                                        keys.remove();
3045                                }
3046                        }
3047
3048                        for (relationship relationElem : relations) {
3049                                if (RelationshipType.call.name().equals(relationElem.getType())) {
3050                                        continue;
3051                                }
3052                                if (RelationshipType.er.name().equals(relationElem.getType())) {
3053                                        continue;
3054                                }
3055                                targetColumn target = relationElem.getTarget();
3056                                String targetParent = target.getParent_id();
3057                                if (isTarget(instance, targetParent, simpleOutput)) {
3058                                        List<Pair<sourceColumn, List<String>>> relationSources = new ArrayList<Pair<sourceColumn, List<String>>>();
3059                                        findSourceRelations(target, instance, targetIdRelationMap, relationElem, relationSources,
3060                                                        new String[] { relationElem.getType() }, simpleOutput);
3061                                        if (relationSources.size() > 0) {
3062                                                Map<sourceColumn, List<Pair<sourceColumn, List<String>>>> columnMap = new LinkedHashMap<sourceColumn, List<Pair<sourceColumn, List<String>>>>();
3063                                                for (Pair<sourceColumn, List<String>> t : relationSources) {
3064                                                        sourceColumn key = ((Pair<sourceColumn, List<String>>) t).first;
3065                                                        if (!columnMap.containsKey(key)) {
3066                                                                columnMap.put(key, new ArrayList<Pair<sourceColumn, List<String>>>());
3067                                                        }
3068                                                        columnMap.get(key).add(t);
3069                                                }
3070                                                Iterator<sourceColumn> iter = columnMap.keySet().iterator();
3071                                                Map<String, List<sourceColumn>> relationSourceMap = new LinkedHashMap<String, List<sourceColumn>>();
3072                                                Set<sourceColumn> passthroughColumns = new HashSet<sourceColumn>();
3073                                                while (iter.hasNext()) {
3074                                                        sourceColumn column = iter.next();
3075                                                        String relationType = mergeRelationType(columnMap.get(column));
3076                                                        if (!relationSourceMap.containsKey(relationType)) {
3077                                                                relationSourceMap.put(relationType, new ArrayList<sourceColumn>());
3078                                                        }
3079                                                        relationSourceMap.get(relationType).add(column);
3080                                                        // Mark columns that reached this target through an assumed passthrough edge.
3081                                                        for (Pair<sourceColumn, List<String>> typePath : columnMap.get(column)) {
3082                                                                if (typePath.second != null
3083                                                                        && typePath.second.contains(EffectType.external_script_passthrough.name())) {
3084                                                                        passthroughColumns.add(column);
3085                                                                        break;
3086                                                                }
3087                                                        }
3088                                                }
3089
3090                                                Iterator<String> sourceIter = relationSourceMap.keySet().iterator();
3091                                                while (sourceIter.hasNext()) {
3092                                                        String relationType = sourceIter.next();
3093                                                        relationship simpleRelation = (relationship) relationElem.clone();
3094                                                        List<sourceColumn> groupSources = relationSourceMap.get(relationType);
3095                                                        simpleRelation.setSources(groupSources);
3096                                                        simpleRelation.setType(relationType);
3097                                                        simpleRelation.setId(String.valueOf(++ModelBindingManager.get().RELATION_ID));
3098                                                        // If any source in this group reached the target through an assumed external-
3099                                                        // script passthrough edge, tag the flattened edge so simple output does not
3100                                                        // present assumed lineage as proven.
3101                                                        for (sourceColumn groupSource : groupSources) {
3102                                                                if (passthroughColumns.contains(groupSource)) {
3103                                                                        simpleRelation.setEffectType(EffectType.external_script_passthrough.name());
3104                                                                        break;
3105                                                                }
3106                                                        }
3107                                                        simpleRelations.add(simpleRelation);
3108                                                }
3109                                        }
3110                                }
3111                        }
3112                }
3113
3114                simple.setProcedures(instance.getProcedures());
3115                simple.setPackages(instance.getPackages());
3116                simple.setProcesses(instance.getProcesses());
3117                simple.setErrors(instance.getErrors());
3118
3119                List<table> tables = new ArrayList<table>();
3120                for (table t : instance.getTables()) {
3121                        if (!SQLUtil.isTempTable(t)) {
3122                                tables.add(t);
3123                        }
3124                        else {
3125                                if (option.isIgnoreTemporaryTable()) {
3126                                        continue;
3127                                }
3128                                else {
3129                                        tables.add(t);
3130                                }
3131                        }
3132                }
3133                simple.setStages(instance.getStages());
3134                simple.setSequences(instance.getSequences());
3135                simple.setDatasources(instance.getDatasources());
3136                simple.setStreams(instance.getStreams());
3137                simple.setPaths(instance.getPaths());
3138                simple.setTables(tables);
3139                simple.setViews(instance.getViews());
3140                if(option.isSimpleShowVariable()) {
3141                        simple.setVariables(instance.getVariables());
3142                }
3143                else if(option.isSimpleShowCursor()) {
3144                        simple.setVariables(instance.getVariables().stream().filter(t->SubType.cursor.name().equals(t.getSubType())).collect(Collectors.toList()));
3145                }
3146        else{
3147            simple.setVariables(new ArrayList());
3148        }
3149
3150        if (option.isSimpleRetainIntermediate() && instance.getVariables() != null) {
3151                        for (table var : instance.getVariables()) {
3152                                if (var.isTarget()) {
3153                    simple.getVariables().add(var);
3154                                }
3155                        }
3156                }
3157                if (instance.getResultsets() != null) {
3158                        List<table> resultSets = new ArrayList<table>();
3159                        for (int i = 0; i < instance.getResultsets().size(); i++) {
3160                                table resultSet = instance.getResultsets().get(i);
3161                                if (isTargetResultSet(instance, resultSet.getId(), simpleOutput)) {
3162                                        // special handle function #524 #296
3163                                        resultSets.add(resultSet);
3164                                }
3165                                else if (option.isSimpleRetainIntermediate() && (resultSet.isFunction() || resultSet.getProcedureId() != null)) {
3166                                        resultSet.setIsTarget("true");
3167                                        resultSets.add(resultSet);
3168                                }
3169                        }
3170                        simple.setResultsets(resultSets);
3171                }
3172
3173                List<table> functions = new ArrayList<table>();
3174                if (option.isShowCallRelation()) {
3175                        for (int i = 0; i < relations.size(); i++) {
3176                                relationship relationElem = relations.get(i);
3177                                if (!RelationshipType.call.name().equals(relationElem.getType())) {
3178                                        continue;
3179                                }
3180                                simpleRelations.add(relationElem);
3181                                for (sourceColumn callee : relationElem.getCallees()) {
3182                                        String calleeId = callee.getId();
3183                                        if (resultSetMap.containsKey(calleeId)) {
3184                                                table function = resultSetMap.get(calleeId);
3185                                                function.setIsTarget("true");
3186                                                functions.add(function);
3187                                        }
3188                                }
3189                        }
3190                }
3191
3192                if (option.isShowERDiagram()) {
3193                        for (int i = 0; i < relations.size(); i++) {
3194                                relationship relationElem = relations.get(i);
3195                                if (!RelationshipType.er.name().equals(relationElem.getType())) {
3196                                        continue;
3197                                }
3198                                simpleRelations.add(relationElem);
3199                                for (sourceColumn callee : relationElem.getCallees()) {
3200                                        String calleeId = callee.getId();
3201                                        if (resultSetMap.containsKey(calleeId)) {
3202                                                table function = resultSetMap.get(calleeId);
3203                                                function.setIsTarget("true");
3204                                                functions.add(function);
3205                                        }
3206                                }
3207                        }
3208                }
3209                
3210                if (option.isSimpleRetainIntermediate() && !option.isShowCallRelation()) {
3211                        for (int i = 0; i < relations.size(); i++) {
3212                                relationship relationElem = relations.get(i);
3213                                if (RelationshipType.call.name().equals(relationElem.getType())) {
3214                                        simpleRelations.add(relationElem);
3215                                }
3216                        }
3217                }
3218
3219                if (!functions.isEmpty()) {
3220                        if (simple.getResultsets() == null) {
3221                                simple.setResultsets(functions);
3222                        } else {
3223                                simple.getResultsets().addAll(functions);
3224                        }
3225                }
3226
3227                simple.setRelationships(simpleRelations);
3228                simple.setOrientation(instance.getOrientation());
3229
3230                targetTables.clear();
3231                resultSetMap.clear();
3232                tableMap.clear();
3233                viewMap.clear();
3234                cursorMap.clear();
3235                variableMap.clear();
3236                fileMap.clear();
3237                stageMap.clear();
3238                dataSourceMap.clear();
3239                databaseMap.clear();
3240                schemaMap.clear();
3241                streamMap.clear();
3242                return simple;
3243        }
3244
3245        private void findSourceRelations(targetColumn target, dataflow instance, Map<String, Set<relationship>> sourceIdRelationMap,
3246                        relationship targetRelation, List<Pair<sourceColumn, List<String>>> relationSources, String[] pathTypes, boolean simpleOutput) {
3247                findStarSourceRelations(target, instance, null, sourceIdRelationMap, targetRelation, relationSources, pathTypes,
3248                                new HashSet<String>(), new LinkedHashSet<transform>(), new LinkedHashSet<candidateTable>(), 0, simpleOutput);
3249        }
3250
3251        private void findStarSourceRelations(targetColumn target, dataflow instance, targetColumn starRelationTarget,
3252                        Map<String, Set<relationship>> sourceIdRelationMap, relationship targetRelation,
3253                        List<Pair<sourceColumn, List<String>>> relationSources, String[] pathTypes, Set<String> paths,
3254                        Set<transform> transforms, Set<candidateTable> candidateTables, int level, boolean simpleOutput) {
3255                if (targetRelation != null && targetRelation.getSources() != null) {
3256                        
3257                        //获取source为*的Column Parent
3258                        String starParentId = null;
3259                        for (int i = 0; i < targetRelation.getSources().size(); i++) {
3260                                sourceColumn source = targetRelation.getSources().get(i);
3261                                if (starRelationTarget != null && "*".equals(source.getColumn())) {
3262                                        starParentId = source.getParent_id();
3263                                }
3264                        }
3265                        
3266                        for (int i = 0; i < targetRelation.getSources().size(); i++) {
3267                                sourceColumn source = targetRelation.getSources().get(i);
3268                                if (starRelationTarget != null && !"*".equals(source.getColumn())
3269                                                && !DlineageUtil.getIdentifierNormalColumnName(starRelationTarget.getColumn())
3270                                                                .equals(DlineageUtil.getIdentifierNormalColumnName(source.getColumn()))) {
3271                                        table parent = allMap.get(source.getParent_id());
3272                                        if (parent != null && isFunction(parent)) {
3273                                                // function返回值未知,不对星号做处理
3274                                        } 
3275                                        else if (parent == null) {
3276                                                continue;
3277                                        } else if(starParentId!=null && starParentId.equals(parent.getId())){
3278                                                //如果source和 * column的parent相同,则跳过
3279                                                continue;
3280                                        }
3281                                }
3282
3283                                String sourceColumnId = source.getId();
3284                                String sourceParentId = source.getParent_id();
3285                                if (sourceParentId == null || sourceColumnId == null) {
3286                                        continue;
3287                                }
3288                                if (isTarget(instance, sourceParentId, simpleOutput)) {
3289                                        List<transform> transforms2 = new ArrayList<transform>(transforms.size());
3290                                        transforms2.addAll(transforms);
3291                                        Collections.reverse(transforms2);
3292                                        
3293                                        List<candidateTable> candidateTables2 = new ArrayList<candidateTable>(candidateTables.size());
3294                                        candidateTables2.addAll(candidateTables);
3295                                        
3296                                        sourceColumn sourceColumnCopy = DlineageUtil.copySourceColumn(source);
3297                                        for (transform t : transforms2) {
3298                                                sourceColumnCopy.addTransform(t);
3299                                        }
3300                                        
3301                                        for (candidateTable t : candidateTables2) {
3302                                                sourceColumnCopy.addCandidateParent(t);
3303                                        }
3304                                        
3305                                        if(Boolean.TRUE.equals(target.isStruct()) && Boolean.TRUE.equals(source.isStruct())) {
3306                                                List<String> targetColumns = SQLUtil.parseNames(target.getColumn());
3307                                                List<String> sourceColumns = SQLUtil.parseNames(source.getColumn());
3308                                                if(!DlineageUtil.getIdentifierNormalColumnName(targetColumns.get(targetColumns.size()-1))
3309                                                                .equals(DlineageUtil.getIdentifierNormalColumnName(sourceColumns.get(sourceColumns.size()-1)))) {
3310                                                        continue;
3311                                                }
3312                                        }
3313                                        relationSources.add(new Pair<sourceColumn, List<String>>(sourceColumnCopy, Arrays.asList(pathTypes)));
3314                                } else {
3315                                        Set<relationship> sourceRelations = sourceIdRelationMap
3316                                                        .get(source.getParent_id() + "." + source.getId());
3317                                        if (sourceRelations != null) {
3318                                                if (paths.contains(source.getParent_id() + "." + source.getId())) {
3319                                                        continue;
3320                                                } else {
3321                                                        paths.add(source.getParent_id() + "." + source.getId());
3322                                                        if (source.getTransforms() != null) {
3323                                                                transforms.addAll(source.getTransforms());
3324                                                        }
3325                                                        if (source.getCandidateParents() != null) {
3326                                                                candidateTables.addAll(source.getCandidateParents());
3327                                                        }
3328                                                }
3329                                                for (relationship relation : sourceRelations) {
3330                                                        LinkedHashSet<transform> transforms2 = new LinkedHashSet<transform>(transforms.size());
3331                                                        transforms2.addAll(transforms);
3332                                                        LinkedHashSet<candidateTable> candidateTables2 = new LinkedHashSet<candidateTable>(candidateTables.size());
3333                                                        candidateTables2.addAll(candidateTables);
3334                                                        // Carry an assumed external-script passthrough marker down the path
3335                                                        // (alongside the relationship type) so a flattened simple-output edge
3336                                                        // that traversed it can be tagged and never look proven. The marker is
3337                                                        // ignored by getRelationType, so type computation is unaffected.
3338                                                        boolean passthroughHop = EffectType.external_script_passthrough.name()
3339                                                                        .equals(relation.getEffectType());
3340                                                        String[] types = new String[pathTypes.length + (passthroughHop ? 2 : 1)];
3341                                                        int typeOffset = 0;
3342                                                        types[typeOffset++] = relation.getType();
3343                                                        if (passthroughHop) {
3344                                                                types[typeOffset++] = EffectType.external_script_passthrough.name();
3345                                                        }
3346                                                        System.arraycopy(pathTypes, 0, types, typeOffset, pathTypes.length);
3347                                                        if (!"*".equals(source.getColumn())) {
3348                                                                findStarSourceRelations(target, instance, null, sourceIdRelationMap, relation, relationSources,
3349                                                                                types, paths, transforms2, candidateTables2, level + 1, simpleOutput);
3350                                                        } else {
3351                                                                findStarSourceRelations(target, instance,
3352                                                                                starRelationTarget == null ? targetRelation.getTarget() : starRelationTarget,
3353                                                                                sourceIdRelationMap, relation, relationSources, types, paths, transforms, candidateTables2,
3354                                                                                level + 1, simpleOutput);
3355                                                        }
3356                                                }
3357                                        }
3358                                }
3359                        }
3360                }
3361        }
3362
3363        private Map<String, Boolean> targetTables = new HashMap<String, Boolean>();
3364        private Map<String, table> resultSetMap = new HashMap<String, table>();
3365        private Map<String, table> tableMap = new HashMap<String, table>();
3366        private Map<String, table> viewMap = new HashMap<String, table>();
3367        private Map<String, table> cursorMap = new HashMap<String, table>();
3368        private Map<String, table> variableMap = new HashMap<String, table>();
3369        private Map<String, table> fileMap = new HashMap<String, table>();
3370        private Map<String, table> stageMap = new HashMap<String, table>();
3371        private Map<String, table> sequenceMap = new HashMap<String, table>();
3372        private Map<String, table> dataSourceMap = new HashMap<String, table>();
3373        private Map<String, table> databaseMap = new HashMap<String, table>();
3374        private Map<String, table> schemaMap = new HashMap<String, table>();
3375        private Map<String, table> streamMap = new HashMap<String, table>();
3376        private Map<String, table> allMap = new HashMap<String, table>();
3377
3378        private boolean isTarget(dataflow instance, String targetParentId, boolean simpleOutput) {
3379                if (targetTables.containsKey(targetParentId))
3380                        return targetTables.get(targetParentId);
3381                if (isTable(instance, targetParentId)) {
3382                        targetTables.put(targetParentId, true);
3383                        return true;
3384                } else if (isView(instance, targetParentId)) {
3385                        targetTables.put(targetParentId, true);
3386                        return true;
3387                } else if (isFile(instance, targetParentId)) {
3388                        targetTables.put(targetParentId, true);
3389                        return true;
3390                } else if (isDatabase(instance, targetParentId)) {
3391                        targetTables.put(targetParentId, true);
3392                        return true;
3393                } else if (isSchema(instance, targetParentId)) {
3394                        targetTables.put(targetParentId, true);
3395                        return true;
3396                } else if (isStage(instance, targetParentId)) {
3397                        targetTables.put(targetParentId, true);
3398                        return true;
3399                } else if (isSequence(instance, targetParentId)) {
3400                        targetTables.put(targetParentId, true);
3401                        return true;
3402                } else if (isDataSource(instance, targetParentId)) {
3403                        targetTables.put(targetParentId, true);
3404                        return true;
3405                } else if (isStream(instance, targetParentId)) {
3406                        targetTables.put(targetParentId, true);
3407                        return true;
3408                } else if (isCursor(instance, targetParentId) && option.isSimpleShowCursor()) {
3409                        targetTables.put(targetParentId, true);
3410                        return true;
3411                } else if ((isVariable(instance, targetParentId) || isCursor(instance, targetParentId)) && option.isSimpleShowVariable()) {
3412                        targetTables.put(targetParentId, true);
3413                        return true;
3414                } else if ((isVariable(instance, targetParentId) || isCursor(instance, targetParentId)) && option.isSimpleRetainIntermediate()) {
3415                        table var = variableMap.get(targetParentId.toLowerCase());
3416                        if (var != null && var.isTarget()) {
3417                                targetTables.put(targetParentId, true);
3418                                return true;
3419                        }
3420                } else if (isTargetResultSet(instance, targetParentId, simpleOutput)) {
3421                        targetTables.put(targetParentId, true);
3422                        return true;
3423                }
3424                targetTables.put(targetParentId, false);
3425                return false;
3426        }
3427
3428        private boolean isTargetResultSet(dataflow instance, String targetParent, boolean simpleOutput) {
3429                if (resultSetMap.containsKey(targetParent.toLowerCase())) {
3430                        table result = resultSetMap.get(targetParent.toLowerCase());
3431                        boolean isTarget = result.isTarget();
3432                        Option option = ModelBindingManager.getGlobalOption();
3433                        if (option != null && option.isSqlflowIgnoreFunction() && isFunction(result)) {
3434                                return false;
3435                        }
3436                        if (isTarget && simpleOutput) {
3437                                if (option != null && option.isSimpleShowFunction() && isFunction(result)) {
3438                                        return true;
3439
3440                                } else if (option != null && option.isSimpleShowTopSelectResultSet()) {
3441                                        return true;
3442                                }
3443                                if (option != null && option.isSimpleRetainIntermediate() && (isFunction(result) || result.getProcedureId() != null)) {
3444                                        return true;
3445                                }
3446                                if (ResultSetType.of(result.getType()) != null && option.containsResultSetType(ResultSetType.of(result.getType()))) {
3447                                        return true;
3448                                }
3449                        } else
3450                                return isTarget;
3451                }
3452                return false;
3453        }
3454
3455        private boolean isFunction(table resultSet) {
3456                if("function".equals(resultSet.getType())){
3457                        return true;
3458                }
3459                else if("resultset".equals(resultSet.getType()) && "function".equals(resultSet.getSubType())){
3460                        return true;
3461                }
3462                return false;
3463        }
3464
3465        private boolean isView(dataflow instance, String targetParent) {
3466                if (viewMap.containsKey(targetParent.toLowerCase())) {
3467                        return true;
3468                }
3469                return false;
3470        }
3471
3472        private boolean isCursor(dataflow instance, String targetParent) {
3473                if (cursorMap.containsKey(targetParent.toLowerCase())) {
3474                        return true;
3475                }
3476                return false;
3477        }
3478
3479        private boolean isVariable(dataflow instance, String targetParent) {
3480                if (variableMap.containsKey(targetParent.toLowerCase())) {
3481                        return true;
3482                }
3483                return false;
3484        }
3485
3486        private boolean isFile(dataflow instance, String targetParent) {
3487                if (fileMap.containsKey(targetParent.toLowerCase())) {
3488                        return true;
3489                }
3490                return false;
3491        }
3492
3493        private boolean isStage(dataflow instance, String targetParent) {
3494                if (stageMap.containsKey(targetParent.toLowerCase())) {
3495                        return true;
3496                }
3497                return false;
3498        }
3499        
3500        private boolean isSequence(dataflow instance, String targetParent) {
3501                if (sequenceMap.containsKey(targetParent.toLowerCase())) {
3502                        return true;
3503                }
3504                return false;
3505        }
3506
3507        private boolean isDataSource(dataflow instance, String targetParent) {
3508                if (dataSourceMap.containsKey(targetParent.toLowerCase())) {
3509                        return true;
3510                }
3511                return false;
3512        }
3513
3514        private boolean isDatabase(dataflow instance, String targetParent) {
3515                if (databaseMap.containsKey(targetParent.toLowerCase())) {
3516                        return true;
3517                }
3518                return false;
3519        }
3520
3521        private boolean isSchema(dataflow instance, String targetParent) {
3522                if (schemaMap.containsKey(targetParent.toLowerCase())) {
3523                        return true;
3524                }
3525                return false;
3526        }
3527
3528        private boolean isStream(dataflow instance, String targetParent) {
3529                if (streamMap.containsKey(targetParent.toLowerCase())) {
3530                        return true;
3531                }
3532                return false;
3533        }
3534
3535        private boolean isTable(dataflow instance, String targetParent) {
3536                if (tableMap.containsKey(targetParent.toLowerCase())) {
3537                        if (SQLUtil.isTempTable(tableMap.get(targetParent))) {
3538                                if (option.isIgnoreTemporaryTable()) {
3539                                        return false;
3540                                }
3541                        }
3542                        if (tableMap.get(targetParent).isFunction()) {
3543                                if (option != null && option.isSimpleShowFunction()) {
3544                                        return true;
3545                                } else {
3546                                        return false;
3547                                }
3548                        }
3549                        if (SubType.synonym.name().equals(tableMap.get(targetParent).getSubType())) {
3550                                if (option != null && option.isSimpleShowSynonym()) {
3551                                        return true;
3552                                } else {
3553                                        return false;
3554                                }
3555                        }
3556                        return true;
3557                } else {
3558                        return false;
3559                }
3560        }
3561
3562        private void init() {
3563                metadataErrors.clear();
3564                sqlInfoMap.clear();
3565                errorInfos.clear();
3566                dynamicSqlSites.clear();
3567                cteSelectRelationKeys.clear();
3568                authoritativeEvidenceCollector.reset();
3569                authoritativeLineageEvidence = AuthoritativeLineageEvidence.empty();
3570                dynamicEvalCache.clear();
3571                routineCatalog = null;
3572                crossFileParseCache.clear();
3573                plsqlEvaluatedTexts.clear();
3574                dataflow = null;
3575                dataflowString = null;
3576                ModelBindingManager.removeGlobalDatabase();
3577                ModelBindingManager.removeGlobalSchema();
3578                ModelBindingManager.removeGlobalVendor();
3579                ModelBindingManager.removeGlobalSQLEnv();
3580                ModelBindingManager.removeGlobalHash();
3581                appendResultSets.clear();
3582                appendStarColumns.clear();
3583                appendTableStarColumns.clear();
3584                normalizedColumnNameCache.clear();
3585                columnNameIndexCache.clear();
3586                resultSetColumnLookupCache.clear();
3587                tableColumnLookupCache.clear();
3588                modelManager.TABLE_COLUMN_ID = option.getStartId();
3589                modelManager.RELATION_ID = option.getStartId();
3590                modelManager.DISPLAY_ID.clear();
3591                modelManager.DISPLAY_NAME.clear();
3592                tableIds.clear();
3593                clickhouseExternalSourceModels.clear();
3594                ModelBindingManager.setGlobalVendor(option.getVendor());
3595                modelManager.reset();
3596        }
3597
3598        private String getErrorMessage(TSyntaxError error, String errorType) {
3599                String s = "", hint = "Syntax error";
3600                if (ErrorInfo.SYNTAX_HINT.equals(errorType)) {
3601                        hint = "Syntax hint";
3602                }
3603                if (error.hint.length() > 0)
3604                        hint = error.hint;
3605                s = s + hint + "(" + error.errorno + ") near: " + error.tokentext;
3606                s = s + "(" + error.lineNo;
3607                s = s + "," + error.columnNo + ")";
3608                return s;
3609        }
3610
3611//      boolean OLD_ENABLE_RESOLVER = TBaseType.isEnableResolver();
3612        private void analyzeAndOutputResult(TGSqlParser sqlparser) {
3613                try {
3614                        accessedSubqueries.clear();
3615                        accessedStatements.clear();
3616                        stmtStack.clear();
3617                        viewDDLMap.clear();
3618                        procedureDDLMap.clear();
3619                        structObjectMap.clear();
3620                        try {
3621                                if(sqlenv!=null) {
3622                                        sqlparser.setSqlEnv(sqlenv);
3623                                }
3624                                int result = sqlparser.parse();
3625                                if (result != 0) {
3626                                        ArrayList<TSyntaxError> errors = sqlparser.getSyntaxErrors();
3627                                        if ((errors == null || errors.isEmpty())
3628                                                        && sqlparser.getErrormessage() != null
3629                                                        && !sqlparser.getErrormessage().isEmpty()) {
3630                                                // A refusal with no syntax errors -- the trial size cap
3631                                                // is the one that behaves this way. Recording nothing
3632                                                // here turned the rejection into an ordinary EMPTY
3633                                                // lineage result, which reads as "this SQL has no
3634                                                // dependencies" rather than "this SQL was not analyzed".
3635                                                ErrorInfo errorInfo = new ErrorInfo();
3636                                                errorInfo.setErrorType(ErrorInfo.SYNTAX_ERROR);
3637                                                errorInfo.setErrorMessage(sqlparser.getErrormessage());
3638                                                errorInfo.setStartPosition(new Pair3<Long, Long, String>(1L, 1L,
3639                                                                ModelBindingManager.getGlobalHash()));
3640                                                errorInfo.setEndPosition(new Pair3<Long, Long, String>(1L, 1L,
3641                                                                ModelBindingManager.getGlobalHash()));
3642                                                errorInfo.fillInfo(this);
3643                                                errorInfos.add(errorInfo);
3644                                        }
3645                                        if (errors != null && !errors.isEmpty()) {
3646                                                for (int i = 0; i < errors.size(); i++) {
3647                                                        TSyntaxError error = errors.get(i);
3648                                                        ErrorInfo errorInfo = new ErrorInfo();
3649                                                        errorInfo.setErrorType(ErrorInfo.SYNTAX_ERROR);
3650                                                        errorInfo.setErrorMessage(getErrorMessage(error, ErrorInfo.SYNTAX_ERROR));
3651                                                        errorInfo.setStartPosition(new Pair3<Long, Long, String>(error.lineNo, error.columnNo,
3652                                                                        ModelBindingManager.getGlobalHash()));
3653                                                        String[] segments = error.tokentext.split("\n");
3654                                                        if (segments.length <= 1) {
3655                                                                errorInfo.setEndPosition(new Pair3<Long, Long, String>(error.lineNo,
3656                                                                                error.columnNo + error.tokentext.length(),
3657                                                                                ModelBindingManager.getGlobalHash()));
3658                                                        } else {
3659                                                                errorInfo.setEndPosition(
3660                                                                                new Pair3<Long, Long, String>(error.lineNo + segments.length - 1,
3661                                                                                                (long) segments[segments.length - 1].length() + 1,
3662                                                                                                ModelBindingManager.getGlobalHash()));
3663                                                        }
3664                                                        ;
3665                                                        errorInfo.fillInfo(this);
3666                                                        errorInfos.add(errorInfo);
3667                                                }
3668                                        }
3669                                }
3670
3671                                if (option.getHandleListener() != null) {
3672                                        option.getHandleListener().endParse(result == 0);
3673                                }
3674                        } catch (Exception e) {
3675                                logger.error("analyze sql failed.", e);
3676                                if (option.getHandleListener() != null) {
3677                                        option.getHandleListener().endParse(false);
3678                                }
3679                                ErrorInfo errorInfo = new ErrorInfo();
3680                                errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
3681                                if (e.getMessage() == null) {
3682                                        if (e.getStackTrace() != null && e.getStackTrace().length > 0) {
3683                                                errorInfo
3684                                                                .setErrorMessage(e.getClass().getSimpleName() + ": " + e.getStackTrace()[0].toString());
3685                                        } else {
3686                                                errorInfo.setErrorMessage(e.getClass().getSimpleName());
3687                                        }
3688                                } else {
3689                                        errorInfo.setErrorMessage(e.getClass().getSimpleName() + ": " + e.getMessage());
3690                                }
3691                                errorInfo.fillInfo(this);
3692                                errorInfos.add(errorInfo);
3693                                return;
3694                        }
3695
3696                        
3697                        TSQLResolver2 resolver = sqlparser.getResolver2();
3698                        if (resolver != null && option.getVendor() == EDbVendor.dbvbigquery) {
3699                                ScopeBuildResult buildResult = resolver.getScopeBuildResult();
3700                                List<TObjectName> columns = buildResult.getAllColumnReferences();
3701                                for (TObjectName col : columns) {
3702                                        structObjectMap.putIfAbsent(col.getSourceTable(), new TObjectNameList());
3703                                        structObjectMap.get(col.getSourceTable()).addObjectName(col);
3704                                }
3705                        }
3706                        
3707                        if (option.getHandleListener() != null) {
3708                                option.getHandleListener().startAnalyzeDataFlow(sqlparser);
3709                        }
3710
3711                        ModelBindingManager.setSchemaUnsetStatements(collectSchemaUnsetStatements(sqlparser));
3712
3713                        for (int i = 0; i < sqlparser.getSqlstatements().size(); i++) {
3714                                if (option.getHandleListener() != null && option.getHandleListener().isCanceled()) {
3715                                        break;
3716                                }
3717
3718                                TCustomSqlStatement stmt = sqlparser.getSqlstatements().get(i);
3719                                if (stmt.getErrorCount() == 0) {
3720                                        if (stmt.getParentStmt() == null) {
3721                                                modelManager.collectSqlHash(stmt);
3722                                                if (stmt instanceof TUseDatabase || stmt instanceof TUseSchema
3723                                                                || stmt instanceof TCreateTableSqlStatement
3724                                                                || stmt instanceof TCreateExternalDataSourceStmt || stmt instanceof TCreateStageStmt
3725                                                                || stmt instanceof TMssqlCreateType
3726                                                                || stmt instanceof TMssqlDeclare
3727                                                                || stmt instanceof TPlsqlCreateType_Placeholder
3728                                                                || stmt instanceof TPlsqlCreateType
3729                                                                || stmt instanceof TPlsqlTableTypeDefStmt
3730                                                                || (stmt instanceof TCreateFunctionStmt && hasDb2ReturnStmt((TCreateFunctionStmt)stmt))
3731                                                                || (stmt instanceof TMssqlCreateFunction
3732                                                                                && (((TMssqlCreateFunction) stmt).getReturnTableDefinitions() != null
3733                                                                                                || ((TMssqlCreateFunction) stmt).getReturnStmt() != null))) {
3734                                                        boolean listen = false;
3735                                                        if (option.getHandleListener() != null && !accessedStatements.contains(stmt)) {
3736                                                                option.getHandleListener().startAnalyzeStatment(stmt);
3737                                                                listen = true;
3738                                                        }
3739                                                        analyzeCustomSqlStmt(stmt);
3740                                                        if (listen && option.getHandleListener() != null) {
3741                                                                option.getHandleListener().endAnalyzeStatment(stmt);
3742                                                        }
3743                                                }
3744                                        }
3745                                }
3746                        }
3747
3748                        for (int i = 0; i < sqlparser.sqlstatements.size(); i++) {
3749                                if (option.getHandleListener() != null && option.getHandleListener().isCanceled()) {
3750                                        break;
3751                                }
3752
3753                                TCustomSqlStatement stmt = sqlparser.getSqlstatements().get(i);
3754                                if (stmt.getErrorCount() == 0) {
3755                                        if (stmt.getParentStmt() == null) {
3756                                                if (stmt instanceof TUseDatabase 
3757                                                                || stmt instanceof TUseSchema
3758                                                                || stmt instanceof TCreateViewSqlStatement 
3759                                                                || stmt instanceof TCreateSynonymStmt 
3760                                                                || stmt instanceof TStoredProcedureSqlStatement) {
3761                                                        boolean listen = false;
3762                                                        if (option.getHandleListener() != null && !accessedStatements.contains(stmt)) {
3763                                                                option.getHandleListener().startAnalyzeStatment(stmt);
3764                                                                listen = true;
3765                                                        }
3766                                                        if (stmt instanceof TUseDatabase || stmt instanceof TUseSchema) {
3767                                                                analyzeCustomSqlStmt(stmt);
3768                                                        } else if (stmt instanceof TCreateViewSqlStatement) {
3769                                                                TCreateViewSqlStatement view = (TCreateViewSqlStatement) stmt;
3770                                                                if(view.getViewName()!=null) {
3771                                                                        viewDDLMap.put(DlineageUtil.getTableFullName(view.getViewName().toString()), view);
3772                                                                }
3773                                                        } else if (stmt instanceof TCreateSynonymStmt) {
3774                                                                TCreateSynonymStmt synonym = (TCreateSynonymStmt) stmt;
3775                                                                if(synonym.getSynonymName()!=null) {
3776                                                                        viewDDLMap.put(DlineageUtil.getTableFullName(synonym.getSynonymName().toString()), synonym);
3777                                                                }
3778                                                        } else if (stmt instanceof TStoredProcedureSqlStatement) {
3779                                                                TStoredProcedureSqlStatement procedure = (TStoredProcedureSqlStatement) stmt;
3780                                                                if(procedure.getStoredProcedureName() == null) {
3781                                                                        continue;
3782                                                                }
3783                                                                procedureDDLMap.put(DlineageUtil.getProcedureNameWithArgNum(procedure), procedure);
3784                                                        }
3785                                                        if (listen && option.getHandleListener() != null) {
3786                                                                option.getHandleListener().endAnalyzeStatment(stmt);
3787                                                        }
3788                                                }
3789                                        }
3790                                }
3791                        }
3792
3793                        for (int i = 0; i < sqlparser.sqlstatements.size(); i++) {
3794                                if (option.getHandleListener() != null && option.getHandleListener().isCanceled()) {
3795                                        break;
3796                                }
3797
3798                                TCustomSqlStatement stmt = sqlparser.getSqlstatements().get(i);
3799                                if (stmt.getErrorCount() == 0) {
3800                                        if (stmt.getParentStmt() == null) {
3801                                                if (stmt instanceof TUseDatabase 
3802                                                                || stmt instanceof TUseSchema
3803                                                                || stmt instanceof TCreateViewSqlStatement 
3804                                                                || stmt instanceof TCreateSynonymStmt) {
3805                                                        boolean listen = false;
3806                                                        if (option.getHandleListener() != null && !accessedStatements.contains(stmt)) {
3807                                                                option.getHandleListener().startAnalyzeStatment(stmt);
3808                                                                listen = true;
3809                                                        }
3810                                                        analyzeCustomSqlStmt(stmt);
3811                                                        if (listen && option.getHandleListener() != null) {
3812                                                                option.getHandleListener().endAnalyzeStatment(stmt);
3813                                                        }
3814                                                }
3815                                        }
3816                                }
3817                        }
3818
3819                        for (int i = 0; i < sqlparser.sqlstatements.size(); i++) {
3820                                if (option.getHandleListener() != null && option.getHandleListener().isCanceled()) {
3821                                        break;
3822                                }
3823
3824                                TCustomSqlStatement stmt = sqlparser.getSqlstatements().get(i);
3825                                if (stmt.getErrorCount() == 0) {
3826                                        if (stmt.getParentStmt() == null) {
3827                                                if (stmt instanceof TUseDatabase || stmt instanceof TUseSchema
3828                                                                || stmt instanceof TStoredProcedureSqlStatement) {
3829                                                        boolean listen = false;
3830                                                        if (option.getHandleListener() != null && !accessedStatements.contains(stmt)) {
3831                                                                option.getHandleListener().startAnalyzeStatment(stmt);
3832                                                                listen = true;
3833                                                        }
3834                                                        if (stmt instanceof TPlsqlCreateTrigger)
3835                                                                continue;
3836                                                        analyzeCustomSqlStmt(stmt);
3837                                                        if (listen && option.getHandleListener() != null) {
3838                                                                option.getHandleListener().endAnalyzeStatment(stmt);
3839                                                        }
3840                                                }
3841                                        }
3842                                }
3843                        }
3844
3845                        for (int i = 0; i < sqlparser.sqlstatements.size(); i++) {
3846                                if (option.getHandleListener() != null && option.getHandleListener().isCanceled()) {
3847                                        break;
3848                                }
3849
3850                                TCustomSqlStatement stmt = sqlparser.getSqlstatements().get(i);
3851
3852                                if (option.isIgnoreTopSelect()) {
3853                                        if ((stmt instanceof TSelectSqlStatement && ((TSelectSqlStatement)stmt).getIntoClause() == null && ((TSelectSqlStatement)stmt).getIntoTableClause() == null ) || stmt instanceof TRedshiftDeclare
3854                                                        || stmt instanceof TRedshiftDeclare) {
3855                                                continue;
3856                                        }
3857                                }
3858
3859                                if (stmt.getErrorCount() == 0) {
3860                                        if (stmt.getParentStmt() == null) {
3861                                                if (!(stmt instanceof TCreateViewSqlStatement) && !(stmt instanceof TCreateStageStmt)
3862                                                                && !(stmt instanceof TCreateExternalDataSourceStmt)
3863                                                                && !(stmt instanceof TCreateViewSqlStatement) && !(stmt instanceof TMssqlDeclare)
3864                                                                && !(stmt instanceof TMssqlCreateFunction
3865                                                                                && ((TMssqlCreateFunction) stmt).getReturnTableDefinitions() != null)) {
3866                                                        boolean listen = false;
3867                                                        if (option.getHandleListener() != null && !accessedStatements.contains(stmt)) {
3868                                                                option.getHandleListener().startAnalyzeStatment(stmt);
3869                                                                listen = true;
3870                                                        }
3871                                                        analyzeCustomSqlStmt(stmt);
3872                                                        if (listen && option.getHandleListener() != null) {
3873                                                                option.getHandleListener().endAnalyzeStatment(stmt);
3874                                                        }
3875                                                }
3876                                        }
3877                                }
3878                        }
3879
3880                        if (option.getHandleListener() != null) {
3881                                option.getHandleListener().endAnalyzeDataFlow(sqlparser);
3882                        }
3883
3884                        // Finalize pipelined function stitching after all passes
3885                        if (!modelManager.getPendingPipelinedCallSites().isEmpty() && sqlparser.getSqlstatements().size() > 0 && pipelinedAnalyzer != null) {
3886                                TCustomSqlStatement lastStmt = sqlparser.getSqlstatements().get(sqlparser.getSqlstatements().size() - 1);
3887                                stmtStack.push(lastStmt);
3888                                try {
3889                                        pipelinedAnalyzer.stitchPendingCallSites();
3890                                } catch (Exception ex) {
3891                                        // Don't let pipelined stitching failure break main flow
3892                                } finally {
3893                                        stmtStack.pop();
3894                                }
3895                        }
3896                } catch (Throwable e) {
3897                        logger.error("analyze sql failed.", e);
3898                        ErrorInfo errorInfo = new ErrorInfo();
3899                        errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
3900                        if (e.getMessage() == null) {
3901                                if (e.getStackTrace() != null && e.getStackTrace().length > 0) {
3902                                        errorInfo.setErrorMessage(e.getClass().getSimpleName() + ": " + e.getStackTrace()[0].toString());
3903                                } else {
3904                                        errorInfo.setErrorMessage(e.getClass().getSimpleName());
3905                                }
3906                        } else {
3907                                errorInfo.setErrorMessage(e.getClass().getSimpleName() + ": " + e.getMessage());
3908                        }
3909                        errorInfo.fillInfo(this);
3910                        errorInfos.add(errorInfo);
3911                }
3912
3913        }
3914
3915        private boolean hasDb2ReturnStmt(TCreateFunctionStmt stmt) {
3916                if (stmt.getReturnStmt() != null) {
3917                        return true;
3918                }
3919                if (stmt.getBodyStatements() != null) {
3920                        for (int i = 0; i < stmt.getBodyStatements().size(); i++) {
3921                                if(stmt.getBodyStatements().get(i) instanceof TDb2ReturnStmt) {
3922                                        return true;
3923                                }
3924                        }
3925                }
3926                return false;
3927        }
3928
3929
3930        /**
3931         * The statements that a Snowflake {@code USE <database>} left with no current
3932         * schema: everything after such a switch, up to the next schema switch.
3933         *
3934         * <p>Computed once, in statement order, because the analysis below walks the
3935         * list several times and replays the {@code USE} statements in every pass. A
3936         * running flag would still be set when a later pass reached a statement that
3937         * PRECEDES the switch, and would strip that statement's already-exact
3938         * qualification. Keyed by statement, the answer does not depend on the pass.
3939         *
3940         * <p>Only Snowflake can populate this, so no other vendor's output moves.
3941         * See GitHub #715 / MantisBT 4677.
3942         */
3943        private Set<TCustomSqlStatement> collectSchemaUnsetStatements(TGSqlParser sqlparser) {
3944                Set<TCustomSqlStatement> unset = new HashSet<TCustomSqlStatement>();
3945                if (sqlparser == null || sqlparser.getSqlstatements() == null) {
3946                        return unset;
3947                }
3948                boolean schemaUnset = false;
3949                for (int i = 0; i < sqlparser.getSqlstatements().size(); i++) {
3950                        TCustomSqlStatement stmt = sqlparser.getSqlstatements().get(i);
3951                        if (stmt == null) {
3952                                continue;
3953                        }
3954                        if (schemaUnset) {
3955                                unset.add(stmt);
3956                        }
3957                        if (stmt.dbvendor != EDbVendor.dbvsnowflake) {
3958                                continue;
3959                        }
3960                        if (stmt instanceof TUseSchema || stmt.sqlstatementtype == ESqlStatementType.sstSetSchema) {
3961                                schemaUnset = false;
3962                        } else if (stmt instanceof TUseDatabase) {
3963                                schemaUnset = !((TUseDatabase) stmt).isSchema();
3964                        }
3965                }
3966                return unset;
3967        }
3968
3969        private void analyzeCustomSqlStmt(TCustomSqlStatement stmt) {
3970                if (stmt != null && dynamicFoldDepth == 0 && anyDynamicTemplateMarked) {
3971                        // A STATIC statement analyzed after a fold marked something: its references
3972                        // withdraw marks on shared nodes (see the field javadoc).
3973                        applyStaticTemplateVetoes(stmt);
3974                }
3975                if (!accessedStatements.contains(stmt)) {
3976                        accessedStatements.add(stmt);
3977                } else if (!(stmt instanceof TUseDatabase || stmt instanceof TUseSchema)) {
3978                        return;
3979                }
3980
3981                ArrayList<TSyntaxError> errors = stmt.getSyntaxHints();
3982                if (errors != null && !errors.isEmpty()) {
3983                        for (int i = 0; i < errors.size(); i++) {
3984                                TSyntaxError error = errors.get(i);
3985                                ErrorInfo errorInfo = new ErrorInfo();
3986                                errorInfo.setErrorType(ErrorInfo.SYNTAX_HINT);
3987                                errorInfo.setErrorMessage(getErrorMessage(error, ErrorInfo.SYNTAX_HINT));
3988                                errorInfo.setStartPosition(new Pair3<Long, Long, String>(error.lineNo, error.columnNo,
3989                                                ModelBindingManager.getGlobalHash()));
3990                                String[] segments = error.tokentext.split("\n");
3991                                if (segments.length == 1) {
3992                                        errorInfo.setEndPosition(new Pair3<Long, Long, String>(error.lineNo,
3993                                                        error.columnNo + error.tokentext.length(), ModelBindingManager.getGlobalHash()));
3994                                } else {
3995                                        errorInfo.setEndPosition(new Pair3<Long, Long, String>(error.lineNo + segments.length - 1,
3996                                                        (long) segments[segments.length - 1].length() + 1, ModelBindingManager.getGlobalHash()));
3997                                }
3998                                errorInfo.fillInfo(this);
3999                                errorInfos.add(errorInfo);
4000                        }
4001                }
4002
4003                if (option.getAnalyzeMode() == AnalyzeMode.dynamic) {
4004                        if (!(stmt instanceof TStoredProcedureSqlStatement
4005                                        || stmt instanceof TExecuteSqlStatement
4006                                        || stmt instanceof TMssqlExecute
4007                                        || stmt instanceof TExecImmeStmt)) {
4008                                return;
4009                        }
4010                }
4011
4012                if(DlineageUtil.getTopStmt(stmt) == stmt){
4013                        modelManager.collectSqlHash(stmt);
4014                }
4015
4016                try {
4017                        if (stmt instanceof TUseDatabase) {
4018                                if (((TUseDatabase) stmt).getDatabaseName() != null) {
4019                                        // Snowflake: a database switch invalidates the schema carried
4020                                        // over from the PREVIOUS database. Leaving it attached makes
4021                                        // later unqualified names resolve as <newdb>.<oldschema>, a
4022                                        // pair that need not exist. Clear rather than assign PUBLIC:
4023                                        // that the old schema is invalid is provable from the
4024                                        // statement, that PUBLIC exists in the new database is not.
4025                                        // Matches TDDLSQLEnv.analyzeUseDatabase and
4026                                        // DatabaseContextTracker.processUseDatabase. See GitHub #715.
4027                                        //
4028                                        // The clearing happens HERE, in the statement-by-statement
4029                                        // analysis, and NOT in TUseDatabase.doParseStatement: the whole
4030                                        // batch is parsed before any of it is analyzed and one TSQLEnv
4031                                        // is shared across it, so a parse-time clear would also apply
4032                                        // to the statements BEFORE the switch and would downgrade their
4033                                        // already-exact names to unqualified ones. The env fallback is
4034                                        // handled per statement instead - see
4035                                        // ModelBindingManager.setSchemaUnsetStatements.
4036                                        if (stmt.dbvendor == EDbVendor.dbvsnowflake
4037                                                        && !((TUseDatabase) stmt).isSchema()) {
4038                                                ModelBindingManager.removeGlobalSchema();
4039                                        }
4040                                        ModelBindingManager.setGlobalDatabase(((TUseDatabase) stmt).getDatabaseName().toString());
4041                                }
4042                        } else if (stmt instanceof TUseSchema) {
4043                                if (((TUseSchema) stmt).getSchemaName() != null) {
4044                                        String schemaName = ((TUseSchema) stmt).getSchemaName().toString();
4045                                        List<String> splits = SQLUtil.parseNames(schemaName);
4046                                        if (splits.size() == 1) {
4047                                                ModelBindingManager.setGlobalSchema(schemaName);
4048                                        } else if (splits.size() > 1) {
4049                                                ModelBindingManager.setGlobalSchema(splits.get(splits.size() - 1));
4050                                                ModelBindingManager.setGlobalDatabase(splits.get(splits.size() - 2));
4051                                        }
4052                                }
4053                        } else if (stmt instanceof TPlsqlRecordTypeDefStmt) {
4054                                this.stmtStack.push(stmt);
4055                                this.analyzePlsqlRecordTypeDefStmt((TPlsqlRecordTypeDefStmt) stmt);
4056                                this.stmtStack.pop();
4057                        } else if (stmt instanceof TPlsqlCreateType_Placeholder) {
4058                                this.stmtStack.push(stmt);
4059                                TPlsqlCreateType_Placeholder placeholder = (TPlsqlCreateType_Placeholder) stmt;
4060                                if (placeholder.getObjectStatement() != null && pipelinedAnalyzer != null) {
4061                                        this.pipelinedAnalyzer.indexObjectType(placeholder.getObjectStatement());
4062                                }
4063                                if (placeholder.getNestedTableStatement() != null) {
4064                                        if (pipelinedAnalyzer != null) {
4065                                                this.pipelinedAnalyzer.indexCollectionType(placeholder.getNestedTableStatement());
4066                                        }
4067                                        this.analyzePlsqlTableTypeDefStmt(placeholder.getNestedTableStatement());
4068                                }
4069                                this.stmtStack.pop();
4070                        } else if (stmt instanceof TPlsqlCreateType) {
4071                                this.stmtStack.push(stmt);
4072                                if (pipelinedAnalyzer != null) {
4073                                        this.pipelinedAnalyzer.indexObjectType((TPlsqlCreateType) stmt);
4074                                }
4075                                this.stmtStack.pop();
4076                        } else if (stmt instanceof TPlsqlTableTypeDefStmt) {
4077                                this.stmtStack.push(stmt);
4078                                this.analyzePlsqlTableTypeDefStmt((TPlsqlTableTypeDefStmt) stmt);
4079                                if (pipelinedAnalyzer != null) {
4080                                        this.pipelinedAnalyzer.indexCollectionType((TPlsqlTableTypeDefStmt) stmt);
4081                                }
4082                                this.stmtStack.pop();
4083                        } else if (stmt instanceof TStoredProcedureSqlStatement) {
4084                                this.stmtStack.push(stmt);
4085                                this.analyzeStoredProcedureStmt((TStoredProcedureSqlStatement) stmt);
4086                                this.stmtStack.pop();
4087                        } else if (stmt instanceof TCreateTableSqlStatement) {
4088                                stmtStack.push(stmt);
4089                                analyzeCreateTableStmt((TCreateTableSqlStatement) stmt);
4090                                stmtStack.pop();
4091                        } else if (stmt instanceof TCreateStageStmt) {
4092                                stmtStack.push(stmt);
4093                                analyzeCreateStageStmt((TCreateStageStmt) stmt);
4094                                stmtStack.pop();
4095                        } else if (stmt instanceof TCreateExternalDataSourceStmt) {
4096                                stmtStack.push(stmt);
4097                                analyzeCreateExternalDataSourceStmt((TCreateExternalDataSourceStmt) stmt);
4098                                stmtStack.pop();
4099                        } else if (stmt instanceof TCreateStreamStmt) {
4100                                stmtStack.push(stmt);
4101                                analyzeCreateStreamStmt((TCreateStreamStmt) stmt);
4102                                stmtStack.pop();
4103                        } else if (stmt instanceof TSelectSqlStatement) {
4104                                analyzeSelectStmt((TSelectSqlStatement) stmt);
4105                        } else if (stmt instanceof TDropTableSqlStatement) {
4106                                stmtStack.push(stmt);
4107                                analyzeDropTableStmt((TDropTableSqlStatement) stmt);
4108                                stmtStack.pop();
4109                        } else if (stmt instanceof TTruncateStatement) {
4110                                stmtStack.push(stmt);
4111                                analyzeTruncateTableStmt((TTruncateStatement) stmt);
4112                                stmtStack.pop();
4113                        } else if (stmt instanceof TCreateMaterializedSqlStatement) {
4114                                stmtStack.push(stmt);
4115                                TCreateMaterializedSqlStatement view = (TCreateMaterializedSqlStatement) stmt;
4116                                analyzeCreateViewStmt(view, view.getSubquery(), view.getViewAliasClause(), view.getViewName());
4117                                stmtStack.pop();
4118                        } else if (stmt instanceof TCreateViewSqlStatement) {
4119                                stmtStack.push(stmt);
4120                                TCreateViewSqlStatement view = (TCreateViewSqlStatement) stmt;
4121                                analyzeCreateViewStmt(view, view.getSubquery(), view.getViewAliasClause(), view.getViewName());
4122                                stmtStack.pop();
4123                        } else if(stmt instanceof TDb2SqlVariableDeclaration){
4124                                stmtStack.push(stmt);
4125                                analyzeDb2Declare((TDb2SqlVariableDeclaration)stmt);
4126                                stmtStack.pop();
4127                        } else if (stmt instanceof TMssqlCreateType) {
4128                                stmtStack.push(stmt);
4129                                TMssqlCreateType createType = (TMssqlCreateType) stmt;
4130                                analyzeMssqlCreateType(createType);
4131                                stmtStack.pop();
4132                        } else if (stmt instanceof TMssqlDeclare) {
4133                                stmtStack.push(stmt);
4134                                TMssqlDeclare declare = (TMssqlDeclare) stmt;
4135                                analyzeMssqlDeclare(declare);
4136                                stmtStack.pop();
4137                        } else if (stmt instanceof TInsertSqlStatement) {
4138                                stmtStack.push(stmt);
4139                                TInsertSqlStatement insert = (TInsertSqlStatement)stmt;
4140                                analyzeInsertStmt(insert);
4141                                if(insert.getMultiInsertStatements()!=null) {
4142                                        for(int i=0;i<insert.getMultiInsertStatements().size();i++) {
4143                                                analyzeInsertStmt(insert.getMultiInsertStatements().get(i));
4144                                        }
4145                                }
4146                                stmtStack.pop();
4147                        } else if (stmt instanceof TRedshiftCopy) {
4148                                stmtStack.push(stmt);
4149                                analyzeRedshiftCopyStmt((TRedshiftCopy) stmt);
4150                                stmtStack.pop();
4151                        } else if (stmt instanceof TSnowflakeCopyIntoStmt) {
4152                                stmtStack.push(stmt);
4153                                analyzeCopyIntoStmt((TSnowflakeCopyIntoStmt) stmt);
4154                                stmtStack.pop();
4155                        } else if (stmt instanceof TUnloadStmt) {
4156                                stmtStack.push(stmt);
4157                                analyzeUnloadStmt((TUnloadStmt) stmt);
4158                                stmtStack.pop();
4159                        } else if (stmt instanceof TUpdateSqlStatement) {
4160                                stmtStack.push(stmt);
4161                                analyzeUpdateStmt((TUpdateSqlStatement) stmt);
4162                                stmtStack.pop();
4163                        } else if (stmt instanceof TMergeSqlStatement) {
4164                                stmtStack.push(stmt);
4165                                analyzeMergeStmt((TMergeSqlStatement) stmt);
4166                                stmtStack.pop();
4167                        } else if (stmt instanceof TDeleteSqlStatement) {
4168                                stmtStack.push(stmt);
4169                                analyzeDeleteStmt((TDeleteSqlStatement) stmt);
4170                                stmtStack.pop();
4171                        } else if (stmt instanceof TCursorDeclStmt) {
4172                                stmtStack.push(stmt);
4173                                analyzeCursorDeclStmt((TCursorDeclStmt) stmt);
4174                                stmtStack.pop();
4175                        } else if (stmt instanceof TFetchStmt) {
4176                                stmtStack.push(stmt);
4177                                analyzeFetchStmt((TFetchStmt) stmt);
4178                                stmtStack.pop();
4179                        } else if (stmt instanceof TMssqlFetch) {
4180                                stmtStack.push(stmt);
4181                                analyzeFetchStmt((TMssqlFetch) stmt);
4182                                stmtStack.pop();
4183                        } else if (stmt instanceof TForStmt) {
4184                                stmtStack.push(stmt);
4185                                analyzeForStmt((TForStmt) stmt);
4186                                stmtStack.pop();
4187                        } else if (stmt instanceof TOpenforStmt) {
4188                                stmtStack.push(stmt);
4189                                analyzeOpenForStmt((TOpenforStmt) stmt);
4190                                stmtStack.pop();
4191                        } else if (stmt instanceof TLoopStmt) {
4192                                stmtStack.push(stmt);
4193                                analyzeLoopStmt((TLoopStmt) stmt);
4194                                stmtStack.pop();
4195                        } else if (stmt instanceof TAssignStmt) {
4196                                stmtStack.push(stmt);
4197                                analyzeAssignStmt((TAssignStmt) stmt);
4198                                stmtStack.pop();
4199                        } else if (stmt instanceof TSetStmt) {
4200                                stmtStack.push(stmt);
4201                                analyzeSetStmt((TSetStmt) stmt);
4202                                stmtStack.pop();
4203                        } else if (stmt instanceof TMssqlSet) {
4204                                stmtStack.push(stmt);
4205                                analyzeMssqlSetStmt((TMssqlSet) stmt);
4206                                stmtStack.pop();
4207                        } else if (stmt instanceof TVarDeclStmt) {
4208                                stmtStack.push(stmt);
4209                                analyzeVarDeclStmt((TVarDeclStmt) stmt);
4210                                stmtStack.pop();
4211                        } else if (stmt instanceof TCreateDatabaseSqlStatement) {
4212                                stmtStack.push(stmt);
4213                                analyzeCloneDatabaseStmt((TCreateDatabaseSqlStatement) stmt);
4214                                stmtStack.pop();
4215                        } else if (stmt instanceof TCreateSchemaSqlStatement) {
4216                                stmtStack.push(stmt);
4217                                analyzeCloneSchemaStmt((TCreateSchemaSqlStatement) stmt);
4218                                stmtStack.pop();
4219                        } else if (stmt instanceof TAlterTableStatement) {
4220                                stmtStack.push(stmt);
4221                                analyzeAlterTableStmt((TAlterTableStatement) stmt);
4222                                stmtStack.pop();
4223                        } else if (stmt instanceof TAlterViewStatement) {
4224                                stmtStack.push(stmt);
4225                                analyzeAlterViewStmt((TAlterViewStatement) stmt);
4226                                stmtStack.pop();
4227                        } else if (stmt instanceof TRenameStmt) {
4228                                stmtStack.push(stmt);
4229                                analyzeRenameStmt((TRenameStmt) stmt);
4230                                stmtStack.pop();
4231                        } else if (stmt instanceof TCreateSynonymStmt) {
4232                                stmtStack.push(stmt);
4233                                analyzeCreateSynonymStmt((TCreateSynonymStmt) stmt);
4234                                stmtStack.pop();
4235                        } else if (stmt instanceof TLoadDataStmt) {
4236                                stmtStack.push(stmt);
4237                                analyzeLoadDataStmt((TLoadDataStmt) stmt);
4238                                stmtStack.pop();
4239                        } else if (stmt instanceof THiveLoad) {
4240                                stmtStack.push(stmt);
4241                                analyzeHiveLoadStmt((THiveLoad) stmt);
4242                                stmtStack.pop();
4243                        } else if (stmt instanceof TDb2ReturnStmt) {
4244                                stmtStack.push(stmt);
4245                                analyzeDb2ReturnStmt((TDb2ReturnStmt) stmt);
4246                                stmtStack.pop();
4247                        } else if (stmt instanceof TReturnStmt) {
4248                                stmtStack.push(stmt);
4249                                analyzeReturnStmt((TReturnStmt) stmt);
4250                                stmtStack.pop();
4251                        } else if (stmt instanceof TMssqlReturn) {
4252                                stmtStack.push(stmt);
4253                                analyzeMssqlReturnStmt((TMssqlReturn) stmt);
4254                                stmtStack.pop();
4255                        } else if (stmt instanceof TMssqlUpdateText) {
4256                                stmtStack.push(stmt);
4257                                analyzeMssqlUpdateTextStmt((TMssqlUpdateText) stmt);
4258                                stmtStack.pop();
4259                        } else if (stmt instanceof TMssqlWriteText) {
4260                                stmtStack.push(stmt);
4261                                analyzeMssqlWriteTextStmt((TMssqlWriteText) stmt);
4262                                stmtStack.pop();
4263                        } else if (stmt instanceof TMssqlReadText) {
4264                                stmtStack.push(stmt);
4265                                analyzeMssqlReadTextStmt((TMssqlReadText) stmt);
4266                                stmtStack.pop();
4267                        } else if (stmt instanceof TExecuteSqlStatement) {
4268                                String sqlText = ((TExecuteSqlStatement) stmt).getPreparedSqlText();
4269                                if(sqlText == null) {
4270                                        sqlText = ((TExecuteSqlStatement) stmt).getSqlText();
4271                                }
4272                                if (!option.isAnalyzeDynamicSql()) {
4273                                        if (sqlText != null
4274                                                        || ((TExecuteSqlStatement) stmt).getStmtString() != null) {
4275                                                modelManager.collectDynamicSqlHash(stmt);
4276                                                recordDynamicSqlSite(stmt, DynamicSqlSite.Kind.OTHER,
4277                                                                DynamicSqlSite.Status.UNRESOLVED,
4278                                                                "dynamic SQL inner analysis disabled");
4279                                        }
4280                                } else if (sqlText != null) {
4281                                        modelManager.collectDynamicSqlHash(stmt);
4282                                        TGSqlParser sqlparser = createSqlParser(option.getVendor());
4283                                        sqlparser.sqltext = SQLUtil.trimColumnStringQuote(sqlText);
4284                                        int result = sqlparser.parse();
4285                                        if (result != 0) {
4286                                                errors = sqlparser.getSyntaxErrors();
4287                                                if (errors != null && !errors.isEmpty()) {
4288                                                        for (int i = 0; i < errors.size(); i++) {
4289                                                                TSyntaxError error = errors.get(i);
4290                                                                ErrorInfo errorInfo = new ErrorInfo();
4291                                                                errorInfo.setErrorType(ErrorInfo.SYNTAX_ERROR);
4292                                                                errorInfo.setErrorMessage(getErrorMessage(error, ErrorInfo.SYNTAX_ERROR));
4293                                                                errorInfo.setStartPosition(new Pair3<Long, Long, String>(error.lineNo, error.columnNo,
4294                                                                                ModelBindingManager.getGlobalHash()));
4295                                                                String[] segments = error.tokentext.split("\n");
4296                                                                if (segments.length == 1) {
4297                                                                        errorInfo.setEndPosition(new Pair3<Long, Long, String>(error.lineNo,
4298                                                                                        error.columnNo + error.tokentext.length(),
4299                                                                                        ModelBindingManager.getGlobalHash()));
4300                                                                } else {
4301                                                                        errorInfo.setEndPosition(
4302                                                                                        new Pair3<Long, Long, String>(error.lineNo + segments.length - 1,
4303                                                                                                        (long) segments[segments.length - 1].length() + 1,
4304                                                                                                        ModelBindingManager.getGlobalHash()));
4305                                                                }
4306                                                                errorInfo.fillInfo(this);
4307                                                                errorInfos.add(errorInfo);
4308                                                        }
4309                                                }
4310                                        } else if (sqlparser.sqlstatements != null) {
4311                                                dynamicFoldDepth++;
4312                                                try {
4313                                                        for (int i = 0; i < sqlparser.sqlstatements.size(); i++) {
4314                                                                analyzeCustomSqlStmt(sqlparser.sqlstatements.get(i));
4315                                                        }
4316                                                } finally {
4317                                                        dynamicFoldDepth--;
4318                                                }
4319                                        }
4320                                }
4321                                else if (((TExecuteSqlStatement) stmt).getStmtString() != null) {
4322                                        modelManager.collectDynamicSqlHash(stmt);
4323                                }
4324                        } else if (stmt instanceof TMssqlExecute) {
4325                                TMssqlExecute executeStmt = (TMssqlExecute)stmt;
4326                                if (!option.isAnalyzeDynamicSql()
4327                                                && expandsDynamicSql(executeStmt)) {
4328                                        modelManager.collectDynamicSqlHash(stmt);
4329                                        recordDynamicSqlSite(stmt, dynamicKindOf(executeStmt),
4330                                                        DynamicSqlSite.Status.UNRESOLVED,
4331                                                        "dynamic SQL inner analysis disabled");
4332                                } else if (executeStmt.getSqlText() != null) {
4333                                        modelManager.collectDynamicSqlHash(stmt);
4334                                        boolean evalHandled = false;
4335                                        DynamicSqlTrustMode trustMode = option.getDynamicSqlTrustMode();
4336                                        boolean nonLiteralDynamicArg = !isLiteralDynamicArg(executeStmt);
4337                                        gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite evalSite = null;
4338                                        if (nonLiteralDynamicArg) {
4339                                                evalSite = evaluatedDynamicSqlFor(stmt);
4340                                                // Abstract evaluation fully materialized one or more strings
4341                                                // (constant relation rows and/or literal call bindings). Analyze
4342                                                // every distinct concrete variant in stable evidence order.
4343                                                evalHandled = analyzeEvaluatedDynamicSqlVariants(stmt,
4344                                                                dynamicKindOf(executeStmt), true);
4345                                        }
4346                                        boolean evaluatorProofUnavailable = nonLiteralDynamicArg
4347                                                        && (evalSite == null || evalSite.sqlText == null
4348                                                                        || evalSite.provenanceIncomplete);
4349                                        boolean provenanceIncomplete = trustMode == DynamicSqlTrustMode.SHADOW
4350                                                        && evaluatorProofUnavailable;
4351                                        // Hole evidence is a property of the ARGUMENT's provenance as the
4352                                        // evaluator saw it (which variable, assigned from what, where), not
4353                                        // of the text analyzed below - the evaluator and the legacy fold
4354                                        // render the same expression differently (@DeleteWhere vs a dropped
4355                                        // tail), but its holes are the same holes. Hole SPANS inside the
4356                                        // analyzed text remain text-gated (markDynamicTemplateEndpoints).
4357                                        // Whenever the evaluator saw the node its verdict stands, including
4358                                        // "not known" (-1). Only without an evaluator view (no enclosing
4359                                        // procedure) does the legacy rule apply: SHADOW says unknown, LEGACY
4360                                        // keeps its historical zero.
4361                                        DynamicHoleEvidence holeEvidence = dynamicHoleEvidence(evalSite);
4362                                        int trustHoleCount = evalSite != null ? holeEvidence.count
4363                                                        : (provenanceIncomplete ? -1 : 0);
4364                                        boolean trustHoleCountExact = evalSite != null ? holeEvidence.exact
4365                                                        : !provenanceIncomplete;
4366                                        if (!evalHandled) {
4367                                        TGSqlParser sqlparser = createSqlParser(option.getVendor());
4368                                        sqlparser.sqltext = ((TMssqlExecute) stmt).getSqlText();
4369                                        Set<Relationship> shadowBefore = provenanceIncomplete
4370                                                        && trustMode == DynamicSqlTrustMode.SHADOW
4371                                                        ? snapshotDynamicRelationships() : null;
4372                                        int relsBeforeDynamic = modelManager.getRelations().length;
4373                                        int dynamicSitesBefore = dynamicSqlSites.size();
4374                                        // A data-driven argument (variable / expression) is at most PARTIALLY folded:
4375                                        // GSP keeps the literal pieces but drops the unresolved trailing @var, leaving a
4376                                        // legitimately truncated fragment (e.g. "... WHERE " with no predicate -> end of
4377                                        // input). A parse failure on such text says nothing about the real SQL, so we must
4378                                        // not surface it as a unit SYNTAX_ERROR — that would discard the whole procedure's
4379                                        // lineage even though the outer parse succeeded. Only a true compile-time literal
4380                                        // EXEC('...') surfaces inner parse failures (mirrors the EXECUTE IMMEDIATE
4381                                        // isDynamicSQLPartial() guard below).
4382                                        boolean literalArg = isLiteralDynamicArg(executeStmt);
4383                                        // A partial fold leaves the unresolved @var / @param remnant in the folded
4384                                        // text. Decided BEFORE the inner analysis so the endpoints this site
4385                                        // introduces can be told apart from the ones that already existed, and so
4386                                        // a fully-resolved fold pays nothing (snapshot returns null).
4387                                        boolean maybePartialFold = !literalArg
4388                                                        && ((TMssqlExecute) stmt).getSqlText().matches("(?s).*@\\w+.*");
4389                                        // The conservative @\w+ test is not the only way a hole can be present:
4390                                        // the evaluator may know this fold has value-unknown holes whose legacy
4391                                        // rendering carries no '@' (UPPER(@X) folds to the bare function name).
4392                                        // Snapshot whenever EITHER tier could fire, so provenance is never wasted.
4393                                        Set<Table> templateTablesBefore = snapshotTemplateCandidateTables(maybePartialFold
4394                                                        || hasValueUnknownHole(evalSite));
4395                                        boolean foldStatementsAnalyzed = false;
4396                                        int result = sqlparser.parse();
4397                                        if (result != 0) {
4398                                                errors = literalArg ? sqlparser.getSyntaxErrors() : null;
4399                                                if (errors != null && !errors.isEmpty()) {
4400                                                        for (int i = 0; i < errors.size(); i++) {
4401                                                                TSyntaxError error = errors.get(i);
4402                                                                ErrorInfo errorInfo = new ErrorInfo();
4403                                                                errorInfo.setErrorType(ErrorInfo.SYNTAX_ERROR);
4404                                                                errorInfo.setErrorMessage(getErrorMessage(error, ErrorInfo.SYNTAX_ERROR));
4405                                                                errorInfo.setStartPosition(new Pair3<Long, Long, String>(error.lineNo, error.columnNo,
4406                                                                                ModelBindingManager.getGlobalHash()));
4407                                                                String[] segments = error.tokentext.split("\n");
4408                                                                if (segments.length == 1) {
4409                                                                        errorInfo.setEndPosition(new Pair3<Long, Long, String>(error.lineNo,
4410                                                                                        error.columnNo + error.tokentext.length(),
4411                                                                                        ModelBindingManager.getGlobalHash()));
4412                                                                } else {
4413                                                                        errorInfo.setEndPosition(
4414                                                                                        new Pair3<Long, Long, String>(error.lineNo + segments.length - 1,
4415                                                                                                        (long) segments[segments.length - 1].length() + 1,
4416                                                                                                        ModelBindingManager.getGlobalHash()));
4417                                                                }
4418                                                                errorInfo.fillInfo(this);
4419                                                                errorInfos.add(errorInfo);
4420                                                        }
4421                                                }
4422                                        } else if (sqlparser.sqlstatements != null) {
4423                                                foldStatementsAnalyzed = true;
4424                                                dynamicFoldDepth++;
4425                                                try {
4426                                                        for (int i = 0; i < sqlparser.sqlstatements.size(); i++) {
4427                                                                analyzeCustomSqlStmt(sqlparser.sqlstatements.get(i));
4428                                                        }
4429                                                } finally {
4430                                                        dynamicFoldDepth--;
4431                                                }
4432                                        }
4433                                        // getSqlText()!=null means GSP folded the dynamic text. RESOLVED only when the argument
4434                                        // was a compile-time literal AND it parsed; a variable/expression argument is data-driven
4435                                        // (often only partially folded, e.g. an unresolved @parameter left in) -> UNRESOLVED.
4436                                        // NOTE: a variable
4437                                        // argument whose folded text fails to parse deliberately stays UNRESOLVED,
4438                                        // not PARSE_ERROR — the fold is flow-insensitive (the variable may hold a
4439                                        // different value at runtime), so a parse failure on folded text says
4440                                        // nothing definitive about the real SQL. Only a compile-time literal
4441                                        // argument earns PARSE_ERROR.
4442                                        // Land the placeholder verdict on the model NODES, not only on the site
4443                                        // status: exact hole spans when the evaluator described this very text,
4444                                        // literal-aligned spans when it described the same fold's other rendering,
4445                                        // otherwise the conservative @\w+ scan of the legacy fold.
4446                                        if (foldStatementsAnalyzed) {
4447                                                markDynamicTemplateEndpoints(templateTablesBefore, evalSite, sqlparser);
4448                                        }
4449                                        int[] dynCounts = classifyDynamicSiteLineage(relsBeforeDynamic, maybePartialFold);
4450                                        int unprovenCount = countNewDynamicRelationships(shadowBefore);
4451                                        if (!literalArg) {
4452                                                // Truthful status: a fold that yielded clean lineage into base tables
4453                                                // — or a pure dynamic SELECT whose only product is its projection —
4454                                                // is RESOLVED (with counts). Placeholder-contaminated, partial-target,
4455                                                // or empty outcomes stay UNRESOLVED with an accurate reason.
4456                                                boolean usable = dynCounts[2] == 0
4457                                                                && (dynCounts[0] > 0 || (dynCounts[1] == 0 && dynCounts[3] > 0));
4458                                                if (provenanceIncomplete && trustMode == DynamicSqlTrustMode.SHADOW) {
4459                                                        recordDynamicSqlSite(stmt, dynamicKindOf(executeStmt),
4460                                                                        unprovenCount > 0 ? DynamicSqlSite.Status.PARTIAL
4461                                                                                        : DynamicSqlSite.Status.UNRESOLVED,
4462                                                                dynamicTrustReason("SHADOW retained legacy publication behavior",
4463                                                                                unprovenCount, trustHoleCount, trustHoleCountExact),
4464                                                                dynCounts[0], dynCounts[1], unprovenCount,
4465                                                                trustHoleCount, trustHoleCountExact, trustMode,
4466                                                                relsBeforeDynamic, dynamicSitesBefore, null, null,
4467                                                                holeEvidence.fragments, holeEvidence.materializedText);
4468                                                } else {
4469                                                        recordDynamicSqlSite(stmt, dynamicKindOf(executeStmt),
4470                                                                        usable ? DynamicSqlSite.Status.RESOLVED : DynamicSqlSite.Status.UNRESOLVED,
4471                                                                        usable ? null
4472                                                                                        : (dynCounts[2] != 0
4473                                                                                                        ? "dynamic SQL references a runtime-built object name; lineage source is a placeholder"
4474                                                                                                        : dynamicReasonOf(executeStmt)),
4475                                                                        dynCounts[0], dynCounts[1], 0,
4476                                                                        trustHoleCount, trustHoleCountExact, trustMode,
4477                                                                        relsBeforeDynamic, dynamicSitesBefore, null, null,
4478                                                                        holeEvidence.fragments, holeEvidence.materializedText);
4479                                                }
4480                                        } else {
4481                                                recordDynamicSqlSite(stmt, dynamicKindOf(executeStmt),
4482                                                                result != 0 ? DynamicSqlSite.Status.PARSE_ERROR : DynamicSqlSite.Status.RESOLVED,
4483                                                                result != 0 ? "inner dynamic SQL failed to parse" : null,
4484                                                                dynCounts[0], dynCounts[1], 0, 0, true, trustMode,
4485                                                                relsBeforeDynamic, dynamicSitesBefore, null, null,
4486                                                                Collections.<UnresolvedFragment>emptyList(), executeStmt.getSqlText());
4487                                        }
4488                                        }
4489                                } else if (executeStmt.getModuleName() != null) {
4490                                        // getSqlText()==null here: a folded literal would have set sqlText. So this is either an
4491                                        // opaque sp_executesql (data-driven argument), a dynamic proc name (EXEC @var), or an
4492                                        // ordinary static procedure call. Record the dynamic ones; leave static EXEC proc alone.
4493                                        boolean evalHandledSite = false;
4494                                        if (isSpExecutesql(executeStmt.getModuleName())) {
4495                                                evalHandledSite = analyzeEvaluatedDynamicSqlVariants(stmt,
4496                                                                DynamicSqlSite.Kind.SP_EXECUTESQL, false);
4497                                                if (!evalHandledSite) {
4498                                                        recordUnresolvedDynamicSqlSite(stmt, executeStmt,
4499                                                                        DynamicSqlSite.Kind.SP_EXECUTESQL);
4500                                                }
4501                                        } else if (executeStmt.getModuleName().toString().startsWith("@")) {
4502                                                // EXEC @var is SQL Server's @module_name_var form: the variable
4503                                                // holds a PROCEDURE NAME, not SQL text (parenthesized EXEC(@var)
4504                                                // arrives as metExecStringCmd below) - never evaluate it as SQL.
4505                                                recordDynamicSqlSite(stmt, DynamicSqlSite.Kind.OTHER,
4506                                                                DynamicSqlSite.Status.UNRESOLVED, "EXEC target procedure name is a runtime variable");
4507                                        }
4508                                        if (!evalHandledSite) {
4509                                                // When evaluation already analyzed the materialized batch, the
4510                                                // legacy path would re-execute the same argument and duplicate
4511                                                // every process/edge - skip it.
4512                                                stmtStack.push(stmt);
4513                                                analyzeMssqlExecute(executeStmt);
4514                                                stmtStack.pop();
4515                                        }
4516                                } else if (executeStmt.getExecType() == TBaseType.metExecStringCmd) {
4517                                        // EXEC(<expr>) whose string was not folded: try the no-bindings abstract
4518                                        // evaluation (template/constant-built strings); otherwise honestly UNRESOLVED.
4519                                        if (!analyzeEvaluatedDynamicSqlVariants(stmt,
4520                                                        DynamicSqlSite.Kind.EXEC_STRING, false)) {
4521                                                recordUnresolvedDynamicSqlSite(stmt, executeStmt,
4522                                                                DynamicSqlSite.Kind.EXEC_STRING);
4523                                        }
4524                                }
4525                        } else if (stmt instanceof TExecImmeStmt) {
4526                                TExecImmeStmt execImmeStmt = (TExecImmeStmt) stmt;
4527                                modelManager.collectDynamicSqlHash(stmt);
4528                                if (!option.isAnalyzeDynamicSql()) {
4529                                        recordDynamicSqlSite(stmt,
4530                                                        DynamicSqlSite.Kind.EXECUTE_IMMEDIATE,
4531                                                        DynamicSqlSite.Status.UNRESOLVED,
4532                                                        "dynamic SQL inner analysis disabled");
4533                                        return;
4534                                }
4535                                synchronized (DataFlowAnalyzer.class) {
4536                                        int relsBeforeDynamic = modelManager.getRelations().length;
4537                                        int dynamicSitesBefore = dynamicSqlSites.size();
4538                                        boolean analyzedInner = false;
4539                                        boolean innerParseFailed = false;
4540                                        // PL/SQL call-site binding (plan Phase 5): when the token fold left
4541                                        // this site partial or empty, interpret the file's statements (plus
4542                                        // literal callers harvested from other files of a multi-file
4543                                        // analysis) so parameter-dependent templates can fold fully.
4544                                        // getDynamicStatements() below then reads the evaluated text.
4545                                        maybeEvaluatePlsqlDynamicSql(execImmeStmt);
4546                                        DynamicSqlTrustMode trustMode = option.getDynamicSqlTrustMode();
4547                                        boolean legacyPartialFold = execImmeStmt.isDynamicSQLPartial();
4548                                        boolean shadowIncomplete = legacyPartialFold
4549                                                        && execImmeStmt.getEvaluatedDynamicSQLs().isEmpty();
4550                                        Set<Relationship> shadowBefore = shadowIncomplete
4551                                                        && trustMode == DynamicSqlTrustMode.SHADOW
4552                                                        ? snapshotDynamicRelationships() : null;
4553                                        TStatementList stmts = execImmeStmt.getDynamicStatements();
4554                                        if (stmts != null && stmts.size() > 0) {
4555                                                dynamicFoldDepth++;
4556                                                try {
4557                                                        for (int i = 0; i < stmts.size(); i++) {
4558                                                                analyzeCustomSqlStmt(stmts.get(i));
4559                                                        }
4560                                                } finally {
4561                                                        dynamicFoldDepth--;
4562                                                }
4563                                                analyzedInner = true;
4564                                        }
4565
4566                                        // Only re-parse the raw dynamic SQL when getDynamicStatements()
4567                                        // produced nothing. getDynamicStatements() already parses and
4568                                        // analyzes the fragment with coordinates remapped back to the
4569                                        // original file position; re-parsing it here with a fresh parser
4570                                        // would analyze the same SQL a second time and emit duplicate
4571                                        // hints/lineage carrying fragment-relative (un-remapped)
4572                                        // coordinates. This fallback still covers the case where
4573                                        // getDynamicStatements() failed (e.g. dynamic SQL syntax error),
4574                                        // reporting the parse error below.
4575                                        String dynamicSql = (stmts == null || stmts.size() == 0) ? execImmeStmt.getDynamicSQL() : null;
4576                                        if (!SQLUtil.isEmpty(dynamicSql)) {
4577                                                TGSqlParser sqlparser = createSqlParser(option.getVendor());
4578                                                sqlparser.sqltext = dynamicSql;
4579                                                int result = sqlparser.parse();
4580                                                        // This fresh parse runs the dynamic SQL un-padded, so its
4581                                                        // coordinates are relative to the fragment (first char at
4582                                                        // line 1). Shift them back to the position the fragment
4583                                                        // occupies in the original file, mirroring the shift
4584                                                        // TExecImmeStmt.getDynamicStatements() applies; otherwise
4585                                                        // fallback parse errors for EXECUTE IMMEDIATE text located
4586                                                        // later in the file would report fragment-relative lines.
4587                                                        if (execImmeStmt.getDynamicStringExpr() != null
4588                                                                        && execImmeStmt.getDynamicStringExpr().getPlainTextLineNo() != -1) {
4589                                                                int deltaLine = (int) execImmeStmt.getDynamicStringExpr().getPlainTextLineNo() - 1;
4590                                                                int deltaColumn = (int) execImmeStmt.getDynamicStringExpr().getPlainTextColumnNo();
4591                                                                if ((deltaLine != 0 || deltaColumn != 0) && sqlparser.getSyntaxErrors() != null) {
4592                                                                        for (int ei = 0; ei < sqlparser.getSyntaxErrors().size(); ei++) {
4593                                                                                TSyntaxError dynErr = sqlparser.getSyntaxErrors().get(ei);
4594                                                                                if (dynErr == null) continue;
4595                                                                                boolean onFirstLine = (dynErr.lineNo == 1);
4596                                                                                dynErr.lineNo += deltaLine;
4597                                                                                if (onFirstLine) dynErr.columnNo += deltaColumn;
4598                                                                        }
4599                                                                }
4600                                                        }
4601                                                if (result != 0) {
4602                                                        innerParseFailed = true;
4603                                                        // a partially resolved value contains placeholder identifiers
4604                                                        // for unknowable parts; a parse failure on such text says
4605                                                        // nothing about the real SQL, so don't report it
4606                                                        errors = execImmeStmt.isDynamicSQLPartial() ? null : sqlparser.getSyntaxErrors();
4607                                                        if (errors != null && !errors.isEmpty()) {
4608                                                                for (int i = 0; i < errors.size(); i++) {
4609                                                                        TSyntaxError error = errors.get(i);
4610                                                                        ErrorInfo errorInfo = new ErrorInfo();
4611                                                                        errorInfo.setErrorType(ErrorInfo.SYNTAX_ERROR);
4612                                                                        errorInfo.setErrorMessage(getErrorMessage(error, ErrorInfo.SYNTAX_ERROR));
4613                                                                        errorInfo.setStartPosition(new Pair3<Long, Long, String>(error.lineNo, error.columnNo,
4614                                                                                        ModelBindingManager.getGlobalHash()));
4615                                                                        String[] segments = error.tokentext.split("\n");
4616                                                                        if (segments.length == 1) {
4617                                                                                errorInfo.setEndPosition(new Pair3<Long, Long, String>(error.lineNo,
4618                                                                                                error.columnNo + error.tokentext.length(),
4619                                                                                                ModelBindingManager.getGlobalHash()));
4620                                                                        } else {
4621                                                                                errorInfo.setEndPosition(
4622                                                                                                new Pair3<Long, Long, String>(error.lineNo + segments.length - 1,
4623                                                                                                                (long) segments[segments.length - 1].length() + 1,
4624                                                                                                                ModelBindingManager.getGlobalHash()));
4625                                                                        }
4626                                                                        errorInfo.fillInfo(this);
4627                                                                        errorInfos.add(errorInfo);
4628                                                                }
4629                                                        }
4630                                                } else if (sqlparser.sqlstatements != null) {
4631                                                        dynamicFoldDepth++;
4632                                                        try {
4633                                                                for (int i = 0; i < sqlparser.sqlstatements.size(); i++) {
4634                                                                        analyzeCustomSqlStmt(sqlparser.sqlstatements.get(i));
4635                                                                }
4636                                                        } finally {
4637                                                                dynamicFoldDepth--;
4638                                                        }
4639                                                        analyzedInner = true;
4640                                                }
4641                                        }
4642                                        // Site accounting mirrors the T-SQL EXEC branch: every
4643                                        // EXECUTE IMMEDIATE is a dynamic site; status reflects what
4644                                        // the analyzer actually did with it.
4645                                        boolean partialFold = legacyPartialFold;
4646                                        int[] dynCounts = classifyDynamicSiteLineage(relsBeforeDynamic, partialFold);
4647                                        int unprovenCount = countNewDynamicRelationships(shadowBefore);
4648                                        boolean fullyKnown = analyzedInner && !partialFold;
4649                                        boolean usableLineage = dynCounts[2] == 0
4650                                                        && (dynCounts[0] > 0 || (dynCounts[1] == 0 && dynCounts[3] > 0));
4651                                        if (shadowIncomplete && trustMode == DynamicSqlTrustMode.SHADOW) {
4652                                                DynamicSqlSite.Status status = analyzedInner && unprovenCount > 0
4653                                                                ? DynamicSqlSite.Status.PARTIAL : DynamicSqlSite.Status.UNRESOLVED;
4654                                                recordDynamicSqlSite(stmt, DynamicSqlSite.Kind.EXECUTE_IMMEDIATE, status,
4655                                                                dynamicTrustReason("SHADOW retained legacy publication behavior",
4656                                                                                unprovenCount, -1, false),
4657                                                                dynCounts[0], dynCounts[1], unprovenCount,
4658                                                                -1, false, trustMode,
4659                                                                relsBeforeDynamic, dynamicSitesBefore);
4660                                        } else if (analyzedInner && (fullyKnown || usableLineage)) {
4661                                                recordDynamicSqlSite(stmt, DynamicSqlSite.Kind.EXECUTE_IMMEDIATE,
4662                                                                DynamicSqlSite.Status.RESOLVED, null, dynCounts[0], dynCounts[1],
4663                                                                0, 0, true, trustMode,
4664                                                                relsBeforeDynamic, dynamicSitesBefore);
4665                                        } else if (innerParseFailed && !partialFold) {
4666                                                recordDynamicSqlSite(stmt, DynamicSqlSite.Kind.EXECUTE_IMMEDIATE,
4667                                                                DynamicSqlSite.Status.PARSE_ERROR, "inner dynamic SQL failed to parse",
4668                                                                dynCounts[0], dynCounts[1], 0, 0, true, trustMode,
4669                                                                relsBeforeDynamic, dynamicSitesBefore);
4670                                        } else if (innerParseFailed) {
4671                                                recordDynamicSqlSite(stmt, DynamicSqlSite.Kind.EXECUTE_IMMEDIATE,
4672                                                                DynamicSqlSite.Status.UNRESOLVED,
4673                                                                "partially folded dynamic SQL failed to parse; unresolved parts remain",
4674                                                                dynCounts[0], dynCounts[1], 0, -1, false, trustMode,
4675                                                                relsBeforeDynamic, dynamicSitesBefore);
4676                                        } else if (analyzedInner) {
4677                                                recordDynamicSqlSite(stmt, DynamicSqlSite.Kind.EXECUTE_IMMEDIATE,
4678                                                                DynamicSqlSite.Status.UNRESOLVED,
4679                                                                        dynCounts[2] != 0
4680                                                                                        ? "dynamic SQL references a runtime-built object name; lineage source is a placeholder"
4681                                                                                        : "dynamic SQL only partially folded; no resolved lineage produced",
4682                                                                dynCounts[0], dynCounts[1], 0, -1, false, trustMode,
4683                                                                relsBeforeDynamic, dynamicSitesBefore);
4684                                        } else {
4685                                                boolean runtimeTextUnavailable = !analyzedInner
4686                                                                && execImmeStmt.getDynamicSQL() == null;
4687                                                boolean unresolvedFragments = partialFold || runtimeTextUnavailable;
4688                                                recordDynamicSqlSite(stmt, DynamicSqlSite.Kind.EXECUTE_IMMEDIATE,
4689                                                                DynamicSqlSite.Status.UNRESOLVED,
4690                                                                "dynamic SQL string is a runtime value; no statically-known text to analyze",
4691                                                                dynCounts[0], dynCounts[1], 0,
4692                                                                unresolvedFragments ? -1 : 0, !unresolvedFragments, trustMode,
4693                                                                relsBeforeDynamic, dynamicSitesBefore);
4694                                        }
4695                                }
4696                        } else if (stmt instanceof TCallStatement) {
4697                                TCallStatement callStmt = (TCallStatement)stmt;
4698                                stmtStack.push(stmt);
4699                                analyzeCallStmt(callStmt);
4700                                stmtStack.pop();
4701                        } else if (stmt instanceof TDb2CallStmt) {
4702                                TDb2CallStmt db2CallStmt = (TDb2CallStmt) stmt;
4703                                stmtStack.push(stmt);
4704                                analyzeDb2CallStmt(db2CallStmt);
4705                                stmtStack.pop();
4706                        } else if (stmt instanceof TBasicStmt) {
4707                                TBasicStmt oracleBasicStmt = (TBasicStmt) stmt;
4708                                stmtStack.push(stmt);
4709                                analyzeOracleBasicStmt(oracleBasicStmt);
4710                                stmtStack.pop();
4711                        } else if (stmt instanceof TIfStmt) {
4712                                TIfStmt ifStmt = (TIfStmt) stmt;
4713                                stmtStack.push(stmt);
4714                                analyzeIfStmt(ifStmt);
4715                                stmtStack.pop();
4716                        } else if (stmt instanceof TElsifStmt) {
4717                                TElsifStmt elsIfStmt = (TElsifStmt) stmt;
4718                                stmtStack.push(stmt);
4719                                analyzeElsIfStmt(elsIfStmt);
4720                                stmtStack.pop();
4721                        } else if (stmt instanceof TMssqlIfElse) {
4722                                TMssqlIfElse ifStmt = (TMssqlIfElse) stmt;
4723                                stmtStack.push(stmt);
4724                                analyzeMssqlIfElseStmt(ifStmt);
4725                                stmtStack.pop();
4726                        } else if (stmt.getStatements() != null && stmt.getStatements().size() > 0) {
4727                                for (int i = 0; i < stmt.getStatements().size(); i++) {
4728                                        analyzeCustomSqlStmt(stmt.getStatements().get(i));
4729                                }
4730                        } else if (stmt instanceof TCreateIndexSqlStatement) {
4731                                stmtStack.push(stmt);
4732                                analyzeCreateIndexStageStmt((TCreateIndexSqlStatement) stmt);
4733                                stmtStack.pop();
4734                        } else if (stmt instanceof gudusoft.gsqlparser.stmt.mdx.TMdxSelect) {
4735                                stmtStack.push(stmt);
4736                                analyzeMdxSelectStmt((gudusoft.gsqlparser.stmt.mdx.TMdxSelect) stmt);
4737                                stmtStack.pop();
4738                        } else if (stmt instanceof TPowerQueryDocumentStmt) {
4739                                stmtStack.push(stmt);
4740                                analyzePowerQueryDocumentStmt((TPowerQueryDocumentStmt) stmt);
4741                                stmtStack.pop();
4742                        }
4743                } catch (Exception e) {
4744            StringBuffer errorMessage = new StringBuffer();
4745            errorMessage.append("analyze sql stmt failed, ");
4746            if (stmt.getStartToken() != null) {
4747                errorMessage.append("line: " + stmt.getStartToken().lineNo + ", column: " + stmt.getStartToken().columnNo).append(", ");
4748            }
4749            if (stmt.getGsqlparser() != null && !SQLUtil.isEmpty(stmt.getGsqlparser().sqlfilename)) {
4750                errorMessage.append("file: "+ stmt.getGsqlparser().sqlfilename).append(", ");
4751            }
4752            if (stmt.toString() != null) {
4753                errorMessage.append("sql:\n" + stmt.toString());
4754            }
4755            logger.error(errorMessage.toString(), e);
4756                        ErrorInfo errorInfo = new ErrorInfo();
4757                        errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
4758                        if (e.getMessage() == null) {
4759                                if (e.getStackTrace() != null && e.getStackTrace().length > 0) {
4760                                        errorInfo.setErrorMessage(e.getClass().getSimpleName() + ": " + e.getStackTrace()[0].toString());
4761                                } else {
4762                                        errorInfo.setErrorMessage(e.getClass().getSimpleName());
4763                                }
4764                        } else {
4765                                errorInfo.setErrorMessage(e.getClass().getSimpleName() + ": " + e.getMessage());
4766                        }
4767                        errorInfo.setStartPosition(new Pair3<Long, Long, String>(stmt.getStartToken().lineNo,
4768                                        stmt.getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
4769                        String[] segments = stmt.getEndToken().getAstext().split("\n", -1);
4770                        if (segments.length == 1) {
4771                                errorInfo.setEndPosition(new Pair3<Long, Long, String>(stmt.getEndToken().lineNo,
4772                                                stmt.getEndToken().columnNo + stmt.getEndToken().getAstext().length(),
4773                                                ModelBindingManager.getGlobalHash()));
4774                        } else {
4775                                errorInfo.setEndPosition(new Pair3<Long, Long, String>(stmt.getEndToken().lineNo + segments.length - 1,
4776                                                (long) segments[segments.length - 1].length() + 1, ModelBindingManager.getGlobalHash()));
4777                        }
4778                        errorInfo.fillInfo(this);
4779                        errorInfos.add(errorInfo);
4780                }
4781        }
4782
4783        private void analyzePlsqlRecordTypeDefStmt(TPlsqlRecordTypeDefStmt stmt) {
4784                TObjectName typeName = stmt.getTypeName();
4785                Variable variable = modelFactory.createVariable(typeName);
4786                variable.setSubType(SubType.record_type);
4787        
4788                if (stmt.getFieldDeclarations() != null) {
4789                        for (int i = 0; i < stmt.getFieldDeclarations().size(); i++) {
4790                                TParameterDeclaration param = stmt.getFieldDeclarations().getParameterDeclarationItem(i);
4791                                String dataTypeName = param.getDataType().getDataTypeName();
4792                                TObjectName columnName = param.getParameterName();      
4793                                TableColumn variableProperty = modelFactory.createTableColumn(variable, columnName, true);
4794                                if(dataTypeName.indexOf(".")!=-1) {
4795                                        String tableName = dataTypeName.substring(0, dataTypeName.lastIndexOf("."));
4796                                        Table table = modelFactory.createTableByName(tableName, true);
4797                                        if(table!=null) {
4798                                                TableColumn tableColumn = modelFactory.createInsertTableColumn(table, dataTypeName);
4799                                                if (tableColumn != null) {
4800                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
4801                                                        relation.setEffectType(EffectType.rowtype);
4802                                                        relation.setTarget(new TableColumnRelationshipElement(variableProperty));
4803                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
4804                                                }
4805                                        }
4806                                }
4807                        }
4808                        variable.setDetermined(true);
4809                }
4810                else {
4811                        TObjectName starColumn = new TObjectName();
4812                        starColumn.setString("*");
4813                }
4814        }
4815        
4816        private void analyzePlsqlTableTypeDefStmt(TPlsqlTableTypeDefStmt stmt) {
4817                TTypeName typeName = stmt.getElementDataType();
4818                if (typeName != null && typeName.toString().toUpperCase().indexOf("ROWTYPE") != -1) {
4819                        Variable cursorVariable = modelFactory.createVariable(stmt.getTypeName());
4820                        cursorVariable.setSubType(SubType.record_type);
4821
4822                        Table variableTable = modelFactory.createTableByName(typeName.getDataTypeName(), false);
4823                        if(!variableTable.isCreateTable()) {
4824                                TObjectName starColumn1 = new TObjectName();
4825                                starColumn1.setString("*");
4826                                TableColumn variableTableStarColumn = modelFactory.createTableColumn(variableTable, starColumn1, true);
4827                                variableTableStarColumn.setShowStar(false);
4828                                variableTableStarColumn.setExpandStar(true);
4829
4830                                TObjectName starColumn = new TObjectName();
4831                                starColumn.setString("*");
4832                                TableColumn variableProperty = modelFactory.createTableColumn(cursorVariable, starColumn, true);
4833                                variableProperty.setShowStar(false);
4834                                variableProperty.setExpandStar(true);
4835
4836                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
4837                                dataflowRelation.setEffectType(EffectType.rowtype);
4838                                dataflowRelation.addSource(new TableColumnRelationshipElement(variableTableStarColumn));
4839                                dataflowRelation.setTarget(new TableColumnRelationshipElement(variableProperty));
4840                        } else {
4841                                for (TableColumn sourceColumn : variableTable.getColumns()) {
4842                                        String columnName = sourceColumn.getName();
4843                                        TObjectName targetColumn = new TObjectName();
4844                                        targetColumn.setString(columnName);
4845                                        TableColumn variableProperty = modelFactory.createTableColumn(cursorVariable, targetColumn, true);
4846                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
4847                                        dataflowRelation.setEffectType(EffectType.rowtype);
4848                                        dataflowRelation.addSource(new TableColumnRelationshipElement(sourceColumn));
4849                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(variableProperty));
4850                                }
4851                        }
4852                } 
4853        }
4854
4855        private void analyzeMssqlCreateType(TMssqlCreateType createType) {
4856
4857        }
4858
4859        /**
4860         * UPDATETEXT table.dest_column dest_text_ptr {NULL|offset} {NULL|del_len}
4861         * [WITH LOG] [inserted_data | table.src_column src_text_ptr] (Mantis #4607).
4862         * The written column is a real data-write target: emit inserted_data (variable
4863         * or literal) -&gt; dest column, and for the column-copy form the proven
4864         * src_column -&gt; dest_column edge.
4865         */
4866        private void analyzeMssqlUpdateTextStmt(TMssqlUpdateText stmt) {
4867                TableColumn destColumn = createTextPtrTargetColumn(stmt.getTargetTable(), stmt.getDestColumnName());
4868                if (destColumn == null) {
4869                        return;
4870                }
4871                List<TObjectName> sourceNames = new ArrayList<TObjectName>();
4872                if (stmt.getInsertedVariable() != null) {
4873                        sourceNames.add(stmt.getInsertedVariable());
4874                }
4875                if (stmt.getSourceColumnName() != null && stmt.getCopySourceTable() != null) {
4876                        // make sure the copy-source table has a model so the edge resolves
4877                        modelFactory.createTable(stmt.getCopySourceTable());
4878                        sourceNames.add(stmt.getSourceColumnName());
4879                }
4880                if (!sourceNames.isEmpty()) {
4881                        analyzeDataFlowRelation(destColumn, sourceNames, EffectType.update, null);
4882                }
4883                if (stmt.getInsertedLiteral() != null) {
4884                        List<TParseTreeNode> constants = new ArrayList<TParseTreeNode>();
4885                        constants.add(stmt.getInsertedLiteral());
4886                        analyzeConstantDataFlowRelation(destColumn, constants, EffectType.update, null);
4887                }
4888        }
4889
4890        /** WRITETEXT table.column text_ptr [WITH LOG] data (Mantis #4607): data -&gt; column. */
4891        private void analyzeMssqlWriteTextStmt(TMssqlWriteText stmt) {
4892                TableColumn destColumn = createTextPtrTargetColumn(stmt.getTargetTable(), stmt.getDestColumnName());
4893                if (destColumn == null) {
4894                        return;
4895                }
4896                if (stmt.getWriteDataVariable() != null) {
4897                        List<TObjectName> sourceNames = new ArrayList<TObjectName>();
4898                        sourceNames.add(stmt.getWriteDataVariable());
4899                        analyzeDataFlowRelation(destColumn, sourceNames, EffectType.update, null);
4900                }
4901                if (stmt.getWriteDataLiteral() != null) {
4902                        List<TParseTreeNode> constants = new ArrayList<TParseTreeNode>();
4903                        constants.add(stmt.getWriteDataLiteral());
4904                        analyzeConstantDataFlowRelation(destColumn, constants, EffectType.update, null);
4905                }
4906        }
4907
4908        /**
4909         * READTEXT table.column text_ptr offset size [HOLDLOCK] (Mantis #4607): the
4910         * column is read into the client stream; model the output as the statement's
4911         * result set with the read column so the read is visible in the lineage.
4912         */
4913        private void analyzeMssqlReadTextStmt(TMssqlReadText stmt) {
4914                TObjectName columnRef = stmt.getSourceColumnName();
4915                if (columnRef == null || columnRef.getSourceTable() == null) {
4916                        return;
4917                }
4918                Table tableModel = modelFactory.createTable(columnRef.getSourceTable());
4919                if (tableModel == null) {
4920                        return;
4921                }
4922                TableColumn sourceColumn = modelFactory.createTableColumn(tableModel, columnRef, false);
4923                if (sourceColumn == null) {
4924                        return;
4925                }
4926                ResultSet resultSet = modelFactory.createResultSet(stmt, true);
4927                ResultColumn resultColumn = modelFactory.createResultColumn(resultSet, columnRef);
4928                if (resultColumn == null) {
4929                        return;
4930                }
4931                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
4932                relation.setEffectType(EffectType.select);
4933                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
4934                relation.addSource(new TableColumnRelationshipElement(sourceColumn));
4935        }
4936
4937        /**
4938         * The write target of an UPDATETEXT/WRITETEXT statement: the model column of
4939         * the fused table.column reference, or null when the statement carries no
4940         * table-qualified column.
4941         */
4942        private TableColumn createTextPtrTargetColumn(TTable targetTable, TObjectName columnRef) {
4943                if (targetTable == null || columnRef == null) {
4944                        return null;
4945                }
4946                Table tableModel = modelFactory.createTable(targetTable);
4947                if (tableModel == null) {
4948                        return null;
4949                }
4950                return modelFactory.createTableColumn(tableModel, columnRef, false);
4951        }
4952
4953        private void analyzeMssqlExecute(TMssqlExecute executeStmt) {
4954                if (executeStmt.getModuleName() != null) {
4955                        TObjectName module = executeStmt.getModuleName();
4956                        if(module.toString().toLowerCase().endsWith("sp_rename")) {
4957                                String oldTableName = SQLUtil.trimColumnStringQuote(executeStmt.getParameters().getExecParameter(0).toString());        
4958                                Table oldNameTableModel = modelFactory.createTableByName(oldTableName, true);
4959                                List<String> oldTableNames = SQLUtil.parseNames(oldNameTableModel.getName());
4960                                TObjectName oldStarColumn = new TObjectName();
4961                                oldStarColumn.setString("*");
4962                                TableColumn oldTableStarColumn = modelFactory.createTableColumn(oldNameTableModel, oldStarColumn, true);
4963                                
4964                                String newTableName = SQLUtil.trimColumnStringQuote(executeStmt.getParameters().getExecParameter(1).toString());
4965                                List<String> newTableNames = SQLUtil.parseNames(newTableName);
4966                                if (oldTableNames.size() > newTableNames.size()) {
4967                                        for (int i = oldTableNames.size() - newTableNames.size() - 1; i >= 0; i--) {
4968                                                newTableName = (oldTableNames.get(i) + ".") + newTableName;
4969                                        }
4970                                }
4971
4972                                Table newNameTableModel = modelFactory.createTableByName(newTableName, true);
4973                                TObjectName newStarColumn = new TObjectName();
4974                                newStarColumn.setString("*");
4975                                TableColumn newTableStarColumn = modelFactory.createTableColumn(newNameTableModel, newStarColumn, true);
4976                                
4977                                Process process = modelFactory.createProcess(executeStmt);
4978                                newNameTableModel.addProcess(process);
4979                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
4980                                relation.setEffectType(EffectType.rename_table);
4981                                relation.setTarget(new TableColumnRelationshipElement(newTableStarColumn));
4982                                relation.addSource(new TableColumnRelationshipElement(oldTableStarColumn));
4983                                relation.setProcess(process);
4984                                
4985                                if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) {
4986                                        oldTableStarColumn.setShowStar(false);
4987                                        relation.setShowStarRelation(false);
4988                                }
4989
4990                                if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) {
4991                                        newTableStarColumn.setShowStar(false);
4992                                        relation.setShowStarRelation(false);
4993                                }
4994                        }
4995                        else if(module.toString().toLowerCase().endsWith("sp_executesql")) {
4996                                String sql = SQLUtil.trimColumnStringQuote(executeStmt.getParameters().getExecParameter(0).toString());
4997                                if(sql.startsWith("N")) {
4998                                        sql =  SQLUtil.trimColumnStringQuote(sql.substring(1));
4999                                }
5000                                executeDynamicSql(sql);
5001                        }
5002                        else {
5003                                // sp_execute_external_script runs an R / Python / Java / ONNX script
5004                                // in-database (SQL Server 2016+ Machine Learning Services). Its
5005                                // @input_data_1 parameter is a SQL string literal whose SELECT reads
5006                                // real tables to build the InputDataSet the script consumes. Recover
5007                                // that source-table -> result-set lineage instead of dropping it
5008                                // (MantisBT #4606). This runs IN ADDITION to the generic call-edge
5009                                // handling below, so the call to sp_execute_external_script is still
5010                                // recorded.
5011                                if (option.isAnalyzeDynamicSql()
5012                                                && isSpExecuteExternalScript(module)) {
5013                                        analyzeExternalScriptInputData(executeStmt);
5014                                }
5015                                int argumentSize = executeStmt.getParameters() == null ? 0 : executeStmt.getParameters().size();
5016                                String procedureNameWithArgSize = module.toString() + "(" + argumentSize + ")";
5017                                if (argumentSize <= 0  || !DlineageUtil.supportFunctionOverride(option.getVendor())) {
5018                                        procedureNameWithArgSize = module.toString();
5019                                }
5020                                if (procedureDDLMap.containsKey(procedureNameWithArgSize)) {
5021                                        analyzeCustomSqlStmt(procedureDDLMap.get(procedureNameWithArgSize));
5022                                }
5023                                Procedure procedure = modelFactory.createProcedureByName(module, executeStmt.getParameters() == null ? 0 : executeStmt.getParameters().size());
5024                                String procedureParent = getProcedureParentName(executeStmt);
5025                                if (procedureParent != null) {
5026                                        Procedure caller = modelManager
5027                                                        .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
5028                                        if (caller != null) {
5029                                                CallRelationship callRelation = modelFactory.createCallRelation();
5030                                                callRelation.setCallObject(executeStmt);
5031                                                callRelation.setTarget(new ProcedureRelationshipElement(caller));
5032                                                callRelation.addSource(new ProcedureRelationshipElement(procedure));
5033                                                if(isBuiltInFunctionName(module) || isKeyword(module)){
5034                                                        callRelation.setBuiltIn(true);
5035                                                }
5036                                        }
5037                                }
5038
5039                                if (procedure.getArguments() != null) {
5040                                        for (int i = 0; i < procedure.getArguments().size(); i++) {
5041                                                Argument argument = procedure.getArguments().get(i);
5042                                                Variable variable = modelFactory.createVariable(procedure, argument.getName(), false);
5043                                                if (variable != null) {
5044                                                        if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
5045                                                                Transform transform = new Transform();
5046                                                                transform.setType(Transform.FUNCTION);
5047                                                                transform.setCode(module);
5048                                                                compositeBindingColumn(variable).setTransform(transform);
5049                                                        }
5050                                                        Process process = modelFactory.createProcess(executeStmt);
5051                                                        variable.addProcess(process);
5052                                                        analyzeFunctionArgumentsDataFlowRelation(compositeBindingColumn(variable), executeStmt, argument.getName(), i, process);
5053                                                }
5054                                        }
5055                                }
5056                        }
5057                }
5058        }
5059
5060        private void analyzeDropTableStmt(TDropTableSqlStatement stmt) {
5061                TTable dropTable = stmt.getTargetTable();
5062                if(dropTable == null) {
5063                        return;
5064                }
5065                Table tableModel = modelManager.getTableByName(DlineageUtil.getTableFullName(dropTable.getTableName().toString()));
5066                if(tableModel!=null) {
5067                        modelManager.dropTable(tableModel);
5068                }
5069                
5070                if(option.getAnalyzeMode() == AnalyzeMode.crud) {
5071                        tableModel = modelFactory.createTable(dropTable);
5072                        CrudRelationship crudRelationship = modelFactory.createCrudRelation();
5073                        crudRelationship.setTarget(new TableRelationshipElement(tableModel));
5074                        crudRelationship.setEffectType(EffectType.drop_table);
5075                }
5076        }
5077        
5078        private void analyzeTruncateTableStmt(TTruncateStatement stmt) {
5079                if(option.getAnalyzeMode() == AnalyzeMode.crud) {
5080                        TObjectName table = stmt.getTableName();
5081                        Table tableModel = modelFactory.createTableByName(table);
5082                        CrudRelationship crudRelationship = modelFactory.createCrudRelation();
5083                        crudRelationship.setTarget(new TableRelationshipElement(tableModel));
5084                        crudRelationship.setEffectType(EffectType.truncate_table);
5085                }
5086        }
5087
5088        private void analyzeCallStmt(TCallStatement callStmt) {
5089                if (callStmt.getRoutineExpr() != null && callStmt.getRoutineExpr().getFunctionCall() != null) {
5090
5091                        TFunctionCall functionCall = callStmt.getRoutineExpr().getFunctionCall();
5092                        Procedure callee = modelManager.getProcedureByName(
5093                                        DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
5094                        if (callee == null && procedureDDLMap.containsKey(DlineageUtil.getFunctionNameWithArgNum(functionCall))) {
5095                                analyzeCustomSqlStmt(procedureDDLMap.get(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
5096                                callee = modelManager.getProcedureByName(DlineageUtil
5097                                                .getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
5098                        }
5099                        if (callee != null) {
5100                                String procedureParent = getProcedureParentName(callStmt);
5101                                if (procedureParent != null) {
5102                                        Procedure caller = modelManager
5103                                                        .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
5104                                        if (caller != null) {
5105                                                CallRelationship callRelation = modelFactory.createCallRelation();
5106                                                callRelation.setCallObject(callStmt);
5107                                                callRelation.setTarget(new ProcedureRelationshipElement(caller));
5108                                                callRelation.addSource(new ProcedureRelationshipElement(callee));
5109                                                if (isBuiltInFunctionName(functionCall.getFunctionName())
5110                                                                || isKeyword(functionCall.getFunctionName())) {
5111                                                        callRelation.setBuiltIn(true);
5112                                                }
5113                                        }
5114                                }
5115                                if (callee.getArguments() != null) {
5116                                        for (int i = 0; i < callee.getArguments().size(); i++) {
5117                                                Argument argument = callee.getArguments().get(i);
5118                                                Variable variable = resolveFormalVariable(callee, argument);
5119                                                if (variable != null) {
5120                                                        if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
5121                                                                Transform transform = new Transform();
5122                                                                transform.setType(Transform.FUNCTION);
5123                                                                transform.setCode(functionCall);
5124                                                                compositeBindingColumn(variable).setTransform(transform);
5125                                                        }
5126                                                        Process process = modelFactory.createProcess(callStmt);
5127                                                        variable.addProcess(process);
5128                                                        analyzeFunctionArgumentsDataFlowRelation(compositeBindingColumn(variable), functionCall, i,
5129                                                                        process);
5130                                                }
5131                                        }
5132                                }
5133                        } else {
5134                                Function function = (Function) createFunction(functionCall);
5135                                String procedureParent = getProcedureParentName(callStmt);
5136                                if (procedureParent != null) {
5137                                        Procedure caller = modelManager
5138                                                        .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
5139                                        if (caller != null) {
5140                                                CallRelationship callRelation = modelFactory.createCallRelation();
5141                                                callRelation.setCallObject(callStmt);
5142                                                callRelation.setTarget(new ProcedureRelationshipElement(caller));
5143                                                callRelation.addSource(new FunctionRelationshipElement(function));
5144                                                if (isBuiltInFunctionName(functionCall.getFunctionName())
5145                                                                || isKeyword(functionCall.getFunctionName())) {
5146                                                        callRelation.setBuiltIn(true);
5147                                                }
5148                                        }
5149                                }
5150                        }
5151                }
5152                else if (callStmt.getRoutineName() != null) {
5153                        TObjectName function = callStmt.getRoutineName();
5154                        String functionName = function.toString();
5155                        Procedure callee = modelManager.getProcedureByName(
5156                                        DlineageUtil.getIdentifierNormalTableName(functionName));
5157                        if (callee == null && procedureDDLMap.containsKey(functionName)) {
5158                                analyzeCustomSqlStmt(procedureDDLMap.get(functionName));
5159                                callee = modelManager.getProcedureByName(functionName);
5160                        }
5161                        if (callee != null) {
5162                                String procedureParent = getProcedureParentName(callStmt);
5163                                if (procedureParent != null) {
5164                                        Procedure caller = modelManager
5165                                                        .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
5166                                        if (caller != null) {
5167                                                CallRelationship callRelation = modelFactory.createCallRelation();
5168                                                callRelation.setCallObject(callStmt);
5169                                                callRelation.setTarget(new ProcedureRelationshipElement(caller));
5170                                                callRelation.addSource(new ProcedureRelationshipElement(callee));
5171                                                if (isBuiltInFunctionName(function)
5172                                                                || isKeyword(function)) {
5173                                                        callRelation.setBuiltIn(true);
5174                                                }
5175                                        }
5176                                }
5177                                if (callee.getArguments() != null) {
5178                                        for (int i = 0; i < callee.getArguments().size(); i++) {
5179                                                Argument argument = callee.getArguments().get(i);
5180                                                Variable variable = resolveFormalVariable(callee, argument);
5181                                                if (variable != null) {
5182                                                        if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
5183                                                                Transform transform = new Transform();
5184                                                                transform.setType(Transform.FUNCTION);
5185                                                                transform.setCode(callStmt);
5186                                                                compositeBindingColumn(variable).setTransform(transform);
5187                                                        }
5188                                                        Process process = modelFactory.createProcess(callStmt);
5189                                                        variable.addProcess(process);
5190                                                        analyzeFunctionArgumentsDataFlowRelation(compositeBindingColumn(variable), callStmt, i,
5191                                                                        process);
5192                                                }
5193                                        }
5194                                }
5195                        }
5196                }
5197        }
5198
5199        /**
5200         * DB2's CALL node is {@link TDb2CallStmt} (a sibling of {@link TCallStatement},
5201         * not a subclass), so the generic call dispatch never reaches it. Resolve its
5202         * caller/callee and emit the call relationship exactly as the
5203         * {@link #analyzeCallStmt} getRoutineName path does, using the DB2 accessors
5204         * getProcedureName()/getParameters().
5205         */
5206        private void analyzeDb2CallStmt(TDb2CallStmt callStmt) {
5207                if (callStmt.getProcedureName() == null) {
5208                        return;
5209                }
5210                TObjectName function = callStmt.getProcedureName();
5211                String functionName = function.toString();
5212                // DB2 supports procedure overloading by arity, so overloadable procedures
5213                // register under a "name(argCount)" key (see getFunctionNameWithArgNum).
5214                // Resolve by name+argCount first to bind the correct overload, then fall
5215                // back to the plain name for the single-definition / non-overloaded case.
5216                String nameWithArgNum = functionName;
5217                if (callStmt.getParameters() != null
5218                                && DlineageUtil.supportFunctionOverride(ModelBindingManager.getGlobalVendor())) {
5219                        nameWithArgNum = functionName + "(" + callStmt.getParameters().size() + ")";
5220                }
5221                Procedure callee = modelManager.getProcedureByName(
5222                                DlineageUtil.getIdentifierNormalTableName(nameWithArgNum));
5223                if (callee == null && !nameWithArgNum.equals(functionName)) {
5224                        callee = modelManager.getProcedureByName(
5225                                        DlineageUtil.getIdentifierNormalTableName(functionName));
5226                }
5227                if (callee == null && procedureDDLMap.containsKey(nameWithArgNum)) {
5228                        analyzeCustomSqlStmt(procedureDDLMap.get(nameWithArgNum));
5229                        callee = modelManager.getProcedureByName(nameWithArgNum);
5230                }
5231                if (callee == null && procedureDDLMap.containsKey(functionName)) {
5232                        analyzeCustomSqlStmt(procedureDDLMap.get(functionName));
5233                        callee = modelManager.getProcedureByName(functionName);
5234                }
5235                if (callee != null) {
5236                        String procedureParent = getProcedureParentName(callStmt);
5237                        if (procedureParent != null) {
5238                                Procedure caller = modelManager
5239                                                .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
5240                                if (caller != null) {
5241                                        CallRelationship callRelation = modelFactory.createCallRelation();
5242                                        callRelation.setCallObject(callStmt);
5243                                        callRelation.setTarget(new ProcedureRelationshipElement(caller));
5244                                        callRelation.addSource(new ProcedureRelationshipElement(callee));
5245                                        if (isBuiltInFunctionName(function) || isKeyword(function)) {
5246                                                callRelation.setBuiltIn(true);
5247                                        }
5248                                }
5249                        }
5250                        if (callee.getArguments() != null) {
5251                                for (int i = 0; i < callee.getArguments().size(); i++) {
5252                                        Argument argument = callee.getArguments().get(i);
5253                                        Variable variable = resolveFormalVariable(callee, argument);
5254                                        if (variable != null) {
5255                                                if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
5256                                                        Transform transform = new Transform();
5257                                                        transform.setType(Transform.FUNCTION);
5258                                                        transform.setCode(callStmt);
5259                                                        compositeBindingColumn(variable).setTransform(transform);
5260                                                }
5261                                                Process process = modelFactory.createProcess(callStmt);
5262                                                variable.addProcess(process);
5263                                                analyzeFunctionArgumentsDataFlowRelation(compositeBindingColumn(variable), callStmt, i,
5264                                                                process);
5265                                        }
5266                                }
5267                        }
5268                }
5269        }
5270
5271        private boolean analyzeCustomFunctionCall(TFunctionCall functionCall) {
5272                Procedure callee = modelManager.getProcedureByName(
5273                                DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
5274                if(callee == null && procedureDDLMap.containsKey(DlineageUtil.getFunctionNameWithArgNum(functionCall))) {
5275                        analyzeCustomSqlStmt(procedureDDLMap.get(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
5276                        callee = modelManager.getProcedureByName(
5277                                        DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
5278                }
5279                if (callee != null) {
5280                        String procedureParent = getProcedureParentName(stmtStack.peek());
5281                        if (procedureParent != null) {
5282                                Procedure caller = modelManager
5283                                                .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
5284                                if (caller != null) {
5285                                        CallRelationship callRelation = modelFactory.createCallRelation();
5286                                        callRelation.setCallObject(functionCall);
5287                                        callRelation.setTarget(new ProcedureRelationshipElement(caller));
5288                                        callRelation.addSource(new ProcedureRelationshipElement(callee));
5289                                        if(isBuiltInFunctionName(functionCall.getFunctionName()) || isKeyword(functionCall.getFunctionName())){
5290                                                callRelation.setBuiltIn(true);
5291                                        }
5292                                }
5293                        }
5294                        if (callee.getArguments() != null) {
5295                                for (int i = 0; i < callee.getArguments().size(); i++) {
5296                                        Argument argument = callee.getArguments().get(i);
5297                                        Variable variable = resolveFormalVariable(callee, argument);
5298                                        if(variable!=null) {
5299                                                if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
5300                                                        Transform transform = new Transform();
5301                                                        transform.setType(Transform.FUNCTION);
5302                                                        transform.setCode(functionCall);
5303                                                        compositeBindingColumn(variable).setTransform(transform);
5304                                                }
5305                                                Process process = modelFactory.createProcess(functionCall);
5306                                                variable.addProcess(process);
5307                                                analyzeFunctionArgumentsDataFlowRelation(compositeBindingColumn(variable), functionCall, i, process);
5308                                        }
5309                                }
5310                        }
5311                        return true;
5312                }
5313                return false;
5314        }
5315
5316        private void analyzeOracleBasicStmt(TBasicStmt oracleBasicStmt) {
5317                if (oracleBasicStmt.getExpr() == null || oracleBasicStmt.getExpr().getFunctionCall() == null) {
5318                        return;
5319                }
5320
5321                TFunctionCall functionCall = oracleBasicStmt.getExpr().getFunctionCall();
5322                recordDbmsSqlSiteIfParseCall(oracleBasicStmt, functionCall);
5323                Procedure callee = modelManager.getProcedureByName(
5324                                DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
5325                if(callee == null && procedureDDLMap.containsKey(DlineageUtil.getFunctionNameWithArgNum(functionCall))) {
5326                        analyzeCustomSqlStmt(procedureDDLMap.get(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
5327                        callee = modelManager.getProcedureByName(
5328                                        DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
5329                }
5330                if (callee != null) {
5331                        String procedureParent = getProcedureParentName(oracleBasicStmt);
5332                        if (procedureParent != null) {
5333                                Procedure caller = modelManager
5334                                                .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
5335                                if (caller != null) {
5336                                        CallRelationship callRelation = modelFactory.createCallRelation();
5337                                        callRelation.setCallObject(functionCall);
5338                                        callRelation.setTarget(new ProcedureRelationshipElement(caller));
5339                                        callRelation.addSource(new ProcedureRelationshipElement(callee));
5340                                        if (functionChecker.isOraclePredefinedPackageFunction(functionCall.getFunctionName().toString())
5341                                                        || (isBuiltInFunctionName(functionCall.getFunctionName())
5342                                                        || isKeyword(functionCall.getFunctionName()))) {
5343                                                callRelation.setBuiltIn(true);
5344                                        }
5345                                }
5346                        }
5347                        if (callee.getArguments() != null) {
5348                                for (int i = 0; i < callee.getArguments().size(); i++) {
5349                                        Argument argument = callee.getArguments().get(i);
5350                                        Variable variable = resolveFormalVariable(callee, argument);
5351                                        if(variable!=null) {
5352                                                if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
5353                                                        Transform transform = new Transform();
5354                                                        transform.setType(Transform.FUNCTION);
5355                                                        transform.setCode(functionCall);
5356                                                        compositeBindingColumn(variable).setTransform(transform);
5357                                                }
5358                                                Process process = modelFactory.createProcess(functionCall);
5359                                                variable.addProcess(process);
5360                                                analyzeFunctionArgumentsDataFlowRelation(compositeBindingColumn(variable), functionCall, i, process);
5361                                        }
5362                                }
5363                        }
5364                } else {
5365                        Object functionModel = createFunction(functionCall);
5366                        if (!(functionModel instanceof Function)) {
5367                                return;
5368                        }
5369                        Function function = (Function) functionModel;
5370                        String procedureParent = getProcedureParentName(oracleBasicStmt);
5371                        if (procedureParent != null) {
5372                                Procedure caller = modelManager
5373                                                .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
5374                                if (caller != null) {
5375                                        CallRelationship callRelation = modelFactory.createCallRelation();
5376                                        callRelation.setCallObject(functionCall);
5377                                        callRelation.setTarget(new ProcedureRelationshipElement(caller));
5378                                        callRelation.addSource(new FunctionRelationshipElement(function));
5379                                        if (functionChecker.isOraclePredefinedPackageFunction(functionCall.getFunctionName().toString())
5380                                                        || (isBuiltInFunctionName(functionCall.getFunctionName()) || isKeyword(functionCall.getFunctionName()))) {
5381                                                callRelation.setBuiltIn(true);
5382                                        }
5383                                }
5384                        }
5385                }
5386        }
5387
5388        private void analyzeMssqlIfElseStmt(TMssqlIfElse ifStmt) {
5389                if (ifStmt.getCondition() != null) {
5390                        columnsInExpr visitor = new columnsInExpr();
5391                        ifStmt.getCondition().inOrderTraverse(visitor);
5392                        List<TParseTreeNode> functions = visitor.getFunctions();
5393
5394                        if (functions != null && !functions.isEmpty()) {
5395                                for (int i = 0; i < functions.size(); i++) {
5396                                        if (!(functions.get(i) instanceof TFunctionCall))
5397                                                continue;
5398                                        TFunctionCall functionCall = (TFunctionCall) functions.get(i);
5399                                        Procedure callee = modelManager.getProcedureByName(DlineageUtil
5400                                                        .getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
5401                                        if(callee == null && procedureDDLMap.containsKey(DlineageUtil.getFunctionNameWithArgNum(functionCall))) {
5402                                                analyzeCustomSqlStmt(procedureDDLMap.get(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
5403                                                callee = modelManager.getProcedureByName(
5404                                                                DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
5405                                        }
5406                                        if (callee != null) {
5407                                                String procedureParent = getProcedureParentName(ifStmt);
5408                                                if (procedureParent != null) {
5409                                                        Procedure caller = modelManager
5410                                                                        .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
5411                                                        if (caller != null) {
5412                                                                CallRelationship callRelation = modelFactory.createCallRelation();
5413                                                                callRelation.setCallObject(functionCall);
5414                                                                callRelation.setTarget(new ProcedureRelationshipElement(caller));
5415                                                                callRelation.addSource(new ProcedureRelationshipElement(callee));
5416                                                                if (isBuiltInFunctionName(functionCall.getFunctionName()) || isKeyword(functionCall.getFunctionName())) {
5417                                                                        callRelation.setBuiltIn(true);
5418                                                                }
5419                                                        }
5420                                                }
5421                                                if (callee.getArguments() != null) {
5422                                                        for (int j = 0; j < callee.getArguments().size(); j++) {
5423                                                                Argument argument = callee.getArguments().get(j);
5424                                                                Variable variable = resolveFormalVariable(callee, argument);
5425                                                                if(variable!=null) {
5426                                                                        if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
5427                                                                                Transform transform = new Transform();
5428                                                                                transform.setType(Transform.FUNCTION);
5429                                                                                transform.setCode(functionCall);
5430                                                                                compositeBindingColumn(variable).setTransform(transform);
5431                                                                        }
5432                                                                        Process process = modelFactory.createProcess(functionCall);
5433                                                                        variable.addProcess(process);
5434                                                                        analyzeFunctionArgumentsDataFlowRelation(compositeBindingColumn(variable), functionCall, j, process);
5435                                                                }
5436                                                        }
5437                                                }
5438                                        } else {
5439                                                Object functionModel = createFunction(functionCall);
5440                                                if (!(functionModel instanceof Function)) {
5441                                                        continue;
5442                                                }
5443                                                Function function = (Function) functionModel;
5444                                                String procedureParent = getProcedureParentName(ifStmt);
5445                                                if (procedureParent != null) {
5446                                                        Procedure caller = modelManager
5447                                                                        .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
5448                                                        if (caller != null) {
5449                                                                CallRelationship callRelation = modelFactory.createCallRelation();
5450                                                                callRelation.setCallObject(functionCall);
5451                                                                callRelation.setTarget(new ProcedureRelationshipElement(caller));
5452                                                                callRelation.addSource(new FunctionRelationshipElement(function));
5453                                                                if (isBuiltInFunctionName(functionCall.getFunctionName()) || isKeyword(functionCall.getFunctionName())) {
5454                                                                        callRelation.setBuiltIn(true);
5455                                                                }
5456                                                        }
5457                                                }
5458                                        }
5459                                }
5460                        }
5461                }
5462
5463                if(ifStmt.getStmt()!=null) {
5464                        analyzeCustomSqlStmt(ifStmt.getStmt());
5465                }
5466        
5467                if (ifStmt.getElseStmt() != null) {
5468                        analyzeCustomSqlStmt(ifStmt.getElseStmt());
5469                }
5470        }
5471        
5472        private void analyzeIfStmt(TIfStmt ifStmt) {
5473                if (ifStmt.getCondition() != null) {
5474                        columnsInExpr visitor = new columnsInExpr();
5475                        ifStmt.getCondition().inOrderTraverse(visitor);
5476                        List<TParseTreeNode> functions = visitor.getFunctions();
5477
5478                        if (functions != null && !functions.isEmpty()) {
5479                                for (int i = 0; i < functions.size(); i++) {
5480                                        if (!(functions.get(i) instanceof TFunctionCall))
5481                                                continue;
5482                                        TFunctionCall functionCall = (TFunctionCall) functions.get(i);
5483                                        Procedure callee = modelManager.getProcedureByName(DlineageUtil
5484                                                        .getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
5485                                        if(callee == null && procedureDDLMap.containsKey(DlineageUtil.getFunctionNameWithArgNum(functionCall))) {
5486                                                analyzeCustomSqlStmt(procedureDDLMap.get(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
5487                                                callee = modelManager.getProcedureByName(
5488                                                                DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
5489                                        }
5490                                        if (callee != null) {
5491                                                String procedureParent = getProcedureParentName(ifStmt);
5492                                                if (procedureParent != null) {
5493                                                        Procedure caller = modelManager
5494                                                                        .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
5495                                                        if (caller != null) {
5496                                                                CallRelationship callRelation = modelFactory.createCallRelation();
5497                                                                callRelation.setCallObject(functionCall);
5498                                                                callRelation.setTarget(new ProcedureRelationshipElement(caller));
5499                                                                callRelation.addSource(new ProcedureRelationshipElement(callee));
5500                                                                if (isBuiltInFunctionName(functionCall.getFunctionName()) || isKeyword(functionCall.getFunctionName())) {
5501                                                                        callRelation.setBuiltIn(true);
5502                                                                }
5503                                                        }
5504                                                }
5505                                                if (callee.getArguments() != null) {
5506                                                        for (int j = 0; j < callee.getArguments().size(); j++) {
5507                                                                Argument argument = callee.getArguments().get(j);
5508                                                                Variable variable = resolveFormalVariable(callee, argument);
5509                                                                if(variable!=null) {
5510                                                                        if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
5511                                                                                Transform transform = new Transform();
5512                                                                                transform.setType(Transform.FUNCTION);
5513                                                                                transform.setCode(functionCall);
5514                                                                                compositeBindingColumn(variable).setTransform(transform);
5515                                                                        }
5516                                                                        Process process = modelFactory.createProcess(functionCall);
5517                                                                        variable.addProcess(process);
5518                                                                        analyzeFunctionArgumentsDataFlowRelation(compositeBindingColumn(variable), functionCall, j, process);
5519                                                                }
5520                                                        }
5521                                                }
5522                                        } else {
5523                                                Object functionModel = createFunction(functionCall);
5524                                                if (!(functionModel instanceof Function)) {
5525                                                        continue;
5526                                                }
5527                                                Function function = (Function) functionModel;
5528                                                String procedureParent = getProcedureParentName(ifStmt);
5529                                                if (procedureParent != null) {
5530                                                        Procedure caller = modelManager
5531                                                                        .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
5532                                                        if (caller != null) {
5533                                                                CallRelationship callRelation = modelFactory.createCallRelation();
5534                                                                callRelation.setCallObject(functionCall);
5535                                                                callRelation.setTarget(new ProcedureRelationshipElement(caller));
5536                                                                callRelation.addSource(new FunctionRelationshipElement(function));
5537                                                                if (isBuiltInFunctionName(functionCall.getFunctionName()) || isKeyword(functionCall.getFunctionName())) {
5538                                                                        callRelation.setBuiltIn(true);
5539                                                                }
5540                                                        }
5541                                                }
5542                                        }
5543                                }
5544                        }
5545                }
5546
5547                if (ifStmt.getThenStatements() != null) {
5548                        for (int i = 0; i < ifStmt.getThenStatements().size(); ++i) {
5549                                analyzeCustomSqlStmt(ifStmt.getThenStatements().get(i));
5550                                ResultSet returnResult = modelFactory.createResultSet(ifStmt.getThenStatements().get(i), false);
5551                                if(returnResult!=null){
5552                                        for(ResultColumn resultColumn: returnResult.getColumns()){
5553                                                analyzeFilterCondition(resultColumn, ifStmt.getCondition(), null, null, EffectType.function);
5554                                        }
5555                                }
5556                        }
5557                }
5558
5559                if (ifStmt.getElseifStatements() != null) {
5560                        for (int i = 0; i < ifStmt.getElseifStatements().size(); ++i) {
5561                                analyzeCustomSqlStmt(ifStmt.getElseifStatements().get(i));
5562                        }
5563                }
5564
5565                if (ifStmt.getElseStatements() != null) {
5566                        for (int i = 0; i < ifStmt.getElseStatements().size(); ++i) {
5567                                analyzeCustomSqlStmt(ifStmt.getElseStatements().get(i));
5568                        }
5569                }
5570        }
5571
5572        private void analyzeElsIfStmt(TElsifStmt elsIfStmt) {
5573                if (elsIfStmt.getCondition() != null) {
5574                        columnsInExpr visitor = new columnsInExpr();
5575                        elsIfStmt.getCondition().inOrderTraverse(visitor);
5576                        List<TParseTreeNode> functions = visitor.getFunctions();
5577
5578                        if (functions != null && !functions.isEmpty()) {
5579                                for (int i = 0; i < functions.size(); i++) {
5580                                        if (!(functions.get(i) instanceof TFunctionCall))
5581                                                continue;
5582                                        TFunctionCall functionCall = (TFunctionCall) functions.get(i);
5583                                        Procedure callee = modelManager.getProcedureByName(DlineageUtil
5584                                                        .getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
5585                                        if(callee == null && procedureDDLMap.containsKey(DlineageUtil.getFunctionNameWithArgNum(functionCall))) {
5586                                                analyzeCustomSqlStmt(procedureDDLMap.get(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
5587                                                callee = modelManager.getProcedureByName(
5588                                                                DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
5589                                        }
5590                                        if (callee != null) {
5591                                                String procedureParent = getProcedureParentName(elsIfStmt);
5592                                                if (procedureParent != null) {
5593                                                        Procedure caller = modelManager
5594                                                                        .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
5595                                                        if (caller != null) {
5596                                                                CallRelationship callRelation = modelFactory.createCallRelation();
5597                                                                callRelation.setCallObject(functionCall);
5598                                                                callRelation.setTarget(new ProcedureRelationshipElement(caller));
5599                                                                callRelation.addSource(new ProcedureRelationshipElement(callee));
5600                                                                if(isBuiltInFunctionName(functionCall.getFunctionName()) || isKeyword(functionCall.getFunctionName())){
5601                                                                        callRelation.setBuiltIn(true);
5602                                                                }
5603                                                        }
5604                                                }
5605                                                if (callee.getArguments() != null) {
5606                                                        for (int j = 0; j < callee.getArguments().size(); j++) {
5607                                                                Argument argument = callee.getArguments().get(j);
5608                                                                Variable variable = resolveFormalVariable(callee, argument);
5609                                                                if(variable!=null) {
5610                                                                        if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
5611                                                                                Transform transform = new Transform();
5612                                                                                transform.setType(Transform.FUNCTION);
5613                                                                                transform.setCode(functionCall);
5614                                                                                compositeBindingColumn(variable).setTransform(transform);
5615                                                                        }
5616                                                                        Process process = modelFactory.createProcess(functionCall);
5617                                                                        variable.addProcess(process);
5618                                                                        analyzeFunctionArgumentsDataFlowRelation(compositeBindingColumn(variable), functionCall, j, process);
5619                                                                }
5620                                                        }
5621                                                }
5622                                        } else {
5623                                                Function function = (Function)createFunction(functionCall);
5624                                                String procedureParent = getProcedureParentName(elsIfStmt);
5625                                                if (procedureParent != null) {
5626                                                        Procedure caller = modelManager
5627                                                                        .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
5628                                                        if (caller != null) {
5629                                                                CallRelationship callRelation = modelFactory.createCallRelation();
5630                                                                callRelation.setCallObject(functionCall);
5631                                                                callRelation.setTarget(new ProcedureRelationshipElement(caller));
5632                                                                callRelation.addSource(new FunctionRelationshipElement(function));
5633                                                                if(isBuiltInFunctionName(functionCall.getFunctionName()) || isKeyword(functionCall.getFunctionName())){
5634                                                                        callRelation.setBuiltIn(true);
5635                                                                }
5636                                                        }
5637                                                }
5638                                        }
5639                                }
5640                        }
5641                }
5642
5643                if (elsIfStmt.getThenStatements() != null) {
5644                        for (int i = 0; i < elsIfStmt.getThenStatements().size(); ++i) {
5645                                ResultSet returnResult = modelFactory.createResultSet(elsIfStmt.getThenStatements().get(i), false);
5646                                if (returnResult != null) {
5647                                        for (ResultColumn resultColumn : returnResult.getColumns()) {
5648                                                analyzeFilterCondition(resultColumn, elsIfStmt.getCondition(), null, null, EffectType.function);
5649                                        }
5650                                }
5651                                analyzeCustomSqlStmt(elsIfStmt.getThenStatements().get(i));
5652                        }
5653                }
5654        }
5655
5656        private void analyzeCloneTableStmt(TCreateTableSqlStatement stmt) {
5657                if (stmt.getCloneSourceTable() != null) {
5658                        Table sourceTable = modelFactory.createTableByName(stmt.getCloneSourceTable());
5659                        Table cloneTable = modelFactory.createTableByName(stmt.getTableName());
5660                        // Clone creates the target. Parse fact, set regardless of source
5661                        // resolution (the gated setFromDDL below stays as-is). See
5662                        // dlineage-authoritative-endpoint-classification.md.
5663                        cloneTable.setCreatedInSql(true);
5664                        cloneTable.setEndpointIntroduction(EndpointIntroduction.CLONE_TABLE);
5665                        Process process = modelFactory.createProcess(stmt);
5666                        cloneTable.addProcess(process);
5667
5668                        if (sourceTable.isDetermined()) {
5669                                for (int k = 0; k < sourceTable.getColumns().size(); k++) {
5670                                        TableColumn sourceColumn = sourceTable.getColumns().get(k);
5671                                        TObjectName objectName = new TObjectName();
5672                                        objectName.setString(sourceColumn.getName());
5673                                        TableColumn tableColumn = modelFactory.createTableColumn(cloneTable, objectName, true);
5674                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
5675                                        dataflowRelation.setEffectType(EffectType.clone_table);
5676                                        dataflowRelation
5677                                                        .addSource(new TableColumnRelationshipElement(sourceColumn));
5678                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(tableColumn));
5679                                        dataflowRelation.setProcess(process);
5680                                }
5681                                cloneTable.setDetermined(true);
5682                                cloneTable.setFromDDL(true);
5683                        } else {
5684                                TObjectName sourceName = new TObjectName();
5685                                sourceName.setString("*");
5686                                TableColumn sourceTableColumn = modelFactory.createTableColumn(sourceTable, sourceName, false);
5687                                TObjectName targetName = new TObjectName();
5688                                targetName.setString("*");
5689                                TableColumn targetTableColumn = modelFactory.createTableColumn(cloneTable, targetName, false);
5690                                if(sourceTableColumn!=null && targetTableColumn!=null) {
5691                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
5692                                        dataflowRelation.setEffectType(EffectType.clone_table);
5693                                        dataflowRelation
5694                                                        .addSource(new TableColumnRelationshipElement(sourceTableColumn));
5695                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(targetTableColumn));
5696                                        dataflowRelation.setProcess(process);
5697                                }
5698                        }
5699                }
5700        }
5701
5702        private void analyzeCloneDatabaseStmt(TCreateDatabaseSqlStatement stmt) {
5703                if (stmt.getCloneSourceDb() != null) {
5704                        Database sourceDatabase = modelFactory.createDatabase(stmt.getCloneSourceDb());
5705                        Database cloneDatabase = modelFactory.createDatabase(stmt.getDatabaseName());
5706                        Process process = modelFactory.createProcess(stmt);
5707                        cloneDatabase.addProcess(process);
5708                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
5709                        relation.setEffectType(EffectType.clone_database);
5710                        relation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(cloneDatabase.getRelationRows()));
5711                        relation.addSource(
5712                                        new RelationRowsRelationshipElement<TableRelationRows>(sourceDatabase.getRelationRows()));
5713                        relation.setProcess(process);
5714                }
5715        }
5716
5717        private void analyzeCloneSchemaStmt(TCreateSchemaSqlStatement stmt) {
5718                if (stmt.getCloneSourceSchema() != null) {
5719                        Schema sourceSchema = modelFactory.createSchema(stmt.getCloneSourceSchema());
5720                        Schema cloneSchema = modelFactory.createSchema(stmt.getSchemaName());
5721                        Process process = modelFactory.createProcess(stmt);
5722                        cloneSchema.addProcess(process);
5723                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
5724                        relation.setEffectType(EffectType.clone_schema);
5725                        relation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(cloneSchema.getRelationRows()));
5726                        relation.addSource(new RelationRowsRelationshipElement<TableRelationRows>(sourceSchema.getRelationRows()));
5727                        relation.setProcess(process);
5728                }
5729        }
5730
5731        /**
5732         * Record one dynamic-SQL site. Span is the statement extent (1-based, end-exclusive), tagged with
5733         * the per-statement file hash, mirroring the {@link ErrorInfo} position convention. When the
5734         * {@code reportDynamicSqlSitesAsErrors} option is on, non-RESOLVED sites are also mirrored into an
5735         * {@link ErrorInfo} so legacy {@code getErrorMessages()} consumers can see them.
5736         */
5737        private void recordDynamicSqlSite(TCustomSqlStatement stmt, DynamicSqlSite.Kind kind,
5738                        DynamicSqlSite.Status status, String reason) {
5739                recordDynamicSqlSite(stmt, kind, status, reason, 0, 0);
5740        }
5741
5742        private void recordDynamicSqlSite(TCustomSqlStatement stmt, DynamicSqlSite.Kind kind,
5743                        DynamicSqlSite.Status status, String reason, int resolvedCount, int partialCount) {
5744                DynamicSqlTrustMode trustMode = option == null ? DynamicSqlTrustMode.LEGACY
5745                                : option.getDynamicSqlTrustMode();
5746                // This compatibility overload has no fragment provenance. Never turn
5747                // that absence into an exact zero; callers with evidence use the full
5748                // overload below.
5749                recordDynamicSqlSite(stmt, kind, status, reason, resolvedCount, partialCount,
5750                                0, -1, false, trustMode);
5751        }
5752
5753        private void recordDynamicSqlSite(TCustomSqlStatement stmt, DynamicSqlSite.Kind kind,
5754                        DynamicSqlSite.Status status, String reason, int resolvedCount, int partialCount,
5755                        int unprovenCount, int unresolvedHoleCount,
5756                        boolean holeCountExact, DynamicSqlTrustMode trustMode) {
5757                recordDynamicSqlSite(stmt, kind, status, reason, resolvedCount, partialCount,
5758                                unprovenCount, unresolvedHoleCount, holeCountExact, trustMode,
5759                                -1, -1, null);
5760        }
5761
5762        private void recordDynamicSqlSite(TCustomSqlStatement stmt, DynamicSqlSite.Kind kind,
5763                        DynamicSqlSite.Status status, String reason, int resolvedCount, int partialCount,
5764                        int unprovenCount, int unresolvedHoleCount,
5765                        boolean holeCountExact, DynamicSqlTrustMode trustMode,
5766                        int relationshipsBefore, int nestedSitesBefore) {
5767                recordDynamicSqlSite(stmt, kind, status, reason, resolvedCount, partialCount,
5768                                unprovenCount, unresolvedHoleCount, holeCountExact, trustMode,
5769                                relationshipsBefore, nestedSitesBefore, null);
5770        }
5771
5772        private void recordDynamicSqlSite(TCustomSqlStatement stmt, DynamicSqlSite.Kind kind,
5773                        DynamicSqlSite.Status status, String reason, int resolvedCount, int partialCount,
5774                        int unprovenCount, int unresolvedHoleCount,
5775                        boolean holeCountExact, DynamicSqlTrustMode trustMode,
5776                        int relationshipsBefore, int nestedSitesBefore,
5777                        List<Long> explicitlyProvenRelationshipIds) {
5778                recordDynamicSqlSite(stmt, kind, status, reason, resolvedCount, partialCount,
5779                                unprovenCount, unresolvedHoleCount, holeCountExact, trustMode,
5780                                relationshipsBefore, nestedSitesBefore,
5781                                explicitlyProvenRelationshipIds, null);
5782        }
5783
5784        private void recordDynamicSqlSite(TCustomSqlStatement stmt, DynamicSqlSite.Kind kind,
5785                        DynamicSqlSite.Status status, String reason, int resolvedCount, int partialCount,
5786                        int unprovenCount, int unresolvedHoleCount,
5787                        boolean holeCountExact, DynamicSqlTrustMode trustMode,
5788                        int relationshipsBefore, int nestedSitesBefore,
5789                        List<Long> explicitlyProvenRelationshipIds, String diagnostic) {
5790                recordDynamicSqlSite(stmt, kind, status, reason, resolvedCount, partialCount,
5791                                unprovenCount, unresolvedHoleCount, holeCountExact, trustMode,
5792                                relationshipsBefore, nestedSitesBefore,
5793                                explicitlyProvenRelationshipIds, diagnostic,
5794                                Collections.<UnresolvedFragment>emptyList(), null);
5795        }
5796
5797        /**
5798         * An opaque site the evaluator could not turn into analyzable text: UNRESOLVED with
5799         * the argument-shape reason, carrying whatever hole evidence the evaluator gathered
5800         * on the way (the variable that stopped the fold, its assignment, its span).
5801         */
5802        private void recordUnresolvedDynamicSqlSite(TCustomSqlStatement stmt, TMssqlExecute executeStmt,
5803                        DynamicSqlSite.Kind kind) {
5804                DynamicHoleEvidence holeEvidence = dynamicHoleEvidence(evaluatedDynamicSqlFor(stmt));
5805                recordDynamicSqlSite(stmt, kind, DynamicSqlSite.Status.UNRESOLVED, dynamicReasonOf(executeStmt),
5806                                0, 0, 0, holeEvidence.count, holeEvidence.exact, option.getDynamicSqlTrustMode(),
5807                                -1, -1, null, null, holeEvidence.fragments, holeEvidence.materializedText);
5808        }
5809
5810        private void recordDynamicSqlSite(TCustomSqlStatement stmt, DynamicSqlSite.Kind kind,
5811                        DynamicSqlSite.Status status, String reason, int resolvedCount, int partialCount,
5812                        int unprovenCount, int unresolvedHoleCount,
5813                        boolean holeCountExact, DynamicSqlTrustMode trustMode,
5814                        int relationshipsBefore, int nestedSitesBefore,
5815                        List<Long> explicitlyProvenRelationshipIds, String diagnostic,
5816                        List<UnresolvedFragment> unresolvedFragments, String materializedText) {
5817                if (stmt == null || stmt.getStartToken() == null || stmt.getEndToken() == null) {
5818                        return;
5819                }
5820                List<Long> observedRelationshipIds = dynamicRelationshipIdsSince(
5821                                relationshipsBefore, nestedSitesBefore);
5822                List<Long> provenRelationshipIds = Collections.emptyList();
5823                if (explicitlyProvenRelationshipIds != null) {
5824                        LinkedHashSet<Long> observed = new LinkedHashSet<Long>(observedRelationshipIds);
5825                        provenRelationshipIds = new ArrayList<Long>();
5826                        for (Long id : explicitlyProvenRelationshipIds) {
5827                                if (id != null && observed.contains(id)
5828                                                && !provenRelationshipIds.contains(id)) {
5829                                        provenRelationshipIds.add(id);
5830                                }
5831                        }
5832                } else if (status == DynamicSqlSite.Status.RESOLVED
5833                                && unprovenCount == 0
5834                                && holeCountExact && unresolvedHoleCount == 0) {
5835                        provenRelationshipIds = provenDynamicRelationshipIdsSince(
5836                                        relationshipsBefore, nestedSitesBefore);
5837                }
5838                Pair3<Long, Long, String> start = new Pair3<Long, Long, String>(stmt.getStartToken().lineNo,
5839                                stmt.getStartToken().columnNo, ModelBindingManager.getGlobalHash());
5840                Pair3<Long, Long, String> end;
5841                String[] segments = stmt.getEndToken().getAstext().split("\n", -1);
5842                if (segments.length == 1) {
5843                        end = new Pair3<Long, Long, String>(stmt.getEndToken().lineNo,
5844                                        stmt.getEndToken().columnNo + stmt.getEndToken().getAstext().length(),
5845                                        ModelBindingManager.getGlobalHash());
5846                } else {
5847                        end = new Pair3<Long, Long, String>(stmt.getEndToken().lineNo + segments.length - 1,
5848                                        (long) segments[segments.length - 1].length() + 1, ModelBindingManager.getGlobalHash());
5849                }
5850                dynamicSqlSites.add(new DynamicSqlSite(kind, status, reason, start, end,
5851                                resolvedCount, partialCount, unprovenCount,
5852                                unresolvedHoleCount, holeCountExact, trustMode,
5853                                observedRelationshipIds, provenRelationshipIds, diagnostic,
5854                                unresolvedFragments, materializedText));
5855
5856                if (status != DynamicSqlSite.Status.RESOLVED && option != null
5857                                && option.isReportDynamicSqlSitesAsErrors()) {
5858                        ErrorInfo errorInfo = new ErrorInfo();
5859                        errorInfo.setErrorType(ErrorInfo.DYNAMIC_SQL_UNRESOLVED);
5860                        errorInfo.setErrorMessage(reason != null ? reason : "dynamic SQL not resolved");
5861                        errorInfo.setStartPosition(start);
5862                        errorInfo.setEndPosition(end);
5863                        errorInfo.fillInfo(this);
5864                        errorInfos.add(errorInfo);
5865                }
5866        }
5867
5868        /**
5869         * Relationship IDs created by one dynamic site, excluding relationship
5870         * deltas already attributed to nested dynamic sites. The IDs match the
5871         * non-simple XML model emitted by this analyzer instance.
5872         */
5873        private List<Long> dynamicRelationshipIdsSince(int beforeCount, int nestedSitesBefore) {
5874                if (beforeCount < 0) {
5875                        return Collections.emptyList();
5876                }
5877                Set<Long> nested = new HashSet<Long>();
5878                if (nestedSitesBefore >= 0) {
5879                        for (int i = Math.min(nestedSitesBefore, dynamicSqlSites.size());
5880                                        i < dynamicSqlSites.size(); i++) {
5881                                nested.addAll(dynamicSqlSites.get(i).getObservedRelationshipIds());
5882                        }
5883                }
5884                Relationship[] relationships = modelManager.getRelations();
5885                List<Long> ids = new ArrayList<Long>();
5886                for (int i = Math.min(beforeCount, relationships.length);
5887                                i < relationships.length; i++) {
5888                        Relationship relationship = relationships[i];
5889                        if (relationship instanceof AbstractRelationship) {
5890                                Long id = Long.valueOf(((AbstractRelationship) relationship).getId());
5891                                if (!nested.contains(id)) {
5892                                        ids.add(id);
5893                                }
5894                        }
5895                }
5896                return ids;
5897        }
5898
5899        /**
5900         * Per-edge proof filter for a complete materialization. Placeholder-tainted
5901         * relations and every downstream relation that consumes their target stay
5902         * observation-only; independent relations from the same site remain proven.
5903         */
5904        private List<Long> provenDynamicRelationshipIdsSince(
5905                        int beforeCount, int nestedSitesBefore) {
5906                List<Long> observedIds = dynamicRelationshipIdsSince(beforeCount, nestedSitesBefore);
5907                if (observedIds.isEmpty()) {
5908                        return observedIds;
5909                }
5910                Set<Long> observed = new HashSet<Long>(observedIds);
5911                Set<Long> nestedUnproven = new HashSet<Long>();
5912                if (nestedSitesBefore >= 0) {
5913                        for (int i = Math.min(nestedSitesBefore, dynamicSqlSites.size());
5914                                        i < dynamicSqlSites.size(); i++) {
5915                                DynamicSqlSite nestedSite = dynamicSqlSites.get(i);
5916                                nestedUnproven.addAll(nestedSite.getObservedRelationshipIds());
5917                                nestedUnproven.removeAll(nestedSite.getProvenRelationshipIds());
5918                        }
5919                }
5920
5921                Set<Object> taintedElements = Collections.newSetFromMap(
5922                                new IdentityHashMap<Object, Boolean>());
5923                Set<Relationship> taintedRelationships = Collections.newSetFromMap(
5924                                new IdentityHashMap<Relationship, Boolean>());
5925                Relationship[] relationships = modelManager.getRelations();
5926                Map<Long, Relationship> relationshipsById = new HashMap<Long, Relationship>();
5927                for (Relationship relationship : relationships) {
5928                        if (!(relationship instanceof AbstractRelationship)) continue;
5929                        Long id = Long.valueOf(((AbstractRelationship) relationship).getId());
5930                        relationshipsById.put(id, relationship);
5931                        if (nestedUnproven.contains(id)) {
5932                                addDynamicTargetElement(taintedElements, relationship);
5933                        }
5934                }
5935
5936                boolean changed;
5937                do {
5938                        changed = false;
5939                        for (Relationship relationship : relationships) {
5940                                if (!(relationship instanceof AbstractRelationship)
5941                                                || !observed.contains(Long.valueOf(
5942                                                                ((AbstractRelationship) relationship).getId()))
5943                                                || taintedRelationships.contains(relationship)) {
5944                                        continue;
5945                                }
5946                                if (hasUnresolvedDynamicEndpoint(relationship)
5947                                                || hasDynamicSourceElement(taintedElements, relationship)) {
5948                                        taintedRelationships.add(relationship);
5949                                        changed |= addDynamicTargetElement(taintedElements, relationship);
5950                                }
5951                        }
5952                } while (changed);
5953
5954                List<Long> proven = new ArrayList<Long>();
5955                for (Long id : observedIds) {
5956                        Relationship relationship = relationshipsById.get(id);
5957                        if (relationship != null && !taintedRelationships.contains(relationship)) {
5958                                proven.add(id);
5959                        }
5960                }
5961                return proven;
5962        }
5963
5964        private boolean hasDynamicSourceElement(Set<Object> elements, Relationship relationship) {
5965                if (relationship.getSources() == null) return false;
5966                for (Object value : relationship.getSources()) {
5967                        if (value instanceof RelationshipElement
5968                                        && elements.contains(((RelationshipElement<?>) value).getElement())) {
5969                                return true;
5970                        }
5971                }
5972                return false;
5973        }
5974
5975        private boolean addDynamicTargetElement(Set<Object> elements, Relationship relationship) {
5976                if (relationship.getTarget() == null
5977                                || relationship.getTarget().getElement() == null) {
5978                        return false;
5979                }
5980                return elements.add(relationship.getTarget().getElement());
5981        }
5982
5983        private boolean hasUnresolvedDynamicEndpoint(Relationship relationship) {
5984                if (hasPlaceholderSourceTable(relationship, false)) {
5985                        return true;
5986                }
5987                if (relationship.getSources() != null) {
5988                        for (Object value : relationship.getSources()) {
5989                                if (value instanceof RelationRowsRelationshipElement) {
5990                                        Object rows = ((RelationRowsRelationshipElement<?>) value).getElement();
5991                                        if (rows instanceof TableRelationRows
5992                                                        && isRuntimeObjectPlaceholder(
5993                                                                        ((TableRelationRows) rows).getHolder())) {
5994                                                return true;
5995                                        }
5996                                }
5997                        }
5998                }
5999                Object target = relationship.getTarget() == null
6000                                ? null : relationship.getTarget().getElement();
6001                if (target instanceof TableColumn) {
6002                        return isRuntimeObjectPlaceholder(((TableColumn) target).getTable());
6003                }
6004                return target instanceof TableRelationRows
6005                                && isRuntimeObjectPlaceholder(((TableRelationRows) target).getHolder());
6006        }
6007
6008        private boolean isRuntimeObjectPlaceholder(Table table) {
6009                if (table == null || table.isPseudo()) return true;
6010                if (table.isVariable()) return false;
6011                String name = table.getName();
6012                return name != null && name.trim().startsWith("@");
6013        }
6014
6015        /** Identity snapshot used only by SHADOW observability; it never drives removal. */
6016        private Set<Relationship> snapshotDynamicRelationships() {
6017                Set<Relationship> snapshot = Collections.newSetFromMap(
6018                                new IdentityHashMap<Relationship, Boolean>());
6019                Collections.addAll(snapshot, modelManager.getRelations());
6020                return snapshot;
6021        }
6022
6023        /** Exact number of currently published relationships not present in {@code before}. */
6024        private int countNewDynamicRelationships(Set<Relationship> before) {
6025                if (before == null) {
6026                        return 0;
6027                }
6028                int count = 0;
6029                for (Relationship relation : modelManager.getRelations()) {
6030                        if (!before.contains(relation)) {
6031                                count++;
6032                        }
6033                }
6034                return count;
6035        }
6036
6037        /**
6038         * One unresolved hole of a dynamic fold, located by CHARACTER SPAN in the analyzed text
6039         * rather than by its rendered spelling. Locating by text would over-mark: a hole rendering
6040         * as {@code SRC} also occurs inside the unrelated literal {@code SRC_COPY} in the same batch.
6041         *
6042         * <p>A hole here is always a fragment whose VALUE is unknown. An {@code INEXACT_TRANSFORM}
6043         * fragment is deliberately NOT one: there the evaluator did compute the value from literals
6044         * ({@code REPLACE(tmpl, '{SRC}', 'dbo.SalesOrders')}) and only its byte-exactness against
6045         * vendor runtime semantics is unproven. Treating it as a hole would brand every fully
6046         * template-resolved name a runtime template — the opposite of this feature's purpose.
6047         */
6048        private static final class FoldHole {
6049                /** Character span [start, end) in the analyzed text; start == end for a splice point. */
6050                final int start;
6051                final int end;
6052                FoldHole(int start, int end) {
6053                        this.start = start;
6054                        this.end = end;
6055                }
6056
6057                /** True when this hole can affect any character of [from, to) — splice points included. */
6058                boolean touches(int from, int to) {
6059                        if (start == end) {
6060                                return from <= start && start <= to;
6061                        }
6062                        return start < to && end > from;
6063                }
6064        }
6065
6066        /** Conservative fold-hole spelling: the unresolved {@code @variable} remnant a partial fold leaves behind. */
6067        private static final Pattern DYNAMIC_FOLD_PLACEHOLDER = Pattern.compile("@\\w+");
6068
6069        /**
6070         * Depth of dynamic-fold inner analysis currently on the stack. Statements analyzed at
6071         * depth 0 are STATIC statements of the unit; their table references veto dynamic-template
6072         * marks on the same name (see {@link #applyStaticTemplateVetoes}), which is what makes the
6073         * shared-node contract independent of statement ORDER — a static reference after the fold
6074         * withdraws the mark exactly as one before the fold prevents it.
6075         */
6076        private int dynamicFoldDepth = 0;
6077
6078        /** True once any endpoint of this analysis carries a dynamic-template mark. */
6079        private boolean anyDynamicTemplateMarked = false;
6080
6081        /**
6082         * Name-registry keys of every table referenced by a STATIC statement after the first
6083         * template mark. A fold never marks (and a later collection unmarks) a name in this set.
6084         */
6085        private final Set<String> staticTemplateVetoKeys = new HashSet<String>();
6086
6087        /** Statements already veto-walked (identity), so repeated re-analysis stays O(1). */
6088        private final Set<TCustomSqlStatement> staticVetoWalkedStmts =
6089                        Collections.newSetFromMap(new IdentityHashMap<TCustomSqlStatement, Boolean>());
6090
6091        /**
6092         * Collects {@code stmt}'s static table references into the veto set and withdraws any
6093         * template mark already sitting on a node they share. Runs only at fold depth 0 and only
6094         * once a mark exists, so the common no-dynamic-SQL unit pays nothing.
6095         */
6096        private void applyStaticTemplateVetoes(TCustomSqlStatement stmt) {
6097                // Walk each statement object once: the veto keys accumulate, so a repeat walk of a
6098                // re-analyzed procedure body adds nothing but O(AST) cost per call.
6099                if (!staticVetoWalkedStmts.add(stmt)) {
6100                        return;
6101                }
6102                TemplateTableCollector collector = new TemplateTableCollector();
6103                try {
6104                        if (stmt instanceof TSelectSqlStatement) {
6105                                collector.preVisit((TSelectSqlStatement) stmt);
6106                        }
6107                        stmt.acceptChildren(collector);
6108                } catch (RuntimeException ex) {
6109                        // This is an enrichment-only walk over an error-recovered AST. Keep every
6110                        // static veto collected before the malformed slot, and never let a missing
6111                        // annotation discard the statement's real lineage.
6112                        logger.warn("Static template-veto walk was incomplete; keeping recovered lineage", ex);
6113                }
6114                for (TTable table : collector.tables) {
6115                        if (table.getSubquery() != null || table.getCTE() != null
6116                                        || table.getTableName() == null) {
6117                                continue;
6118                        }
6119                        vetoStaticTemplateName(table.getTableName().toString());
6120                }
6121                for (TObjectName intoTarget : collector.intoTargets) {
6122                        vetoStaticTemplateName(intoTarget.toString());
6123                }
6124        }
6125
6126        /**
6127         * Records one static reference under the SAME name-registry keys the model-reuse lookups
6128         * use ({@code ModelFactory.createTableByName}), so the veto matches exactly the reuse that
6129         * creates a shared node.
6130         */
6131        private void vetoStaticTemplateName(String rawName) {
6132                if (SQLUtil.isEmpty(rawName)) {
6133                        return;
6134                }
6135                String[] keys = { DlineageUtil.getTableFullName(rawName),
6136                                DlineageUtil.getTableFullNameWithDefaultSchema(rawName) };
6137                for (String key : keys) {
6138                        if (SQLUtil.isEmpty(key)) {
6139                                continue;
6140                        }
6141                        staticTemplateVetoKeys.add(key);
6142                        Table bound = modelManager.getTableByName(key);
6143                        if (bound != null && bound.isDynamicTemplate()) {
6144                                bound.clearDynamicTemplate();
6145                        }
6146                }
6147        }
6148
6149        /**
6150         * A delimited identifier whose ENTIRE content is an {@code @variable} remnant —
6151         * {@code [@S]} / {@code "@S"}, the shape a {@code '... [' + @S + '] ...'} splice folds to.
6152         * Deliberately anchored to the whole content: {@code [weird@name]} does NOT match, because
6153         * a delimited name that merely CONTAINS an {@code @} is an ordinary legal identifier.
6154         */
6155        private static final Pattern DYNAMIC_FOLD_DELIMITED_PLACEHOLDER =
6156                        Pattern.compile("\\[(@\\w+)\\]|\"(@\\w+)\"");
6157
6158        /**
6159         * Identity snapshot of the endpoints that already existed before a dynamic fold was
6160         * analyzed, so {@link #markDynamicTemplateEndpoints} can restrict itself to the nodes
6161         * THIS fold introduced. Returns {@code null} when the fold cannot carry a hole, which is
6162         * the signal to skip marking entirely (and pay nothing on the fully-resolved path).
6163         */
6164        private Set<Table> snapshotTemplateCandidateTables(boolean foldMayCarryHoles) {
6165                if (!foldMayCarryHoles) {
6166                        return null;
6167                }
6168                Set<Table> snapshot = Collections.newSetFromMap(new IdentityHashMap<Table, Boolean>());
6169                snapshot.addAll(modelManager.getTablesByName());
6170                return snapshot;
6171        }
6172
6173        /**
6174         * Exact hole spans for {@code analyzedText}, from the evaluator's fragment provenance.
6175         * Empty unless the site describes THIS text — spans taken from a different materialization
6176         * would point at arbitrary offsets of the string actually analyzed.
6177         */
6178        /** True when the site carries fragments that provably render its own text. */
6179        private static boolean hasCompleteFragmentProvenance(
6180                        gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite site) {
6181                if (site == null || site.sqlText == null || site.fragments.isEmpty()) {
6182                        return false;
6183                }
6184                StringBuilder rendered = new StringBuilder(site.sqlText.length());
6185                for (gudusoft.gsqlparser.dlineage.dynamicsql.SqlFragment f : site.fragments) {
6186                        rendered.append(f.text);
6187                }
6188                return rendered.toString().equals(site.sqlText);
6189        }
6190
6191        /** True when any fragment of the site stands in for a value nobody knows. */
6192        private static boolean hasValueUnknownHole(
6193                        gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite site) {
6194                if (site == null) {
6195                        return false;
6196                }
6197                for (gudusoft.gsqlparser.dlineage.dynamicsql.SqlFragment f : site.fragments) {
6198                        if (!f.isLiteral() && f.valueUnknown) {
6199                                return true;
6200                        }
6201                }
6202                return false;
6203        }
6204
6205        /**
6206         * Value-unknown hole spans of a site whose text IS the analyzed text. A hole whose value
6207         * the evaluator computed from literals ({@code SqlFragment.valueUnknown} false) is an
6208         * unproven RENDERING of a known name, not a runtime template, and yields no span.
6209         */
6210        private List<FoldHole> exactFoldHoles(
6211                        gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite site,
6212                        String analyzedText) {
6213                List<FoldHole> holes = new ArrayList<FoldHole>();
6214                if (site == null || site.sqlText == null || analyzedText == null
6215                                || !site.sqlText.equals(analyzedText)) {
6216                        return holes;
6217                }
6218                List<int[]> spans = gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlEdgeProof
6219                                .holeSpansOf(site);
6220                List<gudusoft.gsqlparser.dlineage.dynamicsql.SqlFragment> fragments =
6221                                gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlEdgeProof.holesOf(site);
6222                for (int i = 0; i < spans.size() && i < fragments.size(); i++) {
6223                        if (!fragments.get(i).valueUnknown) {
6224                                continue;
6225                        }
6226                        int[] span = spans.get(i);
6227                        holes.add(new FoldHole(span[0], span[1]));
6228                }
6229                return holes;
6230        }
6231
6232        /**
6233         * Maps the site's hole positions into a DIFFERENT rendering of the same fold — the legacy
6234         * token fold renders a hole as the variable/function name ({@code [pre_UPPER]}) where the
6235         * evaluator renders {@code [pre_@X]} — by aligning the LITERAL fragments, which both
6236         * renderings share, against the analyzed text. The literals are matched twice, greedily
6237         * from the left and greedily from the right; only when both passes agree is the alignment
6238         * forced, and the gaps between matched literals are then provably the holes. Returns
6239         * {@code null} when alignment is ambiguous or impossible — the caller falls back to the
6240         * conservative tier rather than guess.
6241         */
6242        private List<FoldHole> alignedFoldHoles(
6243                        gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite site,
6244                        String analyzedText) {
6245                if (site == null || analyzedText == null) {
6246                        return null;
6247                }
6248                List<gudusoft.gsqlparser.dlineage.dynamicsql.SqlFragment> fragments = site.fragments;
6249                // A value-KNOWN hole's text IS its value (that is what valueUnknown=false means), so
6250                // it anchors the alignment exactly like a literal. Without this, a known hole
6251                // adjacent to an unknown one would smear "templated" across every segment the
6252                // combined gap touches — [' + UPPER(@D) + '].[dbo].[x] must report db, not
6253                // db|schema|name. If the legacy rendering of the known hole differs, the anchor
6254                // match fails and the alignment is refused rather than guessed.
6255                List<String> literals = new ArrayList<String>();
6256                for (gudusoft.gsqlparser.dlineage.dynamicsql.SqlFragment f : fragments) {
6257                        if (isAlignmentAnchor(f) && !f.text.isEmpty()) {
6258                                literals.add(f.text);
6259                        }
6260                }
6261                int n = literals.size();
6262                int[] posForward = new int[n];
6263                int cursor = 0;
6264                for (int i = 0; i < n; i++) {
6265                        int at = analyzedText.indexOf(literals.get(i), cursor);
6266                        if (at < 0) {
6267                                return null;
6268                        }
6269                        posForward[i] = at;
6270                        cursor = at + literals.get(i).length();
6271                }
6272                int[] posBackward = new int[n];
6273                cursor = analyzedText.length();
6274                for (int i = n - 1; i >= 0; i--) {
6275                        int at = analyzedText.lastIndexOf(literals.get(i), cursor - literals.get(i).length());
6276                        if (at < 0) {
6277                                return null;
6278                        }
6279                        posBackward[i] = at;
6280                        cursor = at;
6281                }
6282                for (int i = 0; i < n; i++) {
6283                        if (posForward[i] != posBackward[i]) {
6284                                return null;
6285                        }
6286                }
6287                // Walk the fragments, attributing each inter-literal gap to the hole fragments
6288                // between the anchors. A non-empty gap with no hole, or leftover text at either
6289                // end, means the analyzed text is not this fold — refuse.
6290                List<FoldHole> holes = new ArrayList<FoldHole>();
6291                int pos = 0;
6292                int literalIndex = 0;
6293                boolean pendingHole = false;
6294                for (gudusoft.gsqlparser.dlineage.dynamicsql.SqlFragment f : fragments) {
6295                        if (isAlignmentAnchor(f)) {
6296                                if (f.text.isEmpty()) {
6297                                        continue;
6298                                }
6299                                int at = posForward[literalIndex++];
6300                                if (at < pos) {
6301                                        return null;
6302                                }
6303                                if (at > pos && !pendingHole) {
6304                                        return null;
6305                                }
6306                                if (pendingHole) {
6307                                        holes.add(new FoldHole(pos, at));
6308                                }
6309                                pendingHole = false;
6310                                pos = at + f.text.length();
6311                        } else {
6312                                pendingHole = true;
6313                        }
6314                }
6315                if (pos < analyzedText.length() && !pendingHole) {
6316                        return null;
6317                }
6318                if (pendingHole) {
6319                        holes.add(new FoldHole(pos, analyzedText.length()));
6320                }
6321                return holes;
6322        }
6323
6324        /** Literal fragments AND value-known holes anchor the alignment; only value-unknown holes gap. */
6325        private static boolean isAlignmentAnchor(gudusoft.gsqlparser.dlineage.dynamicsql.SqlFragment f) {
6326                return f.isLiteral() || !f.valueUnknown;
6327        }
6328
6329        /**
6330         * Fallback for a legacy token fold that carries no fragment provenance: the unresolved
6331         * {@code @variable} remnants left in the text. Restricted to what is PROVABLE — a T-SQL
6332         * family dialect (where an undelimited leading {@code @} cannot begin a legal object name,
6333         * the same reasoning {@link #isPlaceholderNamedTable} relies on), and only occurrences
6334         * outside identifier delimiters and string literals. A bracketed {@code [weird@name]} is a
6335         * legal literal identifier even inside a folded batch, so it is deliberately not a hole
6336         * here; such a name is only marked when exact provenance says so.
6337         */
6338        private List<FoldHole> conservativeFoldHoles(String analyzedText) {
6339                List<FoldHole> holes = new ArrayList<FoldHole>();
6340                if (analyzedText == null || !Table.isTsqlFamilyVendor(option.getVendor())) {
6341                        return holes;
6342                }
6343                Matcher matcher = DYNAMIC_FOLD_PLACEHOLDER.matcher(analyzedText);
6344                while (matcher.find()) {
6345                        if (!isInsideDelimiterOrLiteral(analyzedText, matcher.start())) {
6346                                holes.add(new FoldHole(matcher.start(), matcher.end()));
6347                        }
6348                }
6349                Matcher delimited = DYNAMIC_FOLD_DELIMITED_PLACEHOLDER.matcher(analyzedText);
6350                while (delimited.find()) {
6351                        // The delimiters themselves are literal fold text; only the content is the hole,
6352                        // so a '#'/'@' that survives outside them still classifies the endpoint.
6353                        if (!isInsideDelimiterOrLiteral(analyzedText, delimited.start())) {
6354                                holes.add(new FoldHole(delimited.start() + 1, delimited.end() - 1));
6355                        }
6356                }
6357                return holes;
6358        }
6359
6360        /** True when {@code index} sits inside a delimited identifier or a string literal. */
6361        private static boolean isInsideDelimiterOrLiteral(String text, int index) {
6362                boolean inBracket = false, inDoubleQuote = false, inSingleQuote = false;
6363                for (int i = 0; i < index && i < text.length(); i++) {
6364                        char c = text.charAt(i);
6365                        if (inBracket) {
6366                                if (c == ']') {
6367                                        inBracket = false;
6368                                }
6369                        } else if (inDoubleQuote) {
6370                                if (c == '"') {
6371                                        inDoubleQuote = false;
6372                                }
6373                        } else if (inSingleQuote) {
6374                                if (c == '\'') {
6375                                        inSingleQuote = false;
6376                                }
6377                        } else if (c == '[') {
6378                                inBracket = true;
6379                        } else if (c == '"') {
6380                                inDoubleQuote = true;
6381                        } else if (c == '\'') {
6382                                inSingleQuote = true;
6383                        }
6384                }
6385                return inBracket || inDoubleQuote || inSingleQuote;
6386        }
6387
6388        /**
6389         * Land the placeholder verdict on the MODEL NODES a partially-folded dynamic site just
6390         * produced (docs/tmp/dynamic-template-endpoint-provenance.md): every endpoint whose
6391         * identifier OVERLAPS an unresolved hole is marked {@link EndpointKind#DYNAMIC_TEMPLATE}
6392         * and told WHICH name segments are templated, so a consumer never has to sniff {@code @}
6393         * out of a name — which it cannot do soundly, because the fold keeps the original
6394         * delimiters and {@code [weird@name]} is a legal literal identifier.
6395         *
6396         * <p>Overlap is decided on CHARACTER SPANS of this parse's own tokens against the hole
6397         * spans (exact tier) or the provable {@code @\w+} remnants (conservative tier) — never on
6398         * rendered-text coincidence, which marks unrelated literals that happen to contain a
6399         * hole's spelling.
6400         *
6401         * <p>Never marks outside a fold context ({@code before == null} short-circuits), and never
6402         * marks a plain endpoint that already existed before this fold — a static statement's
6403         * {@code dbo.[weird@name]} stays exactly what it was. <b>Known limitation:</b> when a
6404         * static statement and a folded statement name the SAME object, the model carries one
6405         * shared node; it is left unmarked, because marking it would make the static reference
6406         * wrong. Per-reference template provenance would be needed to state both facts.
6407         *
6408         * <p>Oracle {@code EXECUTE IMMEDIATE} / DBMS_SQL legacy folds splice a BARE identifier
6409         * ({@code '... FROM ' || p_tab}), indistinguishable from a real table name and carrying no
6410         * fragment provenance, so they are deliberately NOT marked rather than marked on a guess.
6411         */
6412        private void markDynamicTemplateEndpoints(Set<Table> before,
6413                        gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite site,
6414                        TGSqlParser analyzedParser) {
6415                if (before == null || analyzedParser == null || analyzedParser.sqlstatements == null) {
6416                        return;
6417                }
6418                String analyzedText = analyzedParser.sqltext;
6419                List<FoldHole> holes;
6420                if (hasCompleteFragmentProvenance(site)) {
6421                        // Exact provenance is AUTHORITATIVE in both directions: zero value-unknown
6422                        // holes means every part of the string is a known value, so nothing here is
6423                        // runtime-templated and the conservative heuristic must NOT run — it would
6424                        // mark a legitimate literal-derived [@S].
6425                        if (!hasValueUnknownHole(site)) {
6426                                return;
6427                        }
6428                        if (site.sqlText.equals(analyzedText)) {
6429                                holes = exactFoldHoles(site, analyzedText);
6430                        } else {
6431                                // The analyzed text is the legacy token fold's rendering of the same
6432                                // fold; map the holes across via literal alignment, falling back to the
6433                                // conservative remnant scan when the alignment cannot be proven.
6434                                holes = alignedFoldHoles(site, analyzedText);
6435                                if (holes == null) {
6436                                        holes = conservativeFoldHoles(analyzedText);
6437                                }
6438                        }
6439                } else {
6440                        holes = conservativeFoldHoles(analyzedText);
6441                }
6442                if (holes.isEmpty()) {
6443                        return;
6444                }
6445                TemplateTableCollector collector = new TemplateTableCollector();
6446                for (int i = 0; i < analyzedParser.sqlstatements.size(); i++) {
6447                        TCustomSqlStatement stmt = analyzedParser.sqlstatements.get(i);
6448                        if (stmt == null) {
6449                                continue;
6450                        }
6451                        try {
6452                                // acceptChildren does not preVisit the statement itself, so a top-level
6453                                // SELECT ... INTO <target> would lose its write endpoint.
6454                                if (stmt instanceof TSelectSqlStatement) {
6455                                        collector.preVisit((TSelectSqlStatement) stmt);
6456                                }
6457                                stmt.acceptChildren(collector);
6458                        } catch (RuntimeException ex) {
6459                                // A clean inner parse can still contain recovery shells. The collector is
6460                                // stateless, so keep endpoints reached before the bad slot and continue with
6461                                // later statements. A warning is intentional: marks for this site may be
6462                                // incomplete, but enrichment must never erase otherwise valid lineage.
6463                                logger.warn("Dynamic template endpoint walk was incomplete; keeping recovered lineage", ex);
6464                        }
6465                }
6466                for (TTable table : collector.tables) {
6467                        if (table.getSubquery() != null || table.getCTE() != null) {
6468                                continue;
6469                        }
6470                        markDynamicTemplateEndpoint(before, table.getTableName(),
6471                                        templateTableModelOf(table), analyzedText, holes);
6472                }
6473                for (TObjectName intoTarget : collector.intoTargets) {
6474                        // SELECT ... INTO targets are bound by NAME (ModelFactory.createTableByName),
6475                        // not against the parsed node, so they resolve through the name registry.
6476                        Table model = modelManager.getTableByName(
6477                                        DlineageUtil.getTableFullName(intoTarget.toString()));
6478                        markDynamicTemplateEndpoint(before, intoTarget, model, analyzedText, holes);
6479                }
6480        }
6481
6482        /** Collects the endpoint-bearing name nodes of an analyzed dynamic-fold parse. */
6483        private static final class TemplateTableCollector extends TParseTreeVisitor {
6484                final List<TTable> tables = new ArrayList<TTable>();
6485                final List<TObjectName> intoTargets = new ArrayList<TObjectName>();
6486
6487                @Override
6488                public void preVisit(TTable table) {
6489                        if (table != null) {
6490                                tables.add(table);
6491                        }
6492                }
6493
6494                @Override
6495                public void preVisit(TSelectSqlStatement select) {
6496                        if (select == null) {
6497                                return;
6498                        }
6499                        if (select.getIntoTableClause() != null
6500                                        && select.getIntoTableClause().getTableName() != null) {
6501                                intoTargets.add(select.getIntoTableClause().getTableName());
6502                        }
6503                        // T-SQL SELECT ... INTO <target> arrives as an INTO clause carrying an
6504                        // expression list, the same shape analyzeSelectIntoClause() reads.
6505                        TIntoClause intoClause = select.getIntoClause();
6506                        if (intoClause == null || intoClause.getExprList() == null) {
6507                                return;
6508                        }
6509                        for (int i = 0; i < intoClause.getExprList().size(); i++) {
6510                                TExpression expr = intoClause.getExprList().getExpression(i);
6511                                if (expr != null && expr.getObjectOperand() != null) {
6512                                        intoTargets.add(expr.getObjectOperand());
6513                                }
6514                        }
6515                }
6516        }
6517
6518        /** Per-endpoint half of {@link #markDynamicTemplateEndpoints}: span overlap + part labelling. */
6519        private void markDynamicTemplateEndpoint(Set<Table> before, TObjectName tableName,
6520                        Table model, String analyzedText, List<FoldHole> holes) {
6521                if (tableName == null || model == null || tableName.getStartToken() == null
6522                                || tableName.getEndToken() == null) {
6523                        return;
6524                }
6525                int from = (int) tableName.getStartToken().offset;
6526                int to = (int) (tableName.getEndToken().offset
6527                                + tableName.getEndToken().toString().length());
6528                if (from < 0 || to <= from || to > analyzedText.length()) {
6529                        return;
6530                }
6531                List<int[]> segments = nameSegmentSpans(analyzedText, from, to);
6532                if (segments.isEmpty()) {
6533                        return;
6534                }
6535                int last = segments.size() - 1;
6536                StringBuilder parts = new StringBuilder();
6537                boolean identityTemplated = false;
6538                for (int i = 0; i < segments.size(); i++) {
6539                        int[] segment = segments.get(i);
6540                        boolean touched = false;
6541                        for (FoldHole hole : holes) {
6542                                if (!hole.touches(segment[0], segment[1])) {
6543                                        continue;
6544                                }
6545                                touched = true;
6546                                // The hole reaching the FIRST character of the object-name segment means the
6547                                // '#'/'@' prefix that would classify this endpoint as temp / table-variable is
6548                                // itself unknown runtime text ('INSERT INTO ' + @tab folds to a table named
6549                                // @tab, which really names a CATALOG object).
6550                                if (i == last
6551                                                && hole.touches(firstNameChar(analyzedText, segment),
6552                                                                firstNameChar(analyzedText, segment) + 1)) {
6553                                        identityTemplated = true;
6554                                }
6555                        }
6556                        if (!touched) {
6557                                continue;
6558                        }
6559                        String part = templatedPartLabel(i, last);
6560                        if (part == null) {
6561                                continue;
6562                        }
6563                        if (parts.length() > 0) {
6564                                parts.append('|');
6565                        }
6566                        parts.append(part);
6567                }
6568                if (parts.length() == 0) {
6569                        return;
6570                }
6571                if (model.hasSubquery() || model.isPseudo() || model.isVariable()) {
6572                        return;
6573                }
6574                // A plain endpoint that predates this fold is shared with a static reference; marking
6575                // it would make that reference wrong. An endpoint already known to be a template may
6576                // accumulate more segments from a second fold.
6577                if (before.contains(model) && !model.isDynamicTemplate()) {
6578                        return;
6579                }
6580                // The same contract, order-independent: a name a STATIC statement referenced at any
6581                // earlier point of the unit is never marked (see staticTemplateVetoKeys).
6582                if (staticTemplateVetoKeys.contains(DlineageUtil.getTableFullName(tableName.toString()))
6583                                || staticTemplateVetoKeys.contains(
6584                                                DlineageUtil.getTableFullNameWithDefaultSchema(tableName.toString()))) {
6585                        return;
6586                }
6587                model.markDynamicTemplate(parts.toString(), identityTemplated);
6588                anyDynamicTemplateMarked = true;
6589        }
6590
6591        /** Offset of a segment's first NAME character, skipping an opening identifier delimiter. */
6592        private static int firstNameChar(String text, int[] segment) {
6593                int i = segment[0];
6594                if (i < text.length()) {
6595                        char c = text.charAt(i);
6596                        if (c == '[' || c == '"' || c == '`') {
6597                                return i + 1;
6598                        }
6599                }
6600                return i;
6601        }
6602
6603        /**
6604         * Splits the source span of a qualified name into one span per identifier segment, on the
6605         * dots that sit OUTSIDE identifier delimiters (so {@code [a.b].c} is two segments).
6606         */
6607        private static List<int[]> nameSegmentSpans(String text, int from, int to) {
6608                List<int[]> segments = new ArrayList<int[]>();
6609                int segmentStart = from;
6610                boolean inBracket = false, inDoubleQuote = false, inBacktick = false;
6611                for (int i = from; i < to; i++) {
6612                        char c = text.charAt(i);
6613                        if (inBracket) {
6614                                if (c == ']') {
6615                                        inBracket = false;
6616                                }
6617                        } else if (inDoubleQuote) {
6618                                if (c == '"') {
6619                                        inDoubleQuote = false;
6620                                }
6621                        } else if (inBacktick) {
6622                                if (c == '`') {
6623                                        inBacktick = false;
6624                                }
6625                        } else if (c == '[') {
6626                                inBracket = true;
6627                        } else if (c == '"') {
6628                                inDoubleQuote = true;
6629                        } else if (c == '`') {
6630                                inBacktick = true;
6631                        } else if (c == '.') {
6632                                segments.add(new int[] { segmentStart, i });
6633                                segmentStart = i + 1;
6634                        }
6635                }
6636                segments.add(new int[] { segmentStart, to });
6637                return segments;
6638        }
6639
6640        /**
6641         * Outermost-first segment labels for a qualified name, counted from the RIGHT so a
6642         * 2-, 3- or 4-part name labels consistently. Anything deeper than server is unlabelled.
6643         */
6644        private String templatedPartLabel(int index, int lastIndex) {
6645                switch (lastIndex - index) {
6646                case 0:
6647                        return "name";
6648                case 1:
6649                        return "schema";
6650                case 2:
6651                        return "db";
6652                case 3:
6653                        return "server";
6654                default:
6655                        return null;
6656                }
6657        }
6658
6659        /** The dlineage endpoint model bound to a parsed table, across the binding maps. */
6660        private Table templateTableModelOf(TTable table) {
6661                Object model = modelManager.getModel(table);
6662                if (!(model instanceof Table)) {
6663                        model = modelManager.getCreateModel(table);
6664                }
6665                if (!(model instanceof Table)) {
6666                        model = modelManager.getInsertModel(table);
6667                }
6668                if (!(model instanceof Table)) {
6669                        model = modelManager.getUpdateModel(table);
6670                }
6671                if (!(model instanceof Table)) {
6672                        model = modelManager.getMergeModel(table);
6673                }
6674                return model instanceof Table ? (Table) model : null;
6675        }
6676
6677        /**
6678         * What a site can say about its holes. {@code exact} is true only when the evaluator
6679         * saw the site's argument: then {@code count == fragments.size()}, and
6680         * {@code materializedText} is the argument as far as it folded (holes rendered as
6681         * their placeholder text). Otherwise {@code count} is {@code -1} and the list is
6682         * empty - "not known", never an invented zero.
6683         */
6684        private static final class DynamicHoleEvidence {
6685                final int count;
6686                final boolean exact;
6687                final List<UnresolvedFragment> fragments;
6688                final String materializedText;
6689
6690                DynamicHoleEvidence(int count, boolean exact, List<UnresolvedFragment> fragments,
6691                                String materializedText) {
6692                        this.count = count;
6693                        this.exact = exact;
6694                        this.fragments = fragments;
6695                        this.materializedText = materializedText;
6696                }
6697        }
6698
6699        private static final DynamicHoleEvidence NO_HOLE_EVIDENCE = new DynamicHoleEvidence(-1, false,
6700                        Collections.<UnresolvedFragment>emptyList(), null);
6701
6702        private DynamicHoleEvidence dynamicHoleEvidence(
6703                        gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite site) {
6704                if (site == null) {
6705                        return NO_HOLE_EVIDENCE;
6706                }
6707                if (!site.provenanceIncomplete) {
6708                        return new DynamicHoleEvidence(0, true, Collections.<UnresolvedFragment>emptyList(),
6709                                        site.sqlText);
6710                }
6711                if (site.fragments.isEmpty()) {
6712                        // The value did not reduce and the producer left no provenance (a channel
6713                        // without fragments, or a value that never touched a variable).
6714                        return new DynamicHoleEvidence(-1, false, Collections.<UnresolvedFragment>emptyList(),
6715                                        site.sqlText);
6716                }
6717                List<UnresolvedFragment> holes = new ArrayList<UnresolvedFragment>();
6718                for (gudusoft.gsqlparser.dlineage.dynamicsql.SqlFragment fragment : site.fragments) {
6719                        UnresolvedFragment hole = UnresolvedFragment.of(fragment);
6720                        if (hole != null) {
6721                                holes.add(hole);
6722                        }
6723                }
6724                // Only a reduced argument has a materialization; rendering the provenance of an
6725                // unreduced one would fabricate text (the arguments of an unfoldable call glued
6726                // together). Its holes are still reported.
6727                String text = site.sqlText;
6728                if (site.sqlText == null && holes.isEmpty()) {
6729                        // The value did not reduce, yet nothing in its provenance is a hole: the
6730                        // producer that stopped the fold left no fragment (an undecidable CASE, a
6731                        // resource bound, a malformed call). "Not known" - never an exact zero.
6732                        return new DynamicHoleEvidence(-1, false, Collections.<UnresolvedFragment>emptyList(), text);
6733                }
6734                return new DynamicHoleEvidence(holes.size(), true, holes, text);
6735        }
6736
6737        /**
6738         * True when the materialized text is nothing but value-unknown holes - a bare
6739         * {@code @sql} whose value the evaluator could not fold. Parsing that as SQL proves
6740         * nothing about the runtime statement, so such a site is reported as its
6741         * argument shape (with the hole evidence) instead of as a parse failure of text
6742         * that was never SQL.
6743         */
6744        private static boolean isWholeTextHole(
6745                        gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite site) {
6746                if (site == null || site.sqlText == null || site.fragments.isEmpty()) {
6747                        return false;
6748                }
6749                for (gudusoft.gsqlparser.dlineage.dynamicsql.SqlFragment fragment : site.fragments) {
6750                        if (fragment.isLiteral() || !fragment.valueUnknown) {
6751                                return false;
6752                        }
6753                }
6754                return true;
6755        }
6756
6757        private String dynamicTrustReason(String prefix, int unprovenCount,
6758                        int unresolvedHoleCount, boolean holeCountExact) {
6759                String holes = holeCountExact ? unresolvedHoleCount + " unresolved fragment(s)"
6760                                : "unresolved fragment count unavailable from the producer";
6761                return prefix + "; " + holes
6762                                + (unprovenCount >= 0 ? "; " + unprovenCount + " unproven candidate relationship(s)" : "");
6763        }
6764
6765        /**
6766         * Classify the lineage edges a folded dynamic site produced. {@code relsBefore} is the relation
6767         * snapshot taken immediately before the folded SQL was analyzed; the tail of the current relation set
6768         * is this site's delta (relationHolder is insertion-ordered). counts[0]=resolved (edge into a real
6769         * target column), counts[1]=partial (edge into a T-SQL variable / placeholder target). Edges whose
6770         * target is an intermediate result-set are ignored — only final-target edges are counted.
6771         */
6772        /**
6773         * String-building functions whose names show up as fabricated source "tables" when a partial
6774         * fold leaves a runtime-built object name in the re-parsed text (e.g.
6775         * {@code EXEC('... FROM ' + QUOTENAME(@tab))}).
6776         */
6777        private static final java.util.Set<String> DYNAMIC_NAME_BUILDER_FUNCTIONS = new HashSet<String>(
6778                        Arrays.asList("QUOTENAME", "REPLACE", "CONCAT", "FORMAT", "UPPER", "LOWER", "TRIM", "LTRIM", "RTRIM"));
6779
6780        /**
6781         * Returns {@code {resolvedCount, partialCount, placeholderSeen, resultSetEdgeCount}} for the
6782         * lineage edges added after {@code beforeCount}.
6783         *
6784         * <ul>
6785         *   <li>{@code resolvedCount} — edges into a real base-table target column with no placeholder
6786         *       contamination anywhere in the new edge set.</li>
6787         *   <li>{@code partialCount} — edges into a variable / placeholder target, plus (when the edge
6788         *       set is contaminated) every edge that would otherwise have counted as resolved, so the
6789         *       public counts never claim usable lineage a placeholder actually feeds.</li>
6790         *   <li>{@code placeholderSeen} — 1 when any new edge draws from a fabricated placeholder table:
6791         *       a raw name starting with '@' (delimited {@code [@x]} identifiers are legal object names
6792         *       and are NOT placeholders), or — during a partial fold only — a variable-backed source
6793         *       table (Oracle {@code ' ... FROM ' || p_tab}) or a table named after a string-building
6794         *       function ({@code QUOTENAME} etc.).</li>
6795         *   <li>{@code resultSetEdgeCount} — clean edges into a result-set column (a folded dynamic
6796         *       {@code SELECT}'s projection). These contribute to the site-resolution decision but stay
6797         *       out of {@code resolvedCount}, which is documented as base-table targets only.</li>
6798         * </ul>
6799         */
6800        private int[] classifyDynamicSiteLineage(int beforeCount, boolean partialFold) {
6801                int resolved = 0;
6802                int partial = 0;
6803                int resultSetEdges = 0;
6804                boolean placeholderSeen = false;
6805                Relationship[] after = modelManager.getRelations();
6806                for (int i = beforeCount; i < after.length; i++) {
6807                        Relationship r = after[i];
6808                        if (r == null) {
6809                                continue;
6810                        }
6811                        if (hasPlaceholderSourceTable(r, partialFold)) {
6812                                placeholderSeen = true;
6813                        }
6814                        if (r.getTarget() instanceof ResultColumnRelationshipElement) {
6815                                resultSetEdges++;
6816                        } else if (r.getTarget() instanceof TableColumnRelationshipElement) {
6817                                TableColumn targetColumn = ((TableColumnRelationshipElement) r.getTarget()).getElement();
6818                                Table targetTable = targetColumn == null ? null : targetColumn.getTable();
6819                                if (isPlaceholderNamedTable(targetTable, partialFold)) {
6820                                        // Runtime-built INSERT destination (e.g. 'INSERT INTO '+QUOTENAME(@tab)+...)
6821                                        // — the target object is unknown, so the whole site is contaminated.
6822                                        placeholderSeen = true;
6823                                        partial++;
6824                                } else if (isUnresolvedTargetTable(targetTable)) {
6825                                        partial++;
6826                                } else {
6827                                        resolved++;
6828                                }
6829                        }
6830                }
6831                if (placeholderSeen) {
6832                        // Contaminated edges must not be surfaced as usable lineage
6833                        // (producedLineage() reads resolvedCount).
6834                        partial += resolved;
6835                        resolved = 0;
6836                        resultSetEdges = 0;
6837                }
6838                return new int[] { resolved, partial, placeholderSeen ? 1 : 0, resultSetEdges };
6839        }
6840
6841        /**
6842         * Oracle DBMS_SQL resolution (plan Phase 5 slice 4; supersedes the Phase 1
6843         * visibility-only recording): a {@code DBMS_SQL.PARSE(c, <sql>, ...)} call is a
6844         * dynamic-SQL execution site. The SQL-text argument is folded through the shared
6845         * literal-bindable whitelist — inline literals, literal-composed locals, and
6846         * parameters bound at vetted literal call sites (same-file or cross-file via the
6847         * routine catalog) — and each folded text is analyzed through the ordinary inner-
6848         * SQL path. Anything that does not fold keeps an honest UNRESOLVED diagnostic;
6849         * PARSE_ERROR is reserved for compile-time literal text, mirroring the EXECUTE
6850         * IMMEDIATE status contract. Recording keys off PARSE, the call that carries the
6851         * SQL text, so a PARSE/EXECUTE pair maps to one site.
6852         */
6853        private void recordDbmsSqlSiteIfParseCall(TBasicStmt stmt, TFunctionCall functionCall) {
6854                if (functionCall == null || functionCall.getFunctionName() == null
6855                                || option.getVendor() != EDbVendor.dbvoracle) {
6856                        return;
6857                }
6858                String calledName = functionCall.getFunctionName().toString();
6859                if (!SQLUtil.compareIdentifier(EDbVendor.dbvoracle, ESQLDataObjectType.dotProcedure,
6860                                calledName, "DBMS_SQL.PARSE")
6861                                && !SQLUtil.compareIdentifier(EDbVendor.dbvoracle, ESQLDataObjectType.dotProcedure,
6862                                                calledName, "SYS.DBMS_SQL.PARSE")) {
6863                        return;
6864                }
6865                if (!option.isAnalyzeDynamicSql()) {
6866                        recordDynamicSqlSite(stmt, DynamicSqlSite.Kind.DBMS_SQL,
6867                                        DynamicSqlSite.Status.UNRESOLVED,
6868                                        "dynamic SQL inner analysis disabled");
6869                        return;
6870                }
6871                boolean literalSql = false;
6872                DynamicSqlTrustMode trustMode = option.getDynamicSqlTrustMode();
6873                TExpression sqlArg = null;
6874                if (functionCall.getArgs() != null && functionCall.getArgs().size() >= 2) {
6875                        // Positional form: PARSE(c, <sql>, <lang>). Named association may reorder the
6876                        // arguments (statement => '...'), so resolve the named form first.
6877                        boolean namedForm = false;
6878                        for (int i = 0; i < functionCall.getArgs().size(); i++) {
6879                                TExpression arg = functionCall.getArgs().getExpression(i);
6880                                if (arg != null && arg.getExpressionType() == EExpressionType.ref_arrow_t) {
6881                                        namedForm = true;
6882                                        if (arg.getLeftOperand() != null
6883                                                        && SQLUtil.sameName(EDbVendor.dbvoracle, ESQLDataObjectType.dotColumn,
6884                                                                        arg.getLeftOperand().toString(), "statement")) {
6885                                                sqlArg = arg.getRightOperand();
6886                                                break;
6887                                        }
6888                                }
6889                        }
6890                        if (sqlArg == null && !namedForm) {
6891                                sqlArg = functionCall.getArgs().getExpression(1);
6892                        }
6893                        literalSql = sqlArg != null && sqlArg.getExpressionType() == EExpressionType.simple_constant_t;
6894                }
6895                if (sqlArg != null) {
6896                        TCustomSqlStatement routine = null;
6897                        for (int i = stmtStack.size() - 1; i >= 0; i--) {
6898                                if (stmtStack.get(i) instanceof TStoredProcedureSqlStatement) {
6899                                        routine = stmtStack.get(i);
6900                                        break;
6901                                }
6902                        }
6903                        List<String> foldedTexts;
6904                        try {
6905                                foldedTexts = gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver
6906                                                .foldDbmsSqlTexts(routine, sqlArg, routineCatalog, 4);
6907                        } catch (RuntimeException ex) {
6908                                foldedTexts = new ArrayList<String>();
6909                        }
6910                        if (!foldedTexts.isEmpty()) {
6911                                Set<Relationship> shadowBefore = trustMode == DynamicSqlTrustMode.SHADOW
6912                                                ? snapshotDynamicRelationships() : null;
6913                                int dynamicSitesBefore = dynamicSqlSites.size();
6914                                int relsBeforeDynamic = modelManager.getRelations().length;
6915                                int errorsBeforeDynamic = errorInfos.size();
6916                                boolean analyzedAny = false;
6917                                boolean variantFailed = false;
6918                                List<Long> provenVariantRelationshipIds = new ArrayList<Long>();
6919                                for (String foldedText : foldedTexts) {
6920                                        int relationshipsBeforeVariant = modelManager.getRelations().length;
6921                                        int dynamicSitesBeforeVariant = dynamicSqlSites.size();
6922                                        int errorsBeforeVariant = errorInfos.size();
6923                                        TGSqlParser sqlparser = createSqlParser(option.getVendor());
6924                                        sqlparser.sqltext = foldedText;
6925                                        int result;
6926                                        try {
6927                                                result = sqlparser.parse();
6928                                        } catch (RuntimeException ex) {
6929                                                variantFailed = true;
6930                                                continue;
6931                                        }
6932                                        if (result != 0 || sqlparser.sqlstatements == null) {
6933                                                variantFailed = true;
6934                                                continue;
6935                                        }
6936                                        try {
6937                                                dynamicFoldDepth++;
6938                                                try {
6939                                                        for (int i = 0; i < sqlparser.sqlstatements.size(); i++) {
6940                                                                analyzeCustomSqlStmt(sqlparser.sqlstatements.get(i));
6941                                                        }
6942                                                } finally {
6943                                                        dynamicFoldDepth--;
6944                                                }
6945                                                analyzedAny = true;
6946                                                if (errorInfos.size() == errorsBeforeVariant
6947                                                                && !hasIncompleteDynamicSqlSitesSince(dynamicSitesBeforeVariant)) {
6948                                                        provenVariantRelationshipIds.addAll(provenDynamicRelationshipIdsSince(
6949                                                                        relationshipsBeforeVariant, dynamicSitesBeforeVariant));
6950                                                }
6951                                        } catch (RuntimeException ex) {
6952                                                // An analyzer crash on folded text must neither surface as an
6953                                                // AnalyzeError blessed into output nor masquerade as resolved.
6954                                                variantFailed = true;
6955                                        }
6956                                }
6957                                // Inner analysis handlers may swallow their own crashes into
6958                                // errorInfos (never rethrowing to the catch above). Errors raised
6959                                // while analyzing SYNTHESIZED text carry misleading fragment-
6960                                // relative coordinates - roll them back and treat the variant as
6961                                // failed so the site status stays honest.
6962                                if (errorInfos.size() > errorsBeforeDynamic) {
6963                                        while (errorInfos.size() > errorsBeforeDynamic) {
6964                                                errorInfos.remove(errorInfos.size() - 1);
6965                                        }
6966                                        variantFailed = true;
6967                                }
6968                                int[] dynCounts = classifyDynamicSiteLineage(relsBeforeDynamic, false);
6969                                boolean nestedIncomplete = trustMode != DynamicSqlTrustMode.LEGACY
6970                                                && hasIncompleteDynamicSqlSitesSince(dynamicSitesBefore);
6971                                boolean aggregateIncomplete = variantFailed || nestedIncomplete;
6972                                int shadowUnproven = trustMode == DynamicSqlTrustMode.SHADOW && aggregateIncomplete
6973                                                ? Math.max(0, countNewDynamicRelationships(shadowBefore)
6974                                                                - provenVariantRelationshipIds.size()) : 0;
6975                                if (analyzedAny && !aggregateIncomplete) {
6976                                        recordDynamicSqlSite(stmt, DynamicSqlSite.Kind.DBMS_SQL,
6977                                                        DynamicSqlSite.Status.RESOLVED, null, dynCounts[0], dynCounts[1],
6978                                                        0, 0, true, trustMode,
6979                                                        relsBeforeDynamic, dynamicSitesBefore,
6980                                                        provenVariantRelationshipIds);
6981                                        return;
6982                                }
6983                                if (analyzedAny) {
6984                                        // Some accepted variants could not be analyzed - the aggregate
6985                                        // picture is incomplete. SHADOW retains the historical candidates
6986                                        // and labels the aggregate as lacking per-edge proof.
6987                                        DynamicSqlSite.Status status = trustMode == DynamicSqlTrustMode.SHADOW
6988                                                        && (shadowUnproven > 0 || !provenVariantRelationshipIds.isEmpty())
6989                                                        ? DynamicSqlSite.Status.PARTIAL
6990                                                        : DynamicSqlSite.Status.UNRESOLVED;
6991                                        String reason = nestedIncomplete
6992                                                        ? "nested dynamic SQL in a folded DBMS_SQL variant is incomplete"
6993                                                        : "some folded DBMS_SQL variants failed to parse or analyze; lineage may be incomplete";
6994                                        recordDynamicSqlSite(stmt, DynamicSqlSite.Kind.DBMS_SQL, status,
6995                                                        trustMode == DynamicSqlTrustMode.SHADOW
6996                                    ? dynamicTrustReason(reason + "; SHADOW retained historical candidates",
6997                                                                                        shadowUnproven, nestedIncomplete ? -1 : 0,
6998                                                                                        !nestedIncomplete)
6999                                                                        : reason,
7000                                                        dynCounts[0], dynCounts[1], shadowUnproven,
7001                                                        nestedIncomplete ? -1 : 0, !nestedIncomplete, trustMode,
7002                                                        relsBeforeDynamic, dynamicSitesBefore,
7003                                                        provenVariantRelationshipIds);
7004                                        return;
7005                                }
7006                                if (variantFailed && literalSql) {
7007                                        recordDynamicSqlSite(stmt, DynamicSqlSite.Kind.DBMS_SQL,
7008                                                        DynamicSqlSite.Status.PARSE_ERROR,
7009                                                        "DBMS_SQL.PARSE literal SQL failed to parse", 0, 0,
7010                                                        0, 0, true, trustMode,
7011                                                        relsBeforeDynamic, dynamicSitesBefore);
7012                                        return;
7013                                }
7014                                if (variantFailed) {
7015                                        recordDynamicSqlSite(stmt, DynamicSqlSite.Kind.DBMS_SQL,
7016                                                        DynamicSqlSite.Status.UNRESOLVED,
7017                                                        "folded DBMS_SQL text failed to parse", 0, 0,
7018                                                        0, 0, true, trustMode,
7019                                                        relsBeforeDynamic, dynamicSitesBefore);
7020                                        return;
7021                                }
7022                        }
7023                }
7024                String unresolvedReason = literalSql ? "DBMS_SQL.PARSE with literal SQL; inner SQL not analyzed"
7025                                : "DBMS_SQL.PARSE with runtime SQL text; inner SQL not analyzed";
7026                if (trustMode == DynamicSqlTrustMode.LEGACY) {
7027                        recordDynamicSqlSite(stmt, DynamicSqlSite.Kind.DBMS_SQL,
7028                                        DynamicSqlSite.Status.UNRESOLVED, unresolvedReason);
7029                } else {
7030                        boolean runtimeText = !literalSql;
7031                        if (runtimeText && trustMode == DynamicSqlTrustMode.SHADOW) {
7032                                unresolvedReason = dynamicTrustReason("SHADOW observed no materialized DBMS_SQL candidate",
7033                                                0, -1, false);
7034                        }
7035                        recordDynamicSqlSite(stmt, DynamicSqlSite.Kind.DBMS_SQL,
7036                                        DynamicSqlSite.Status.UNRESOLVED, unresolvedReason, 0, 0,
7037                                        0,
7038                                        runtimeText ? -1 : 0, !runtimeText, trustMode);
7039                }
7040        }
7041
7042        private boolean hasIncompleteDynamicSqlSitesSince(int start) {
7043                return hasIncompleteDynamicSqlSitesSince(dynamicSqlSites, start);
7044        }
7045
7046        private static boolean hasIncompleteDynamicSqlSitesSince(List<DynamicSqlSite> sites, int start) {
7047                for (int i = Math.max(0, start); i < sites.size(); i++) {
7048                        DynamicSqlSite site = sites.get(i);
7049                        if (site.getStatus() != DynamicSqlSite.Status.RESOLVED) {
7050                                return true;
7051                        }
7052                }
7053                return false;
7054        }
7055
7056        /** True when any source of this edge is a fabricated placeholder table (see caller javadoc). */
7057        private boolean hasPlaceholderSourceTable(Relationship r, boolean partialFold) {
7058                if (r.getSources() == null) {
7059                        return false;
7060                }
7061                for (Object source : r.getSources()) {
7062                        if (!(source instanceof TableColumnRelationshipElement)) {
7063                                continue;
7064                        }
7065                        TableColumn sourceColumn = ((TableColumnRelationshipElement) source).getElement();
7066                        Table sourceTable = sourceColumn == null ? null : sourceColumn.getTable();
7067                        if (sourceTable == null || sourceTable.isPseudo()) {
7068                                continue;
7069                        }
7070                        if (sourceTable.isVariable()) {
7071                                // A variable-backed source inside a PARTIAL fold is the folded remnant of a
7072                                // runtime-built OBJECT NAME only when it stands in table position — its
7073                                // "columns" then carry real column names distinct from the variable's own
7074                                // name ('... FROM ' || p_tab reparsed as table p_tab with column s1). A
7075                                // scalar variable used as a value/predicate operand surfaces as a
7076                                // single-column variable whose column IS the variable name — legitimate,
7077                                // not contamination.
7078                                if (partialFold && sourceColumn != null && sourceColumn.getName() != null
7079                                                && sourceTable.getName() != null
7080                                                && !DlineageUtil.sameName(ESQLDataObjectType.dotColumn,
7081                                                                sourceColumn.getName(), sourceTable.getName())) {
7082                                        return true;
7083                                }
7084                                continue;
7085                        }
7086                        if (isPlaceholderNamedTable(sourceTable, partialFold)) {
7087                                return true;
7088                        }
7089                }
7090                return false;
7091        }
7092
7093        /**
7094         * A table whose NAME betrays a runtime-built object: a raw name starting with '@' (fold
7095         * artifact; delimited {@code [@x]} / {@code "@x"} are legal object names and do not match), or —
7096         * during a partial fold — a name equal to a string-building function left behind by the folder
7097         * ({@code QUOTENAME(...)} etc.). Applied to both edge sources and INSERT targets.
7098         */
7099        private boolean isPlaceholderNamedTable(Table table, boolean partialFold) {
7100                if (table == null) {
7101                        return false;
7102                }
7103                String name = table.getName() == null ? "" : table.getName().trim();
7104                if (name.startsWith("@")) {
7105                        return true;
7106                }
7107                // Locale-stable fold: default-locale toUpperCase turns "ltrim" into "LTRİM" under tr_TR.
7108                return partialFold && DYNAMIC_NAME_BUILDER_FUNCTIONS.contains(name.toUpperCase(Locale.ENGLISH));
7109        }
7110
7111        /**
7112         * A folded dynamic site's target is "unresolved" when it is a T-SQL variable / placeholder rather
7113         * than a real object. When GSP only partially folds (e.g. {@code EXEC sp_executesql @sql} →
7114         * {@code INSERT INTO @p SELECT ...}), the re-parsed target is a table literally named {@code @p};
7115         * its {@code isVariable()} flag is not set, so the reliable signal is a name starting with '@'.
7116         */
7117        private boolean isUnresolvedTargetTable(Table table) {
7118                if (table == null) {
7119                        return true;
7120                }
7121                if (table.isVariable() || table.isPseudo()) {
7122                        return true;
7123                }
7124                String name = table.getName();
7125                if (name != null) {
7126                        // Raw-name check only: delimited [@x] / "@x" are legal permanent-object names,
7127                        // while the fold artifact keeps a bare leading '@'.
7128                        if (name.trim().startsWith("@")) {
7129                                return true;
7130                        }
7131                }
7132                return false;
7133        }
7134
7135        /** True when a module name resolves to sp_executesql, ignoring schema qualifier and brackets/quotes. */
7136        private boolean isSpExecutesql(TObjectName moduleName) {
7137                if (moduleName == null) {
7138                        return false;
7139                }
7140                String s = moduleName.toString();
7141                int dot = s.lastIndexOf('.');
7142                if (dot >= 0) {
7143                        s = s.substring(dot + 1);
7144                }
7145                s = s.replace("[", "").replace("]", "").replace("\"", "").replace("`", "").trim();
7146                return "sp_executesql".equalsIgnoreCase(s);
7147        }
7148
7149        /**
7150         * True when a module name is a <em>local</em> invocation of the system
7151         * procedure sp_execute_external_script whose input query therefore executes in
7152         * the caller's current database context.
7153         *
7154         * <p>Uses the parser's structured object/schema segmentation plus the
7155         * identifier-equality facade ({@link SQLUtil#sameName}) so vendor delimiter
7156         * rules are honored: a naive {@code lastIndexOf('.')} or delimiter strip would
7157         * mistake a delimited identifier that merely contains a dot
7158         * ({@code [dbo.sp_execute_external_script]}) or one with escaped delimiters
7159         * ({@code [sp_execute_external_]]script]}, {@code "sp_execute_external_""script"})
7160         * for the system procedure.
7161         *
7162         * <p>Only the unqualified form or a bare {@code sys} schema is accepted, and any
7163         * server or database qualifier is rejected. A {@code dbo.} (or other user
7164         * schema) qualifier denotes a user procedure, and a server/database qualifier
7165         * (e.g. a four-part {@code Srv.Db.sys.sp_execute_external_script} linked-server
7166         * call) means the input query resolves in a remote context this pass does not
7167         * model. In both cases the argument is left unanalyzed rather than analyzed
7168         * against the wrong objects.
7169         */
7170        private boolean isSpExecuteExternalScript(TObjectName moduleName) {
7171                if (moduleName == null || moduleName.getObjectToken() == null) {
7172                        return false;
7173                }
7174                EDbVendor vendor = option.getVendor();
7175                if (!SQLUtil.sameName(vendor, gudusoft.gsqlparser.sqlenv.ESQLDataObjectType.dotProcedure,
7176                                moduleName.getObjectToken().toString(), "sp_execute_external_script")) {
7177                        return false;
7178                }
7179                // A server or database qualifier changes (or makes remote) the context in
7180                // which the input query resolves; only handle the local, unqualified call.
7181                if (moduleName.getServerToken() != null || moduleName.getDatabaseToken() != null) {
7182                        return false;
7183                }
7184                TSourceToken schemaTok = moduleName.getSchemaToken();
7185                if (schemaTok == null) {
7186                        return true;
7187                }
7188                return SQLUtil.sameName(vendor, gudusoft.gsqlparser.sqlenv.ESQLDataObjectType.dotSchema,
7189                                schemaTok.toString(), "sys");
7190        }
7191
7192        /**
7193         * Recover lineage for the query passed to sp_execute_external_script via its
7194         * input dataset (MantisBT #4606). The input value is a SQL string literal whose
7195         * SELECT reads real tables; analyze it as a normal statement so its
7196         * source-table -&gt; result-set lineage is emitted.
7197         *
7198         * <p>The input dataset is the {@code @input_data_1} parameter (named form) or,
7199         * for a positional call, the third argument — the documented signature is
7200         * {@code (@language, @script, @input_data_1, ...)}. Only that one parameter
7201         * carries a query; its siblings {@code @input_data_1_name},
7202         * {@code @input_data_1_order_by_columns} and
7203         * {@code @input_data_1_partition_by_columns} are identifiers / column lists,
7204         * NOT queries — analyzing them would attach lineage to the wrong objects, so
7205         * named matching is exact and positional matching is restricted to the third
7206         * slot.
7207         *
7208         * <p>Only a plain string literal is materialized. A variable-valued argument is
7209         * left unanalyzed: the parse-time symbol table is not available in this pass,
7210         * so its value cannot be statically resolved here (the same unresolved-value
7211         * limitation as the general dynamic-SQL case).
7212         *
7213         * <p>The script body is opaque (arbitrary Python/R can reorder, drop, or
7214         * synthesize columns), so by default the input query's result set is NOT
7215         * connected to the {@code WITH RESULT SETS} output columns — that mapping is
7216         * unprovable. Only when {@link Option#isAssumeExternalScriptPassthrough()} is
7217         * explicitly enabled does {@link #buildExternalScriptPassthrough} add that
7218         * connection, tagged {@link EffectType#external_script_passthrough} to keep it
7219         * distinguishable from proven lineage.
7220         */
7221        private void analyzeExternalScriptInputData(TMssqlExecute executeStmt) {
7222                TExecParameterList params = executeStmt.getParameters();
7223                if (params == null) {
7224                        return;
7225                }
7226                // T-SQL allows a positional prefix followed by named arguments; once an
7227                // argument is named, every later one must be named too. So a positional
7228                // argument is only valid while no NAMED argument has appeared before it.
7229                // The documented signature is (@language, @script, @input_data_1, ...), so
7230                // the third positional argument (index 2) is @input_data_1.
7231                boolean namedBefore = false;
7232                for (int i = 0; i < params.size(); i++) {
7233                        TExecParameter param = params.getExecParameter(i);
7234                        TObjectName pname = param.getParameterName();
7235                        boolean isInputData1;
7236                        if (pname != null) {
7237                                String nm = pname.toString();
7238                                if (nm.startsWith("@")) {
7239                                        nm = nm.substring(1);
7240                                }
7241                                isInputData1 = "input_data_1".equalsIgnoreCase(nm.trim()); // non-identifier-compare: T-SQL system-proc parameter name
7242                                namedBefore = true;
7243                        } else {
7244                                isInputData1 = !namedBefore && i == 2;
7245                        }
7246                        if (!isInputData1 || param.getParameterValue() == null) {
7247                                continue;
7248                        }
7249                        String sql = materializeTSqlStringLiteral(param.getParameterValue().toString());
7250                        if (sql != null && sql.trim().length() > 0) {
7251                                ResultSet inputResultSet = analyzeExternalScriptInputQuery(sql);
7252                                // Opt-in only: bridge the opaque script by assuming the input dataset
7253                                // is passed through to the WITH RESULT SETS output columns positionally
7254                                // (EffectType.external_script_passthrough). Off by default because the
7255                                // script may reorder/drop/add columns, so these edges are assumed, not
7256                                // proven.
7257                                if (option.isAssumeExternalScriptPassthrough() && inputResultSet != null) {
7258                                        buildExternalScriptPassthrough(executeStmt, inputResultSet);
7259                                }
7260                        }
7261                        // Only @input_data_1 carries a query; nothing else to look at.
7262                        return;
7263                }
7264        }
7265
7266        /**
7267         * Parse and analyze an sp_execute_external_script input query, returning the
7268         * ResultSet model of its (last) top-level SELECT, or null. Behaves like
7269         * {@link #executeDynamicSql(String)} but hands back the result set so the
7270         * optional passthrough model can wire it to the WITH RESULT SETS output.
7271         */
7272        private ResultSet analyzeExternalScriptInputQuery(String sql) {
7273                TGSqlParser sqlparser = createSqlParser(option.getVendor());
7274                sqlparser.sqltext = sql;
7275                if (sqlparser.parse() != 0) {
7276                        return null;
7277                }
7278                ResultSet resultSet = null;
7279                // Dynamic inner analysis: its references are NOT static statements of the unit
7280                // and must not veto dynamic-template marks (see dynamicFoldDepth).
7281                dynamicFoldDepth++;
7282                try {
7283                        for (int i = 0; i < sqlparser.sqlstatements.size(); i++) {
7284                                TCustomSqlStatement stmt = sqlparser.sqlstatements.get(i);
7285                                analyzeCustomSqlStmt(stmt);
7286                                if (stmt instanceof TSelectSqlStatement) {
7287                                        Object model = modelManager.getModel(stmt);
7288                                        if (model instanceof ResultSet) {
7289                                                resultSet = (ResultSet) model;
7290                                        }
7291                                }
7292                        }
7293                } finally {
7294                        dynamicFoldDepth--;
7295                }
7296                return resultSet;
7297        }
7298
7299        /**
7300         * Under the opt-in passthrough assumption, wire the external-script input
7301         * dataset to its declared {@code WITH RESULT SETS} output columns positionally,
7302         * emitting {@link EffectType#external_script_passthrough} edges and binding the
7303         * synthesized output ResultSet to the EXEC statement so an
7304         * {@code INSERT ... EXEC} consumer can inherit it (see the INSERT..EXEC path in
7305         * {@link #analyzeInsertStmt}).
7306         */
7307        private void buildExternalScriptPassthrough(TMssqlExecute executeStmt, ResultSet inputResultSet) {
7308                TColumnDefinitionList outputColumns = firstInlineResultSetColumns(executeStmt);
7309                if (outputColumns == null || outputColumns.size() == 0) {
7310                        // No declared output columns (e.g. WITH RESULT SETS NONE / UNDEFINED):
7311                        // there is nothing to pass the input through to.
7312                        return;
7313                }
7314                List<ResultColumn> inputColumns = inputResultSet.getColumns();
7315                // A star projection ("SELECT *") has unknown width, so positional
7316                // correspondence to the declared output cannot be established -- decline
7317                // rather than mis-map a single star column onto the first output column.
7318                for (ResultColumn inputColumn : inputColumns) {
7319                        if (inputColumn != null && "*".equals(inputColumn.getName())) {
7320                                return;
7321                        }
7322                }
7323                ResultSet outputResultSet = modelFactory.createResultSet(executeStmt, false);
7324                Process process = modelFactory.createProcess(executeStmt);
7325                // Materialize the FULL declared output schema (so every WITH RESULT SETS
7326                // column is available to a downstream INSERT..EXEC), preserving cardinality
7327                // by position via the raw ResultColumn constructor (the name-keyed factory
7328                // overload would collapse duplicate declared names). Passthrough edges are
7329                // added only for positions that have a matching input column.
7330                for (int i = 0; i < outputColumns.size(); i++) {
7331                        TObjectName outputName = outputColumns.getColumn(i).getColumnName();
7332                        if (outputName == null) {
7333                                continue;
7334                        }
7335                        ResultColumn outputColumn = new ResultColumn(outputResultSet, outputName);
7336                        if (i < inputColumns.size() && inputColumns.get(i) != null) {
7337                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7338                                relation.setEffectType(EffectType.external_script_passthrough);
7339                                relation.setTarget(new ResultColumnRelationshipElement(outputColumn));
7340                                relation.addSource(new ResultColumnRelationshipElement(inputColumns.get(i)));
7341                                relation.setProcess(process);
7342                        }
7343                }
7344        }
7345
7346        /** The column list of the first inline {@code WITH RESULT SETS ((...))} definition of an EXEC, or null. */
7347        private gudusoft.gsqlparser.nodes.TColumnDefinitionList firstInlineResultSetColumns(TMssqlExecute executeStmt) {
7348                gudusoft.gsqlparser.nodes.mssql.TExecuteOption execOption = executeStmt.getExecuteOption();
7349                if (!(execOption instanceof gudusoft.gsqlparser.nodes.mssql.TResultSetsExecuteOption)) {
7350                        return null;
7351                }
7352                for (gudusoft.gsqlparser.nodes.mssql.TResultSetDefinition definition
7353                                : ((gudusoft.gsqlparser.nodes.mssql.TResultSetsExecuteOption) execOption).getDefinitions()) {
7354                        if (definition instanceof gudusoft.gsqlparser.nodes.mssql.TInlineResultSetDefinition) {
7355                                return ((gudusoft.gsqlparser.nodes.mssql.TInlineResultSetDefinition) definition)
7356                                                .getColumnDefinitionList();
7357                        }
7358                }
7359                return null;
7360        }
7361
7362        /**
7363         * Result sets that an {@code INSERT ... EXEC <proc>} can consume: the result
7364         * sets of the procedure's trailing SELECT(s), or — when the procedure has no
7365         * such SELECT — the {@code WITH RESULT SETS} output(s) synthesized for its
7366         * internal sp_execute_external_script call. The latter exist only when the
7367         * passthrough assumption is enabled ({@link Option#isAssumeExternalScriptPassthrough()}),
7368         * so with the option off this reduces to the previous SELECT-only behavior.
7369         */
7370        private List<ResultSet> collectCalleeResultSets(TStoredProcedureSqlStatement procedureStmt) {
7371                List<ResultSet> resultSets = new ArrayList<ResultSet>();
7372                List<TSelectSqlStatement> selects = getLastSelectStmt(procedureStmt);
7373                if (selects != null) {
7374                        for (TSelectSqlStatement select : selects) {
7375                                Object model = modelManager.getModel(select);
7376                                if (model instanceof ResultSet) {
7377                                        resultSets.add((ResultSet) model);
7378                                }
7379                        }
7380                }
7381                // A procedure may return an ordinary SELECT result set AND an external-script
7382                // result set; INSERT..EXEC consumes every returned (compatible) result set, so
7383                // include the passthrough output(s) as well, not only when no SELECT exists.
7384                // externalScriptOutputResultSets is empty unless the passthrough option is on,
7385                // so with the option off this reduces to the previous SELECT-only behavior.
7386                resultSets.addAll(externalScriptOutputResultSets(procedureStmt));
7387                return resultSets;
7388        }
7389
7390        /**
7391         * Result sets an {@code INSERT ... EXEC <target>} consumes. When the target is a
7392         * user stored procedure, this is {@link #collectCalleeResultSets}. When it is a
7393         * DIRECT call to sp_execute_external_script (no wrapping procedure), the
7394         * synthesized WITH RESULT SETS output is bound to the EXEC statement itself, so
7395         * it is fetched directly. Empty unless the passthrough assumption is enabled.
7396         */
7397        private List<ResultSet> calleeResultSetsForInsertExec(TMssqlExecute executeStmt, Procedure procedure) {
7398                if (procedure != null && procedure.getProcedureObject() instanceof TStoredProcedureSqlStatement) {
7399                        return collectCalleeResultSets((TStoredProcedureSqlStatement) procedure.getProcedureObject());
7400                }
7401                if (isSpExecuteExternalScript(executeStmt.getModuleName())) {
7402                        Object model = modelManager.getModel(executeStmt);
7403                        if (model instanceof ResultSet) {
7404                                return java.util.Collections.singletonList((ResultSet) model);
7405                        }
7406                }
7407                return new ArrayList<ResultSet>();
7408        }
7409
7410        /**
7411         * WITH RESULT SETS output result sets bound to sp_execute_external_script calls
7412         * inside a procedure body (the passthrough model binds one to each such EXEC).
7413         * Empty unless the passthrough assumption is enabled.
7414         */
7415        private List<ResultSet> externalScriptOutputResultSets(TStoredProcedureSqlStatement procedureStmt) {
7416                final List<ResultSet> resultSets = new ArrayList<ResultSet>();
7417                procedureStmt.acceptChildren(new TParseTreeVisitor() {
7418                        @Override
7419                        public void preVisit(TMssqlExecute execute) {
7420                                if (isSpExecuteExternalScript(execute.getModuleName())) {
7421                                        Object model = modelManager.getModel(execute);
7422                                        if (model instanceof ResultSet) {
7423                                                resultSets.add((ResultSet) model);
7424                                        }
7425                                }
7426                        }
7427                });
7428                return resultSets;
7429        }
7430
7431        /**
7432         * Materialize a T-SQL string literal: strip the {@code N'...'} / {@code '...'}
7433         * wrapper and unescape doubled single quotes ({@code ''} -&gt; {@code '}).
7434         * Unlike {@link TBaseType#getStringInsideLiteral(String)} this does NOT treat
7435         * {@code \'} as an escape — backslash is an ordinary character in T-SQL, so a
7436         * source such as an {@code OPENROWSET} path must survive unchanged. Returns
7437         * null when the argument is not a plain string literal (e.g. a variable).
7438         */
7439        private static String materializeTSqlStringLiteral(String raw) {
7440                if (raw == null) {
7441                        return null;
7442                }
7443                String s = raw.trim();
7444                int start = -1;
7445                if (s.length() >= 2 && s.charAt(0) == '\'') {
7446                        start = 1;
7447                } else if (s.length() >= 3 && (s.charAt(0) == 'N' || s.charAt(0) == 'n') && s.charAt(1) == '\'') {
7448                        start = 2;
7449                }
7450                if (start < 0 || s.charAt(s.length() - 1) != '\'') {
7451                        return null;
7452                }
7453                return s.substring(start, s.length() - 1).replace("''", "'");
7454        }
7455
7456        /** True when this EXEC form would expand SQL text rather than invoke an
7457         * ordinary statically named stored procedure. */
7458        private boolean expandsDynamicSql(TMssqlExecute e) {
7459                return e.getSqlText() != null
7460                                || e.getExecType() == TBaseType.metExecStringCmd
7461                                || isSpExecutesql(e.getModuleName());
7462        }
7463
7464        /** Kind of a {@link TMssqlExecute} dynamic site: sp_executesql vs the EXEC(string) family. */
7465        private DynamicSqlSite.Kind dynamicKindOf(TMssqlExecute e) {
7466                if (e.getExecType() == TBaseType.metExecStringCmd) {
7467                        return DynamicSqlSite.Kind.EXEC_STRING;
7468                }
7469                if (isSpExecutesql(e.getModuleName())) {
7470                        return DynamicSqlSite.Kind.SP_EXECUTESQL;
7471                }
7472                return DynamicSqlSite.Kind.OTHER;
7473        }
7474
7475        /** The dynamic-SQL argument expression of a {@link TMssqlExecute}, or null. */
7476        private TExpression dynamicArgOf(TMssqlExecute e) {
7477                if (e.getExecType() == TBaseType.metExecStringCmd) {
7478                        if (e.getStringValues() != null && e.getStringValues().size() > 0) {
7479                                return e.getStringValues().getExpression(0);
7480                        }
7481                        return null;
7482                }
7483                if (e.getParameters() != null && e.getParameters().size() > 0) {
7484                        TExecParameter p = e.getParameters().getExecParameter(0);
7485                        if (p != null) {
7486                                return p.getParameterValue();
7487                        }
7488                }
7489                return null;
7490        }
7491
7492        /**
7493         * True only when the dynamic-SQL argument is a compile-time string literal. A variable or
7494         * expression argument is data-driven: even when GSP partially folds it (e.g. {@code @sql} →
7495         * {@code INSERT INTO @t SELECT ...} with an unresolved parameter left in), the site is NOT
7496         * fully resolved, so it must not be reported RESOLVED.
7497         */
7498        private boolean isLiteralDynamicArg(TMssqlExecute e) {
7499                TExpression arg = dynamicArgOf(e);
7500                return arg != null && arg.getExpressionType() == EExpressionType.simple_constant_t;
7501        }
7502
7503        /** Refine the UNRESOLVED reason by inspecting the dynamic argument expression. */
7504        private String dynamicReasonOf(TMssqlExecute e) {
7505                TExpression arg = dynamicArgOf(e);
7506                if (arg == null) {
7507                        return "dynamic SQL target could not be resolved";
7508                }
7509                if (arg.getExpressionType() == EExpressionType.simple_object_name_t && arg.getObjectOperand() != null
7510                                && arg.getObjectOperand().getDbObjectType() == EDbObjectType.variable) {
7511                        return "argument is a runtime variable";
7512                }
7513                if (arg.getExpressionType() == EExpressionType.simple_constant_t) {
7514                        return "inline literal not folded; no lineage produced";
7515                }
7516                return "argument is a runtime expression or partially-folded variable";
7517        }
7518
7519        private void executeDynamicSql(String sql) {
7520                TGSqlParser sqlparser = createSqlParser(option.getVendor());
7521                sqlparser.sqltext = sql;
7522                int result = sqlparser.parse();
7523                if (result == 0) {
7524                        dynamicFoldDepth++;
7525                        try {
7526                                for (int i = 0; i < sqlparser.sqlstatements.size(); i++) {
7527                                        analyzeCustomSqlStmt(sqlparser.sqlstatements.get(i));
7528                                }
7529                        } finally {
7530                                dynamicFoldDepth--;
7531                        }
7532                }
7533        }
7534
7535        /**
7536         * Callback invoked from snowflake.js (the script shim run by
7537         * {@link #extractSnowflakeSQLFromProcedure}) for every SQL text a
7538         * JavaScript procedure body passes to snowflake.execute() /
7539         * snowflake.createStatement(). The script resolves this method by name
7540         * at runtime, so it must stay public and keep this exact signature —
7541         * renaming it to executeDynamicSql broke JavaScript-procedure lineage
7542         * with "analyzer.extractSnowflakeSQL is not a function" (Mantis 4590).
7543         */
7544        public void extractSnowflakeSQL(String sql) {
7545                executeDynamicSql(sql);
7546        }
7547
7548        private void extractSnowflakeSQLFromProcedure(TCreateProcedureStmt procedure) {
7549                Map<String, String> argMap = new LinkedHashMap<String, String>();
7550                if (procedure.getParameterDeclarations() != null) {
7551                        for (int i = 0; i < procedure.getParameterDeclarations().size(); i++) {
7552                                TParameterDeclaration def = procedure.getParameterDeclarations().getParameterDeclarationItem(i);
7553                                argMap.put(def.getParameterName().toString(), def.getDataType().getDataTypeName());
7554                        }
7555                }
7556                StringBuilder buffer = new StringBuilder();
7557                buffer.append("(function(");
7558                String[] args = argMap.keySet().toArray(new String[0]);
7559                for (int i = 0; i < args.length; i++) {
7560                        buffer.append(args[i].toUpperCase());
7561                        if (i < args.length - 1) {
7562                                buffer.append(",");
7563                        }
7564                }
7565                buffer.append("){\n");
7566
7567                int start = -1;
7568                int end = -1;
7569                boolean dollar = false;
7570                boolean quote = false;
7571                if (procedure.getRoutineBody().indexOf("$$") != -1) {
7572                        start = procedure.getRoutineBody().indexOf("$$") + 2;
7573                        end = procedure.getRoutineBody().lastIndexOf("$$") - 1;
7574                        dollar = true;
7575                } else if (procedure.getRoutineBody().indexOf("'") != -1) {
7576                        start = procedure.getRoutineBody().indexOf("'") + 1;
7577                        end = procedure.getRoutineBody().lastIndexOf("'");
7578                        quote = true;
7579                }
7580                String body = procedure.getRoutineBody().substring(start, end);
7581                if (dollar && body.indexOf("`") != -1) {
7582                        Pattern pattern = Pattern.compile("`.+?`", Pattern.CASE_INSENSITIVE | Pattern.DOTALL);
7583                        Matcher matcher = pattern.matcher(body);
7584                        StringBuffer replaceBuffer = new StringBuffer();
7585                        while (matcher.find()) {
7586                                String condition = matcher.group().replace("\r\n", "\n").replace("'", "\\\\'")
7587                                                .replace("\n", "\\\\n'\n+'").replace("`", "'").replace("$", "RDS_CHAR_DOLLAR");
7588                                matcher.appendReplacement(replaceBuffer, condition);
7589                        }
7590                        matcher.appendTail(replaceBuffer);
7591                        body = replaceBuffer.toString().replace("RDS_CHAR_DOLLAR", "$");
7592                }
7593                if (quote && body.indexOf("'") != -1) {
7594                        body = body.replace("''", "'");
7595                }
7596                buffer.append(body);
7597                buffer.append("})(");
7598                for (int i = 0; i < args.length; i++) {
7599                        String type = argMap.get(args[i]);
7600                        if (type.equalsIgnoreCase("VARCHAR")) {
7601                                buffer.append("'pseudo'");
7602                        } else if (type.equalsIgnoreCase("STRING")) {
7603                                buffer.append("'pseudo'");
7604                        } else if (type.equalsIgnoreCase("CHAR")) {
7605                                buffer.append("'pseudo'");
7606                        } else if (type.equalsIgnoreCase("CHARACTER")) {
7607                                buffer.append("'pseudo'");
7608                        } else if (type.equalsIgnoreCase("TEXT")) {
7609                                buffer.append("'pseudo'");
7610                        } else if (type.equalsIgnoreCase("BINARY")) {
7611                                buffer.append("'pseudo'");
7612                        } else if (type.equalsIgnoreCase("VARBINARY")) {
7613                                buffer.append("'pseudo'");
7614                        } else if (type.equalsIgnoreCase("BOOLEAN")) {
7615                                buffer.append(true);
7616                        } else if (type.equalsIgnoreCase("FLOAT")) {
7617                                buffer.append("1.0");
7618                        } else if (type.equalsIgnoreCase("FLOAT4")) {
7619                                buffer.append("1.0");
7620                        } else if (type.equalsIgnoreCase("FLOAT8")) {
7621                                buffer.append("1.0");
7622                        } else if (type.equalsIgnoreCase("DOUBLE")) {
7623                                buffer.append("1.0");
7624                        } else if (type.equalsIgnoreCase("DOUBLE PRECISION")) {
7625                                buffer.append("1.0");
7626                        } else if (type.equalsIgnoreCase("REAL")) {
7627                                buffer.append("1.0");
7628                        } else if (type.equalsIgnoreCase("NUMBER")) {
7629                                buffer.append("1.0");
7630                        } else if (type.equalsIgnoreCase("DECIMAL")) {
7631                                buffer.append("1.0");
7632                        } else if (type.equalsIgnoreCase("NUMERIC")) {
7633                                buffer.append("1.0");
7634                        } else if (type.equalsIgnoreCase("INT")) {
7635                                buffer.append("1");
7636                        } else if (type.equalsIgnoreCase("INTEGER")) {
7637                                buffer.append("1");
7638                        } else if (type.equalsIgnoreCase("BIGINT")) {
7639                                buffer.append("1");
7640                        } else if (type.equalsIgnoreCase("SMALLINT")) {
7641                                buffer.append("1");
7642                        } else if (type.equalsIgnoreCase("DATE")) {
7643                                buffer.append("new Date()");
7644                        } else if (type.equalsIgnoreCase("DATETIME")) {
7645                                buffer.append("new Date()");
7646                        } else if (type.equalsIgnoreCase("TIME")) {
7647                                buffer.append("new Date()");
7648                        } else if (type.equalsIgnoreCase("TIMESTAMP")) {
7649                                buffer.append("new Date()");
7650                        } else if (type.equalsIgnoreCase("TIMESTAMP_LTZ")) {
7651                                buffer.append("new Date()");
7652                        } else if (type.equalsIgnoreCase("TIMESTAMP_NTZ")) {
7653                                buffer.append("new Date()");
7654                        } else if (type.equalsIgnoreCase("TIMESTAMP_TZ")) {
7655                                buffer.append("new Date()");
7656                        } else if (type.equalsIgnoreCase("VARIANT")) {
7657                                buffer.append("{}");
7658                        } else if (type.equalsIgnoreCase("OBJECT")) {
7659                                buffer.append("{}");
7660                        } else if (type.equalsIgnoreCase("ARRAY")) {
7661                                buffer.append("[]");
7662                        } else if (type.equalsIgnoreCase("GEOGRAPHY")) {
7663                                buffer.append("{}");
7664                        }
7665                        if (i < args.length - 1) {
7666                                buffer.append(",");
7667                        }
7668                }
7669                buffer.append(");");
7670
7671                try {
7672                        ScriptEngineManager scriptEngineManager = new ScriptEngineManager();
7673                        ScriptEngine nashorn = scriptEngineManager.getEngineByName("nashorn");
7674                        if (nashorn == null) {
7675                                // The Nashorn engine was removed from the JDK in Java 15, so the
7676                                // JavaScript body cannot be executed to extract the SQL it runs.
7677                                // Try the body as plain SQL; otherwise keep the procedure model
7678                                // (with its declared language) and skip the body (Mantis 4590).
7679                                TGSqlParser sqlparser = createSqlParser(option.getVendor());
7680                                sqlparser.sqltext = body;
7681                                if (sqlparser.parse() == 0) {
7682                                        dynamicFoldDepth++;
7683                                        try {
7684                                                for (int i = 0; i < sqlparser.sqlstatements.size(); i++) {
7685                                                        analyzeCustomSqlStmt(sqlparser.sqlstatements.get(i));
7686                                                }
7687                                        } finally {
7688                                                dynamicFoldDepth--;
7689                                        }
7690                                }
7691                                return;
7692                        }
7693                        nashorn.put("analyzer", this);
7694                        nashorn.eval(new InputStreamReader(
7695                                        getClass().getResourceAsStream("/gudusoft/gsqlparser/parser/snowflake/snowflake.js")));
7696                        nashorn.eval(new StringReader(buffer.toString()));
7697                } catch (ScriptException e) {
7698                        TGSqlParser sqlparser = createSqlParser(option.getVendor());
7699                        sqlparser.sqltext = body;
7700                        int result = sqlparser.parse();
7701                        if (result == 0) {
7702                                dynamicFoldDepth++;
7703                                try {
7704                                        for (int i = 0; i < sqlparser.sqlstatements.size(); i++) {
7705                                                analyzeCustomSqlStmt(sqlparser.sqlstatements.get(i));
7706                                        }
7707                                } finally {
7708                                        dynamicFoldDepth--;
7709                                }
7710                                return;
7711                        }
7712                        ErrorInfo errorInfo = new ErrorInfo();
7713                        errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
7714                        errorInfo.setErrorMessage("Invoke script error: " + e.getMessage());
7715                        errorInfo.setStartPosition(new Pair3<Long, Long, String>(procedure.getStartToken().lineNo,
7716                                        procedure.getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
7717                        errorInfo.setEndPosition(new Pair3<Long, Long, String>(procedure.getEndToken().lineNo,
7718                                        procedure.getEndToken().columnNo + procedure.getEndToken().getAstext().length(),
7719                                        ModelBindingManager.getGlobalHash()));
7720                        errorInfos.add(errorInfo);
7721                }
7722        }
7723
7724        private void analyzeHiveLoadStmt(THiveLoad stmt) {
7725                if (stmt.getPath() != null && stmt.getTable() != null) {
7726                        Table uriFile = modelFactory.createTableByName(stmt.getPath(), true);
7727                        uriFile.setPath(true);
7728                        uriFile.setCreateTable(true);
7729                        TObjectName fileUri = new TObjectName();
7730                        fileUri.setString("uri=" + stmt.getPath());
7731                        TableColumn fileUriColumn = modelFactory.createFileUri(uriFile, fileUri);
7732
7733                        Table tableModel = modelFactory.createTable(stmt.getTable());
7734                        Process process = modelFactory.createProcess(stmt);
7735                        tableModel.addProcess(process);
7736
7737                        TPartitionExtensionClause p = stmt.getTable().getPartitionExtensionClause();
7738                        if (p.getKeyValues() != null && p.getKeyValues().size() > 0) {
7739                                for (int i = 0; i < p.getKeyValues().size(); i++) {
7740                                        TExpression expression = p.getKeyValues().getExpression(i);
7741                                        if (expression.getLeftOperand().getExpressionType() == EExpressionType.simple_object_name_t) {
7742                                                modelFactory.createTableColumn(tableModel, expression.getLeftOperand().getObjectOperand(),
7743                                                                true);
7744                                        }
7745                                }
7746                        }
7747
7748                        for (int j = 0; j < tableModel.getColumns().size(); j++) {
7749                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7750                                relation.addSource(new TableColumnRelationshipElement(fileUriColumn));
7751                                relation.setTarget(new TableColumnRelationshipElement(tableModel.getColumns().get(j)));
7752                                relation.setProcess(process);
7753                        }
7754                }
7755        }
7756        
7757        private void analyzeLoadDataStmt(TLoadDataStmt stmt) {
7758                
7759        }
7760
7761        private void analyzeUnloadStmt(TUnloadStmt unloadStmt) {
7762                if (unloadStmt.getSelectSqlStatement() != null && unloadStmt.getS3() != null) {
7763
7764                        Table uriFile = modelFactory.createTableByName(unloadStmt.getS3(), true);
7765                        uriFile.setPath(true);
7766                        uriFile.setCreateTable(true);
7767                        TObjectName fileUri = new TObjectName();
7768                        fileUri.setString("uri=" + unloadStmt.getS3());
7769                        TableColumn fileUriColumn = modelFactory.createFileUri(uriFile, fileUri);
7770
7771                        Process process = modelFactory.createProcess(unloadStmt);
7772                        uriFile.addProcess(process);
7773
7774                        TCustomSqlStatement stmt = unloadStmt.getSelectSqlStatement();
7775                        analyzeCustomSqlStmt(stmt);
7776                        if (stmt instanceof TSelectSqlStatement) {
7777                                TSelectSqlStatement select = (TSelectSqlStatement) stmt;
7778                                ResultSet resultSetModel = (ResultSet) modelManager.getModel(select);
7779                                if (resultSetModel != null) {
7780                                        for (int j = 0; j < resultSetModel.getColumns().size(); j++) {
7781                                                ResultColumn resultColumn = resultSetModel.getColumns().get(j);
7782                                                if (resultColumn.hasStarLinkColumn()
7783                                                                && resultColumn.getStarLinkColumnNames().size() > 0) {
7784                                                        for (int k = 0; k < resultColumn.getStarLinkColumnNames().size(); k++) {
7785                                                                ResultColumn expandStarColumn = modelFactory.createResultColumn(resultSetModel,
7786                                                                                resultColumn.getStarLinkColumnName(k), false);
7787                                                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
7788                                                                dataflowRelation.setEffectType(EffectType.unload);
7789                                                                dataflowRelation
7790                                                                                .addSource(new ResultColumnRelationshipElement(expandStarColumn));
7791                                                                dataflowRelation.setTarget(new TableColumnRelationshipElement(fileUriColumn));
7792                                                                dataflowRelation.setProcess(process);
7793                                                        }
7794                                                }
7795                                                if (!resultColumn.hasStarLinkColumn() || resultColumn.isShowStar()) {
7796                                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
7797                                                        dataflowRelation.setEffectType(EffectType.unload);
7798                                                        dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn));
7799                                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(fileUriColumn));
7800                                                        dataflowRelation.setProcess(process);
7801                                                }
7802                                        }
7803                                }
7804                        }
7805                }
7806
7807        }
7808
7809        private void analyzeCopyIntoStmt(TSnowflakeCopyIntoStmt stmt) {
7810                if (stmt.getTableName() != null) {
7811                        if (stmt.getStageLocation() != null) {
7812                                Table intoTable = modelManager
7813                                                .getTableByName(DlineageUtil.getTableFullName(stmt.getTableName().toString()));
7814                                if (intoTable == null) {
7815                                        intoTable = modelFactory.createTableByName(stmt.getTableName(), false);
7816                                        TObjectName starColumn = new TObjectName();
7817                                        starColumn.setString("*");
7818                                        TableColumn column = modelFactory.createTableColumn(intoTable, starColumn, false);
7819                                        if (column != null) {
7820                                                column.setExpandStar(false);
7821                                                column.setPseduo(true);
7822                                        }
7823                                }
7824                                Process process = modelFactory.createProcess(stmt);
7825                                intoTable.addProcess(process);
7826
7827                                TObjectName stageName = stmt.getStageLocation().getStageName();
7828                                if (stageName == null || stmt.getStageLocation().getTableName() != null) {
7829                                        stageName = stmt.getStageLocation().getTableName();
7830                                }
7831                                if (stageName != null) {
7832                                        String stageFullName = DlineageUtil.getTableFullName(stageName.toString());
7833                                        Table stage = modelManager.getTableByName(stageFullName);
7834                                        if (stage == null) {
7835                                                stage = modelFactory.createStage(stageName);
7836                                                stage.setCreateTable(true);
7837                                                stage.setStage(true);
7838                                                String stagePath = stmt.getStageLocation().getPath() == null ? null
7839                                                                : stmt.getStageLocation().getPath().toString();
7840                                                if (stagePath != null) {
7841                                                        stage.setLocation(stagePath);
7842                                                        TObjectName location = new TObjectName();
7843                                                        location.setString(stagePath);
7844                                                        modelFactory.createStageLocation(stage, location);
7845                                                } else {
7846                                                        stage.setLocation("unknownPath");
7847                                                        TObjectName location = new TObjectName();
7848                                                        location.setString("unknownPath");
7849                                                        modelFactory.createStageLocation(stage, location);
7850                                                }
7851                                        }
7852
7853                                        if (stage != null && intoTable != null) {
7854                                                if (intoTable != null && !intoTable.getColumns().isEmpty() && !stage.getColumns().isEmpty()) {
7855                                                        for (int i = 0; i < intoTable.getColumns().size(); i++) {
7856                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7857                                                                relation.addSource(new TableColumnRelationshipElement(stage.getColumns().get(0)));
7858                                                                relation.setTarget(new TableColumnRelationshipElement(intoTable.getColumns().get(i)));
7859                                                                relation.setProcess(process);
7860                                                        }
7861                                                }
7862                                        }
7863                                } else if (stmt.getStageLocation().getExternalLocation() != null) {
7864                                        Table pathModel = modelFactory.createTableByName(stmt.getStageLocation().getExternalLocation(),
7865                                                        true);
7866                                        pathModel.setPath(true);
7867                                        pathModel.setCreateTable(true);
7868                                        TableColumn fileUriColumn = modelFactory.createFileUri(pathModel,
7869                                                        stmt.getStageLocation().getExternalLocation());
7870                                        if (intoTable != null) {
7871                                                if (intoTable != null && !intoTable.getColumns().isEmpty()) {
7872                                                        for (int i = 0; i < intoTable.getColumns().size(); i++) {
7873                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7874                                                                relation.addSource(new TableColumnRelationshipElement(fileUriColumn));
7875                                                                relation.setTarget(new TableColumnRelationshipElement(intoTable.getColumns().get(i)));
7876                                                                relation.setProcess(process);
7877                                                        }
7878                                                }
7879                                        }
7880                                }
7881                        }
7882                        else if (stmt.getSubQuery() != null) {
7883                                analyzeSelectStmt(stmt.getSubQuery());
7884                                
7885                                Table intoTable = modelManager
7886                                                .getTableByName(DlineageUtil.getTableFullName(stmt.getTableName().toString()));
7887                                if (intoTable == null) {
7888                                        intoTable = modelFactory.createTableByName(stmt.getTableName(), false);
7889                                        TObjectName starColumn = new TObjectName();
7890                                        starColumn.setString("*");
7891                                        TableColumn column = modelFactory.createTableColumn(intoTable, starColumn, false);
7892                                        if (column != null) {
7893                                                column.setExpandStar(true);
7894                                                column.setPseduo(true);
7895                                        }
7896
7897                                        Process process = modelFactory.createProcess(stmt);
7898                                        intoTable.addProcess(process);
7899
7900                                        ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt.getSubQuery());
7901                                        if (resultSetModel != null && column != null) {
7902                                                for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
7903                                                        ResultColumn resultColumn = resultSetModel.getColumns().get(i);
7904                                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
7905                                                        dataflowRelation.setEffectType(EffectType.copy);
7906                                                        dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn));
7907                                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(column));
7908                                                        dataflowRelation.setProcess(process);
7909                                                }
7910                                        }
7911                                }
7912                        }
7913                }
7914        }
7915
7916        private void analyzeRedshiftCopyStmt(TRedshiftCopy stmt) {
7917                if (stmt.getTableName() != null && stmt.getFromSource() != null) {
7918
7919                        Table intoTable = modelManager
7920                                        .getTableByName(DlineageUtil.getTableFullName(stmt.getTableName().toString()));
7921                        if (intoTable == null) {
7922                                intoTable = modelFactory.createTableByName(stmt.getTableName(), false);
7923                                if (stmt.getColumnList() == null || stmt.getColumnList().size() == 0) {
7924                                        TObjectName starColumn = new TObjectName();
7925                                        starColumn.setString("*");
7926                                        TableColumn column = modelFactory.createTableColumn(intoTable, starColumn, false);
7927                                        if (column != null) {
7928                                                column.setExpandStar(false);
7929                                                column.setPseduo(true);
7930                                        }
7931                                } else {
7932                                        for (TObjectName columnName : stmt.getColumnList()) {
7933                                                modelFactory.createTableColumn(intoTable, columnName, true);
7934                                        }
7935                                }
7936                        }
7937                        Process process = modelFactory.createProcess(stmt);
7938                        intoTable.addProcess(process);
7939
7940                        if (stmt.getFromSource() != null) {
7941                                Table pathModel = modelFactory.createTableByName(stmt.getFromSource(), true);
7942                                pathModel.setPath(true);
7943                                pathModel.setCreateTable(true);
7944                                TObjectName fileUri = new TObjectName();
7945                                fileUri.setString(stmt.getFromSource());
7946                                TableColumn fileUriColumn = modelFactory.createFileUri(pathModel, fileUri);
7947                                if (intoTable != null) {
7948                                        if (intoTable != null && !intoTable.getColumns().isEmpty()) {
7949                                                for (int i = 0; i < intoTable.getColumns().size(); i++) {
7950                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
7951                                                        relation.addSource(new TableColumnRelationshipElement(fileUriColumn));
7952                                                        relation.setTarget(new TableColumnRelationshipElement(intoTable.getColumns().get(i)));
7953                                                        relation.setProcess(process);
7954                                                }
7955                                        }
7956                                }
7957                        }
7958                }
7959        }
7960
7961        private void analyzeCreateIndexExpressionOperand(TExpression expr, Table tableModel){
7962                if(expr == null) return;
7963                Deque<TExpression> stack = new ArrayDeque<>();
7964                stack.push(expr);
7965                while (!stack.isEmpty()) {
7966                        TExpression current = stack.pop();
7967                        if (current == null) continue;
7968                        TObjectName columnObj = current.getObjectOperand();
7969                        if (columnObj != null) {
7970                                TableColumn tableConstraint = modelFactory.createTableColumn(tableModel, columnObj, true);
7971                                tableConstraint.setIndexKey(true);
7972                        } else {
7973                                if (current.getRightOperand() != null) {
7974                                        stack.push(current.getRightOperand());
7975                                }
7976                                if (current.getLeftOperand() != null) {
7977                                        stack.push(current.getLeftOperand());
7978                                }
7979                        }
7980                }
7981        }
7982        private void analyzeCreateIndexStageStmt(TCreateIndexSqlStatement stmt) {
7983        if(stmt.getTableName() == null){
7984            return;
7985        }
7986                Table tableModel = modelFactory.createTableByName(stmt.getTableName());
7987                TOrderByItemList columns = stmt.getColumnNameList();
7988        if(columns!=null) {
7989            for (int i = 0; i < columns.size(); i++) {
7990                TExpression expr = columns.getOrderByItem(i).getSortKey();
7991                analyzeCreateIndexExpressionOperand(expr, tableModel);
7992            }
7993        }
7994        }
7995
7996        private void analyzeCreateSynonymStmt(TCreateSynonymStmt stmt) {
7997                TObjectName sourceTableName = stmt.getForName();
7998                if(sourceTableName == null) {
7999                        return;
8000                }
8001                TCustomSqlStatement createView = viewDDLMap
8002                                .get(DlineageUtil.getTableFullName(sourceTableName.toString()));
8003                if (createView != null) {
8004                        analyzeCustomSqlStmt(createView);
8005                }
8006                Table sourceTableModel = modelFactory.createTableByName(sourceTableName);
8007                Process process = modelFactory.createProcess(stmt);
8008                sourceTableModel.addProcess(process);
8009                if(stmt.getSynonymName()!=null) {
8010                        Table synonymTableModel = modelFactory.createTableByName(stmt.getSynonymName());
8011                        synonymTableModel.setSubType(SubType.synonym);
8012                        if(sourceTableModel.isCreateTable()) {
8013                                synonymTableModel.setCreateTable(true);
8014                                for(TableColumn sourceTableColumn: new ArrayList<>(sourceTableModel.getColumns())) {
8015                                        TObjectName columnName = new TObjectName();
8016                                        columnName.setString(sourceTableColumn.getName());
8017                                        TableColumn synonymTableColumn = new TableColumn(synonymTableModel, columnName);
8018                                        synonymTableModel.addColumn(synonymTableColumn);
8019                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8020                                        relation.setEffectType(EffectType.create_synonym);
8021                                        relation.setTarget(new TableColumnRelationshipElement(synonymTableColumn));
8022                                        relation.addSource(new TableColumnRelationshipElement(sourceTableColumn));
8023                                        relation.setProcess(process);
8024                                }
8025                        }
8026                        else {
8027                                TObjectName synonymStarColumn = new TObjectName();
8028                                synonymStarColumn.setString("*");
8029                                TableColumn synonymTableStarColumn = modelFactory.createTableColumn(synonymTableModel,
8030                                                synonymStarColumn, true);
8031                                TObjectName sourceStarColumn = new TObjectName();
8032                                sourceStarColumn.setString("*");
8033                                TableColumn sourceTableStarColumn = modelFactory.createTableColumn(sourceTableModel, sourceStarColumn, true);
8034                                
8035                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8036                                relation.setEffectType(EffectType.create_synonym);
8037                                relation.setTarget(new TableColumnRelationshipElement(synonymTableStarColumn));
8038                                relation.addSource(new TableColumnRelationshipElement(sourceTableStarColumn));
8039                                relation.setProcess(process);
8040                        }
8041                }
8042        }
8043
8044        /**
8045         * For every Table model that maps to a {@link TSQLSynonyms} in the current
8046         * {@link #sqlenv}, materialize the synonym's base table as a separate Table
8047         * model and emit per-column {@code fdd}/{@code synonym} relations from the
8048         * synonym's columns to the base table's matching columns. Lets lineage flow
8049         * through synonyms defined in the metadata SQLEnv (e.g. loaded via
8050         * {@code TDDLSQLEnv} or sqlflow metadata) the same way it does for
8051         * {@code CREATE SYNONYM} DDL inside the analyzed script.
8052         */
8053        private void materializeSqlEnvSynonyms() {
8054                if (sqlenv == null) {
8055                        return;
8056                }
8057                List<Table> snapshot = new ArrayList<Table>(modelManager.getTablesByName());
8058
8059                Map<String, TSQLSynonyms> synonymCache = new HashMap<>();
8060                List<TSQLCatalog> catalogs = sqlenv.getCatalogList();
8061                if (catalogs != null) {
8062                        for (TSQLCatalog catalog : catalogs) {
8063                                if (catalog == null) {
8064                                        continue;
8065                                }
8066                                List<TSQLSchema> schemas = catalog.getSchemaList();
8067                                if (schemas == null) {
8068                                        continue;
8069                                }
8070                                for (TSQLSchema schema : schemas) {
8071                                        if (schema == null) {
8072                                                continue;
8073                                        }
8074                                        List<TSQLSchemaObject> schemaObjects = schema.getSchemaObjectList();
8075                                        if (schemaObjects == null || schemaObjects.isEmpty()) {
8076                                                continue;
8077                                        }
8078                                        for (TSQLSchemaObject schemaObject : schemaObjects) {
8079                                                if (schemaObject instanceof TSQLSynonyms) {
8080                                                        TSQLSynonyms synonym = (TSQLSynonyms) schemaObject;
8081                                                        String qualifiedName = synonym.getQualifiedName();
8082                                                        String normalizedName = DlineageUtil.getIdentifierNormalTableName(qualifiedName);
8083                                                        synonymCache.put(normalizedName, synonym);
8084                                                }
8085                                        }
8086                                }
8087                        }
8088                }
8089
8090                for (Table synonymTableModel : snapshot) {
8091                        if (synonymTableModel == null) {
8092                                continue;
8093                        }
8094                        if (SubType.synonym.equals(synonymTableModel.getSubType())) {
8095                                continue;
8096                        }
8097                        String qualifiedName = ModelFactory.getQualifiedTableName(synonymTableModel);
8098                        String normalizedName = DlineageUtil.getIdentifierNormalTableName(qualifiedName);
8099                        TSQLSynonyms synonym = synonymCache.get(normalizedName);
8100                        if (synonym == null) {
8101                                continue;
8102                        }
8103
8104                        String baseSqlTableQualifiedName = synonym.getBaseTableQualifiedName();
8105                        if (baseSqlTableQualifiedName == null) {
8106                                continue;
8107                        }
8108
8109                        synonymTableModel.setSubType(SubType.synonym);
8110
8111                        TObjectName baseObjectName = new TObjectName();
8112                        baseObjectName.setString(baseSqlTableQualifiedName);
8113                        Table baseTableModel = modelFactory.createTableByName(baseObjectName);
8114                        if (baseTableModel == null || baseTableModel == synonymTableModel) {
8115                                continue;
8116                        }
8117
8118                        for (TableColumn synonymColumn : new ArrayList<TableColumn>(synonymTableModel.getColumns())) {
8119                                String colName = synonymColumn.getName();
8120                                if (SQLUtil.isEmpty(colName) || "*".equals(colName)) {
8121                                        continue;
8122                                }
8123                                TableColumn baseColumn = null;
8124                                for (TableColumn bc : baseTableModel.getColumns()) {
8125                                        if (bc.getName() != null && SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotColumn, bc.getName(), colName)) {
8126                                                baseColumn = bc;
8127                                                break;
8128                                        }
8129                                }
8130                                if (baseColumn == null) {
8131                                        TObjectName columnName = new TObjectName();
8132                                        columnName.setString(colName);
8133                                        baseColumn = new TableColumn(baseTableModel, columnName);
8134                                        baseTableModel.addColumn(baseColumn);
8135                                }
8136                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8137                                relation.setEffectType(EffectType.synonym);
8138                                relation.setTarget(new TableColumnRelationshipElement(synonymColumn));
8139                                relation.addSource(new TableColumnRelationshipElement(baseColumn));
8140                        }
8141                }
8142        }
8143
8144        private void analyzeRenameStmt(TRenameStmt stmt) {
8145                TObjectName oldTableName = stmt.getOldName();
8146                TObjectName newTableName = stmt.getNewName();
8147                
8148                Table oldNameTableModel = modelFactory.createTableByName(oldTableName);
8149                TObjectName oldStarColumn = new TObjectName();
8150                oldStarColumn.setString("*");
8151                TableColumn oldTableStarColumn = modelFactory.createTableColumn(oldNameTableModel, oldStarColumn, true);
8152                
8153                Table newNameTableModel = modelFactory.createTableByName(newTableName);
8154                TObjectName newStarColumn = new TObjectName();
8155                newStarColumn.setString("*");
8156                TableColumn newTableStarColumn = modelFactory.createTableColumn(newNameTableModel, newStarColumn, true);
8157                
8158                Process process = modelFactory.createProcess(stmt);
8159                newNameTableModel.addProcess(process);
8160                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8161                relation.setEffectType( EffectType.rename_table);
8162                relation.setTarget(new TableColumnRelationshipElement(newTableStarColumn));
8163                relation.addSource(new TableColumnRelationshipElement(oldTableStarColumn));
8164                relation.setProcess(process);
8165                
8166                if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) {
8167                        oldTableStarColumn.setShowStar(false);
8168                        relation.setShowStarRelation(false);
8169                }
8170
8171                if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) {
8172                        newTableStarColumn.setShowStar(false);
8173                        relation.setShowStarRelation(false);
8174                }
8175        }
8176        
8177        private void analyzeAlterTableStmt(TAlterTableStatement stmt) {
8178                TTable oldNameTable = stmt.getTargetTable();
8179                if (oldNameTable == null) {
8180                        return;
8181                }
8182
8183                Table oldNameTableModel = modelFactory.createTable(oldNameTable);
8184                TObjectName oldStarColumn = new TObjectName();
8185                oldStarColumn.setString("*");
8186                TableColumn oldTableStarColumn = modelFactory.createTableColumn(oldNameTableModel, oldStarColumn, true);
8187
8188                for (int i = 0; stmt.getAlterTableOptionList() != null && i < stmt.getAlterTableOptionList().size(); i++) {
8189                        TAlterTableOption option = stmt.getAlterTableOptionList().getAlterTableOption(i);
8190                        if (option.getOptionType() == EAlterTableOptionType.RenameTable
8191                                        || option.getOptionType() == EAlterTableOptionType.swapWith) {
8192                                TObjectName newTableName = option.getNewTableName();
8193                                Stack<TParseTreeNode> list = newTableName.getStartToken().getNodesStartFromThisToken();
8194                                boolean containsTable = false;
8195                                for (int j = 0; j < list.size(); j++) {
8196                                        if (list.get(j) instanceof TTable) {
8197                                                TTable newTableTable = (TTable) list.get(j);
8198                                                Table newNameTableModel = modelFactory.createTable(newTableTable);
8199                                                newNameTableModel.setStarStmt("rename_table");
8200
8201                                                TObjectName newStarColumn = new TObjectName();
8202                                                newStarColumn.setString("*");
8203                                                TableColumn newTableStarColumn = modelFactory.createTableColumn(newNameTableModel,
8204                                                                newStarColumn, true);
8205
8206                                                Process process = modelFactory.createProcess(stmt);
8207                                                newNameTableModel.addProcess(process);
8208                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8209                                                relation.setEffectType(
8210                                                                option.getOptionType() == EAlterTableOptionType.RenameTable ? EffectType.rename_table
8211                                                                                : EffectType.swap_table);                                               
8212                                                if (option.getOptionType() == EAlterTableOptionType.RenameTable) {
8213                                                        relation.setTarget(new TableColumnRelationshipElement(newTableStarColumn));
8214                                                        relation.addSource(new TableColumnRelationshipElement(oldTableStarColumn));
8215                                                } else if (option.getOptionType() == EAlterTableOptionType.swapWith) {
8216                                                        relation.setTarget(new TableColumnRelationshipElement(oldTableStarColumn));
8217                                                        relation.addSource(new TableColumnRelationshipElement(newTableStarColumn));
8218                                                }
8219                                                relation.setProcess(process);
8220                                                containsTable = true;
8221
8222                                                if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) {
8223                                                        oldTableStarColumn.setShowStar(false);
8224                                                        relation.setShowStarRelation(false);
8225                                                }
8226
8227                                                if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) {
8228                                                        newTableStarColumn.setShowStar(false);
8229                                                        relation.setShowStarRelation(false);
8230                                                }
8231                                        }
8232                                }
8233                                if (!containsTable) {
8234                                        Table newNameTableModel = modelFactory.createTableByName(newTableName);
8235                                        newNameTableModel.setStarStmt("rename_table");
8236                                        TObjectName newStarColumn = new TObjectName();
8237                                        newStarColumn.setString("*");
8238                                        TableColumn newTableStarColumn = modelFactory.createTableColumn(newNameTableModel, newStarColumn,
8239                                                        true);
8240
8241                                        Process process = modelFactory.createProcess(stmt);
8242                                        newNameTableModel.addProcess(process);
8243                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8244                                        relation.setEffectType(
8245                                                        option.getOptionType() == EAlterTableOptionType.RenameTable ? EffectType.rename_table
8246                                                                        : EffectType.swap_table);
8247                                        if (option.getOptionType() == EAlterTableOptionType.RenameTable) {
8248                                                relation.setTarget(new TableColumnRelationshipElement(newTableStarColumn));
8249                                                relation.addSource(new TableColumnRelationshipElement(oldTableStarColumn));
8250                                        } else if (option.getOptionType() == EAlterTableOptionType.swapWith) {
8251                                                relation.setTarget(new TableColumnRelationshipElement(oldTableStarColumn));
8252                                                relation.addSource(new TableColumnRelationshipElement(newTableStarColumn));
8253                                        }
8254                                        relation.setProcess(process);
8255                                        if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) {
8256                                                oldTableStarColumn.setShowStar(false);
8257                                                relation.setShowStarRelation(false);
8258                                        }
8259
8260                                        if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) {
8261                                                newTableStarColumn.setShowStar(false);
8262                                                relation.setShowStarRelation(false);
8263                                        }
8264
8265                                }
8266                        }
8267                        else if (option.getOptionType() == EAlterTableOptionType.appendFrom) {
8268                                TObjectName newTableName = option.getSourceTableName();
8269                                Stack<TParseTreeNode> list = newTableName.getStartToken().getNodesStartFromThisToken();
8270                                boolean containsTable = false;
8271                                for (int j = 0; j < list.size(); j++) {
8272                                        if (list.get(j) instanceof TTable) {
8273                                                TTable newTableTable = (TTable) list.get(j);
8274                                                Table newNameTableModel = modelFactory.createTable(newTableTable);
8275                                                newNameTableModel.setStarStmt("append_from");
8276
8277                                                TObjectName newStarColumn = new TObjectName();
8278                                                newStarColumn.setString("*");
8279                                                TableColumn newTableStarColumn = modelFactory.createTableColumn(newNameTableModel,
8280                                                                newStarColumn, true);
8281
8282                                                Process process = modelFactory.createProcess(stmt);
8283                                                newNameTableModel.addProcess(process);
8284                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8285                                                relation.setEffectType(EffectType.append_from);
8286                                                relation.setTarget(new TableColumnRelationshipElement(oldTableStarColumn));
8287                                                relation.addSource(new TableColumnRelationshipElement(newTableStarColumn));
8288                                                relation.setProcess(process);
8289                                                containsTable = true;
8290
8291                                                if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) {
8292                                                        oldTableStarColumn.setShowStar(false);
8293                                                        relation.setShowStarRelation(false);
8294                                                }
8295
8296                                                if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) {
8297                                                        newTableStarColumn.setShowStar(false);
8298                                                        relation.setShowStarRelation(false);
8299                                                }
8300                                        }
8301                                }
8302                                if (!containsTable) {
8303                                        Table newNameTableModel = modelFactory.createTableByName(newTableName);
8304                                        newNameTableModel.setStarStmt("append_from");
8305                                        TObjectName newStarColumn = new TObjectName();
8306                                        newStarColumn.setString("*");
8307                                        TableColumn newTableStarColumn = modelFactory.createTableColumn(newNameTableModel, newStarColumn,
8308                                                        true);
8309
8310                                        Process process = modelFactory.createProcess(stmt);
8311                                        newNameTableModel.addProcess(process);
8312                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8313                                        relation.setEffectType(EffectType.append_from);
8314                                        relation.setTarget(new TableColumnRelationshipElement(oldTableStarColumn));
8315                                        relation.addSource(new TableColumnRelationshipElement(newTableStarColumn));
8316                                        relation.setProcess(process);
8317                                        if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) {
8318                                                oldTableStarColumn.setShowStar(false);
8319                                                relation.setShowStarRelation(false);
8320                                        }
8321
8322                                        if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) {
8323                                                newTableStarColumn.setShowStar(false);
8324                                                relation.setShowStarRelation(false);
8325                                        }
8326
8327                                }
8328                        }
8329                        else if (option.getOptionType() == EAlterTableOptionType.exchangePartition) {
8330                                TObjectName newTableName = option.getNewTableName();
8331                                Stack<TParseTreeNode> list = newTableName.getStartToken().getNodesStartFromThisToken();
8332                                boolean containsTable = false;
8333                                for (int j = 0; j < list.size(); j++) {
8334                                        if (list.get(j) instanceof TTable) {
8335                                                TTable newTableTable = (TTable) list.get(j);
8336                                                Table newNameTableModel = modelFactory.createTable(newTableTable);
8337                                                newNameTableModel.setStarStmt("exchange_partition");
8338
8339                                                TObjectName newStarColumn = new TObjectName();
8340                                                newStarColumn.setString("*");
8341                                                TableColumn newTableStarColumn = modelFactory.createTableColumn(newNameTableModel,
8342                                                                newStarColumn, true);
8343
8344                                                Process process = modelFactory.createProcess(stmt);
8345                                                newNameTableModel.addProcess(process);
8346                                                {
8347                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8348                                                        relation.setEffectType(EffectType.exchange_partition);
8349                                                        relation.setTarget(new TableColumnRelationshipElement(oldTableStarColumn));
8350                                                        relation.addSource(new TableColumnRelationshipElement(newTableStarColumn));
8351                                                        relation.setProcess(process);
8352                                                        if (option.getPartitionName() != null) {
8353                                                                relation.setPartition(option.getPartitionName().toString());
8354                                                        }
8355                                                        if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) {
8356                                                                oldTableStarColumn.setShowStar(false);
8357                                                                relation.setShowStarRelation(false);
8358                                                        }
8359
8360                                                        if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) {
8361                                                                newTableStarColumn.setShowStar(false);
8362                                                                relation.setShowStarRelation(false);
8363                                                        }
8364                                                }
8365                                                {
8366                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8367                                                        relation.setEffectType(EffectType.exchange_partition);
8368                                                        relation.setTarget(new TableColumnRelationshipElement(newTableStarColumn));
8369                                                        relation.addSource(new TableColumnRelationshipElement(oldTableStarColumn));
8370                                                        relation.setProcess(process);
8371                                                        if (option.getPartitionName() != null) {
8372                                                                relation.setPartition(option.getPartitionName().toString());
8373                                                        }
8374                                                        if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) {
8375                                                                oldTableStarColumn.setShowStar(false);
8376                                                                relation.setShowStarRelation(false);
8377                                                        }
8378
8379                                                        if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) {
8380                                                                newTableStarColumn.setShowStar(false);
8381                                                                relation.setShowStarRelation(false);
8382                                                        }
8383                                                }
8384                                                containsTable = true;   
8385                                        }
8386                                }
8387                                if (!containsTable) {
8388                                        Table newNameTableModel = modelFactory.createTableByName(newTableName);
8389                                        newNameTableModel.setStarStmt("exchange_partition");
8390                                        TObjectName newStarColumn = new TObjectName();
8391                                        newStarColumn.setString("*");
8392                                        TableColumn newTableStarColumn = modelFactory.createTableColumn(newNameTableModel, newStarColumn,
8393                                                        true);
8394
8395                                        Process process = modelFactory.createProcess(stmt);
8396                                        newNameTableModel.addProcess(process);
8397                                        {
8398                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8399                                                relation.setEffectType(EffectType.exchange_partition);
8400                                                relation.setTarget(new TableColumnRelationshipElement(oldTableStarColumn));
8401                                                relation.addSource(new TableColumnRelationshipElement(newTableStarColumn));
8402                                                if (option.getPartitionName() != null) {
8403                                                        relation.setPartition(option.getPartitionName().toString());
8404                                                }
8405                                                relation.setProcess(process);
8406                                                if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) {
8407                                                        oldTableStarColumn.setShowStar(false);
8408                                                        relation.setShowStarRelation(false);
8409                                                }
8410
8411                                                if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) {
8412                                                        newTableStarColumn.setShowStar(false);
8413                                                        relation.setShowStarRelation(false);
8414                                                }
8415                                        }
8416                                        {
8417                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8418                                                relation.setEffectType(EffectType.exchange_partition);
8419                                                relation.setTarget(new TableColumnRelationshipElement(newTableStarColumn));
8420                                                relation.addSource(new TableColumnRelationshipElement(oldTableStarColumn));
8421                                                if (option.getPartitionName() != null) {
8422                                                        relation.setPartition(option.getPartitionName().toString());
8423                                                }
8424                                                relation.setProcess(process);
8425                                                if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) {
8426                                                        oldTableStarColumn.setShowStar(false);
8427                                                        relation.setShowStarRelation(false);
8428                                                }
8429
8430                                                if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) {
8431                                                        newTableStarColumn.setShowStar(false);
8432                                                        relation.setShowStarRelation(false);
8433                                                }
8434                                        }
8435                                }
8436                        }
8437                        else if(option.getOptionType() == EAlterTableOptionType.setLocation) {
8438                                TObjectName location = option.getTableLocation();
8439                                Process process = modelFactory.createProcess(stmt);
8440                                process.setType("Set Table Location");
8441                                oldNameTableModel.addProcess(process);
8442                                Table uriFile = modelFactory.createTableByName(location, true);
8443                                uriFile.setPath(true);
8444                                for (int j = 0; j < oldNameTableModel.getColumns().size(); j++) {
8445                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8446                                        TObjectName fileUri = new TObjectName();
8447                                        fileUri.setString("*");
8448                                        TableColumn fileUriColumn = modelFactory.createFileUri(uriFile, fileUri);
8449                                        relation.addSource(new TableColumnRelationshipElement(fileUriColumn));
8450                                        relation.setTarget(new TableColumnRelationshipElement(oldNameTableModel.getColumns().get(j)));
8451                                        relation.setProcess(process);
8452                                }
8453                        }
8454                        else if (option.getOptionType() == EAlterTableOptionType.AddColumn
8455                                        || option.getOptionType() == EAlterTableOptionType.addColumnIfNotExists) {
8456                                if (option.getColumnDefinitionList() != null) {
8457                                        for (TColumnDefinition column : option.getColumnDefinitionList()) {
8458                                                if (column != null && column.getColumnName() != null) {
8459                                                        TableColumn tableColumn = modelFactory.createTableColumn(oldNameTableModel, column.getColumnName(), true);
8460                                                        if(this.option.getAnalyzeMode() == AnalyzeMode.crud) {
8461                                                                CrudRelationship crudRelationship = modelFactory.createCrudRelation();
8462                                                                crudRelationship.setTarget(new TableColumnRelationshipElement(tableColumn));
8463                                                                crudRelationship.setEffectType(EffectType.add_table_column);
8464                                                        }
8465                                                }
8466                                        }
8467                                }
8468                        }
8469                        else if (option.getOptionType() == EAlterTableOptionType.DropColumn && this.option.getAnalyzeMode() == AnalyzeMode.crud) {
8470                                if (option.getColumnNameList() != null) {
8471                                        for (TObjectName column : option.getColumnNameList()) {
8472                                                TableColumn tableColumn = modelFactory.createTableColumn(oldNameTableModel, column, true);
8473                                                CrudRelationship crudRelationship = modelFactory.createCrudRelation();
8474                                                crudRelationship.setTarget(new TableColumnRelationshipElement(tableColumn));
8475                                                crudRelationship.setEffectType(EffectType.drop_table_column);
8476                                        }
8477                                }
8478                        }
8479                        else if(option.getOptionType() == EAlterTableOptionType.AddConstraint || option.getOptionType() == EAlterTableOptionType.AddConstraintFK
8480                                        || option.getOptionType() == EAlterTableOptionType.AddConstraintPK || option.getOptionType() == EAlterTableOptionType.AddConstraintUnique
8481                                        || option.getOptionType() == EAlterTableOptionType.AddConstraintIndex){
8482                                if (option.getTableConstraint() != null) {
8483                                        TConstraint alertTableConstraint = option.getTableConstraint();
8484                                        TPTNodeList<TColumnWithSortOrder> keyNames = alertTableConstraint.getColumnList();
8485                                        if (keyNames == null) {
8486                                                continue;
8487                                        }
8488                                        for (int k = 0; k < keyNames.size(); k++) {
8489                                                TObjectName keyName = keyNames.getElement(k).getColumnName();
8490                                                TObjectName referencedTableName = alertTableConstraint.getReferencedObject();
8491                                                Table tableModel = modelFactory.createTableByName(stmt.getTableName());
8492                                                TableColumn tableConstraint = modelFactory.createTableColumn(tableModel, keyName, true);
8493                                                if(alertTableConstraint.getConstraint_type() == EConstraintType.primary_key){
8494                                                        tableConstraint.setPrimaryKey(true);
8495                                                }
8496                                                else if(alertTableConstraint.getConstraint_type() == EConstraintType.table_index){
8497                                                        tableConstraint.setIndexKey(true);
8498                                                }
8499                                                else if(alertTableConstraint.getConstraint_type() == EConstraintType.unique){
8500                                                        tableConstraint.setUnqiueKey(true);
8501                                                }
8502                                                else if (alertTableConstraint.getConstraint_type() == EConstraintType.foreign_key) {
8503                                                        tableConstraint.setForeignKey(true);
8504                                                        Table referencedTable = modelManager.getTableByName(DlineageUtil.getTableFullName(referencedTableName.toString()));
8505                                                        if (referencedTable == null) {
8506                                                                referencedTable = modelFactory.createTableByName(referencedTableName);
8507                                                        }
8508                                                        TObjectNameList referencedTableColumns = alertTableConstraint.getReferencedColumnList();
8509                                                        if (k == keyNames.size() - 1) {
8510                                                                captureCompositeForeignKey(tableModel, keyNames, referencedTable,
8511                                                                                referencedTableColumns);
8512                                                        }
8513                                                        if (referencedTableColumns != null) {
8514                                                                for (int j = 0; j < referencedTableColumns.size(); j++) {
8515                                                                        TableColumn tableColumn = modelFactory.createTableColumn(referencedTable,
8516                                                                                        referencedTableColumns.getObjectName(j), false);
8517                                                                        if (tableColumn != null) {
8518                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8519                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
8520                                                                                relation.setTarget(new TableColumnRelationshipElement(tableConstraint));
8521                                                                                relation.setEffectType(EffectType.foreign_key);
8522                                                                                Process process = modelFactory.createProcess(stmt);
8523                                                                                relation.setProcess(process);
8524                                                                                if(this.option.isShowERDiagram()){
8525                                                                                        ERRelationship erRelation = modelFactory.createERRelation();
8526                                                                                        erRelation.addSource(new TableColumnRelationshipElement(tableColumn));
8527                                                                                        erRelation.setTarget(new TableColumnRelationshipElement(tableConstraint));
8528                                                                                }
8529                                                                        }
8530                                                                }
8531                                                        }
8532                                                        else{
8533                                                                TableColumn tableColumn = modelFactory.createTableColumn(referencedTable, keyName, false);
8534                                                                if (tableColumn != null) {
8535                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8536                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
8537                                                                        relation.setTarget(new TableColumnRelationshipElement(tableConstraint));
8538                                                                        relation.setEffectType(EffectType.foreign_key);
8539                                                                        if(this.option.isShowERDiagram()){
8540                                                                                ERRelationship erRelation = modelFactory.createERRelation();
8541                                                                                erRelation.addSource(new TableColumnRelationshipElement(tableColumn));
8542                                                                                erRelation.setTarget(new TableColumnRelationshipElement(tableConstraint));
8543                                                                        }
8544                                                                }
8545                                                        }
8546                                                }
8547                                        }
8548                                }
8549                                else if (option.getConstraintList() != null) {
8550                                        for(int iCons=0; iCons<option.getConstraintList().size(); iCons++){
8551                                                TConstraint alertTableConstraint = option.getConstraintList().getConstraint(iCons);
8552                                                TPTNodeList<TColumnWithSortOrder> keyNames = alertTableConstraint.getColumnList();
8553                                                if(keyNames != null){
8554                                                        for (int k = 0; k < keyNames.size(); k++) {
8555                                                                TObjectName keyName = keyNames.getElement(k).getColumnName();
8556                                                                TObjectName referencedTableName = alertTableConstraint.getReferencedObject();
8557                                                                Table tableModel = modelFactory.createTableByName(stmt.getTableName());
8558                                                                TableColumn tableConstraint = modelFactory.createTableColumn(tableModel, keyName, true);
8559                                                                if(alertTableConstraint.getConstraint_type() == EConstraintType.primary_key){
8560                                                                        tableConstraint.setPrimaryKey(true);
8561                                                                }
8562                                                                else if(alertTableConstraint.getConstraint_type() == EConstraintType.table_index){
8563                                                                        tableConstraint.setIndexKey(true);
8564                                                                }
8565                                                                else if(alertTableConstraint.getConstraint_type() == EConstraintType.unique){
8566                                                                        tableConstraint.setUnqiueKey(true);
8567                                                                }
8568                                                                else if (alertTableConstraint.getConstraint_type() == EConstraintType.foreign_key) {
8569                                                                        tableConstraint.setForeignKey(true);
8570                                                                        Table referencedTable = modelManager.getTableByName(DlineageUtil.getTableFullName(referencedTableName.toString()));
8571                                                                        if (referencedTable == null) {
8572                                                                                referencedTable = modelFactory.createTableByName(referencedTableName);
8573                                                                        }
8574                                                                        TObjectNameList referencedTableColumns = alertTableConstraint.getReferencedColumnList();
8575                                                                        if (k == keyNames.size() - 1) {
8576                                                                                captureCompositeForeignKey(tableModel, keyNames, referencedTable,
8577                                                                                                referencedTableColumns);
8578                                                                        }
8579                                                                        if (referencedTableColumns != null) {
8580                                                                                for (int j = 0; j < referencedTableColumns.size(); j++) {
8581                                                                                        TableColumn tableColumn = modelFactory.createTableColumn(referencedTable,
8582                                                                                                        referencedTableColumns.getObjectName(j), false);
8583                                                                                        if (tableColumn != null) {
8584                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8585                                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
8586                                                                                                relation.setTarget(new TableColumnRelationshipElement(tableConstraint));
8587                                                                                                relation.setEffectType(EffectType.foreign_key);
8588                                                                                                Process process = modelFactory.createProcess(stmt);
8589                                                                                                relation.setProcess(process);
8590                                                                                                if(this.option.isShowERDiagram()){
8591                                                                                                        ERRelationship erRelation = modelFactory.createERRelation();
8592                                                                                                        erRelation.addSource(new TableColumnRelationshipElement(tableColumn));
8593                                                                                                        erRelation.setTarget(new TableColumnRelationshipElement(tableConstraint));
8594                                                                                                }
8595                                                                                        }
8596                                                                                }
8597                                                                        }
8598                                                                        else{
8599                                                                                TableColumn tableColumn = modelFactory.createTableColumn(referencedTable, keyName, false);
8600                                                                                if (tableColumn != null) {
8601                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8602                                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
8603                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableConstraint));
8604                                                                                        relation.setEffectType(EffectType.foreign_key);
8605                                                                                        Process process = modelFactory.createProcess(stmt);
8606                                                                                        relation.setProcess(process);
8607                                                                                        if(this.option.isShowERDiagram()){
8608                                                                                                ERRelationship erRelation = modelFactory.createERRelation();
8609                                                                                                erRelation.addSource(new TableColumnRelationshipElement(tableColumn));
8610                                                                                                erRelation.setTarget(new TableColumnRelationshipElement(tableConstraint));
8611                                                                                        }
8612                                                                                }
8613                                                                        }
8614                                                                }
8615                                                        }
8616                                                }
8617                                        }
8618                                }
8619                                else if (option.getIndexCols() != null){
8620                                        TPTNodeList<TColumnWithSortOrder> keyNames = option.getIndexCols();
8621                                        for (int k = 0; k < keyNames.size(); k++) {
8622                                                TObjectName keyName = keyNames.getElement(k).getColumnName();
8623                                                Table tableModel = modelFactory.createTableByName(stmt.getTableName());
8624                                                TableColumn tableConstraint = modelFactory.createTableColumn(tableModel, keyName, true);
8625                                                if(option.getOptionType() == EAlterTableOptionType.AddConstraintPK){
8626                                                        tableConstraint.setPrimaryKey(true);
8627                                                }
8628                                                else if(option.getOptionType() == EAlterTableOptionType.AddConstraintIndex){
8629                                                        tableConstraint.setIndexKey(true);
8630                                                }
8631                                                else if(option.getOptionType() == EAlterTableOptionType.AddConstraintUnique){
8632                                                        tableConstraint.setUnqiueKey(true);
8633                                                }
8634                                                else if (option.getOptionType() == EAlterTableOptionType.AddConstraintFK) {
8635                                                        TObjectName referencedTableName = option.getReferencedObjectName();
8636                                                        tableConstraint.setForeignKey(true);
8637                                                        Table referencedTable = modelManager.getTableByName(DlineageUtil.getTableFullName(referencedTableName.toString()));
8638                                                        if (referencedTable == null) {
8639                                                                referencedTable = modelFactory.createTableByName(referencedTableName);
8640                                                        }
8641                                                        TObjectNameList referencedTableColumns = option.getReferencedColumnList();
8642                                                        if (k == keyNames.size() - 1) {
8643                                                                captureCompositeForeignKey(tableModel, keyNames, referencedTable,
8644                                                                                referencedTableColumns);
8645                                                        }
8646                                                        if (referencedTableColumns != null) {
8647                                                                for (int j = 0; j < referencedTableColumns.size(); j++) {
8648                                                                        TableColumn tableColumn = modelFactory.createTableColumn(referencedTable,
8649                                                                                        referencedTableColumns.getObjectName(j), false);
8650                                                                        if (tableColumn != null) {
8651                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8652                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
8653                                                                                relation.setTarget(new TableColumnRelationshipElement(tableConstraint));
8654                                                                                relation.setEffectType(EffectType.foreign_key);
8655                                                                                Process process = modelFactory.createProcess(stmt);
8656                                                                                relation.setProcess(process);
8657                                                                                if(this.option.isShowERDiagram()){
8658                                                                                        ERRelationship erRelation = modelFactory.createERRelation();
8659                                                                                        erRelation.addSource(new TableColumnRelationshipElement(tableColumn));
8660                                                                                        erRelation.setTarget(new TableColumnRelationshipElement(tableConstraint));
8661                                                                                }
8662                                                                        }
8663                                                                }
8664                                                        }
8665                                                        else{
8666                                                                TableColumn tableColumn = modelFactory.createTableColumn(referencedTable, keyName, false);
8667                                                                if (tableColumn != null) {
8668                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
8669                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
8670                                                                        relation.setTarget(new TableColumnRelationshipElement(tableConstraint));
8671                                                                        relation.setEffectType(EffectType.foreign_key);
8672                                                                        Process process = modelFactory.createProcess(stmt);
8673                                                                        relation.setProcess(process);
8674                                                                        if(this.option.isShowERDiagram()){
8675                                                                                ERRelationship erRelation = modelFactory.createERRelation();
8676                                                                                erRelation.addSource(new TableColumnRelationshipElement(tableColumn));
8677                                                                                erRelation.setTarget(new TableColumnRelationshipElement(tableConstraint));
8678                                                                        }
8679                                                                }
8680                                                        }
8681                                                }
8682                                        }
8683                                }
8684                        }
8685                }
8686        }
8687
8688        private void analyzeAlterViewStmt(TAlterViewStatement stmt) {
8689                if (stmt.getAlterViewOption() == EAlterViewOption.asSelect) {
8690                        analyzeCreateViewStmt(stmt, stmt.getSelectSqlStatement(), null, stmt.getViewName());
8691                } else {
8692                        throw new UnsupportedOperationException("Can't handle this alter view statement case, alter option = "+ stmt.getAlterViewOption().name());
8693                }
8694        }
8695        
8696        private void analyzeDeleteStmt(TDeleteSqlStatement stmt) {
8697                TTable targetTable = stmt.getTargetTable();
8698                if (targetTable == null)
8699                        return;
8700
8701                TTable table = targetTable;
8702                Table tableModel = null;
8703                TSelectSqlStatement projectedSubquery = null;
8704                TObjectNameList exposedColumns = null;
8705                if (targetTable.getCTE() != null) {
8706                        projectedSubquery = targetTable.getCTE().getSubquery();
8707                        exposedColumns = targetTable.getCTE().getColumnList();
8708                } else if (targetTable.getLinkTable() != null
8709                                && targetTable.getLinkTable().getSubquery() != null) {
8710                        projectedSubquery = targetTable.getLinkTable().getSubquery();
8711                        exposedColumns = getProjectedTargetColumns(targetTable.getLinkTable());
8712                } else if (targetTable.getSubquery() != null) {
8713                        projectedSubquery = targetTable.getSubquery();
8714                        exposedColumns = getProjectedTargetColumns(targetTable);
8715                }
8716
8717                if (projectedSubquery != null) {
8718                        CteWriteTargetResolution resolution = resolveCteDeleteTarget(projectedSubquery);
8719                        if (resolution.isWritable()) {
8720                                analyzeProjectedWriteTarget(targetTable, projectedSubquery, exposedColumns, null);
8721                                table = resolution.baseTable;
8722                                tableModel = modelFactory.createTable(table);
8723                                createProjectedWriteImpact(projectedSubquery, tableModel, EffectType.delete, null);
8724                        } else {
8725                                analyzeProjectedWriteTarget(targetTable, projectedSubquery, exposedColumns, null);
8726                                addCteWriteTargetHint(targetTable, resolution.failureReason);
8727                        }
8728                } else {
8729                        tableModel = modelFactory.createTable(table);
8730                }
8731                if (tableModel != null && getTableLinkedColumns(table) != null
8732                                && getTableLinkedColumns(table).size() > 0) {
8733                        for (int j = 0; j < getTableLinkedColumns(table).size(); j++) {
8734                                TObjectName object = getTableLinkedColumns(table).getObjectName(j);
8735
8736                                if (object.getDbObjectType() == EDbObjectType.variable) {
8737                                        continue;
8738                                }
8739
8740                                if (object.getColumnNameOnly().startsWith("@")
8741                                                && (option.getVendor() == EDbVendor.dbvmssql || option.getVendor() == EDbVendor.dbvazuresql)) {
8742                                        continue;
8743                                }
8744
8745                                if (object.getColumnNameOnly().startsWith(":")
8746                                                && (option.getVendor() == EDbVendor.dbvhana || option.getVendor() == EDbVendor.dbvteradata)) {
8747                                        continue;
8748                                }
8749
8750                                if (!isBuiltInFunctionName(object)) {
8751                                        if (object.getSourceTable() == null || object.getSourceTable() == table) {
8752                                                modelFactory.createTableColumn(tableModel, object, false);
8753                                        }
8754                                }
8755                        }
8756                }
8757
8758                if(option.getAnalyzeMode() == AnalyzeMode.crud && tableModel != null) {
8759                        CrudRelationship crudRelationship = modelFactory.createCrudRelation();
8760                        crudRelationship.setTarget(new TableRelationshipElement(tableModel));
8761                        crudRelationship.setEffectType(EffectType.delete);
8762                }
8763                
8764                if (stmt.getWhereClause() != null && stmt.getWhereClause().getCondition() != null) {
8765                        analyzeFilterCondition(tableModel, stmt.getWhereClause().getCondition(), null, JoinClauseType.where,
8766                                        EffectType.delete);
8767                }
8768        }
8769
8770        private TObjectName getProcedureName(TStoredProcedureSqlStatement stmt) {
8771                if (stmt instanceof TTeradataCreateProcedure) {
8772                        return ((TTeradataCreateProcedure) stmt).getProcedureName();
8773                }
8774                return DlineageUtil.getProcedureOrFunctionName(stmt);
8775        }
8776
8777        private void analyzePlsqlCreatePackage(TPlsqlCreatePackage stmt) {
8778                TObjectName procedureName = getProcedureName(stmt);
8779                OraclePackage oraclePackage;
8780                if (procedureName != null) {
8781                        if (this.modelManager.getOraclePackageByName(
8782                                        DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getProcedureNameWithArgs(stmt))) == null) {
8783                                oraclePackage = this.modelFactory.createOraclePackage(stmt);
8784
8785                                if (stmt.getParameterDeclarations() != null) {
8786                                        TParameterDeclarationList parameters = stmt.getParameterDeclarations();
8787
8788                                        for (int i = 0; i < parameters.size(); ++i) {
8789                                                TParameterDeclaration parameter = parameters.getParameterDeclarationItem(i);
8790                                                if (parameter.getParameterName() != null) {
8791                                                        this.modelFactory.createProcedureArgument(oraclePackage, parameter, i + 1);
8792                                                } else if (parameter.getDataType() != null) {
8793                                                        this.modelFactory.createProcedureArgument(oraclePackage, parameter, i + 1);
8794                                                }
8795                                        }
8796                                }
8797
8798                                ModelBindingManager.setGlobalOraclePackage(oraclePackage);
8799                        } else {
8800                                ModelBindingManager.setGlobalOraclePackage(this.modelManager.getOraclePackageByName(
8801                                                DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getProcedureNameWithArgs(stmt))));
8802                        }
8803                        
8804                        try {
8805                                if (stmt.getDeclareStatements() != null) {
8806                                        for (int i = 0; i < stmt.getDeclareStatements().size(); ++i) {
8807                                                analyzeCustomSqlStmt(stmt.getDeclareStatements().get(i));
8808                                        }
8809                                }
8810                        } finally {
8811                                ModelBindingManager.removeGlobalOraclePackage();
8812                        }
8813                }
8814        }
8815
8816        /**
8817         * B1 (routine-summary-scc-design §2.1a): canonical signature descriptor of
8818         * a routine DECLARATION, built once at declaration time. Null when the
8819         * statement carries no resolvable name.
8820         */
8821        private gudusoft.gsqlparser.dlineage.dynamicsql.RoutineIdentity declarationIdentityOf(
8822                        TStoredProcedureSqlStatement stmt) {
8823                TObjectName name = getProcedureName(stmt);
8824                if (name == null || name.getObjectString() == null
8825                                || name.getObjectString().length() == 0) {
8826                        return null;
8827                }
8828                boolean isFunction = stmt instanceof gudusoft.gsqlparser.stmt.TCreateFunctionStmt
8829                                || stmt instanceof gudusoft.gsqlparser.stmt.oracle.TPlsqlCreateFunction;
8830                boolean isTrigger = stmt instanceof TCreateTriggerStmt
8831                                || stmt instanceof TPlsqlCreateTrigger;
8832                String packageName = null;
8833                gudusoft.gsqlparser.dlineage.dynamicsql.RoutineKind kind;
8834                if (isTrigger) {
8835                        kind = gudusoft.gsqlparser.dlineage.dynamicsql.RoutineKind.TRIGGER;
8836                } else if (ModelBindingManager.getGlobalOraclePackage() != null) {
8837                        packageName = ModelBindingManager.getGlobalOraclePackage().getName();
8838                        kind = isFunction
8839                                        ? gudusoft.gsqlparser.dlineage.dynamicsql.RoutineKind.PACKAGE_MEMBER_FUNCTION
8840                                        : gudusoft.gsqlparser.dlineage.dynamicsql.RoutineKind.PACKAGE_MEMBER_PROCEDURE;
8841                } else {
8842                        kind = isFunction
8843                                        ? gudusoft.gsqlparser.dlineage.dynamicsql.RoutineKind.FUNCTION
8844                                        : gudusoft.gsqlparser.dlineage.dynamicsql.RoutineKind.PROCEDURE;
8845                }
8846                // mssql numbered procedures: the parser drops ";N" from the AST name;
8847                // recover it from the token stream so grp;1 / grp;2 keep DISTINCT
8848                // identity scope keys (per-overload pools — codex-B3-r2 finding 6).
8849                String overloadDiscriminator = option.getVendor() == EDbVendor.dbvmssql
8850                                ? gudusoft.gsqlparser.dlineage.dynamicsql.RoutineCatalog
8851                                                .numberedGroupAfter(name)
8852                                : null;
8853                return gudusoft.gsqlparser.dlineage.dynamicsql.RoutineIdentity.of(
8854                                option.getVendor(), kind, name.getDatabaseString(),
8855                                name.getSchemaString(), packageName, name.getObjectString(),
8856                                stmt.getParameterDeclarations(), overloadDiscriminator);
8857        }
8858
8859        private void analyzeStoredProcedureStmt(TStoredProcedureSqlStatement stmt) {
8860
8861                if (stmt instanceof TPlsqlCreatePackage) {
8862                        analyzePlsqlCreatePackage((TPlsqlCreatePackage) stmt);
8863                        return;
8864                }
8865                
8866                ModelBindingManager.setGlobalProcedure(stmt);
8867
8868                boolean pushedIdentityScope = false;
8869                try {
8870                        Procedure procedure = null;
8871
8872                        TObjectName procedureName = getProcedureName(stmt);
8873
8874                        // Honor procedure-exclusion patterns/names for procedures defined in actual SQL
8875                        // (CREATE PROCEDURE). Previously exclusion was only applied to procedures loaded
8876                        // from metadata JSON; a CREATE PROCEDURE parsed from SQL still produced its
8877                        // procedure node plus all of its inner-statement processes and relationships
8878                        // regardless of the exclusion configuration (MantisBT 4533).
8879                        //
8880                        // analyzeStoredProcedureStmt also handles functions and triggers (every dialect's
8881                        // CREATE FUNCTION / CREATE TRIGGER class extends TStoredProcedureSqlStatement), which
8882                        // are NOT procedures and must never be dropped by procedure exclusion. Gate on the
8883                        // statement type rather than enumerating classes: across all dialects the procedure
8884                        // statement types are exactly those whose ESqlStatementType name contains "procedure"
8885                        // (e.g. sstcreateprocedure, sstoraclecreateprocedure, sstmssqlcreateprocedure),
8886                        // while function/trigger types contain "function"/"trigger".
8887                        boolean isProcedureStmt = stmt.sqlstatementtype != null
8888                                        && stmt.sqlstatementtype.name().toLowerCase().contains("procedure");
8889                        if (procedureName != null && isProcedureStmt
8890                                        && DlineageUtil.isProcedureExcluded(procedureName.toString())) {
8891                                return;
8892                        }
8893
8894                        if (procedureName != null) {
8895                                procedure = this.modelFactory.createProcedure(stmt);
8896                                if (procedure != null) {
8897                                        modelManager.bindModel(stmt, procedure);
8898                                }
8899                                if (ModelBindingManager.getGlobalOraclePackage() != null) {
8900                                        ModelBindingManager.getGlobalOraclePackage().addProcedure(procedure);
8901                                        procedure.setParentPackage(ModelBindingManager.getGlobalOraclePackage());
8902                                        // Canonical scope key of this member's variables: the same
8903                                        // value createVariable(String) keys them under while the
8904                                        // body is analyzed (fullName/name are display forms and
8905                                        // must not be touched).
8906                                        procedure.setVariableScopeKey(DlineageUtil.getProcedureParentName(stmt));
8907                                } else if (procedure != null) {
8908                                        // Non-package routines: body references register their
8909                                        // variables under the getTableFullName-composed scope (the
8910                                        // parent-statement branch of getProcedureParentName), which
8911                                        // under a USE database context differs BOTH from the raw
8912                                        // declared name and from the display name — leaving nested
8913                                        // EXEC argument binding without a matching key. Capture the
8914                                        // body-side composition here, under the declaration's own
8915                                        // ambient context.
8916                                        procedure.setVariableScopeKey(DlineageUtil.getTableFullName(
8917                                                        DlineageUtil.getProcedureParentName(stmt)));
8918                                }
8919                                if (procedure != null) {
8920                                        // B1 (routine-summary-scc-design §3): additive identity
8921                                        // metadata. The identityScopeKey discriminates same-named
8922                                        // overloads; LEGACY paths never consult it (B3's
8923                                        // identity-first lookup is the consumer).
8924                                        gudusoft.gsqlparser.dlineage.dynamicsql.RoutineIdentity routineIdentity =
8925                                                        declarationIdentityOf(stmt);
8926                                        procedure.setRoutineIdentity(routineIdentity);
8927                                        if (procedure.getVariableScopeKey() != null && routineIdentity != null) {
8928                                                procedure.setIdentityScopeKey(DlineageUtil.identityScopeKey(
8929                                                                procedure.getVariableScopeKey(), routineIdentity));
8930                                        }
8931                                        // B3 (routine-summary-scc-design §3): object-level
8932                                        // per-overload variable pools. Mode-gated — LEGACY never
8933                                        // pushes, so the identity pool path stays cold there.
8934                                        if (option.isIdentityFirstVariablePools()
8935                                                        && procedure.getIdentityScopeKey() != null) {
8936                                                modelFactory.pushIdentityScope(procedure.getIdentityScopeKey());
8937                                                pushedIdentityScope = true;
8938                                        }
8939                                }
8940                                if (stmt.getParameterDeclarations() != null) {
8941                                        TParameterDeclarationList parameters = stmt.getParameterDeclarations();
8942        
8943                                        for (int i = 0; i < parameters.size(); ++i) {
8944                                                TParameterDeclaration parameter = parameters.getParameterDeclarationItem(i);
8945                                                Argument argument = null;
8946                                                TObjectName argumentName = null;
8947                                                if (parameter.getParameterName() != null) {
8948                                                        argument = this.modelFactory.createProcedureArgument(procedure, parameter, i + 1);
8949                                                        argumentName = parameter.getParameterName();
8950                                                } else if (parameter.getDataType() != null) {
8951                                                        argument = this.modelFactory.createProcedureArgument(procedure, parameter, i + 1);
8952                                                }
8953                                                
8954                                                if (argument != null) {
8955                                                        if (argumentName == null) {
8956                                                                argumentName = new TObjectName();
8957                                                                argumentName.setString(argument.getName());
8958                                                        }
8959                                                        Variable variable = modelFactory.createVariable(argument.getName());
8960                                                        // B1: index the formal under the overload-
8961                                                        // discriminating identity scope too (lookup-map
8962                                                        // only; consumed by B3's identity-first path).
8963                                                        modelFactory.bindVariableUnderIdentityScope(
8964                                                                        procedure, argument.getName(), variable);
8965                                                        if (argument.getMode() == EParameterMode.in || argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
8966                                                                variable.setSubType(SubType.of(argument.getMode().name()));
8967                                                        } else {
8968                                                                variable.setSubType(SubType.argument);
8969                                                        }
8970                                                        TableColumn parameterField;
8971                                                        if(isSimpleDataType(argument.getDataType())){
8972                                                                parameterField = modelFactory.createTableColumn(variable, argumentName, true);
8973                                                        }
8974                                                        else {
8975                                                                TObjectName variableProperties = new TObjectName();
8976                                                                variableProperties.setString("*");
8977                                                                parameterField = modelFactory.createTableColumn(variable, variableProperties, true);
8978                                                        }
8979                                                        if (shouldCollectAuthoritativeLineageEvidence()) {
8980                                                                authoritativeEvidenceCollector.bindRoutineParameter(argument, parameterField);
8981                                                        }
8982                                                }
8983                                        }
8984                                }
8985                        }
8986        
8987                        if (stmt instanceof TCreateTriggerStmt) {
8988                                TCreateTriggerStmt trigger = (TCreateTriggerStmt) stmt;
8989        
8990                                if (trigger.getFunctionCall() != null) {
8991                                        modelFactory.createProcedureFromFunctionCall(trigger.getFunctionCall());
8992                                }
8993        
8994                                if (trigger.getTables() != null) {
8995                                        for (int i = 0; i < trigger.getTables().size(); i++) {
8996                                                Table tableModel = this.modelFactory.createTriggerOnTable(trigger.getTables().getTable(i));
8997                                        }
8998                                }
8999                        }
9000        
9001                        if (stmt instanceof TPlsqlCreateTrigger && ((TPlsqlCreateTrigger) stmt).getTriggeringClause()!=null
9002                                        && ((TPlsqlCreateTrigger) stmt).getTriggeringClause().getEventClause() instanceof TDmlEventClause) {
9003                                TPlsqlCreateTrigger trigger = (TPlsqlCreateTrigger) stmt;
9004                                TDmlEventClause clause = (TDmlEventClause) ((TPlsqlCreateTrigger) stmt).getTriggeringClause()
9005                                                .getEventClause();
9006                                Table sourceTable = modelFactory.createTableByName(clause.getTableName());
9007
9008                                bindTriggerCorrelationNames(trigger, sourceTable);
9009        
9010                                for (TTriggerEventItem item : clause.getEventItems()) {
9011                                        if (item instanceof TDmlEventItem) {
9012                                                if (((TDmlEventItem) item).getColumnList() != null) {
9013                                                        for (TObjectName column : ((TDmlEventItem) item).getColumnList()) {
9014                                                                modelFactory.createTableColumn(sourceTable, column, true);
9015                                                        }
9016                                                }
9017                                        }
9018                                }
9019        
9020                                for (TCustomSqlStatement subStmt : ((TPlsqlCreateTrigger) stmt).getStatements()) {
9021                                        if (!(subStmt instanceof TCommonBlock))
9022                                                continue;
9023                                        for (TCustomSqlStatement blockSubStmt : ((TCommonBlock) subStmt).getStatements()) {
9024                                                if (!(blockSubStmt instanceof TBasicStmt))
9025                                                        continue;
9026                                                TBasicStmt basicStmt = (TBasicStmt) blockSubStmt;
9027                                                TExpression expression = basicStmt.getExpr();
9028                                                if (expression != null && expression.getExpressionType() == EExpressionType.function_t) {
9029                                                        Procedure targetProcedure = modelManager.getProcedureByName(DlineageUtil
9030                                                                        .getTableFullName(expression.getFunctionCall().getFunctionName().toString()));
9031                                                        if (targetProcedure == null) {
9032                                                                targetProcedure = modelManager
9033                                                                                .getProcedureByName(DlineageUtil.getTableFullName(procedure.getSchema() + "."
9034                                                                                                + expression.getFunctionCall().getFunctionName().toString()));
9035                                                        }
9036                                                        if (targetProcedure != null && expression.getFunctionCall().getArgs() != null && expression
9037                                                                        .getFunctionCall().getArgs().size() == targetProcedure.getArguments().size()) {
9038                                                                for (int j = 0; j < expression.getFunctionCall().getArgs().size(); j++) {
9039                                                                        TExpression columnExpr = expression.getFunctionCall().getArgs().getExpression(j);
9040                                                                        if (columnExpr.getExpressionType() == EExpressionType.simple_object_name_t) {
9041                                                                                TObjectName columnObject = columnExpr.getObjectOperand();
9042                                                                                if (columnObject.toString().indexOf(":") != -1) {
9043                                                                                        TableColumn tableColumn = modelFactory.createTableColumn(sourceTable,
9044                                                                                                        columnObject, true);
9045                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
9046                                                                                        relation.setEffectType(EffectType.trigger);
9047                                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
9048                                                                                        relation.setTarget(
9049                                                                                                        new ArgumentRelationshipElement(targetProcedure.getArguments().get(j)));
9050                                                                                        Process process = modelFactory.createProcess(stmt);
9051                                                                                        relation.setProcess(process);
9052                                                                                }
9053                                                                        }
9054                                                                }
9055                                                        }
9056                                                }
9057                                        }
9058                                }
9059                        }
9060        
9061                        if (stmt instanceof TMssqlCreateFunction) {
9062                                TMssqlCreateFunction createFunction = (TMssqlCreateFunction) stmt;
9063                                if (createFunction.getReturnTableVaraible() != null && createFunction.getReturnTableDefinitions() != null) {
9064                                        Variable tableModel = this.modelFactory.createVariable(createFunction.getReturnTableVaraible());
9065                                        tableModel.setVariable(true);
9066                                        tableModel.setCreateTable(true);
9067                                        if (procedure != null) {
9068                                                tableModel.setProcedureId(String.valueOf(procedure.getId()));
9069                                                tableModel.setTarget(true);
9070                                        }
9071                                        String procedureParent = createFunction.getFunctionName().toString();
9072                                        if (procedureParent != null) {
9073                                                tableModel.setParent(procedureParent);
9074                                        }
9075                                        modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent), tableModel);
9076        
9077                                        if (createFunction.getReturnTableDefinitions() != null) {
9078                                                for (int j = 0; j < createFunction.getReturnTableDefinitions().size(); j++) {
9079                                                        TTableElement tableElement = createFunction.getReturnTableDefinitions().getTableElement(j);
9080                                                        TColumnDefinition column = tableElement.getColumnDefinition();
9081                                                        if (column != null && column.getColumnName() != null) {
9082                                                                modelFactory.createTableColumn(tableModel, column.getColumnName(), true);
9083                                                        }
9084                                                }
9085                                        }
9086                                }
9087        
9088                                if (createFunction.getReturnStmt() != null && createFunction.getReturnStmt().getSubquery() != null) {
9089                                        String procedureParent = createFunction.getFunctionName().toString();
9090                                        analyzeSelectStmt(createFunction.getReturnStmt().getSubquery());
9091                                        ResultSet resultSetModel = (ResultSet) modelManager
9092                                                        .getModel(createFunction.getReturnStmt().getSubquery());
9093                                        if (procedure != null) {
9094                                                resultSetModel.setProcedureId(String.valueOf(procedure.getId()));
9095                                                resultSetModel.setTarget(true);
9096                                        }
9097                                        modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent),
9098                                                        resultSetModel);
9099                                }
9100                        } else if (stmt instanceof TCreateFunctionStmt) {
9101                                TCreateFunctionStmt createFunction = (TCreateFunctionStmt) stmt;
9102                                if (createFunction.getReturnDataType() != null
9103                                                && createFunction.getReturnDataType().getColumnDefList() != null) {
9104                                        Table tableModel = this.modelFactory.createTableByName(createFunction.getFunctionName(), true);
9105                                        tableModel.setCreateTable(true);
9106                                        String procedureParent = createFunction.getFunctionName().toString();
9107                                        if (procedureParent != null) {
9108                                                tableModel.setParent(procedureParent);
9109                                        }
9110        
9111                                        modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent), tableModel);
9112                                        for (int j = 0; j < createFunction.getReturnDataType().getColumnDefList().size(); j++) {
9113                                                TColumnDefinition column = createFunction.getReturnDataType().getColumnDefList().getColumn(j);
9114                                                if (column != null && column.getColumnName() != null) {
9115                                                        modelFactory.createTableColumn(tableModel, column.getColumnName(), true);
9116                                                }
9117                                        }
9118        
9119                                        if (createFunction.getSqlQuery() != null) {
9120                                                analyzeSelectStmt(createFunction.getSqlQuery());
9121                                                ResultSet resultSetModel = (ResultSet) modelManager.getModel(createFunction.getSqlQuery());
9122                                                if (resultSetModel != null) {
9123                                                        for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
9124                                                                ResultColumn resultColumn = resultSetModel.getColumns().get(i);
9125                                                                for (int j = 0; j < tableModel.getColumns().size(); j++) {
9126                                                                        TableColumn tableColumn = tableModel.getColumns().get(j);
9127                                                                        if (DlineageUtil.compareColumnIdentifier(getColumnName(resultColumn.getName()),
9128                                                                                        DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
9129                                                                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
9130                                                                                dataflowRelation.setEffectType(EffectType.select);
9131                                                                                dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn));
9132                                                                                dataflowRelation.setTarget(new TableColumnRelationshipElement(tableColumn));
9133                                                                        }
9134                                                                }
9135                                                        }
9136                                                }
9137                                        }
9138                                }
9139                                
9140                                if (createFunction.getReturnStmt() != null && createFunction.getReturnStmt().getSubquery() != null) {
9141                                        String procedureParent = createFunction.getFunctionName().toString();
9142                                        analyzeSelectStmt(createFunction.getReturnStmt().getSubquery());
9143                                        ResultSet resultSetModel = (ResultSet) modelManager
9144                                                        .getModel(createFunction.getReturnStmt().getSubquery());
9145                                        if (procedure != null) {
9146                                                resultSetModel.setProcedureId(String.valueOf(procedure.getId()));
9147                                                resultSetModel.setTarget(true);
9148                                        }
9149                                        modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent),
9150                                                        resultSetModel);
9151                                }
9152                                
9153                                if (createFunction.getReturnStmt() != null && createFunction.getReturnStmt().getReturnExpr() != null) {
9154                                        TExpression returnExpression =  createFunction.getReturnStmt().getReturnExpr();
9155                                        ResultSet returnResult = modelFactory.createResultSet(createFunction.getReturnStmt(), false);
9156                                        ResultColumn resultColumn = modelFactory.createResultColumn(returnResult, returnExpression);
9157                                        if (shouldCollectAuthoritativeLineageEvidence()) {
9158                                                authoritativeEvidenceCollector.bindRoutineReturn(procedure, resultColumn);
9159                                        }
9160                                        
9161                                        TExpression expression = createFunction.getReturnStmt().getReturnExpr();
9162                                        analyzeResultColumnExpressionRelation(resultColumn, expression);
9163        
9164                                        if (procedure != null) {
9165                                                returnResult.setProcedureId(String.valueOf(procedure.getId()));
9166                                                returnResult.setTarget(true);
9167                                        }
9168                                        String procedureParent = createFunction.getFunctionName().toString();
9169                                        modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent),
9170                                                        returnResult);
9171                                }
9172                        }
9173        
9174                        if (stmt instanceof TCreateProcedureStmt) {
9175                                TCreateProcedureStmt createProcedure = (TCreateProcedureStmt) stmt;
9176                                if (EDbVendor.dbvsnowflake == option.getVendor() && createProcedure.getRoutineBodyInConstant() != null) {
9177                                        String language = createProcedure.getProcedureLanguage() == null ? null
9178                                                        : createProcedure.getProcedureLanguage().toString();
9179                                        // Only a JAVASCRIPT body can be executed by the script-based
9180                                        // SQL extractor. Other handler languages (PYTHON, JAVA, SCALA)
9181                                        // are opaque to SQL analysis: keep the procedure model with its
9182                                        // declared language and skip the body (Mantis 4590).
9183                                        if (language == null || language.equalsIgnoreCase("JAVASCRIPT")) { // non-identifier-compare: LANGUAGE keyword, not a db object name
9184                                                extractSnowflakeSQLFromProcedure(createProcedure);
9185                                        }
9186                                }
9187                        }
9188                                
9189                        if (stmt.getStatements().size() > 0) {
9190                                for (int i = 0; i < stmt.getStatements().size(); ++i) {
9191                                        this.analyzeCustomSqlStmt(stmt.getStatements().get(i));
9192                                }
9193                        }
9194
9195                        if (stmt.getBodyStatements().size() > 0) {
9196                                for (int i = 0; i < stmt.getBodyStatements().size(); ++i) {
9197                                        this.analyzeCustomSqlStmt(stmt.getBodyStatements().get(i));
9198                                }
9199                        }
9200                        
9201                        // Detect pipelined functions and build signatures
9202                        if (stmt instanceof TPlsqlCreateFunction && pipelinedAnalyzer != null) {
9203                                try {
9204                                        pipelinedAnalyzer.analyzePipelinedFunction((TPlsqlCreateFunction) stmt);
9205                                } catch (Exception e) {
9206                                        // Don't let pipelined analysis failure break main flow
9207                                }
9208                        }
9209
9210                        if (procedure != null && !getLastSelectStmt(stmt).isEmpty()) {
9211                                for (TSelectSqlStatement select : getLastSelectStmt(stmt)) {
9212                                        ResultSet resultSet = (ResultSet) modelManager.getModel(select);
9213                                        resultSet.setProcedureId(String.valueOf(procedure.getId()));
9214                                        resultSet.setTarget(true);
9215                                        modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedure.getName()),
9216                                                        resultSet);
9217                                }
9218//                              List<Argument> outArgs = new ArrayList<Argument>();
9219//                              for (int i = 0; i < procedure.getArguments().size(); i++) {
9220//                                      Argument argument = procedure.getArguments().get(i);
9221//                                      if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
9222//                                              outArgs.add(argument);
9223//                                      }
9224//                              }
9225//
9226//                              if (resultSet != null && resultSet.getColumns().size() == outArgs.size()) {
9227//                                      for (int i = 0; i < outArgs.size(); i++) {
9228//                                              Argument argument = outArgs.get(i);
9229//                                              Variable variable = modelFactory.createVariable(argument.getName(), false);
9230//                                              if (variable != null) {
9231//                                                      DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
9232//                                                      dataflowRelation.setEffectType(EffectType.output);
9233//                                                      dataflowRelation
9234//                                                                      .addSource(new ResultColumnRelationshipElement(resultSet.getColumns().get(i)));
9235//                                                      dataflowRelation
9236//                                                                      .setTarget(new TableColumnRelationshipElement(variable.getColumns().get(0)));
9237//                                              }
9238//                                      }
9239//
9240//                              }
9241                        }
9242
9243                        if (stmt instanceof TCreateFunctionStmt && ((TCreateFunctionStmt)stmt).getSqlExpression()!=null) {
9244                                TCreateFunctionStmt createFunction = (TCreateFunctionStmt) stmt;
9245                                TExpression returnExpression = createFunction.getSqlExpression();
9246
9247                                ResultSet returnResult = modelFactory.createResultSet(stmt, false);
9248                                ResultColumn resultColumn = modelFactory.createResultColumn(returnResult, returnExpression);
9249                                if (shouldCollectAuthoritativeLineageEvidence()) {
9250                                        authoritativeEvidenceCollector.bindRoutineReturn(procedure, resultColumn);
9251                                }
9252
9253                                columnsInExpr visitor = new columnsInExpr();
9254                                returnExpression.inOrderTraverse(visitor);
9255
9256                                List<TObjectName> objectNames = visitor.getObjectNames();
9257                                List<TParseTreeNode> functions = visitor.getFunctions();
9258                                List<TParseTreeNode> constants = visitor.getConstants();
9259                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
9260
9261                                if (functions != null && !functions.isEmpty()) {
9262                                        analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function);
9263                                }
9264                                if (subquerys != null && !subquerys.isEmpty()) {
9265                                        analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.select);
9266                                }
9267                                if (objectNames != null && !objectNames.isEmpty()) {
9268                                        analyzeDataFlowRelation(resultColumn, objectNames, EffectType.select, functions);
9269                                }
9270                                if (constants != null && !constants.isEmpty()) {
9271                                        analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.select, functions);
9272                                }
9273
9274                                String procedureParent = SQLUtil.trimColumnStringQuote(getProcedureParentName(stmt));
9275                                modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent),
9276                                                returnResult);
9277                        }
9278                } finally {
9279                        if (pushedIdentityScope) {
9280                                modelFactory.popIdentityScope();
9281                        }
9282                        ModelBindingManager.removeGlobalProcedure();
9283                }
9284
9285        }
9286
9287        private boolean isSimpleDataType(TTypeName dataType) {
9288                if (dataType.getDataType() == EDataType.variant_t) {
9289                        return false;
9290                }
9291                if (dataType.getDataType() == EDataType.cursor_t) {
9292                        return false;
9293                }
9294                if (dataType.getDataType() == EDataType.generic_t) {
9295                        return false;
9296                }
9297                if (dataType.getDataType() == EDataType.unknown_t) {
9298                        return false;
9299                }
9300                if (dataType.getDataType() == EDataType.sql_variant_t) {
9301                        return false;
9302                }
9303                if (dataType.getDataType() == EDataType.table_t) {
9304                        return false;
9305                }
9306                if (dataType.getDataType() == EDataType.raw_t) {
9307                        return false;
9308                }
9309                if (dataType.getDataType() == EDataType.resultset_t) {
9310                        return false;
9311                }
9312                if (dataType.getDataType() == EDataType.row_t) {
9313                        return false;
9314                }
9315                if (dataType.getDataType() == EDataType.map_t) {
9316                        return false;
9317                }
9318                if (dataType.getDataType() == EDataType.anyType_t) {
9319                        return false;
9320                }
9321                if (dataType.getDataType() == EDataType.struct_t) {
9322                        return false;
9323                }
9324                if (dataType.getDataType() == EDataType.structType_t) {
9325                        return false;
9326                }
9327                if (dataType.getDataType() == EDataType.mapType_t) {
9328                        return false;
9329                }
9330                return true;
9331        }
9332
9333        /**
9334         * ClickHouse remote()/remoteSecure() expose a real db.table on another
9335         * host as a table; modelling them as plain function nodes dropped the
9336         * upstream node and every edge into it. Returns true when the call was
9337         * modelled — the model is bound to the TTable, so the columns resolver2
9338         * already linked flow through the normal machinery. The model registers
9339         * through the ordinary table registry: a remote db.table shares identity
9340         * semantics with every other ClickHouse table reference.
9341         *
9342         * Only PROVABLE sources are modelled: plain object names or quoted
9343         * string constants, taken verbatim from the SQL text. Runtime
9344         * expressions and non-string constants (NULL, numbers) fall back to the
9345         * generic function node (never publish an unprovable edge).
9346         *
9347         * The PATH functions (file/s3/url/hdfs, their *Cluster variants and
9348         * azureBlobStorage) are deliberately NOT modelled: resource paths are
9349         * case-sensitive identities that do not fit the folded table-name
9350         * registry (bare filenames collide with same-spelling tables, case
9351         * variants coalesce, and Azure's first argument is a credential-bearing
9352         * connection string). Their missing upstream remains an HONEST, VISIBLE
9353         * gap — a generic function node — until a dedicated resource-identity
9354         * layer exists; see external_source_lineage_gap.md.
9355         *
9356         * Generator table functions (numbers(), generateRandom(), zeros(), ...)
9357         * correctly have NO upstream, which the generic node already expresses.
9358         */
9359        private boolean modelClickhouseExternalTableFunction(TTable table, TFunctionCall functionCall) {
9360                if (option.getVendor() != EDbVendor.dbvclickhouse) {
9361                        return false;
9362                }
9363                if (functionCall.getFunctionName() == null || functionCall.getArgs() == null
9364                                || functionCall.getArgs().size() < 2) {
9365                        return false;
9366                }
9367                String functionName = functionCall.getFunctionName().toString();
9368                if (!isClickhouseTableFunction(functionName, "remote")
9369                                && !isClickhouseTableFunction(functionName, "remoteSecure")) {
9370                        return false;
9371                }
9372                // remote(addresses, db, table[, user, password])
9373                // remote(addresses, db.table[, user, password])
9374                TExpressionList args = functionCall.getArgs();
9375                String first = literalTableFunctionArg(args.getExpression(1));
9376                if (first == null) {
9377                        return false;
9378                }
9379                String remoteTableName;
9380                if (first.indexOf('.') > 0) {
9381                        remoteTableName = first;
9382                } else {
9383                        String second = args.size() >= 3 ? literalTableFunctionArg(args.getExpression(2)) : null;
9384                        if (second == null) {
9385                                return false; // db without a provable table argument
9386                        }
9387                        remoteTableName = first + "." + second;
9388                }
9389                Table model = clickhouseExternalSourceModels.get(remoteTableName);
9390                if (model == null) {
9391                        model = modelFactory.createTableFromCreateDDL(table, false, remoteTableName);
9392                        clickhouseExternalSourceModels.put(remoteTableName, model);
9393                }
9394                modelManager.bindCreateModel(table, model);
9395                modelManager.bindModel(table, model);
9396                return true;
9397        }
9398
9399        /** Same-literal remote() calls merge here; cleared in init(). */
9400        private final Map<String, Table> clickhouseExternalSourceModels = new LinkedHashMap<String, Table>();
9401
9402        /**
9403         * ClickHouse function names are CASE-SENSITIVE (FiLe(...) is
9404         * UNKNOWN_FUNCTION natively) — matched through the canonical
9405         * per-object-type comparison, never a case fold.
9406         */
9407        private static boolean isClickhouseTableFunction(String rawName, String candidate) {
9408                return DlineageUtil.sameName(ESQLDataObjectType.dotFunction, rawName, candidate);
9409        }
9410
9411        /** A provably literal identifier-ish argument: a plain object name or a
9412         *  QUOTED STRING constant (quotes stripped); null for anything computed —
9413         *  including non-string constants (NULL, numbers), which are not resource
9414         *  identities. */
9415        private static String literalTableFunctionArg(TExpression arg) {
9416                if (arg == null) {
9417                        return null;
9418                }
9419                if (arg.getObjectOperand() != null) {
9420                        return arg.getObjectOperand().toString();
9421                }
9422                return literalStringConstant(arg);
9423        }
9424
9425        /** The DECODED value of a quoted string constant, or null. Delegates to
9426         *  the shared ClickHouse literal decoder (IdentifierCodec) — the raw
9427         *  spelling is not the value ('db\x2etable' IS db.table), the escape
9428         *  table must match ClickHouse's parseComplexEscapeSequence exactly, and
9429         *  a value that cannot be represented safely returns null so the caller
9430         *  falls back to the generic function node. */
9431        private static String literalStringConstant(TExpression arg) {
9432                if (arg == null || arg.getConstantOperand() == null) {
9433                        return null;
9434                }
9435                return gudusoft.gsqlparser.sqlenv.IdentifierCodec
9436                                .decodeClickHouseStringLiteral(arg.getConstantOperand().toString());
9437        }
9438
9439        protected void analyzeResultColumnExpressionRelation(Object resultColumn, TExpression expression) {
9440                columnsInExpr visitor = new columnsInExpr();
9441                expression.inOrderTraverse(visitor);
9442                List<TObjectName> objectNames = visitor.getObjectNames();
9443                List<TParseTreeNode> functions = visitor.getFunctions();
9444                List<TParseTreeNode> constants = visitor.getConstants();
9445                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
9446
9447                if (functions != null && !functions.isEmpty()) {
9448                        analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function);
9449                }
9450                if (subquerys != null && !subquerys.isEmpty()) {
9451                        analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.select);
9452                }
9453                if (objectNames != null && !objectNames.isEmpty()) {
9454                        analyzeDataFlowRelation(resultColumn, objectNames, EffectType.select, functions);
9455                }
9456                if (constants != null && !constants.isEmpty()) {
9457                        analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.select, functions);
9458                }
9459        }
9460
9461        private void analyzeDb2ReturnStmt(TDb2ReturnStmt stmt) {
9462                if (stmt.getReturnExpr() != null) {
9463                        TExpression returnExpression = stmt.getReturnExpr();
9464                        ResultSet returnResult = modelFactory.createResultSet(stmt, true);
9465                        ResultColumn resultColumn = modelFactory.createResultColumn(returnResult, returnExpression);
9466                        bindCurrentRoutineReturn(resultColumn);
9467                        
9468                        columnsInExpr visitor = new columnsInExpr();
9469                        stmt.getReturnExpr().inOrderTraverse(visitor);
9470                        
9471                        List<TObjectName> objectNames = visitor.getObjectNames();
9472                        List<TParseTreeNode> functions = visitor.getFunctions();
9473                        List<TParseTreeNode> constants = visitor.getConstants();
9474                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
9475
9476                        if (functions != null && !functions.isEmpty()) {
9477                                analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function);
9478                        }
9479                        if (subquerys != null && !subquerys.isEmpty()) {
9480                                analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.select);
9481                        }
9482                        if (objectNames != null && !objectNames.isEmpty()) {
9483                                analyzeDataFlowRelation(resultColumn, objectNames, EffectType.select, functions);
9484                        }
9485                        if (constants != null && !constants.isEmpty()) {
9486                                analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.select, functions);
9487                        }
9488
9489                        String procedureParent = SQLUtil.trimColumnStringQuote(getProcedureParentName(stmt));
9490                        if (procedureParent != null) {
9491                                modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent),
9492                                                returnResult);
9493                        }
9494                }
9495        }
9496
9497        private void analyzeReturnStmt(TReturnStmt stmt) {
9498                if (stmt.getResultColumnList() != null) {
9499                        ResultSet returnResult = modelFactory.createResultSet(stmt, true);
9500                        for (TResultColumn column : stmt.getResultColumnList()) {
9501                                ResultColumn resultColumn = modelFactory.createResultColumn(returnResult, column);
9502                                if (stmt.getResultColumnList().size() == 1) bindCurrentRoutineReturn(resultColumn);
9503
9504                                columnsInExpr visitor = new columnsInExpr();
9505                                column.getExpr().inOrderTraverse(visitor);
9506
9507                                List<TObjectName> objectNames = visitor.getObjectNames();
9508                                List<TParseTreeNode> functions = visitor.getFunctions();
9509                                List<TParseTreeNode> constants = visitor.getConstants();
9510                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
9511
9512                                if (functions != null && !functions.isEmpty()) {
9513                                        analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function);
9514                                }
9515                                if (subquerys != null && !subquerys.isEmpty()) {
9516                                        analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.select);
9517                                }
9518                                if (objectNames != null && !objectNames.isEmpty()) {
9519                                        analyzeDataFlowRelation(resultColumn, objectNames, EffectType.select, functions);
9520                                }
9521                                if (constants != null && !constants.isEmpty()) {
9522                                        analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.select, functions);
9523                                }
9524
9525                        }
9526
9527                        String procedureParent = SQLUtil.trimColumnStringQuote(getProcedureParentName(stmt));
9528                        modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent),
9529                                        returnResult);
9530                }
9531                else if(stmt.getExpression()!=null){
9532                        TExpression returnExpression = stmt.getExpression();
9533                        ResultSet returnResult = modelFactory.createResultSet(stmt, true);
9534                        
9535                        columnsInExpr visitor = null;
9536                        List<TSelectSqlStatement> subquerys = null;
9537                        
9538                        if (returnExpression.getFunctionCall() != null
9539                                        && returnExpression.getFunctionCall().getFunctionName().toString().equalsIgnoreCase("table")
9540                                        && returnExpression.getFunctionCall().getArgs() != null
9541                                        && returnExpression.getFunctionCall().getArgs().size()>0) {
9542                                visitor = new columnsInExpr();
9543                                returnExpression.getFunctionCall().getArgs().getExpression(0).inOrderTraverse(visitor);
9544                                subquerys = visitor.getSubquerys();
9545                                if (subquerys != null && !subquerys.isEmpty()) {
9546                                        analyzeSelectStmt(subquerys.get(0));
9547                                        ResultSet resultSet = (ResultSet) modelManager.getModel(subquerys.get(0));
9548                                        if (resultSet != null && resultSet.getColumns() != null) {
9549                                                for (ResultColumn column : resultSet.getColumns()) {
9550                                                        TObjectName columnName = new TObjectName();
9551                                                        columnName.setString(column.getName());
9552                                                        ResultColumn resultColumn = modelFactory.createResultColumn(returnResult, columnName);
9553                                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
9554                                                        dataflowRelation.setEffectType(EffectType.select);
9555                                                        dataflowRelation.addSource(new ResultColumnRelationshipElement(column));
9556                                                        dataflowRelation.setTarget(new ResultColumnRelationshipElement(resultColumn));
9557                                                }
9558                                        }
9559                                        returnResult.setDetermined(resultSet.isDetermined());
9560                                        String procedureParent = SQLUtil.trimColumnStringQuote(getProcedureParentName(stmt));
9561                                        modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent),
9562                                                        returnResult);
9563                                        return;
9564                                }
9565                        }
9566                        
9567                        ResultColumn resultColumn = null;
9568                        if (returnExpression.getExpressionType() == EExpressionType.simple_object_name_t) {
9569                                TObjectName columnName = new TObjectName();
9570                                columnName.setString("*");
9571                                resultColumn = modelFactory.createResultColumn(returnResult, columnName);
9572                        }
9573                        else {
9574                                resultColumn = modelFactory.createResultColumn(returnResult, returnExpression);
9575                        }
9576
9577                        visitor = new columnsInExpr();
9578                        stmt.getExpression().inOrderTraverse(visitor);
9579
9580                        List<TObjectName> objectNames = visitor.getObjectNames();
9581                        List<TParseTreeNode> functions = visitor.getFunctions();
9582                        List<TParseTreeNode> constants = visitor.getConstants();
9583                        subquerys = visitor.getSubquerys();
9584
9585                        if (functions != null && !functions.isEmpty()) {
9586                                analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function);
9587                        }
9588                        if (subquerys != null && !subquerys.isEmpty()) {
9589                                analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.select);
9590                        }
9591                        if (objectNames != null && !objectNames.isEmpty()) {
9592                                DataFlowRelationship relation = analyzeDataFlowRelation(resultColumn, objectNames, EffectType.select, functions);
9593                                //如果variable对应的不是一个复杂结构,则resultColumn不要设置为*
9594                                if (relation != null && relation.getTarget().getElement() == resultColumn && relation.getSources().size() == 1) {
9595                                        Object column = relation.getSources().iterator().next().getElement();
9596                                        boolean star = true;
9597                                        if (column instanceof TableColumn && ((TableColumn) column).getName().indexOf("*") == -1) {
9598                                                star = false;
9599                                        }
9600                                        if (column instanceof ResultColumn && ((ResultColumn) column).getName().indexOf("*") == -1) {
9601                                                star = false;
9602                                        }
9603                                        if (!star && returnExpression.getExpressionType() == EExpressionType.simple_object_name_t) {
9604                                                resultColumn = modelFactory.createResultColumn(returnResult, returnExpression.getObjectOperand());
9605                                                returnResult.getColumns().clear();
9606                                                returnResult.addColumn(resultColumn);
9607                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
9608                                        }
9609                                }
9610                        }
9611                        if (constants != null && !constants.isEmpty()) {
9612                                analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.select, functions);
9613                        }
9614                        bindCurrentRoutineReturn(resultColumn);
9615
9616                        String procedureParent = SQLUtil.trimColumnStringQuote(getProcedureParentName(stmt));
9617                        if (procedureParent != null) {
9618                                modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent),
9619                                                returnResult);
9620                        }
9621                }
9622        }
9623
9624        private void bindCurrentRoutineReturn(ResultColumn resultColumn) {
9625                if (!shouldCollectAuthoritativeLineageEvidence()) return;
9626                TStoredProcedureSqlStatement declaration = ModelBindingManager.getGlobalProcedure();
9627                Object routine = declaration == null ? null : modelManager.getModel(declaration);
9628                if (routine instanceof Procedure) {
9629                        authoritativeEvidenceCollector.bindRoutineReturn((Procedure) routine, resultColumn);
9630                }
9631        }
9632
9633        private void analyzeMssqlReturnStmt(TMssqlReturn stmt) {
9634                if (stmt.getResultColumnList() != null) {
9635                        ResultSet returnResult = modelFactory.createResultSet(stmt, true);
9636                        for (TResultColumn column : stmt.getResultColumnList()) {
9637                                ResultColumn resultColumn = modelFactory.createResultColumn(returnResult, column);
9638                                if (stmt.getResultColumnList().size() == 1) bindCurrentRoutineReturn(resultColumn);
9639
9640                                columnsInExpr visitor = new columnsInExpr();
9641                                column.getExpr().inOrderTraverse(visitor);
9642
9643                                List<TObjectName> objectNames = visitor.getObjectNames();
9644                                List<TParseTreeNode> functions = visitor.getFunctions();
9645                                List<TParseTreeNode> constants = visitor.getConstants();
9646                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
9647
9648                                if (functions != null && !functions.isEmpty()) {
9649                                        analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function);
9650                                }
9651                                if (subquerys != null && !subquerys.isEmpty()) {
9652                                        analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.select);
9653                                }
9654                                if (objectNames != null && !objectNames.isEmpty()) {
9655                                        analyzeDataFlowRelation(resultColumn, objectNames, EffectType.select, functions);
9656                                }
9657                                if (constants != null && !constants.isEmpty()) {
9658                                        analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.select, functions);
9659                                }
9660
9661                        }
9662
9663                        String procedureParent = SQLUtil.trimColumnStringQuote(getProcedureParentName(stmt));
9664                        modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent),
9665                                        returnResult);
9666                }
9667                else if(stmt.getReturnExpr()!=null){
9668                        TExpression returnExpression = stmt.getReturnExpr();
9669                        ResultSet returnResult = modelFactory.createResultSet(stmt, true);
9670                        
9671                        if (returnExpression.getSubQuery() == null) {
9672                                ResultColumn resultColumn = modelFactory.createResultColumn(returnResult, returnExpression);
9673                                columnsInExpr visitor = new columnsInExpr();
9674                                stmt.getReturnExpr().inOrderTraverse(visitor);
9675
9676                                List<TObjectName> objectNames = visitor.getObjectNames();
9677                                List<TParseTreeNode> functions = visitor.getFunctions();
9678                                List<TParseTreeNode> constants = visitor.getConstants();
9679                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
9680
9681                                if (functions != null && !functions.isEmpty()) {
9682                                        analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function);
9683                                }
9684                                if (subquerys != null && !subquerys.isEmpty()) {
9685                                        analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.select);
9686                                }
9687                                if (objectNames != null && !objectNames.isEmpty()) {
9688                                        analyzeDataFlowRelation(resultColumn, objectNames, EffectType.select, functions);
9689                                }
9690                                if (constants != null && !constants.isEmpty()) {
9691                                        analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.select, functions);
9692                                }
9693                                bindCurrentRoutineReturn(resultColumn);
9694                        }
9695                        else {
9696                                analyzeSelectStmt(returnExpression.getSubQuery());
9697                                ResultSet subResultSet = (ResultSet)modelManager.getModel(returnExpression.getSubQuery());
9698                                if (subResultSet != null) {
9699                                        for (ResultColumn subResultColumn : subResultSet.getColumns()) {
9700                                                TObjectName objectName = new TObjectName();
9701                                                objectName.setString(subResultColumn.getName());
9702                                                ResultColumn resultColumn = modelFactory.createResultColumn(returnResult, objectName);
9703                                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
9704                                                dataflowRelation.setEffectType(EffectType.select);
9705                                                dataflowRelation.addSource(new ResultColumnRelationshipElement(subResultColumn));
9706                                                dataflowRelation.setTarget(new ResultColumnRelationshipElement(resultColumn));
9707
9708                                        }
9709                                        
9710                                        if(subResultSet.getRelationRows().hasRelation()) {
9711                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
9712                                                impactRelation.setEffectType(EffectType.select);
9713                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
9714                                                                subResultSet.getRelationRows()));
9715                                                impactRelation.setTarget(
9716                                                                new RelationRowsRelationshipElement<ResultSetRelationRows>(returnResult.getRelationRows()));
9717                                        }
9718                                }
9719                        }
9720                        
9721                        String procedureParent = SQLUtil.trimColumnStringQuote(getProcedureParentName(stmt));
9722                        if (procedureParent != null) {
9723                                modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent),
9724                                                returnResult);
9725                        }
9726                }
9727                else if (stmt.getSubquery() != null) {
9728                        analyzeSelectStmt(stmt.getSubquery());
9729                        ResultSet subResultSet = (ResultSet)modelManager.getModel(stmt.getSubquery());
9730                        String procedureParent = SQLUtil.trimColumnStringQuote(getProcedureParentName(stmt));
9731                        if (procedureParent != null && subResultSet!=null) {
9732                                modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent),
9733                                                subResultSet);
9734                        }
9735                }
9736        }
9737        
9738        /**
9739         * The column that stands for a composite variable as a WHOLE (#695).
9740         *
9741         * <p>A PL/SQL record variable used to be modelled as one opaque column, so
9742         * "the" column of a variable was always {@code getColumns().get(0)}. Once
9743         * records expand to per-field columns, {@code get(0)} silently means "an
9744         * arbitrary field" — the edge that FILLS the record then lands on one
9745         * field (or, worse, every producer field is fanned onto it). The whole-
9746         * record binding must instead land on the record's own {@code *} column,
9747         * which stays distinct per variable instance and can be expanded once the
9748         * pairing is known.
9749         */
9750        private TableColumn compositeBindingColumn(Variable variable) {
9751                if (variable.getColumns() == null || variable.getColumns().isEmpty()) {
9752                        return tableStarColumn(variable);
9753                }
9754                if (variable.getColumns().size() == 1) {
9755                        return variable.getColumns().get(0);
9756                }
9757                if (variable.isTypeOrderedFields()) {
9758                        // Fields from the TYPE declaration: the star expands onto exactly
9759                        // those fields at serialization — every one is genuinely filled.
9760                        return tableStarColumn(variable);
9761                }
9762                // Consumption-shaped fields (pseudo-columns, expression names) carry no
9763                // per-field meaning; expanding a star over them fans every producer onto
9764                // every artifact. Keep the historical whole-record binding.
9765                return variable.getColumns().get(0);
9766        }
9767
9768        /**
9769         * The {@code *} column of {@code table}, created on first use with the
9770         * expand-star convention (as the %ROWTYPE declaration path does): the
9771         * serializer expands a relation targeting it onto the sibling fields it
9772         * star-links, so the edge always cites DECLARED columns — a relationship
9773         * must never cite a column the model does not declare.
9774         */
9775        private TableColumn tableStarColumn(Table table) {
9776                if (table.getColumns() != null) {
9777                        for (TableColumn existing : table.getColumns()) {
9778                                if ("*".equals(existing.getName())) {
9779                                        return existing;
9780                                }
9781                        }
9782                }
9783                TableColumn starColumn = modelFactory.createTableColumn(table, starObjectName(), true);
9784                starColumn.setShowStar(false);
9785                starColumn.setExpandStar(true);
9786                return starColumn;
9787        }
9788
9789        private TObjectName starObjectName() {
9790                TObjectName starColumn = new TObjectName();
9791                starColumn.setString("*");
9792                return starColumn;
9793        }
9794
9795        private void analyzeFetchStmt(TFetchStmt stmt) {
9796                if (stmt.getVariableNames() != null) {
9797                        for (int i = 0; i < stmt.getVariableNames().size(); i++) {
9798                                TExpression variableExpression = stmt.getVariableNames().getExpression(i);
9799                                if (variableExpression.getExpressionType() == EExpressionType.simple_object_name_t) {
9800                                        TObjectName columnObject = variableExpression.getObjectOperand();
9801                                        if (columnObject.getDbObjectType() == EDbObjectType.variable) {
9802                                                continue;
9803                                        }
9804
9805                                        if (columnObject.getColumnNameOnly().startsWith("@") && (option.getVendor() == EDbVendor.dbvmssql
9806                                                        || option.getVendor() == EDbVendor.dbvazuresql)) {
9807                                                continue;
9808                                        }
9809
9810                                        if (columnObject.getColumnNameOnly().startsWith(":") && (option.getVendor() == EDbVendor.dbvhana
9811                                                        || option.getVendor() == EDbVendor.dbvteradata)) {
9812                                                continue;
9813                                        }
9814
9815                                        Variable cursorVariable = modelFactory.createVariable(columnObject);
9816                                        cursorVariable.setSubType(SubType.record);
9817                                        if (cursorVariable.isDetermined()) {
9818                                                if (stmt.getCursorName() != null) {
9819                                                        String procedureName = DlineageUtil.getProcedureParentName(stmt);
9820                                                        String variableString = stmt.getCursorName().toString();
9821                                                        if (variableString.startsWith(":")) {
9822                                                                variableString = variableString.substring(variableString.indexOf(":") + 1);
9823                                                        }
9824                                                        if (!SQLUtil.isEmpty(procedureName)) {
9825                                                                variableString = procedureName + "."
9826                                                                                + SQLUtil.getIdentifierNormalTableName(variableString);
9827                                                        }
9828                                                        Table cursor = modelManager.getTableByName(DlineageUtil.getTableFullName(variableString));
9829                                                        if (cursor != null) {
9830                                                                // FETCH ... INTO binds POSITIONALLY (#695): the k-th
9831                                                                // cursor column fills the k-th record field, whatever
9832                                                                // either is called. Matching by name paired
9833                                                                // like-named columns that are NOT each other's source
9834                                                                // and, when nothing matched, fanned every cursor
9835                                                                // column onto every field — edges the SQL never
9836                                                                // asserts. A field the cursor's known width cannot
9837                                                                // account for keeps the cursor's own * as its source:
9838                                                                // filled, but by an unresolved component.
9839                                                                List<TableColumn> recordFields = cursorVariable.getColumns();
9840                                                                List<TableColumn> cursorColumns = cursor.getColumns();
9841                                                                if (columnObject.toString().indexOf('.') != -1) {
9842                                                                        // FETCH ... INTO rec.field names its target
9843                                                                        // explicitly (#695): bind THAT field to the
9844                                                                        // cursor column at this INTO position —
9845                                                                        // iterating every field here fanned each
9846                                                                        // cursor column onto the whole record.
9847                                                                        TableColumn namedField = modelFactory.createTableColumn(cursorVariable,
9848                                                                                        columnObject, true);
9849                                                                        if (namedField != null) {
9850                                                                                DataFlowRelationship dottedRelation = modelFactory.createDataFlowRelation();
9851                                                                                dottedRelation.setEffectType(EffectType.cursor);
9852                                                                                if (cursorColumns != null && i < cursorColumns.size()) {
9853                                                                                        dottedRelation.addSource(
9854                                                                                                        new TableColumnRelationshipElement(cursorColumns.get(i)));
9855                                                                                } else {
9856                                                                                        dottedRelation.addSource(
9857                                                                                                        new TableColumnRelationshipElement(tableStarColumn(cursor)));
9858                                                                                }
9859                                                                                dottedRelation.setTarget(new TableColumnRelationshipElement(namedField));
9860                                                                        }
9861                                                                        continue;
9862                                                                }
9863                                                                boolean ordinalPairing = stmt.getVariableNames().size() == 1
9864                                                                                && cursorColumns != null && !cursorColumns.isEmpty()
9865                                                                                && cursorVariable.isTypeOrderedFields();
9866                                                                // A SCALAR at INTO position i pairs with cursor
9867                                                                // column i when the widths agree — positional by
9868                                                                // the INTO list itself, no type order involved.
9869                                                                boolean intoPositional = recordFields.size() == 1
9870                                                                                && cursorColumns != null
9871                                                                                && cursorColumns.size() == stmt.getVariableNames().size();
9872                                                                boolean hasRealField = false;
9873                                                                for (TableColumn field : recordFields) {
9874                                                                        if (field.getName() != null && !"*".equals(field.getName())) {
9875                                                                                hasRealField = true;
9876                                                                                break;
9877                                                                        }
9878                                                                }
9879                                                                int fieldOrdinal = 0;
9880                                                                for (int k = 0; k < recordFields.size(); k++) {
9881                                                                        TableColumn variableProperty = recordFields.get(k);
9882                                                                        if ("*".equals(variableProperty.getName()) && hasRealField) {
9883                                                                                // With real fields present the wildcard is a
9884                                                                                // binding artifact (codex-696 F2). When it is
9885                                                                                // the ONLY column it IS the whole-record
9886                                                                                // binding — skipping it dropped every fill
9887                                                                                // whose consumption comes after the FETCH.
9888                                                                                continue;
9889                                                                        }
9890                                                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
9891                                                                        dataflowRelation.setEffectType(EffectType.cursor);
9892                                                                        if (ordinalPairing && fieldOrdinal < cursorColumns.size()) {
9893                                                                                dataflowRelation.addSource(
9894                                                                                                new TableColumnRelationshipElement(cursorColumns.get(fieldOrdinal)));
9895                                                                        } else if (intoPositional) {
9896                                                                                dataflowRelation.addSource(
9897                                                                                                new TableColumnRelationshipElement(cursorColumns.get(i)));
9898                                                                        } else if (stmt.getVariableNames().size() > 1) {
9899                                                                                dataflowRelation.addSource(
9900                                                                                                new TableColumnRelationshipElement(tableStarColumn(cursor), i));
9901                                                                        } else {
9902                                                                                dataflowRelation.addSource(
9903                                                                                                new TableColumnRelationshipElement(tableStarColumn(cursor)));
9904                                                                        }
9905                                                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(variableProperty));
9906                                                                        fieldOrdinal++;
9907                                                                }
9908                                                        }
9909                                                }
9910                                                
9911                                        } else {
9912                                                TableColumn variableProperty = null;
9913                                                if (stmt.getVariableNames().size() == 1) {
9914                                                        if (cursorVariable.getColumns() == null || cursorVariable.getColumns().isEmpty()) {
9915                                                                TObjectName starColumn = new TObjectName();
9916                                                                starColumn.setString("*");
9917                                                                variableProperty = modelFactory.createTableColumn(cursorVariable, starColumn, true);
9918                                                        } else {
9919                                                                variableProperty = cursorVariable.getColumns().get(0);
9920                                                        }
9921                                                } else {
9922                                                        variableProperty = modelFactory.createTableColumn(cursorVariable, columnObject, true);
9923                                                }
9924
9925                                                if (stmt.getCursorName() != null) {
9926                                                        String procedureName = DlineageUtil.getProcedureParentName(stmt);
9927                                                        String variableString = stmt.getCursorName().toString();
9928                                                        if (variableString.startsWith(":")) {
9929                                                                variableString = variableString.substring(variableString.indexOf(":") + 1);
9930                                                        }
9931                                                        if (!SQLUtil.isEmpty(procedureName)) {
9932                                                                variableString = procedureName + "."
9933                                                                                + SQLUtil.getIdentifierNormalTableName(variableString);
9934                                                        }
9935                                                        Table cursor = modelManager.getTableByName(DlineageUtil.getTableFullName(variableString));
9936                                                        if (cursor != null) {
9937                                                                for (int j = 0; j < cursor.getColumns().size(); j++) {
9938                                                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
9939                                                                        dataflowRelation.setEffectType(EffectType.cursor);
9940                                                                        if (stmt.getVariableNames().size() == 1) {
9941                                                                                dataflowRelation.addSource(
9942                                                                                                new TableColumnRelationshipElement(cursor.getColumns().get(j)));
9943                                                                        } else {
9944                                                                                dataflowRelation.addSource(
9945                                                                                                new TableColumnRelationshipElement(cursor.getColumns().get(j), i));
9946                                                                        }
9947                                                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(variableProperty));
9948                                                                }
9949                                                        }
9950                                                }
9951                                        }
9952                                }
9953                        }
9954                }
9955        }
9956
9957        private void analyzeFetchStmt(TMssqlFetch stmt) {
9958                if (stmt.getVariableNames() != null) {
9959                        for (int i = 0; i < stmt.getVariableNames().size(); i++) {
9960                                TObjectName columnObject = stmt.getVariableNames().getObjectName(i);
9961                                Variable cursorVariable = modelFactory.createVariable(columnObject);
9962                                cursorVariable.setCreateTable(true);
9963                                cursorVariable.setSubType(SubType.record);
9964                                TableColumn variableProperty = null;
9965                                if (stmt.getVariableNames().size() == 1) {
9966                                        if (cursorVariable.getColumns() == null || cursorVariable.getColumns().isEmpty()) {
9967                                                TObjectName starColumn = new TObjectName();
9968                                                starColumn.setString("*");
9969                                                variableProperty = modelFactory.createTableColumn(cursorVariable, starColumn, true);
9970                                        } else {
9971                                                variableProperty = cursorVariable.getColumns().get(0);
9972                                        }
9973                                } else {
9974                                        variableProperty = modelFactory.createTableColumn(cursorVariable, columnObject, true);
9975                                }
9976
9977                                if (stmt.getCursorName() != null) {
9978                                        String procedureName = DlineageUtil.getProcedureParentName(stmt);
9979                                        String variableString = stmt.getCursorName().toString();
9980                                        if (variableString.startsWith(":")) {
9981                                                variableString = variableString.substring(variableString.indexOf(":") + 1);
9982                                        }
9983                                        if (!SQLUtil.isEmpty(procedureName)) {
9984                                                variableString = procedureName + "." + SQLUtil.getIdentifierNormalTableName(variableString);
9985                                        }
9986                                        Table cursor = modelManager
9987                                                        .getTableByName(DlineageUtil.getTableFullName(variableString));
9988                                        if (cursor != null) {
9989                                                for (int j = 0; j < cursor.getColumns().size(); j++) {
9990                                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
9991                                                        dataflowRelation.setEffectType(EffectType.cursor);
9992                                                        if (stmt.getVariableNames().size() == 1) {
9993                                                                dataflowRelation
9994                                                                                .addSource(new TableColumnRelationshipElement(cursor.getColumns().get(j)));
9995                                                        } else {
9996                                                                dataflowRelation
9997                                                                                .addSource(new TableColumnRelationshipElement(cursor.getColumns().get(j), i));
9998                                                        }
9999                                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(variableProperty));
10000                                                }
10001                                        }
10002                                }
10003                        }
10004
10005                }
10006        }
10007
10008        private void analyzeLoopStmt(TLoopStmt stmt) {
10009
10010                if (stmt.getCursorName() != null && stmt.getIndexName() != null) {
10011                        modelManager.bindCursorIndex(stmt.getIndexName(), stmt.getCursorName());
10012                }
10013
10014                if (stmt.getRecordName() != null && stmt.getSubquery() != null) {
10015                        Variable cursorTempTable = modelFactory.createCursor(stmt);
10016                        cursorTempTable.setVariable(true);
10017                        cursorTempTable.setSubType(SubType.cursor);
10018                        modelManager.bindCursorModel(stmt, cursorTempTable);
10019                        analyzeSelectStmt(stmt.getSubquery());
10020
10021                        TableColumn cursorColumn = null;
10022                        if (cursorTempTable.getColumns() == null || cursorTempTable.getColumns().isEmpty()) {
10023                                TObjectName starColumn = new TObjectName();
10024                                starColumn.setString("*");
10025                                cursorColumn = modelFactory.createTableColumn(cursorTempTable, starColumn, true);
10026                        } else {
10027                                cursorColumn = cursorTempTable.getColumns().get(0);
10028                        }
10029
10030
10031                        ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt.getSubquery());
10032                        if (resultSetModel != null) {
10033                                for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
10034                                        ResultColumn resultColumn = resultSetModel.getColumns().get(i);
10035                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
10036                                        dataflowRelation.setEffectType(EffectType.cursor);
10037                                        dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn));
10038                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(cursorColumn));
10039                                }
10040                        }
10041                }
10042
10043                for (int i = 0; i < stmt.getStatements().size(); i++) {
10044                        analyzeCustomSqlStmt(stmt.getStatements().get(i));
10045                }
10046        }
10047
10048        private void analyzeForStmt(TForStmt stmt) {
10049                if (stmt.getSubquery() == null) {
10050                        return;
10051                }
10052
10053                Variable cursorTempTable = modelFactory.createCursor(stmt);
10054                cursorTempTable.setVariable(true);
10055                cursorTempTable.setSubType(SubType.cursor);
10056                cursorTempTable.setCreateTable(true);
10057                modelManager.bindCursorModel(stmt, cursorTempTable);
10058                analyzeSelectStmt(stmt.getSubquery());
10059
10060                TableColumn cursorColumn = null;
10061                if (cursorTempTable.getColumns() == null || cursorTempTable.getColumns().isEmpty()) {
10062                        TObjectName starColumn = new TObjectName();
10063                        starColumn.setString("*");
10064                        cursorColumn = modelFactory.createTableColumn(cursorTempTable, starColumn, true);
10065                } else {
10066                        cursorColumn = cursorTempTable.getColumns().get(0);
10067                }
10068
10069
10070                ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt.getSubquery());
10071                if (resultSetModel != null) {
10072                        for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
10073                                ResultColumn resultColumn = resultSetModel.getColumns().get(i);
10074                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
10075                                dataflowRelation.setEffectType(EffectType.cursor);
10076                                dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn));
10077                                dataflowRelation.setTarget(new TableColumnRelationshipElement(cursorColumn));
10078                        }
10079                }
10080
10081                if (stmt.getStatements() != null && stmt.getStatements().size() > 0) {
10082                        for (int i = 0; i < stmt.getStatements().size(); i++) {
10083                                analyzeCustomSqlStmt(stmt.getStatements().get(i));
10084                        }
10085                }
10086        }
10087
10088        /**
10089         * Resolve a declared name visible from the CURRENT scope (#695): a TYPE
10090         * declared at package level is visible from every nested procedure, but
10091         * variable models are keyed under their declaring scope only, so a
10092         * same-scope lookup misses it and the declaring variable silently falls
10093         * back to one opaque variant column — the record-fill edges then have no
10094         * per-field target and are lost. Walk the scope chain outward
10095         * (procedure → package → global); never create.
10096         */
10097        private Variable resolveVariableInScope(String name) {
10098                if (SQLUtil.isEmpty(name)) {
10099                        return null;
10100                }
10101                Variable direct = modelFactory.createVariable(name, false);
10102                if (direct != null) {
10103                        return direct;
10104                }
10105                String normalized = SQLUtil.getIdentifierNormalTableName(name);
10106                // Lexical chain first (codex-696 F1): a declaration in an ENCLOSING
10107                // procedure shadows the package's. getProcedureParentName() of the
10108                // current statement omits intermediate nesting levels, so probe each
10109                // enclosing stored-procedure statement on the stack, innermost first.
10110                for (int depth = stmtStack.size() - 1; depth >= 0; depth--) {
10111                        String enclosingScope = DlineageUtil.getProcedureParentName(stmtStack.get(depth));
10112                        if (SQLUtil.isEmpty(enclosingScope)) {
10113                                continue;
10114                        }
10115                        Object model = modelManager.getTableByName(DlineageUtil.getTableFullName(
10116                                        DlineageUtil.getTableFullName(enclosingScope) + "." + normalized));
10117                        if (model instanceof Variable) {
10118                                return (Variable) model;
10119                        }
10120                }
10121                String scope = stmtStack.isEmpty() ? null : DlineageUtil.getProcedureParentName(stmtStack.peek());
10122                while (!SQLUtil.isEmpty(scope)) {
10123                        int cut = scope.lastIndexOf('.');
10124                        scope = cut == -1 ? null : scope.substring(0, cut);
10125                        List<String> keys = new ArrayList<String>();
10126                        if (SQLUtil.isEmpty(scope)) {
10127                                keys.add(normalized);
10128                        } else {
10129                                keys.add(DlineageUtil.getTableFullName(scope) + "." + normalized);
10130                                // A declaration at the top level of a package BODY is scoped
10131                                // "<pkg>.<pkg>" (the body's own block), so a sibling procedure
10132                                // walking up to "<pkg>" must probe that block scope too.
10133                                String tail = scope.indexOf('.') == -1 ? scope : scope.substring(scope.lastIndexOf('.') + 1);
10134                                keys.add(DlineageUtil.getTableFullName(scope) + "." + tail + "." + normalized);
10135                        }
10136                        for (String key : keys) {
10137                                Object model = modelManager.getTableByName(DlineageUtil.getTableFullName(key));
10138                                if (model instanceof Variable) {
10139                                        return (Variable) model;
10140                                }
10141                        }
10142                        if (cut == -1) {
10143                                break;
10144                        }
10145                }
10146                return null;
10147        }
10148
10149        /**
10150         * Materialize the fields of a record-typed FORMAL argument from its
10151         * declared type before binding an actual to it (#695). Without this the
10152         * formal is one opaque column and the argument-fill edge either lands on
10153         * an arbitrary field or is lost. Field order follows the TYPE declaration
10154         * unless consumption already created some fields (name-deduped), so
10155         * ordinal use of formal fields must not rely on this method alone.
10156         */
10157        private void materializeArgumentFields(Variable variable, Argument argument) {
10158                if (variable == null || argument == null || argument.getDataType() == null) {
10159                        return;
10160                }
10161                if (variable.isDetermined() && variable.getColumns() != null && variable.getColumns().size() > 1) {
10162                        return;
10163                }
10164                Variable typeModel = resolveVariableInScope(argument.getDataType().getDataTypeName());
10165                if (typeModel == null || typeModel == variable || !typeModel.isDetermined()
10166                                || typeModel.getColumns() == null || typeModel.getColumns().isEmpty()) {
10167                        return;
10168                }
10169                // Fields created by earlier consumption sit BEFORE the type fields we
10170                // append here, so the column list is only in type order when nothing
10171                // preceded it (codex-696 F2). Ordinal pairing must not trust a mixed
10172                // list.
10173                boolean hadRealFields = false;
10174                if (variable.getColumns() != null) {
10175                        for (TableColumn existing : variable.getColumns()) {
10176                                if (existing.getName() != null && !"*".equals(existing.getName())) {
10177                                        hadRealFields = true;
10178                                        break;
10179                                }
10180                        }
10181                }
10182                for (TableColumn typeField : typeModel.getColumns()) {
10183                        if (typeField.getName() == null || "*".equals(typeField.getName())) {
10184                                continue;
10185                        }
10186                        TObjectName fieldName = new TObjectName();
10187                        fieldName.setString(typeField.getName());
10188                        modelFactory.createTableColumn(variable, fieldName, true);
10189                }
10190                // A formal declared before this had only its wildcard column; now that
10191                // the real fields exist, a relation binding the record as a whole must
10192                // expand onto them at serialization instead of citing the bare star.
10193                if (variable.getColumns() != null) {
10194                        for (TableColumn existing : variable.getColumns()) {
10195                                if ("*".equals(existing.getName())) {
10196                                        existing.setShowStar(false);
10197                                        existing.setExpandStar(true);
10198                                }
10199                        }
10200                }
10201                variable.setDetermined(true);
10202                if (!hadRealFields) {
10203                        variable.setTypeOrderedFields(true);
10204                }
10205        }
10206
10207        /**
10208         * The variable model of a callee's FORMAL argument, shaped for binding
10209         * (#695). {@code createVariable(Procedure, name, false)} deliberately
10210         * refuses a record-typed formal it recovers through the declaration-time
10211         * scope key (one wildcard column = nothing exact to bind to). Recover the
10212         * variable anyway, materialize its fields from the declared type, and only
10213         * bind when a real field emerged — otherwise keep the historical refusal.
10214         */
10215        private Variable resolveFormalVariable(Procedure callee, Argument argument) {
10216                Variable variable = modelFactory.createVariable(callee, argument.getName(), false);
10217                if (variable != null) {
10218                        materializeArgumentFields(variable, argument);
10219                        return variable;
10220                }
10221                Variable recovered = modelFactory.findFormalVariable(callee, argument.getName());
10222                if (recovered == null) {
10223                        return null;
10224                }
10225                materializeArgumentFields(recovered, argument);
10226                if (recovered.getColumns() != null) {
10227                        for (TableColumn column : recovered.getColumns()) {
10228                                if (!"*".equals(column.getName())) {
10229                                        return recovered;
10230                                }
10231                        }
10232                }
10233                return null;
10234        }
10235
10236        private void analyzeVarDeclStmt(TVarDeclStmt stmt) {
10237                TTypeName typeName = stmt.getDataType();
10238                if (typeName != null && typeName.toString().toUpperCase().indexOf("ROWTYPE") != -1) {
10239                        Variable cursorVariable = modelFactory.createVariable(stmt.getElementName());
10240                        cursorVariable.setSubType(SubType.record_type);
10241
10242                        Table variableTable = modelFactory.createTableByName(typeName.getDataTypeName(), false);
10243                        if(!variableTable.isCreateTable()) {
10244                                TObjectName starColumn1 = new TObjectName();
10245                                starColumn1.setString("*");
10246                                TableColumn variableTableStarColumn = modelFactory.createTableColumn(variableTable, starColumn1, true);
10247                                variableTableStarColumn.setShowStar(false);
10248                                variableTableStarColumn.setExpandStar(true);
10249
10250                                TObjectName starColumn = new TObjectName();
10251                                starColumn.setString("*");
10252                                TableColumn variableProperty = modelFactory.createTableColumn(cursorVariable, starColumn, true);
10253                                variableProperty.setShowStar(false);
10254                                variableProperty.setExpandStar(true);
10255
10256                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
10257                                dataflowRelation.setEffectType(EffectType.rowtype);
10258                                dataflowRelation.addSource(new TableColumnRelationshipElement(variableTableStarColumn));
10259                                dataflowRelation.setTarget(new TableColumnRelationshipElement(variableProperty));
10260                        } else {
10261                                for (TableColumn sourceColumn : variableTable.getColumns()) {
10262                                        String columnName = sourceColumn.getName();
10263                                        TObjectName targetColumn = new TObjectName();
10264                                        targetColumn.setString(columnName);
10265                                        TableColumn variableProperty = modelFactory.createTableColumn(cursorVariable, targetColumn, true);
10266                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
10267                                        dataflowRelation.setEffectType(EffectType.rowtype);
10268                                        dataflowRelation.addSource(new TableColumnRelationshipElement(sourceColumn));
10269                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(variableProperty));
10270                                }
10271                        }
10272                } else if (stmt.getElementName() != null) {
10273                        Variable variable = modelFactory.createVariable(stmt.getElementName());
10274                        variable.setCreateTable(true);
10275                        variable.setSubType(SubType.record);
10276                        TableColumn tableColumn = null;
10277                        if (stmt.getDataType() != null && (resolveVariableInScope(stmt.getDataType().getDataTypeName())!=null || isCursorType(stmt.getDataType().getDataTypeName()))) {
10278                                Variable cursorVariable = resolveVariableInScope(stmt.getDataType().getDataTypeName());
10279                                if (cursorVariable != null) {
10280                                        if (cursorVariable.getSubType() == SubType.record_type) {
10281                                                variable.setSubType(SubType.record_type);
10282                                        }
10283                                        if(cursorVariable.isDetermined()) {
10284                                                for (int k = 0; k < cursorVariable.getColumns().size(); k++) {
10285                                                        TableColumn sourceColumn = cursorVariable.getColumns().get(k);
10286                                                        TObjectName objectName = new TObjectName();
10287                                                        objectName.setString(sourceColumn.getName());
10288                                                        tableColumn = modelFactory.createTableColumn(variable, objectName, true);
10289                                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
10290                                                        dataflowRelation.setEffectType(EffectType.cursor);
10291                                                        dataflowRelation
10292                                                                        .addSource(new TableColumnRelationshipElement(sourceColumn));
10293                                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(tableColumn));
10294                                                }
10295                                                variable.setDetermined(true);
10296                                                variable.setTypeOrderedFields(true);
10297                                                return;
10298                                        }
10299                                        else {
10300                                                TObjectName objectName = new TObjectName();
10301                                                objectName.setString("*");
10302                                                tableColumn = modelFactory.createTableColumn(variable, objectName, true);
10303                                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
10304                                                dataflowRelation.setEffectType(EffectType.cursor);
10305                                                dataflowRelation
10306                                                                .addSource(new TableColumnRelationshipElement(cursorVariable.getColumns().get(0)));
10307                                                dataflowRelation.setTarget(new TableColumnRelationshipElement(tableColumn));
10308                                        }
10309                                }
10310                                else {
10311                                        TObjectName objectName = new TObjectName();
10312                                        objectName.setString("*");
10313                                        tableColumn = modelFactory.createTableColumn(variable, objectName, true);                                       
10314                                }
10315                        } else {
10316                                tableColumn = modelFactory.createTableColumn(variable, stmt.getElementName(), true);
10317                                tableColumn.setVariant(true);
10318                        }
10319
10320                        if (stmt.getDefaultValue() != null) {
10321                                columnsInExpr visitor = new columnsInExpr();
10322                                stmt.getDefaultValue().inOrderTraverse(visitor);
10323                                List<TObjectName> objectNames = visitor.getObjectNames();
10324                                List<TParseTreeNode> functions = visitor.getFunctions();
10325                                List<TParseTreeNode> constants = visitor.getConstants();
10326                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
10327
10328                                if (functions != null && !functions.isEmpty()) {
10329                                        analyzeFunctionDataFlowRelation(tableColumn, functions, EffectType.function);
10330                                }
10331                                if (subquerys != null && !subquerys.isEmpty()) {
10332                                        analyzeSubqueryDataFlowRelation(tableColumn, subquerys, EffectType.select);
10333                                }
10334                                if (objectNames != null && !objectNames.isEmpty()) {
10335                                        analyzeDataFlowRelation(tableColumn, objectNames, EffectType.select, functions);
10336                                }
10337                                if (constants != null && !constants.isEmpty()) {
10338                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select, functions);
10339                                }
10340                        }
10341                        
10342                        
10343                }
10344        }
10345
10346        private boolean isCursorType(String dataTypeName) {
10347                if (dataTypeName != null && dataTypeName.toLowerCase().contains("cursor")) {
10348                        return true;
10349                }
10350                return false;
10351        }
10352
10353        private void analyzeSetStmt(TSetStmt stmt) {
10354                TExpression right = stmt.getVariableValue();
10355                TObjectName columnObject = stmt.getVariableName();
10356                if (columnObject != null) {
10357                        TableColumn tableColumn = null;
10358                        Variable tableModel;
10359                        if (columnObject.toString().indexOf(".") != -1) {
10360                                List<String> splits = SQLUtil.parseNames(columnObject.toString());
10361                                tableModel = modelFactory.createVariable(splits.get(splits.size() - 2));
10362                        } else {
10363                                tableModel = modelFactory.createVariable(columnObject);
10364                        }
10365                        tableModel.setCreateTable(true);
10366                        tableModel.setSubType(SubType.record);
10367
10368                        if (tableModel.getColumns() == null || tableModel.getColumns().isEmpty()) {
10369                                tableColumn = modelFactory.createTableColumn(tableModel, columnObject, true);
10370                        } else {
10371                                tableColumn = tableModel.getColumns().get(0);
10372                        }
10373
10374                        if (tableColumn != null && right!=null) {
10375                                columnsInExpr visitor = new columnsInExpr();
10376                                right.inOrderTraverse(visitor);
10377                                List<TObjectName> objectNames = visitor.getObjectNames();
10378                                List<TParseTreeNode> functions = visitor.getFunctions();
10379                                List<TParseTreeNode> constants = visitor.getConstants();
10380                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
10381
10382                                if (functions != null && !functions.isEmpty()) {
10383                                        analyzeFunctionDataFlowRelation(tableColumn, functions, EffectType.function);
10384                                }
10385                                if (subquerys != null && !subquerys.isEmpty()) {
10386                                        analyzeSubqueryDataFlowRelation(tableColumn, subquerys, EffectType.select);
10387                                }
10388                                if (objectNames != null && !objectNames.isEmpty()) {
10389                                        analyzeDataFlowRelation(tableColumn, objectNames, EffectType.select, functions);
10390                                }
10391                                if (constants != null && !constants.isEmpty()) {
10392                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select, functions);
10393                                }
10394                                
10395                                if(columnObject.toString().equalsIgnoreCase("search_path") && !constants.isEmpty()) {
10396                                        ModelBindingManager.setGlobalSchema(constants.get(0).toString());
10397                                }
10398                        }
10399                }
10400
10401                if(stmt.getAssignments()!=null){
10402                        for (int i = 0; i < stmt.getAssignments().size(); i++) {
10403                                TSetAssignment assignStmt = stmt.getAssignments().getElement(i);
10404                                analyzeSetAssignmentStmt(assignStmt);
10405                        }
10406                }
10407        }
10408
10409        private void analyzeSetAssignmentStmt(TSetAssignment stmt) {
10410                TExpression right = stmt.getParameterValue();
10411                TObjectName columnObject = stmt.getParameterName();
10412                if (columnObject != null) {
10413                        TableColumn tableColumn = null;
10414                        Variable tableModel;
10415                        if (columnObject.toString().indexOf(".") != -1) {
10416                                List<String> splits = SQLUtil.parseNames(columnObject.toString());
10417                                tableModel = modelFactory.createVariable(splits.get(splits.size() - 2));
10418                        } else {
10419                                tableModel = modelFactory.createVariable(columnObject);
10420                        }
10421                        tableModel.setCreateTable(true);
10422                        tableModel.setSubType(SubType.record);
10423                        if (tableModel.getColumns() == null || tableModel.getColumns().isEmpty()) {
10424                                tableColumn = modelFactory.createTableColumn(tableModel, columnObject, true);
10425                        } else {
10426                                tableColumn = tableModel.getColumns().get(0);
10427                        }
10428
10429                        if (tableColumn != null && right!=null) {
10430                                columnsInExpr visitor = new columnsInExpr();
10431                                right.inOrderTraverse(visitor);
10432                                List<TObjectName> objectNames = visitor.getObjectNames();
10433                                List<TParseTreeNode> functions = visitor.getFunctions();
10434                                List<TParseTreeNode> constants = visitor.getConstants();
10435                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
10436
10437                                if (functions != null && !functions.isEmpty()) {
10438                                        analyzeFunctionDataFlowRelation(tableColumn, functions, EffectType.function);
10439                                }
10440                                if (subquerys != null && !subquerys.isEmpty()) {
10441                                        analyzeSubqueryDataFlowRelation(tableColumn, subquerys, EffectType.select);
10442                                }
10443                                if (objectNames != null && !objectNames.isEmpty()) {
10444                                        analyzeDataFlowRelation(tableColumn, objectNames, EffectType.select, functions);
10445                                }
10446                                if (constants != null && !constants.isEmpty()) {
10447                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select, functions);
10448                                }
10449
10450                                if(columnObject.toString().equalsIgnoreCase("search_path") && !constants.isEmpty()) {
10451                                        ModelBindingManager.setGlobalSchema(constants.get(0).toString());
10452                                }
10453                        }
10454                }
10455        }
10456
10457        private void analyzeMssqlSetStmt(TMssqlSet stmt) {
10458                TExpression right = stmt.getVarExpr();
10459                TObjectName columnObject = stmt.getVarName();
10460                if (columnObject != null) {
10461                        TableColumn tableColumn = null;
10462                        Variable tableModel;
10463                        if (columnObject.toString().indexOf(".") != -1) {
10464                                List<String> splits = SQLUtil.parseNames(columnObject.toString());
10465                                tableModel = modelFactory.createVariable(splits.get(splits.size() - 2));
10466                        } else {
10467                                tableModel = modelFactory.createVariable(columnObject);
10468                        }
10469                        tableModel.setCreateTable(true);
10470                        tableModel.setSubType(SubType.record);
10471                        if (tableModel.getColumns() == null || tableModel.getColumns().isEmpty()) {
10472                                tableColumn = modelFactory.createTableColumn(tableModel, columnObject, true);
10473                        }
10474                        else {
10475                                tableColumn = tableModel.getColumns().get(0);
10476                        }
10477
10478                        if (tableColumn != null && right!=null) {
10479                                columnsInExpr visitor = new columnsInExpr();
10480                                right.inOrderTraverse(visitor);
10481                                List<TObjectName> objectNames = visitor.getObjectNames();
10482                                List<TParseTreeNode> functions = visitor.getFunctions();
10483                                List<TParseTreeNode> constants = visitor.getConstants();
10484                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
10485
10486                                if (functions != null && !functions.isEmpty()) {
10487                                        analyzeFunctionDataFlowRelation(tableColumn, functions, EffectType.function);
10488                                }
10489                                if (subquerys != null && !subquerys.isEmpty()) {
10490                                        analyzeSubqueryDataFlowRelation(tableColumn, subquerys, EffectType.select);
10491                                }
10492                                if (objectNames != null && !objectNames.isEmpty()) {
10493                                        analyzeDataFlowRelation(tableColumn, objectNames, EffectType.select, functions);
10494                                }
10495                                if (constants != null && !constants.isEmpty()) {
10496                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select, functions);
10497                                }
10498
10499                                if(columnObject.toString().equalsIgnoreCase("search_path") && !constants.isEmpty()) {
10500                                        ModelBindingManager.setGlobalSchema(constants.get(0).toString());
10501                                }
10502                        }
10503                        else if(tableColumn!=null && stmt.getSubquery()!=null){
10504                                analyzeCustomSqlStmt(stmt.getSubquery());
10505                                analyzeSubqueryDataFlowRelation(tableColumn, Arrays.asList(stmt.getSubquery()), EffectType.select);
10506                        }
10507                }
10508        }
10509
10510        /**
10511         * Oracle analog of the T-SQL default-path evaluation (plan Phase 5): when the
10512         * parser's token fold left an EXECUTE IMMEDIATE partial (parameter placeholders)
10513         * or empty, run the AST interpreter over the file's statements — same-file
10514         * literal calls bind parameters naturally — plus literal caller statements
10515         * harvested from OTHER files of a multi-file analysis. The interpreter stores
10516         * evaluated text on each TExecImmeStmt (getEvaluatedDynamicSQLs), which
10517         * getDynamicSQL()/getDynamicStatements() then prefer over the token fold; the
10518         * existing placeholder-contamination classifier keeps half-bound results honest.
10519         * Best-effort: any interpreter failure leaves the token-fold behavior intact.
10520         */
10521        private void maybeEvaluatePlsqlDynamicSql(TExecImmeStmt execImmeStmt) {
10522                if (option.getVendor() != EDbVendor.dbvoracle) {
10523                        return;
10524                }
10525                if (!execImmeStmt.getEvaluatedDynamicSQLs().isEmpty()) {
10526                        return;
10527                }
10528                if (execImmeStmt.getDynamicSQL() != null && !execImmeStmt.isDynamicSQLPartial()) {
10529                        // Fully folded already — do not disturb resolved sites.
10530                        return;
10531                }
10532                TGSqlParser owner = execImmeStmt.getGsqlparser();
10533                if (owner == null || owner.getSqlstatements() == null) {
10534                        return;
10535                }
10536                String ownText = owner.sqltext;
10537                if (ownText == null || !plsqlEvaluatedTexts.add(ownText)) {
10538                        return; // one interpreter run per analysis unit
10539                }
10540                try {
10541                        // Definitions plus fully-literal-vetted caller statements only (same-file
10542                        // first, then cross-file): a caller passing a variable would make the
10543                        // interpreter fold the unevaluated operand to an EMPTY string, silently
10544                        // corrupting object names - such callers never enter the list. Empty
10545                        // list = no vetted caller anywhere = nothing to bind, skip evaluation.
10546                        List<TCustomSqlStatement> toInterpret = gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver
10547                                        .plsqlInterpretationStatements(owner.getSqlstatements(), ownText,
10548                                                        routineCatalog, 4);
10549                        if (toInterpret.isEmpty()) {
10550                                return;
10551                        }
10552                        TStatementList combined = new TStatementList();
10553                        for (TCustomSqlStatement stmt : toInterpret) {
10554                                combined.add(stmt);
10555                        }
10556                        TSQLEnv evalEnv = new TSQLEnv(EDbVendor.dbvoracle) {
10557                                @Override
10558                                public void initSQLEnv() {
10559                                }
10560                        };
10561                        new gudusoft.gsqlparser.compiler.TASTEvaluator(combined,
10562                                        new gudusoft.gsqlparser.compiler.TGlobalScope(evalEnv)).eval();
10563                } catch (RuntimeException ex) {
10564                        // interpretation is best-effort; token-fold behavior remains
10565                } catch (java.lang.Error err) {
10566                        if (err instanceof ThreadDeath) {
10567                                throw (ThreadDeath) err;
10568                        }
10569                        // e.g. StackOverflow on pathological nesting — keep the analysis alive
10570                }
10571        }
10572
10573        /**
10574         * Default-path dynamic-SQL evaluation (plan Phase 4a): when the token-level fold
10575         * left an EXEC / sp_executesql / EXEC(@var) site opaque, abstractly evaluate the
10576         * enclosing procedure's string building with no call-site bindings and try to
10577         * materialize this site's SQL. Returns null when no usable text was obtained.
10578         */
10579        private gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite evaluatedDynamicSqlFor(
10580                        TCustomSqlStatement execStmt) {
10581                if (option.getVendor() != EDbVendor.dbvmssql) {
10582                        return null;
10583                }
10584                TCustomSqlStatement proc = null;
10585                for (int i = stmtStack.size() - 1; i >= 0; i--) {
10586                        if (stmtStack.get(i) instanceof TStoredProcedureSqlStatement) {
10587                                proc = stmtStack.get(i);
10588                                break;
10589                        }
10590                }
10591                if (proc == null) {
10592                        return null;
10593                }
10594                List<Map<TCustomSqlStatement, gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite>> variants = dynamicEvalCache
10595                                .get(proc);
10596                if (variants == null) {
10597                        variants = new ArrayList<Map<TCustomSqlStatement, gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite>>();
10598                        try {
10599                                variants.addAll(gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver
10600                                                .materializeSiteVariants(proc, option.getVendor(), null, null));
10601                                // Call-site binding discovery (plan Phase 2/4a): literal EXEC calls to
10602                                // this proc in the same file supply parameter values that can turn a
10603                                // placeholder-bearing template into fully-known SQL.
10604                                List<Map<String, gudusoft.gsqlparser.dlineage.dynamicsql.SqlValue>> sameFileBindings = gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver
10605                                                .discoverCallSiteBindings(proc, 4);
10606                                for (Map<String, gudusoft.gsqlparser.dlineage.dynamicsql.SqlValue> bindings : sameFileBindings) {
10607                                        variants.addAll(gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver
10608                                                        .materializeSiteVariants(proc, option.getVendor(), bindings, null));
10609                                }
10610                                // Cross-file discovery (plan Phase 5): literal EXEC calls in OTHER
10611                                // files of this multi-file analysis. Same-file bindings keep priority;
10612                                // the overall binding-set cap stays at 4.
10613                                if (routineCatalog != null && sameFileBindings.size() < 4) {
10614                                        for (Map<String, gudusoft.gsqlparser.dlineage.dynamicsql.SqlValue> bindings : gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver
10615                                                        .discoverCrossFileCallSiteBindings(proc, routineCatalog,
10616                                                                        4 - sameFileBindings.size())) {
10617                                                variants.addAll(gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver
10618                                                                .materializeSiteVariants(proc, option.getVendor(), bindings, null));
10619                                        }
10620                                }
10621                        } catch (RuntimeException ex) {
10622                                // evaluation is best-effort; fall through with whatever variants exist
10623                        }
10624                        dynamicEvalCache.put(proc, variants);
10625                }
10626                // Prefer the first fully-concrete materialization; otherwise the first
10627                // usable partial text; otherwise the no-bindings entry (carries the reason).
10628                gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite best = null;
10629                for (Map<TCustomSqlStatement, gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite> variant : variants) {
10630                        gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite candidate = variant.get(execStmt);
10631                        if (candidate == null) {
10632                                continue;
10633                        }
10634                        if (candidate.sqlText != null && !candidate.partial) {
10635                                return candidate;
10636                        }
10637                        if (best == null || (best.sqlText == null && candidate.sqlText != null)) {
10638                                best = candidate;
10639                        }
10640                }
10641                return best;
10642        }
10643
10644        /** All distinct usable materializations for one EXEC, in stable variant order. */
10645        private List<gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite> evaluatedDynamicSqlVariantsFor(
10646                        TCustomSqlStatement execStmt, boolean concreteOnly) {
10647                // Builds and caches the variant list on first use.
10648                evaluatedDynamicSqlFor(execStmt);
10649                TCustomSqlStatement proc = null;
10650                for (int i = stmtStack.size() - 1; i >= 0; i--) {
10651                        if (stmtStack.get(i) instanceof TStoredProcedureSqlStatement) {
10652                                proc = stmtStack.get(i);
10653                                break;
10654                        }
10655                }
10656                List<gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite> result =
10657                                new ArrayList<gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite>();
10658                if (proc == null) {
10659                        return result;
10660                }
10661                List<Map<TCustomSqlStatement, gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite>> variants =
10662                                dynamicEvalCache.get(proc);
10663                Set<String> seen = new LinkedHashSet<String>();
10664                for (Map<TCustomSqlStatement, gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite> variant : variants) {
10665                        gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite candidate =
10666                                        variant.get(execStmt);
10667                        if (candidate == null || candidate.sqlText == null
10668                                        || (concreteOnly && candidate.partial)
10669                                        || isWholeTextHole(candidate)) {
10670                                continue;
10671                        }
10672                        String key = candidate.partial + "\u0000" + candidate.sqlText;
10673                        if (seen.add(key)) {
10674                                result.add(candidate);
10675                        }
10676                }
10677                return result;
10678        }
10679
10680        private boolean analyzeEvaluatedDynamicSqlVariants(TCustomSqlStatement stmt,
10681                        DynamicSqlSite.Kind kind, boolean concreteOnly) {
10682                boolean handled = false;
10683                for (gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite site :
10684                                evaluatedDynamicSqlVariantsFor(stmt, concreteOnly)) {
10685                        handled |= analyzeEvaluatedDynamicSql(stmt, kind, site);
10686                }
10687                return handled;
10688        }
10689
10690        /**
10691         * Analyze the materialized text of a dynamic site through the ordinary inner-SQL
10692         * path and record a truthful site status. Returns true when the site was handled
10693         * (recorded) here; false when the text was unusable and the caller should record
10694         * its own UNRESOLVED diagnostic.
10695         */
10696        private boolean analyzeEvaluatedDynamicSql(TCustomSqlStatement stmt, DynamicSqlSite.Kind kind,
10697                        gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite site) {
10698                if (site == null || site.sqlText == null) {
10699                        return false;
10700                }
10701                DynamicSqlTrustMode trustMode = option.getDynamicSqlTrustMode();
10702                boolean provenanceIncomplete = site.provenanceIncomplete;
10703                DynamicHoleEvidence holeEvidence = dynamicHoleEvidence(site);
10704                int holeCount = holeEvidence.count;
10705                boolean holeCountExact = holeEvidence.exact;
10706                Set<Relationship> shadowBefore = provenanceIncomplete && trustMode == DynamicSqlTrustMode.SHADOW
10707                                ? snapshotDynamicRelationships() : null;
10708                int relsBeforeDynamic = modelManager.getRelations().length;
10709                int dynamicSitesBefore = dynamicSqlSites.size();
10710                // This is the ONE place partially-materialized text reaches the ordinary inner-SQL
10711                // path, and the site carries exact provenance — so the endpoints it introduces can be
10712                // marked from real HOLE fragments rather than from a name heuristic.
10713                Set<Table> templateTablesBefore = snapshotTemplateCandidateTables(
10714                                site.partial || provenanceIncomplete);
10715                TGSqlParser sqlparser = createSqlParser(option.getVendor());
10716                sqlparser.sqltext = site.sqlText;
10717                boolean foldStatementsAnalyzed = false;
10718                int result = sqlparser.parse();
10719                if (result == 0 && sqlparser.sqlstatements != null) {
10720                        foldStatementsAnalyzed = true;
10721                        dynamicFoldDepth++;
10722                        try {
10723                                for (int i = 0; i < sqlparser.sqlstatements.size(); i++) {
10724                                        analyzeCustomSqlStmt(sqlparser.sqlstatements.get(i));
10725                                }
10726                        } finally {
10727                                dynamicFoldDepth--;
10728                        }
10729                }
10730                if (foldStatementsAnalyzed) {
10731                        markDynamicTemplateEndpoints(templateTablesBefore, site, sqlparser);
10732                }
10733                int[] dynCounts = classifyDynamicSiteLineage(relsBeforeDynamic, site.partial);
10734                int unprovenCount = countNewDynamicRelationships(shadowBefore);
10735                if (result != 0) {
10736                        // Evaluated text always derives from variables, and the evaluation is
10737                        // flow-insensitive — the variable may hold a different value at runtime.
10738                        // A parse failure on it is therefore data-driven UNRESOLVED, never
10739                        // PARSE_ERROR (which stays reserved for compile-time literal arguments,
10740                        // matching the fold path's contract).
10741                        String reason = site.partial ? "partially evaluated dynamic SQL failed to parse; unresolved parts remain"
10742                                        : "evaluated dynamic SQL failed to parse; value is flow-insensitive";
10743                        if (provenanceIncomplete && trustMode == DynamicSqlTrustMode.SHADOW) {
10744                                reason = dynamicTrustReason(reason + "; SHADOW retained legacy publication behavior",
10745                                                unprovenCount, holeCount, holeCountExact);
10746                        }
10747                        recordDynamicSqlSite(stmt, kind, DynamicSqlSite.Status.UNRESOLVED, reason,
10748                                        dynCounts[0], dynCounts[1], unprovenCount,
10749                                        holeCount, holeCountExact, trustMode,
10750                                        relsBeforeDynamic, dynamicSitesBefore, null, site.diagnostic,
10751                                        holeEvidence.fragments, holeEvidence.materializedText);
10752                        return true;
10753                }
10754                boolean usable = dynCounts[2] == 0
10755                                && (dynCounts[0] > 0 || (dynCounts[1] == 0 && dynCounts[3] > 0));
10756                DynamicSqlSite.Status status = usable ? DynamicSqlSite.Status.RESOLVED
10757                                : DynamicSqlSite.Status.UNRESOLVED;
10758                String reason = usable ? null
10759                                : (dynCounts[2] != 0
10760                                                ? "dynamic SQL references a runtime-built object name; lineage source is a placeholder"
10761                                                : "evaluated dynamic SQL produced no resolved lineage");
10762                if (provenanceIncomplete && trustMode == DynamicSqlTrustMode.SHADOW) {
10763                        status = unprovenCount > 0 ? DynamicSqlSite.Status.PARTIAL
10764                                        : DynamicSqlSite.Status.UNRESOLVED;
10765                        reason = dynamicTrustReason("SHADOW retained legacy publication behavior",
10766                                        unprovenCount, holeCount, holeCountExact);
10767                }
10768                recordDynamicSqlSite(stmt, kind,
10769                                status, reason, dynCounts[0], dynCounts[1], unprovenCount,
10770                                holeCount, holeCountExact, trustMode,
10771                                relsBeforeDynamic, dynamicSitesBefore, null, site.diagnostic,
10772                                holeEvidence.fragments, holeEvidence.materializedText);
10773                return true;
10774        }
10775
10776        /**
10777         * Trigger correlation names (:NEW / :OLD and REFERENCING aliases) denote rows of the
10778         * trigger's ON table. Bind them so name-based lookups from body statements (e.g. the
10779         * assignment target in ":alias.col := expr") resolve to the base table. Only
10780         * ':'-prefixed and trigger-scoped keys are registered - bare alias names could collide
10781         * with real tables elsewhere in the file.
10782         */
10783        private void bindTriggerCorrelationNames(TPlsqlCreateTrigger trigger, Table sourceTable) {
10784                List<String> correlationNames = new ArrayList<String>();
10785                correlationNames.add("NEW");
10786                correlationNames.add("OLD");
10787                if (trigger.getTriggeringClause() != null
10788                                && trigger.getTriggeringClause().getReferencingClause() != null
10789                                && trigger.getTriggeringClause().getReferencingClause().getReferencingItems() != null) {
10790                        for (TTriggerReferencingItem item : trigger.getTriggeringClause().getReferencingClause()
10791                                        .getReferencingItems()) {
10792                                if (item.getCorrelationName() != null) {
10793                                        correlationNames.add(item.getCorrelationName().toString());
10794                                }
10795                        }
10796                }
10797                String scope = null;
10798                if (trigger.getStatements() != null && trigger.getStatements().size() > 0) {
10799                        scope = DlineageUtil.getProcedureParentName(trigger.getStatements().get(0));
10800                }
10801                for (String correlationName : correlationNames) {
10802                        modelManager.bindTableByName(DlineageUtil.getTableFullName(":" + correlationName), sourceTable);
10803                        if (!SQLUtil.isEmpty(scope)) {
10804                                modelManager.bindTableByName(DlineageUtil.getTableFullName(
10805                                                scope + "." + SQLUtil.getIdentifierNormalTableName(correlationName)), sourceTable);
10806                        }
10807                }
10808        }
10809
10810        private void analyzeAssignStmt(TAssignStmt stmt) {
10811                TExpression left = stmt.getLeft();
10812                TExpression right = stmt.getExpression();
10813                TObjectName columnObject = null;
10814                if (left == null) {
10815                        columnObject = stmt.getVariableName();
10816                } else if (left.getExpressionType() == EExpressionType.simple_object_name_t) {
10817                        columnObject = left.getObjectOperand();
10818                }
10819                if (columnObject != null) {
10820                        TableColumn tableColumn = null;
10821                        List<TableColumn> recordFillTargets = null;
10822                        if (columnObject.getDbObjectType() == EDbObjectType.variable || stmt.getVariableName() != null) {
10823                                Variable tableModel;
10824                                if (columnObject.toString().indexOf(".") != -1) {
10825                                        List<String> splits = SQLUtil.parseNames(columnObject.toString());
10826                                        tableModel = modelFactory.createVariable(splits.get(splits.size() - 2));
10827                                } else {
10828                                        tableModel = modelFactory.createVariable(columnObject);
10829                                }
10830                                tableModel.setCreateTable(true);
10831                                tableModel.setSubType(SubType.record);
10832                                if (tableModel.getColumns() == null || tableModel.getColumns().isEmpty()) {
10833                                        tableColumn = modelFactory.createTableColumn(tableModel, columnObject, true);
10834                                } else if (tableModel.getColumns().size() == 1) {
10835                                        tableColumn = tableModel.getColumns().get(0);
10836                                } else {
10837                                        // Whole-record assignment to a field-expanded record (#695):
10838                                        // every field is filled by the right-hand side, so the fill
10839                                        // edge is emitted per field. Binding get(0) instead attached
10840                                        // the whole assignment to one arbitrary field and left the
10841                                        // rest with no upstream at all.
10842                                        tableColumn = tableModel.getColumns().get(0);
10843                                        recordFillTargets = tableModel.getColumns();
10844                                }
10845                        } else {
10846                                List<String> splits = SQLUtil.parseNames(columnObject.toString());
10847                                if (splits.size() > 1) {
10848                                        Table tableModel = modelManager
10849                                                        .getTableByName(DlineageUtil.getTableFullName(splits.get(splits.size() - 2)));
10850                                        if (tableModel == null) {
10851                                                String procedureName = DlineageUtil.getProcedureParentName(stmt);
10852                                                String variableString = splits.get(splits.size() - 2).toString();
10853                                                if (variableString.startsWith(":")) {
10854                                                        variableString = variableString.substring(variableString.indexOf(":") + 1);
10855                                                }
10856                                                if (!SQLUtil.isEmpty(procedureName)) {
10857                                                        variableString = procedureName + "." + SQLUtil.getIdentifierNormalTableName(variableString);
10858                                                }
10859                                                tableModel = modelManager.getTableByName(DlineageUtil.getTableFullName(variableString));
10860                                        }
10861                                        if (tableModel != null) {
10862                                                tableColumn = modelFactory.createTableColumn(tableModel, columnObject, true);
10863                                        }
10864                                }
10865                        }
10866
10867                        if (tableColumn != null && right != null) {
10868                                columnsInExpr visitor = new columnsInExpr();
10869                                right.inOrderTraverse(visitor);
10870                                List<TObjectName> objectNames = visitor.getObjectNames();
10871                                List<TParseTreeNode> functions = visitor.getFunctions();
10872                                List<TParseTreeNode> constants = visitor.getConstants();
10873                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
10874
10875                                List<TableColumn> bindTargets = recordFillTargets != null ? recordFillTargets
10876                                                : Arrays.asList(tableColumn);
10877                                for (TableColumn bindTarget : bindTargets) {
10878                                        Transform transform = new Transform();
10879                                        transform.setType(Transform.EXPRESSION);
10880                                        transform.setCode(right);
10881                                        bindTarget.setTransform(transform);
10882                                        if (functions != null && !functions.isEmpty()) {
10883                                                analyzeFunctionDataFlowRelation(bindTarget, functions, EffectType.function);
10884                                        }
10885                                        if (subquerys != null && !subquerys.isEmpty()) {
10886                                                analyzeSubqueryDataFlowRelation(bindTarget, subquerys, EffectType.select);
10887                                        }
10888                                        if (objectNames != null && !objectNames.isEmpty()) {
10889                                                analyzeDataFlowRelation(bindTarget, objectNames, EffectType.select, functions);
10890                                        }
10891                                        if (constants != null && !constants.isEmpty()) {
10892                                                analyzeConstantDataFlowRelation(bindTarget, constants, EffectType.select, functions);
10893                                        }
10894                                }
10895                        }
10896                }
10897        }
10898
10899        private void analyzeOpenForStmt(TOpenforStmt stmt) {
10900                if (stmt.getSubquery() == null) {
10901                        return;
10902                }
10903
10904                Variable cursorTempTable = modelFactory.createCursor(stmt);
10905                cursorTempTable.setVariable(true);
10906                cursorTempTable.setSubType(SubType.cursor);
10907                modelManager.bindCursorModel(stmt, cursorTempTable);
10908                analyzeSelectStmt(stmt.getSubquery());
10909
10910                TableColumn cursorColumn = null;
10911                if (cursorTempTable.getColumns() == null || cursorTempTable.getColumns().isEmpty()) {
10912                        TObjectName starColumn = new TObjectName();
10913                        starColumn.setString("*");
10914                        cursorColumn = modelFactory.createTableColumn(cursorTempTable, starColumn, true);
10915                } else {
10916                        cursorColumn = cursorTempTable.getColumns().get(0);
10917                }
10918
10919                ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt.getSubquery());
10920                if (resultSetModel != null) {
10921                        for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
10922                                ResultColumn resultColumn = resultSetModel.getColumns().get(i);
10923                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
10924                                dataflowRelation.setEffectType(EffectType.cursor);
10925                                dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn));
10926                                dataflowRelation.setTarget(new TableColumnRelationshipElement(cursorColumn));
10927                        }
10928                }
10929        }
10930
10931        private void analyzeCursorDeclStmt(TCursorDeclStmt stmt) {
10932                if (stmt.getSubquery() == null) {
10933                        return;
10934                }
10935
10936                Variable cursorTempTable = modelFactory.createCursor(stmt);
10937                cursorTempTable.setVariable(true);
10938                cursorTempTable.setSubType(SubType.cursor);
10939                modelManager.bindCursorModel(stmt, cursorTempTable);
10940                analyzeSelectStmt(stmt.getSubquery());
10941                
10942                ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt.getSubquery());
10943                if(resultSetModel!=null && resultSetModel.isDetermined()) {
10944                        for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
10945                                ResultColumn resultColumn = resultSetModel.getColumns().get(i);
10946                                TObjectName columnName = new TObjectName();
10947                                columnName.setString(resultColumn.getName());
10948                                TableColumn cursorColumn = modelFactory.createTableColumn(cursorTempTable, columnName, true);           
10949                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
10950                                dataflowRelation.setEffectType(EffectType.cursor);
10951                                dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn));
10952                                dataflowRelation.setTarget(new TableColumnRelationshipElement(cursorColumn));
10953                        }
10954                }
10955                else {
10956                        TableColumn cursorColumn = null;
10957                        if (cursorTempTable.getColumns() == null || cursorTempTable.getColumns().isEmpty()) {
10958                                TObjectName starColumn = new TObjectName();
10959                                starColumn.setString("*");
10960                                cursorColumn = modelFactory.createTableColumn(cursorTempTable, starColumn, true);
10961                        } else {
10962                                cursorColumn = cursorTempTable.getColumns().get(0);
10963                        }
10964                        if (resultSetModel != null) {
10965                                for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
10966                                        ResultColumn resultColumn = resultSetModel.getColumns().get(i);
10967                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
10968                                        dataflowRelation.setEffectType(EffectType.cursor);
10969                                        dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn));
10970                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(cursorColumn));
10971                                }
10972                        }
10973                }
10974                
10975        }
10976
10977        private void analyzeDb2Declare(TDb2SqlVariableDeclaration stmt) {
10978                TDeclareVariableList variables = stmt.getVariables();
10979                if (variables == null) {
10980                        return;
10981                }
10982                for (int i = 0; i < variables.size(); i++) {
10983                        TDeclareVariable variable = variables.getDeclareVariable(i);
10984                        if (variable.getTableTypeDefinitions() != null && variable.getTableTypeDefinitions().size() > 0) {
10985
10986
10987                                TObjectName tableName = variable.getVariableName();
10988                                TTableElementList columns = variable.getTableTypeDefinitions();
10989
10990                                Table tableModel = modelFactory.createTableByName(tableName, true);
10991                                tableModel.setCreateTable(true);
10992                                String procedureParent = getProcedureParentName(stmt);
10993                                if (procedureParent != null) {
10994                                        tableModel.setParent(procedureParent);
10995                                }
10996                                modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent), tableModel);
10997
10998                                for (int j = 0; j < columns.size(); j++) {
10999                                        TTableElement tableElement = columns.getTableElement(j);
11000                                        TColumnDefinition column = tableElement.getColumnDefinition();
11001                                        if (column != null && column.getColumnName() != null) {
11002                                                modelFactory.createTableColumn(tableModel, column.getColumnName(), true);
11003                                        }
11004                                }
11005                        } else if (variable.getVariableName() != null) {
11006                                Variable cursorVariable = modelFactory.createVariable(variable.getVariableName());
11007                                cursorVariable.setCreateTable(true);
11008                                cursorVariable.setSubType(SubType.record);
11009                                if(variable.getDatatype()!=null && isSimpleDataType(variable.getDatatype())){
11010                                        TableColumn variableProperty = modelFactory.createTableColumn(cursorVariable,
11011                                                        variable.getVariableName(), true);
11012                                }
11013                                else {
11014                                        TObjectName variableProperties = new TObjectName();
11015                                        variableProperties.setString("*");
11016                                        TableColumn variableProperty = modelFactory.createTableColumn(cursorVariable,
11017                                                        variableProperties, true);
11018                                }
11019                        }
11020                }
11021        }
11022
11023        private void analyzeMssqlDeclare(TMssqlDeclare stmt) {
11024                if (stmt.getDeclareType() == EDeclareType.variable) {
11025                        TDeclareVariableList variables = stmt.getVariables();
11026                        if (variables == null) {
11027                                return;
11028                        }
11029                        for (int i = 0; i < variables.size(); i++) {
11030                                TDeclareVariable variable = variables.getDeclareVariable(i);
11031                                if (variable.getTableTypeDefinitions() != null && variable.getTableTypeDefinitions().size() > 0) {
11032
11033
11034                                        TObjectName tableName = variable.getVariableName();
11035                                        TTableElementList columns = variable.getTableTypeDefinitions();
11036
11037                                        Variable tableModel = modelFactory.createVariable(tableName);
11038                                        tableModel.setVariable(true);
11039                    tableModel.setCreateTable(true);
11040                                        String procedureParent = getProcedureParentName(stmt);
11041                                        if (procedureParent != null) {
11042                                                tableModel.setParent(procedureParent);
11043                                        }
11044                                        modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent), tableModel);
11045
11046                                        for (int j = 0; j < columns.size(); j++) {
11047                                                TTableElement tableElement = columns.getTableElement(j);
11048                                                TColumnDefinition column = tableElement.getColumnDefinition();
11049                                                if (column != null && column.getColumnName() != null) {
11050                                                        modelFactory.createTableColumn(tableModel, column.getColumnName(), true);
11051                                                }
11052                                        }
11053                                } else if (variable.getVariableName() != null) {
11054                                        Variable cursorVariable = modelFactory.createVariable(variable.getVariableName());
11055                                        cursorVariable.setCreateTable(true);
11056                                        cursorVariable.setSubType(SubType.record);
11057                                        TableColumn variableProperty = null;
11058                                        if (variable.getDatatype() != null && isSimpleDataType(variable.getDatatype())) {
11059                                                variableProperty = modelFactory.createTableColumn(cursorVariable, variable.getVariableName(),
11060                                                                true);
11061                                        } else {
11062                                                TObjectName variableProperties = new TObjectName();
11063                                                variableProperties.setString("*");
11064                                                variableProperty = modelFactory.createTableColumn(cursorVariable, variableProperties, true);
11065                                        }
11066                                        
11067                                        if (variable.getDefaultValue() != null && variable.getDefaultValue().getSubQuery() != null) {
11068                                                analyzeSelectStmt(variable.getDefaultValue().getSubQuery());
11069                                                ResultSet resultSetModel = (ResultSet) modelManager
11070                                                                .getModel(variable.getDefaultValue().getSubQuery());
11071                                                if (variableProperty != null && resultSetModel != null) {
11072                                                        for (ResultColumn column : resultSetModel.getColumns()) {
11073                                                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
11074                                                                dataflowRelation.setEffectType(EffectType.select);
11075                                                                dataflowRelation.addSource(new ResultColumnRelationshipElement(column));
11076                                                                dataflowRelation.setTarget(new TableColumnRelationshipElement(variableProperty));
11077                                                        }
11078                                                }
11079                                        }
11080                                }
11081                        }
11082                } else if (stmt.getDeclareType() == EDeclareType.cursor) {
11083                        Variable cursorTempTable = modelFactory.createCursor(stmt);
11084                        cursorTempTable.setVariable(true);
11085                        cursorTempTable.setSubType(SubType.cursor);
11086                        modelManager.bindCursorModel(stmt, cursorTempTable);
11087                        analyzeSelectStmt(stmt.getSubquery());
11088                        ResultSet resultSetModel = (ResultSet)modelManager.getModel(stmt.getSubquery());
11089                        if (resultSetModel != null && resultSetModel.isDetermined()) {
11090                                for(ResultColumn resultColumn: resultSetModel.getColumns()){
11091                                        TObjectName starColumn = new TObjectName();
11092                                        starColumn.setString(resultColumn.getName());
11093                                        TableColumn cursorColumn = modelFactory.createTableColumn(cursorTempTable, starColumn, true);
11094                                        DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
11095                                        dataflowRelation.setEffectType(EffectType.cursor);
11096                                        dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn));
11097                                        dataflowRelation.setTarget(new TableColumnRelationshipElement(cursorColumn));
11098                                }
11099                        }
11100                        else {
11101                                TableColumn cursorColumn = null;
11102                                if (cursorTempTable.getColumns() == null || cursorTempTable.getColumns().isEmpty()) {
11103                                        TObjectName starColumn = new TObjectName();
11104                                        starColumn.setString("*");
11105                                        cursorColumn = modelFactory.createTableColumn(cursorTempTable, starColumn, true);
11106                                        cursorColumn.setShowStar(false);
11107                                        cursorColumn.setExpandStar(true);
11108                                } else {
11109                                        cursorColumn = cursorTempTable.getColumns().get(0);
11110                                }
11111
11112                                if (resultSetModel != null) {
11113                                        for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
11114                                                ResultColumn resultColumn = resultSetModel.getColumns().get(i);
11115                                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
11116                                                dataflowRelation.setEffectType(EffectType.cursor);
11117                                                dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn));
11118                                                dataflowRelation.setTarget(new TableColumnRelationshipElement(cursorColumn));
11119                                        }
11120                                }
11121                        }
11122                }
11123        }
11124
11125
11126        private boolean analyzeMssqlJsonDeclare(TMssqlDeclare stmt, String jsonName, Table jsonTable) {
11127                TDeclareVariableList variables = stmt.getVariables();
11128                if (variables == null) {
11129                        return false;
11130                }
11131                for (int i = 0; i < variables.size(); i++) {
11132                        TDeclareVariable variable = variables.getDeclareVariable(i);
11133                        TObjectName variableName = variable.getVariableName();
11134                        if (DlineageUtil.getIdentifierNormalTableName(variableName.toString())
11135                                        .equals(DlineageUtil.getIdentifierNormalTableName(jsonName))) {
11136                                if (variable.getDefaultValue() != null) {
11137                                        Table variableTable = modelFactory.createJsonVariable(variableName);
11138                                        variableTable.setVariable(true);
11139                                        variableTable.setSubType(SubType.scalar);
11140                                        variableTable.setCreateTable(true);
11141                                        TableColumn property = modelFactory.createVariableProperty(variableTable, variable);
11142
11143                                        for (int j = 0; j < jsonTable.getColumns().size(); j++) {
11144                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11145                                                relation.setEffectType(EffectType.select);
11146                                                relation.setTarget(new TableColumnRelationshipElement(jsonTable.getColumns().get(j)));
11147                                                relation.addSource(new TableColumnRelationshipElement(property));
11148                                        }
11149                                }
11150                                return true;
11151                        }
11152                }
11153                return false;
11154        }
11155
11156        private void analyzeCreateTableStmt(TCreateTableSqlStatement stmt) {
11157                if (stmt.getCloneSourceTable() != null) {
11158                        analyzeCloneTableStmt(stmt);
11159                        return;
11160                }
11161
11162                TTable table = stmt.getTargetTable();
11163
11164                boolean hasDefinition = false;
11165
11166                if (stmt.getColumnList() != null && stmt.getColumnList().size() > 0) {
11167                        hasDefinition = true;
11168                }
11169
11170                if (table != null) {
11171                        Table tableModel = modelFactory.createTableFromCreateDDL(table, hasDefinition || (stmt.getSubQuery() == null && hasDefinition)
11172                                        || (stmt.getSubQuery()!=null && stmt.getSubQuery().getSetOperatorType() == ESetOperatorType.none && stmt.getSubQuery().getResultColumnList().toString().indexOf("*") == -1));
11173                        // Authoritative create-effect classification (parse fact, NOT gated on
11174                        // isDetermined() like fromDDL below). CTAS when there is a defining query,
11175                        // plain DDL otherwise. See dlineage-authoritative-endpoint-classification.md.
11176                        tableModel.setCreatedInSql(true);
11177                        tableModel.setEndpointIntroduction(stmt.getSubQuery() != null
11178                                        ? EndpointIntroduction.CTAS : EndpointIntroduction.CREATE_TABLE_DDL);
11179                        if (stmt.isExternal()) {
11180                                tableModel.setExternal(true);
11181                        }
11182
11183                        if (stmt.getSubQuery() != null) {
11184                                Process process = modelFactory.createProcess(stmt);
11185                                tableModel.addProcess(process);
11186                        }
11187
11188                        String procedureParent = getProcedureParentName(stmt);
11189                        if (procedureParent != null) {
11190                                tableModel.setParent(procedureParent);
11191                        }
11192
11193                        if (stmt.isUsingTemplate()) {
11194                                // Snowflake CREATE TABLE ... USING TEMPLATE <query>: the query only
11195                                // infers the column definitions (e.g. ARRAY_AGG(OBJECT_CONSTRUCT(*))
11196                                // over INFER_SCHEMA); it does NOT populate the table. Analyze it so its
11197                                // own sources (the stage / INFER_SCHEMA) still resolve, but do not
11198                                // project its result columns onto the target table or emit CTAS-style
11199                                // data-flow edges that would misrepresent it as data population.
11200                                if (stmt.getSubQuery() != null) {
11201                                        analyzeSelectStmt(stmt.getSubQuery());
11202                                }
11203                                return;
11204                        }
11205
11206                        if (hasDefinition) {
11207                                if(stmt.getSubQuery()!=null) {
11208                                        TSelectSqlStatement subquery = stmt.getSubQuery();
11209                                        analyzeSelectStmt(subquery);
11210                                        Process process = modelFactory.createProcess(stmt);
11211                                        ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt.getSubQuery());
11212                                        if(resultSetModel.isDetermined()){
11213                                                tableModel.setFromDDL(true);
11214                                        }
11215                                        if (resultSetModel != null) {
11216                                                int resultSetSize = resultSetModel.getColumns().size();
11217                                                int stmtColumnSize = stmt.getColumnList().size();
11218                                                int j = 0;
11219                                                int tableColumnSize = stmtColumnSize;
11220                                                if (resultSetModel.isDetermined() && resultSetSize > tableColumnSize) {
11221                                                        tableColumnSize = resultSetSize;
11222                                                }
11223                                                for (int i = 0; i < tableColumnSize && j < resultSetSize; i++) {
11224                                                        ResultColumn resultColumn = resultSetModel.getColumns().get(j);
11225                                                        if (i < stmtColumnSize) {
11226                                                                TObjectName alias = stmt.getColumnList().getColumn(i).getColumnName();
11227                                                                
11228                                                                if (!resultSetModel.getColumns().get(j).getName().contains("*")) {
11229                                                                        j++;
11230                                                                } else {
11231                                                                        if (resultSetSize - j == stmt.getColumnList().size() - i) {
11232                                                                                j++;
11233                                                                        }
11234                                                                }
11235
11236                                                                if (alias != null) {
11237                                                                        TableColumn viewColumn = modelFactory.createTableColumn(tableModel, alias, true);
11238                                                                        if (resultColumn != null) {
11239                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11240                                                                                relation.setEffectType(EffectType.create_table);
11241                                                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
11242                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11243                                                                                relation.setProcess(process);
11244                                                                        }
11245                                                                } else if (resultColumn.getColumnObject() instanceof TObjectName) {
11246                                                                        TableColumn viewColumn = modelFactory.createTableColumn(tableModel,
11247                                                                                        (TObjectName) resultColumn.getColumnObject(), true);
11248                                                                        if (resultColumn != null) {
11249                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11250                                                                                relation.setEffectType(EffectType.create_table);
11251                                                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
11252                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11253                                                                                relation.setProcess(process);
11254                                                                        }
11255                                                                } else if (resultColumn.getColumnObject() instanceof TResultColumn) {
11256                                                                        TableColumn viewColumn = modelFactory.createTableColumn(tableModel,
11257                                                                                        ((TResultColumn) resultColumn.getColumnObject()).getFieldAttr(), true);
11258                                                                        ResultColumn column = (ResultColumn) modelManager
11259                                                                                        .getModel(resultColumn.getColumnObject());
11260                                                                        if (column != null && !column.getStarLinkColumns().isEmpty()) {
11261                                                                                viewColumn.bindStarLinkColumns(column.getStarLinkColumns());
11262                                                                        }
11263                                                                        if (resultColumn != null) {
11264                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11265                                                                                relation.setEffectType(EffectType.create_table);
11266                                                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
11267                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11268                                                                                relation.setProcess(process);
11269                                                                        }
11270                                                                }
11271                                                        }
11272                                                        else if(resultSetModel.isDetermined()){
11273                                                                TObjectName tableName = new TObjectName();
11274                                                                tableName.setString(resultColumn.getName());
11275                                                                TableColumn viewColumn = modelFactory.createTableColumn(tableModel, tableName, true);
11276                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11277                                                                relation.setEffectType(EffectType.create_table);
11278                                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
11279                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11280                                                                relation.setProcess(process);
11281                                                                j++;
11282                                                        }
11283                                                }
11284                                                if (resultSetModel != null && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
11285                                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
11286                                                        impactRelation.setEffectType(EffectType.create_table);
11287                                                        impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
11288                                                                        resultSetModel.getRelationRows()));
11289                                                        impactRelation.setTarget(
11290                                                                        new RelationRowsRelationshipElement<TableRelationRows>(tableModel.getRelationRows()));
11291                                                }
11292                                        }
11293
11294                                        if (subquery.getResultColumnList() == null && subquery.getValueClause() != null
11295                                                        && subquery.getValueClause().getValueRows().size() == stmt.getColumnList().size()) {
11296                                                for (int i = 0; i < stmt.getColumnList().size(); i++) {
11297                                                        TObjectName alias = stmt.getColumnList().getColumn(i).getColumnName();
11298
11299                                                        if (alias != null) {
11300                                                                TableColumn viewColumn = modelFactory.createTableColumn(tableModel, alias, true);
11301
11302                                                                TExpression expression = subquery.getValueClause().getValueRows().getValueRowItem(i).getExpr();
11303
11304                                                                columnsInExpr visitor = new columnsInExpr();
11305                                                                expression.inOrderTraverse(visitor);
11306                                                                List<TObjectName> objectNames = visitor.getObjectNames();
11307                                                                List<TParseTreeNode> functions = visitor.getFunctions();
11308
11309                                                                if (functions != null && !functions.isEmpty()) {
11310                                                                        analyzeFunctionDataFlowRelation(viewColumn, functions, EffectType.select);
11311
11312                                                                }
11313
11314                                                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
11315                                                                if (subquerys != null && !subquerys.isEmpty()) {
11316                                                                        analyzeSubqueryDataFlowRelation(viewColumn, subquerys, EffectType.select);
11317                                                                }
11318
11319                                                                analyzeDataFlowRelation(viewColumn, objectNames, EffectType.select, functions);
11320                                                                List<TParseTreeNode> constants = visitor.getConstants();
11321                                                                analyzeConstantDataFlowRelation(viewColumn, constants, EffectType.select, functions);
11322                                                        }
11323                                                }
11324                                        }
11325                                        
11326                                        return;
11327                                }
11328                                else {
11329                                        for (int i = 0; i < stmt.getColumnList().size(); i++) {
11330                                                TColumnDefinition column = stmt.getColumnList().getColumn(i);
11331                                                if (column.getDatatype() != null && column.getDatatype().getTypeOfList() != null
11332                                                                && column.getDatatype().getTypeOfList().getColumnDefList() != null) {
11333                                                        for (int j = 0; j < column.getDatatype().getTypeOfList().getColumnDefList().size(); j++) {
11334                                                                TObjectName columnName = new TObjectName();
11335                                                                if (column.getDatatype().getDataType() == EDataType.array_t) {
11336//                                                                      columnName.setString(column.getColumnName().getColumnNameOnly() + ".array."
11337//                                                                                      + column.getDatatype().getTypeOfList().getColumnDefList().getColumn(j)
11338//                                                                                                      .getColumnName().getColumnNameOnly());
11339                                                                        columnName.setString(column.getColumnName().getColumnNameOnly() + "."
11340                                                                                        + column.getDatatype().getTypeOfList().getColumnDefList().getColumn(j)
11341                                                                                        .getColumnName().getColumnNameOnly());
11342                                                                } else {
11343                                                                        columnName.setString(column.getColumnName().getColumnNameOnly() + "."
11344                                                                                        + column.getDatatype().getTypeOfList().getColumnDefList().getColumn(j)
11345                                                                                                        .getColumnName().getColumnNameOnly());
11346                                                                }
11347                                                                TableColumn tableColumn = modelFactory.createTableColumn(tableModel, columnName,
11348                                                                                hasDefinition);
11349                                                                tableColumn.setStruct(true);
11350                                                                tableColumn.setColumnIndex(i);
11351                                                                appendTableColumnToSQLEnv(tableModel, tableColumn);
11352                                                                
11353                                                                if(option.getAnalyzeMode() == AnalyzeMode.crud) {
11354                                                                        CrudRelationship crudRelationship = modelFactory.createCrudRelation();
11355                                                                        crudRelationship.setTarget(new TableColumnRelationshipElement(tableColumn));
11356                                                                        crudRelationship.setEffectType(EffectType.create_table);
11357                                                                }
11358                                                        }
11359                                                        continue;
11360                                                }
11361                                                if (column.getDatatype() != null && column.getDatatype().getColumnDefList() != null) {
11362                                                        Stack<TColumnDefinition> columnPaths = new Stack<TColumnDefinition>();
11363                                                        flattenStructColumns(hasDefinition, tableModel, column, columnPaths, i);
11364                                                        continue;
11365                                                }
11366                                                TableColumn tableColumn = modelFactory.createTableColumn(tableModel, column.getColumnName(),
11367                                                                hasDefinition);
11368                                                if (tableColumn == null) {
11369                                                        continue;
11370                                                }
11371                                                
11372                                                if(option.getAnalyzeMode() == AnalyzeMode.crud) {
11373                                                        CrudRelationship crudRelationship = modelFactory.createCrudRelation();
11374                                                        crudRelationship.setTarget(new TableColumnRelationshipElement(tableColumn));
11375                                                        crudRelationship.setEffectType(EffectType.create_table);
11376                                                }
11377                                                
11378                                                if (column.getDatatype() != null && column.getDatatype().getDataType() == EDataType.variant_t) {
11379                                                        if (tableColumn != null) {
11380                                                                tableColumn.setVariant(true);
11381                                                        }
11382                                                }
11383                                                if (column.getDatatype() != null) {
11384                                                        String dType = column.getDatatype().getDataTypeName();
11385                                                        if ((!SQLUtil.isEmpty(dType)) && (dType.indexOf("_") > 0)) {
11386                                                                dType = dType.split("_")[0];
11387                                                        }
11388                                                        tableColumn.setDataType(dType);
11389                                                }
11390
11391                                                // Inline (column-level) PK / FK constraint FLAGS. The
11392                                                // table-level pass below (stmt.getTableConstraints())
11393                                                // never sees inline constraints, so set the flags here.
11394                                                // Without this, "id INTEGER PRIMARY KEY" or
11395                                                // "fk INTEGER REFERENCES t (c)" produced no
11396                                                // isPrimaryKey()/isForeignKey() flag, unlike the
11397                                                // equivalent table-level CONSTRAINT form.
11398                                                // EConstraintType.reference is the inline REFERENCES
11399                                                // form; foreign_key is the (rare) inline FOREIGN KEY.
11400                                                // Only flags are set here (no model objects allocated)
11401                                                // so column ids are unchanged; the FK relationship
11402                                                // edges are emitted in a deferred pass after the column
11403                                                // loop (see "inline FK relationships" below), matching
11404                                                // the table-level pass ordering so ids stay stable.
11405                                                TConstraintList inlineConstraints = column.getConstraints();
11406                                                if (inlineConstraints != null) {
11407                                                        for (int c = 0; c < inlineConstraints.size(); c++) {
11408                                                                EConstraintType inlineType = inlineConstraints.getConstraint(c)
11409                                                                                .getConstraint_type();
11410                                                                if (inlineType == EConstraintType.primary_key) {
11411                                                                        tableColumn.setPrimaryKey(true);
11412                                                                } else if (inlineType == EConstraintType.foreign_key
11413                                                                                || inlineType == EConstraintType.reference) {
11414                                                                        tableColumn.setForeignKey(true);
11415                                                                }
11416                                                        }
11417                                                }
11418
11419                                                appendTableColumnToSQLEnv(tableModel, tableColumn);
11420                                        }
11421                                }
11422                        }
11423
11424                        if (stmt.getExternalTableOption("DATA_SOURCE") != null) {
11425                                String dataSourceName = stmt.getExternalTableOption("DATA_SOURCE");
11426                                Table dataSource = modelManager.getTableByName(DlineageUtil.getTableFullName(dataSourceName));
11427                                if (dataSource != null) {
11428                                        TableColumn dataSourceColumn = dataSource.getColumns().get(0);
11429                                        for (int i = 0; i < tableModel.getColumns().size(); i++) {
11430                                                TableColumn column = tableModel.getColumns().get(i);
11431                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11432                                                relation.setTarget(new TableColumnRelationshipElement(column));
11433                                                relation.addSource(new TableColumnRelationshipElement(dataSourceColumn));
11434
11435                                                appendTableColumnToSQLEnv(tableModel, column);
11436                                        }
11437                                }
11438                        }
11439
11440                        if (stmt.getSubQuery() != null) {
11441
11442                                analyzeSelectStmt(stmt.getSubQuery());
11443
11444                                ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt.getSubQuery());
11445                                if (resultSetModel != null && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
11446                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
11447                                        impactRelation.setEffectType(EffectType.create_table);
11448                                        impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
11449                                                        resultSetModel.getRelationRows()));
11450                                        impactRelation.setTarget(
11451                                                        new RelationRowsRelationshipElement<TableRelationRows>(tableModel.getRelationRows()));
11452                                }
11453                        }
11454
11455                        if (stmt.getSubQuery() != null && !stmt.getSubQuery().isCombinedQuery()) {
11456                                SelectResultSet resultSetModel = (SelectResultSet) modelManager
11457                                                .getModel(stmt.getSubQuery().getResultColumnList());
11458                                if(resultSetModel.isDetermined()){
11459                                        tableModel.setDetermined(true);
11460                                        tableModel.setFromDDL(true);
11461                                }
11462                                for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
11463                                        ResultColumn resultColumn = resultSetModel.getColumns().get(i);
11464                                        if (resultSetModel.isDetermined() && resultColumn.getName().endsWith("*")) {
11465                                                continue;
11466                                        }
11467
11468                                        if (resultColumn.getColumnObject() instanceof TResultColumn) {
11469                                                TResultColumn columnObject = (TResultColumn) resultColumn.getColumnObject();
11470
11471                                                TAliasClause alias = columnObject.getAliasClause();
11472                                                if (alias != null && alias.getAliasName() != null) {
11473                                                        TableColumn tableColumn = null;
11474                                                        if (!hasDefinition) {
11475                                                                tableColumn = modelFactory.createTableColumn(tableModel, alias.getAliasName(),
11476                                                                                !hasDefinition);
11477                                                        } else {
11478                                                                tableColumn = tableModel.getColumns().get(i);
11479                                                        }
11480
11481                                                        if (!tableColumn.getName().endsWith("*") && tableModel.isDetermined()) {
11482                                                                appendTableColumnToSQLEnv(tableModel, tableColumn);
11483                                                        }
11484
11485                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11486                                                        relation.setEffectType(EffectType.create_table);
11487                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
11488                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11489                                                        Process process = modelFactory.createProcess(stmt);
11490                                                        relation.setProcess(process);
11491                                                } else if (columnObject.getFieldAttr() != null || (columnObject.getExpr()!=null && columnObject.getExpr().getExpressionType() == EExpressionType.typecast_t)) {
11492                                                        TableColumn tableColumn = null;
11493                                                        TObjectName columnObj = columnObject.getFieldAttr();
11494                                                        if(columnObj == null) {
11495                                                                columnObj = columnObject.getExpr().getLeftOperand().getObjectOperand();
11496                                                        }
11497                                                        if ((columnObj == null || columnObj.toString().endsWith("*")) && !resultColumn.getName().endsWith("*")) {
11498                                                                columnObj = new TObjectName();
11499                                                                columnObj.setString(resultColumn.getName());
11500                                                        }
11501
11502                                                        if(columnObj == null){
11503                                                                logger.info("Can't handle column " + resultColumn.getName());
11504                                                                continue;
11505                                                        }
11506
11507                                                        if (!hasDefinition) {
11508                                                                tableColumn = modelFactory.createTableColumn(tableModel, columnObj,
11509                                                                                !hasDefinition);
11510                                                                if (tableColumn == null) {
11511                                                                        if (tableModel.getColumns().isEmpty()) {
11512                                                                                logger.info("Add table " + tableModel.getName() + " column " + columnObj.toString() + " failed");
11513                                                                        }
11514                                                                        else if (resultColumn.getName().endsWith("*")) {
11515                                                                                for (int j = 0; j < tableModel.getColumns().size(); j++) {
11516                                                                                        tableColumn = tableModel.getColumns().get(j);
11517                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11518                                                                                        relation.setEffectType(EffectType.create_table);
11519                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
11520                                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11521                                                                                        if (tableColumn.getName().endsWith("*")
11522                                                                                                        && resultColumn.getName().endsWith("*")) {
11523                                                                                                tableModel.setStarStmt("create_table");
11524                                                                                        }
11525                                                                                        Process process = modelFactory.createProcess(stmt);
11526                                                                                        relation.setProcess(process);
11527                                                                                }
11528                                                                        }
11529                                                                        continue;
11530                                                                }
11531                                                                if (!tableColumn.getName().endsWith("*") && tableModel.isDetermined()) {
11532                                                                        appendTableColumnToSQLEnv(tableModel, tableColumn);
11533                                                                }
11534
11535                                                                Object model = modelManager
11536                                                                                .getModel(resultColumn.getColumnObject());
11537                                                                if (model instanceof ResultColumn) {
11538                                                                        ResultColumn column = (ResultColumn) model;
11539                                                                        if (tableColumn.getName().endsWith("*") && column != null
11540                                                                                        && !column.getStarLinkColumns().isEmpty()) {
11541                                                                                tableColumn.bindStarLinkColumns(column.getStarLinkColumns());
11542                                                                        }
11543
11544                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11545                                                                        relation.setEffectType(EffectType.create_table);
11546                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
11547                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11548                                                                        if (tableColumn.getName().endsWith("*") && resultColumn.getName().endsWith("*")) {
11549                                                                                tableModel.setStarStmt("create_table");
11550                                                                        }
11551                                                                        Process process = modelFactory.createProcess(stmt);
11552                                                                        relation.setProcess(process);
11553                                                                }
11554                                                                else if(model instanceof LinkedHashMap) {
11555                                                                        String columnName = getColumnNameOnly(resultColumn.getName());
11556                                                                        LinkedHashMap<String, ResultColumn> resultColumns =  (LinkedHashMap<String, ResultColumn>)model;
11557                                                                        if (columnObj.toString().endsWith("*")) {
11558                                                                                for (String key : resultColumns.keySet()) {
11559                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel, resultColumns.get(key).getName());
11560                                                                                        DataFlowRelationship relation = modelFactory
11561                                                                                                        .createDataFlowRelation();
11562                                                                                        relation.setEffectType(EffectType.create_table);
11563                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
11564                                                                                        relation.addSource(
11565                                                                                                        new ResultColumnRelationshipElement(resultColumns.get(key)));
11566                                                                                        Process process = modelFactory.createProcess(stmt);
11567                                                                                        relation.setProcess(process);
11568                                                                                }
11569                                                                        } else if (resultColumns.containsKey(columnName)) {
11570                                                                                ResultColumn column = resultColumns.get(columnName);
11571                                                                                tableColumn = modelFactory.createInsertTableColumn(tableModel, column.getName());
11572                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11573                                                                                relation.setEffectType(EffectType.create_table);
11574                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
11575                                                                                relation.addSource(new ResultColumnRelationshipElement(column));
11576                                                                                Process process = modelFactory.createProcess(stmt);
11577                                                                                relation.setProcess(process);
11578                                                                        }
11579                                                                }
11580                                                        } else {
11581                                                                if (resultColumn.getName().endsWith("*")) {
11582                                                                        for (int j = 0; j < tableModel.getColumns().size(); j++) {
11583                                                                                tableColumn = tableModel.getColumns().get(j);
11584                                                                                ResultColumn column = (ResultColumn) modelManager
11585                                                                                                .getModel(resultColumn.getColumnObject());
11586                                                                                if (tableColumn.getName().endsWith("*") && column != null
11587                                                                                                && !column.getStarLinkColumns().isEmpty()) {
11588                                                                                        tableColumn.bindStarLinkColumns(column.getStarLinkColumns());
11589                                                                                }
11590
11591                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11592                                                                                relation.setEffectType(EffectType.create_table);
11593                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
11594                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11595                                                                                if (tableColumn.getName().endsWith("*")
11596                                                                                                && resultColumn.getName().endsWith("*")) {
11597                                                                                        tableModel.setStarStmt("create_table");
11598                                                                                        ;
11599                                                                                }
11600                                                                                Process process = modelFactory.createProcess(stmt);
11601                                                                                relation.setProcess(process);
11602                                                                        }
11603                                                                } else {
11604                                                                        tableColumn = tableModel.getColumns().get(i);
11605                                                                        Object model = modelManager
11606                                                                                        .getModel(resultColumn.getColumnObject());
11607                                                                        String columnName = getColumnNameOnly(resultColumn.getName());
11608                                                                        if(model instanceof LinkedHashMap) {
11609                                                                                LinkedHashMap<String, ResultColumn> resultColumns =  (LinkedHashMap<String, ResultColumn>)model;
11610                                                                                if (resultColumns.size() == tableModel.getColumns().size()) {
11611                                                                                        int j = 0;
11612                                                                                        for (String key : resultColumns.keySet()) {
11613                                                                                                if (j == i) {
11614                                                                                                        ResultColumn column = resultColumns.get(key);
11615                                                                                                        DataFlowRelationship relation = modelFactory
11616                                                                                                                        .createDataFlowRelation();
11617                                                                                                        relation.setEffectType(EffectType.create_table);
11618                                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
11619                                                                                                        relation.addSource(new ResultColumnRelationshipElement(column));
11620                                                                                                        Process process = modelFactory.createProcess(stmt);
11621                                                                                                        relation.setProcess(process);
11622                                                                                                }
11623                                                                                                j++;
11624                                                                                        }
11625                                                                                } else if (resultColumns.containsKey(columnName)) {
11626                                                                                        ResultColumn column = resultColumns.get(columnName);
11627                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel, column.getName());
11628                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11629                                                                                        relation.setEffectType(EffectType.create_table);
11630                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
11631                                                                                        relation.addSource(new ResultColumnRelationshipElement(column));
11632                                                                                        Process process = modelFactory.createProcess(stmt);
11633                                                                                        relation.setProcess(process);
11634                                                                                } else {
11635                                                                                        throw new UnsupportedOperationException("Can't handle this star case.");
11636                                                                                }
11637                                                                        }
11638                                                                        else if (model instanceof ResultColumn) {
11639                                                                                ResultColumn column = (ResultColumn) modelManager
11640                                                                                                .getModel(resultColumn.getColumnObject());
11641                                                                                if (tableColumn.getName().endsWith("*") && column != null
11642                                                                                                && !column.getStarLinkColumns().isEmpty()) {
11643                                                                                        tableColumn.bindStarLinkColumns(column.getStarLinkColumns());
11644                                                                                }
11645
11646                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11647                                                                                relation.setEffectType(EffectType.create_table);
11648                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
11649                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11650                                                                                if (tableColumn.getName().endsWith("*")
11651                                                                                                && resultColumn.getName().endsWith("*")) {
11652                                                                                        tableModel.setStarStmt("create_table");
11653                                                                                }
11654                                                                                Process process = modelFactory.createProcess(stmt);
11655                                                                                relation.setProcess(process);
11656                                                                        }
11657                                                                }
11658                                                        }
11659                                                } else {
11660                                                        TableColumn tableColumn = null;
11661                                                        if (!hasDefinition) {
11662                                                                TObjectName columnName = new TObjectName();
11663                                                                columnName.setString(resultColumn.getColumnObject().toString());
11664                                                                tableColumn = modelFactory.createTableColumn(tableModel, columnName, !hasDefinition);
11665                                                                if(tableColumn == null){
11666                                                                        continue;
11667                                                                }
11668                                                        } else {
11669                                                                tableColumn = tableModel.getColumns().get(i);
11670                                                        }
11671                                                        ResultColumn column = (ResultColumn) modelManager.getModel(resultColumn.getColumnObject());
11672                                                        if (column != null && !column.getStarLinkColumns().isEmpty()) {
11673                                                                tableColumn.bindStarLinkColumns(column.getStarLinkColumns());
11674                                                        }
11675
11676                                                        if (!tableColumn.getName().endsWith("*") && tableModel.isDetermined()) {
11677                                                                appendTableColumnToSQLEnv(tableModel, tableColumn);
11678                                                        }
11679
11680                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11681                                                        relation.setEffectType(EffectType.create_table);
11682                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
11683                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11684                                                        Process process = modelFactory.createProcess(stmt);
11685                                                        relation.setProcess(process);
11686                                                }
11687                                        } else if (resultColumn.getColumnObject() instanceof TObjectName) {
11688                                                TableColumn tableColumn = null;
11689                                                if (!hasDefinition) {
11690                                                        tableColumn = modelFactory.createTableColumn(tableModel,
11691                                                                        (TObjectName) resultColumn.getColumnObject(), !hasDefinition);
11692                                                } else {
11693                                                        tableColumn = tableModel.getColumns().get(i);
11694                                                }
11695
11696                                                if (!tableColumn.getName().endsWith("*") && tableModel.isDetermined()) {
11697                                                        appendTableColumnToSQLEnv(tableModel, tableColumn);
11698                                                }
11699
11700                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11701                                                relation.setEffectType(EffectType.create_table);
11702                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
11703                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11704                                                Process process = modelFactory.createProcess(stmt);
11705                                                relation.setProcess(process);
11706                                        }
11707                                }
11708                        } else if (stmt.getSubQuery() != null) {
11709                                SelectSetResultSet resultSetModel = (SelectSetResultSet) modelManager.getModel(stmt.getSubQuery());
11710                                if(resultSetModel.isDetermined()){
11711                                        tableModel.setDetermined(true);
11712                                        tableModel.setFromDDL(true);
11713                                }
11714                                for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
11715                                        ResultColumn resultColumn = resultSetModel.getColumns().get(i);
11716                                        if (resultColumn.getColumnObject() instanceof TResultColumn) {
11717                                                TResultColumn columnObject = (TResultColumn) resultColumn.getColumnObject();
11718
11719                                                TAliasClause alias = columnObject.getAliasClause();
11720                                                if (alias != null && alias.getAliasName() != null) {
11721                                                        TableColumn tableColumn = null;
11722                                                        if (!hasDefinition) {
11723                                                                tableColumn = modelFactory.createTableColumn(tableModel, alias.getAliasName(),
11724                                                                                !hasDefinition);
11725                                                                if (tableColumn == null) {
11726                                                                        continue;
11727                                                                }
11728                                                        } else {
11729                                                                tableColumn = tableModel.getColumns().get(i);
11730                                                        }
11731
11732                                                        if (!tableColumn.getName().endsWith("*") && tableModel.isDetermined()) {
11733                                                                appendTableColumnToSQLEnv(tableModel, tableColumn);
11734                                                        }
11735
11736                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11737                                                        relation.setEffectType(EffectType.create_table);
11738                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
11739                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11740                                                        Process process = modelFactory.createProcess(stmt);
11741                                                        relation.setProcess(process);
11742                                                } else if (columnObject.getFieldAttr() != null || (columnObject.getExpr()!=null && columnObject.getExpr().getExpressionType() == EExpressionType.typecast_t)) {
11743                                                        TableColumn tableColumn = null;
11744                                                        TObjectName columnObj = columnObject.getFieldAttr();
11745                                                        if(columnObj == null) {
11746                                                                columnObj = columnObject.getExpr().getLeftOperand().getObjectOperand();
11747                                                        }
11748                                                        if (!hasDefinition) {
11749                                                                tableColumn = modelFactory.createTableColumn(tableModel, columnObj,
11750                                                                                !hasDefinition);
11751                                                                if (tableColumn == null) {
11752                                                                        continue;
11753                                                                }
11754                                                        } else {
11755                                                                tableColumn = tableModel.getColumns().get(i);
11756                                                        }
11757                                                        ResultColumn column = (ResultColumn) modelManager.getModel(resultColumn.getColumnObject());
11758                                                        if (column != null && !column.getStarLinkColumns().isEmpty()) {
11759                                                                tableColumn.bindStarLinkColumns(column.getStarLinkColumns());
11760                                                        }
11761
11762                                                        if (!tableColumn.getName().endsWith("*") && tableModel.isDetermined()) {
11763                                                                appendTableColumnToSQLEnv(tableModel, tableColumn);
11764                                                        }
11765
11766                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11767                                                        relation.setEffectType(EffectType.create_table);
11768                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
11769                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11770                                                        Process process = modelFactory.createProcess(stmt);
11771                                                        relation.setProcess(process);
11772                                                } else {
11773                                                        ErrorInfo errorInfo = new ErrorInfo();
11774                                                        errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
11775                                                        errorInfo.setErrorMessage("Can't handle the table column " + columnObject.toString());
11776                                                        errorInfo.setStartPosition(new Pair3<Long, Long, String>(
11777                                                                        columnObject.getStartToken().lineNo, columnObject.getStartToken().columnNo,
11778                                                                        ModelBindingManager.getGlobalHash()));
11779                                                        errorInfo.setEndPosition(new Pair3<Long, Long, String>(columnObject.getEndToken().lineNo,
11780                                                                        columnObject.getEndToken().columnNo + columnObject.getEndToken().getAstext().length(),
11781                                                                        ModelBindingManager.getGlobalHash()));
11782                                                        errorInfos.add(errorInfo);
11783                                                        continue;
11784                                                }
11785                                        } else if (resultColumn.getColumnObject() instanceof TObjectName) {
11786                                                TableColumn tableColumn = null;
11787                                                if (!hasDefinition) {
11788                                                        tableColumn = modelFactory.createTableColumn(tableModel,
11789                                                                        (TObjectName) resultColumn.getColumnObject(), !hasDefinition);
11790                                                } else {
11791                                                        tableColumn = tableModel.getColumns().get(i);
11792                                                }
11793
11794                                                if (!tableColumn.getName().endsWith("*") && tableModel.isDetermined()) {
11795                                                        appendTableColumnToSQLEnv(tableModel, tableColumn);
11796                                                }
11797
11798                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11799                                                relation.setEffectType(EffectType.create_table);
11800                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
11801                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
11802                                                Process process = modelFactory.createProcess(stmt);
11803                                                relation.setProcess(process);
11804                                        }
11805                                }
11806                        } else if (stmt.getLikeTableName() != null) {
11807                                Table likeTableModel = modelFactory.createTableByName(stmt.getLikeTableName());
11808
11809                                if(likeTableModel.isCreateTable()){
11810                                        for(TableColumn column: likeTableModel.getColumns()){
11811                                                TObjectName tableColumn = new TObjectName();
11812                                                tableColumn.setString(column.getName());
11813                                                TableColumn createTableColumn = modelFactory.createTableColumn(tableModel, tableColumn, true);
11814                                                createTableColumn.setPrimaryKey(column.getPrimaryKey());
11815                                                createTableColumn.setForeignKey(column.getForeignKey());
11816                                                createTableColumn.setIndexKey(column.getIndexKey());
11817                                                createTableColumn.setUnqiueKey(column.getUnqiueKey());
11818                                                createTableColumn.setDataType(column.getDataType());
11819                                        }
11820                                        tableModel.setCreateTable(true);
11821                                }
11822
11823//                              Process process = modelFactory.createProcess(stmt);
11824//                              process.setType("Like Table");
11825//                              tableModel.addProcess(process);
11826//
11827//
11828//                              DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11829//                              relation.setEffectType(EffectType.like_table);
11830//                              relation.setTarget(
11831//                                              new RelationRowsRelationshipElement<TableRelationRows>(tableModel.getRelationRows()));
11832//                              relation.addSource(
11833//                                              new RelationRowsRelationshipElement<TableRelationRows>(likeTableModel.getRelationRows()));
11834//                              relation.setProcess(process);
11835                        }
11836
11837                        if (stmt.getStageLocation() != null && stmt.getStageLocation().getStageName() != null) {
11838                                tableModel
11839                                                .setLocation(DlineageUtil.getTableFullName(stmt.getStageLocation().getStageName().toString()));
11840                                Process process = modelFactory.createProcess(stmt);
11841                                process.setType("Create External Table");
11842                                tableModel.addProcess(process);
11843                                Table stage = modelManager.getTableByName(DlineageUtil.getTableFullName(tableModel.getLocation()));
11844                                if (stage == null) {
11845                                        stage = modelManager.getTableByName(
11846                                                        DlineageUtil.getTableFullName(stmt.getStageLocation().toString().replaceFirst("@", "")));
11847                                }
11848                                if (stage == null) {
11849                                        stage = modelFactory.createTableByName(stmt.getStageLocation().toString().replaceFirst("@", ""),
11850                                                        false);
11851                                        stage.setCreateTable(false);
11852                                        stage.setStage(true);
11853                                        if (stmt.getStageLocation().getPath() != null) {
11854                                                stage.setLocation(stmt.getStageLocation().getPath().toString());
11855                                                TObjectName location = new TObjectName();
11856                                                location.setString(stmt.getStageLocation().getPath().toString());
11857                                                modelFactory.createStageLocation(stage, location);
11858                                        } else if (stmt.getRegex_pattern() != null) {
11859                                                stage.setLocation(stmt.getRegex_pattern());
11860                                                TObjectName location = new TObjectName();
11861                                                location.setString(stmt.getRegex_pattern());
11862                                                modelFactory.createStageLocation(stage, location);
11863                                        } else {
11864                                                stage.setLocation("unknownPath");
11865                                                TObjectName location = new TObjectName();
11866                                                location.setString("unknownPath");
11867                                                modelFactory.createStageLocation(stage, location);
11868                                        }
11869                                }
11870                                if (stage != null && !stage.getColumns().isEmpty()) {
11871                                        if (!tableModel.getColumns().isEmpty()) {
11872                                                for (int i = 0; i < tableModel.getColumns().size(); i++) {
11873                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11874                                                        relation.addSource(new TableColumnRelationshipElement(stage.getColumns().get(0)));
11875                                                        relation.setTarget(new TableColumnRelationshipElement(tableModel.getColumns().get(i)));
11876                                                        relation.setProcess(process);
11877                                                }
11878                                        } else {
11879                                                TObjectName starColumn = new TObjectName();
11880                                                starColumn.setString("*");
11881                                                TableColumn tableColumn = modelFactory.createTableColumn(tableModel, starColumn, true);
11882                                                tableColumn.setExpandStar(false);
11883                                                tableColumn.setShowStar(true);
11884                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11885                                                relation.addSource(new TableColumnRelationshipElement(stage.getColumns().get(0)));
11886                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
11887                                                relation.setProcess(process);
11888                                        }
11889                                }
11890                        } else if (stmt.getTableOptions() != null && !stmt.getTableOptions().isEmpty()) {
11891                                for (int i = 0; i < stmt.getTableOptions().size(); i++) {
11892                                        TCreateTableOption createTableOption = stmt.getTableOptions().get(i);
11893                                        if (createTableOption.getCreateTableOptionType() != ECreateTableOption.etoBigQueryExternal)
11894                                                continue;
11895                                        List<String> uris = createTableOption.getUris();
11896                                        if (uris == null || uris.isEmpty())
11897                                                continue;
11898                                        Process process = modelFactory.createProcess(stmt);
11899                                        process.setType("Create External Table");
11900                                        tableModel.addProcess(process);
11901                                        if (tableModel.getColumns().isEmpty()) {
11902                                                TObjectName starColumn = new TObjectName();
11903                                                starColumn.setString("*");
11904                                                TableColumn tableColumn = modelFactory.createTableColumn(tableModel, starColumn, true);
11905                                                tableColumn.setExpandStar(false);
11906                                                tableColumn.setShowStar(true);
11907                                        }
11908                                        for (String uri : uris) {
11909                                                Table uriFile = modelFactory.createTableByName(uri, true);
11910                                                uriFile.setPath(true);
11911                                                uriFile.setFileFormat(createTableOption.getFormat());
11912                                                for (int j = 0; j < tableModel.getColumns().size(); j++) {
11913                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11914                                                        if (stmt.getColumnList() != null) {
11915                                                                TObjectName fileUri = new TObjectName();
11916                                                                fileUri.setString(tableModel.getColumns().get(j).getColumnObject().toString());
11917                                                                TableColumn fileUriColumn = modelFactory.createFileUri(uriFile, fileUri);
11918                                                                relation.addSource(new TableColumnRelationshipElement(fileUriColumn));
11919                                                        } else {
11920                                                                TObjectName fileUri = new TObjectName();
11921                                                                fileUri.setString("*");
11922                                                                TableColumn fileUriColumn = modelFactory.createFileUri(uriFile, fileUri);
11923                                                                relation.addSource(new TableColumnRelationshipElement(fileUriColumn));
11924                                                        }
11925                                                        relation.setTarget(new TableColumnRelationshipElement(tableModel.getColumns().get(j)));
11926                                                        relation.setProcess(process);
11927                                                }
11928                                                if (tableModel.getColumns().isEmpty()) {
11929                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11930                                                        TObjectName fileUri = new TObjectName();
11931                                                        fileUri.setString("*");
11932                                                        TableColumn fileUriColumn = modelFactory.createFileUri(uriFile, fileUri);
11933                                                        relation.addSource(new TableColumnRelationshipElement(fileUriColumn));
11934                                                        relation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(
11935                                                                        tableModel.getRelationRows()));
11936                                                        relation.setProcess(process);
11937                                                }
11938                                        }
11939                                }
11940                        } else if (stmt.getSubQuery() == null && stmt.getTableLocation() != null) {
11941                                Process process = modelFactory.createProcess(stmt);
11942                                process.setType("Create External Table");
11943                                tableModel.addProcess(process);
11944                                Table uriFile = modelFactory.createTableByName(stmt.getTableLocation(), true);
11945                                uriFile.setPath(true);
11946                                for (int j = 0; j < tableModel.getColumns().size(); j++) {
11947                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
11948                                        if (stmt.getColumnList() != null) {
11949                                                TObjectName fileUri = new TObjectName();
11950                                                fileUri.setString(tableModel.getColumns().get(j).getColumnObject().toString());
11951                                                TableColumn fileUriColumn = modelFactory.createFileUri(uriFile, fileUri);
11952                                                relation.addSource(new TableColumnRelationshipElement(fileUriColumn));
11953                                        } else {
11954                                                TObjectName fileUri = new TObjectName();
11955                                                fileUri.setString("*");
11956                                                TableColumn fileUriColumn = modelFactory.createFileUri(uriFile, fileUri);
11957                                                relation.addSource(new TableColumnRelationshipElement(fileUriColumn));
11958                                        }
11959                                        relation.setTarget(new TableColumnRelationshipElement(tableModel.getColumns().get(j)));
11960                                        relation.setProcess(process);
11961                                }
11962                        }
11963
11964                        // Inline (column-level) FK relationships. Emitted here, after the
11965                        // column loop, rather than inline in it, so the referenced-column /
11966                        // relation / process model objects allocate their ids AFTER all
11967                        // column ids (matching the table-level pass below) and column ids
11968                        // stay stable. Mirrors the table-level FK emission: source =
11969                        // referenced column, target = this FK column, effect = foreign_key,
11970                        // plus the optional ER relationship. EConstraintType.reference is
11971                        // the inline REFERENCES form; foreign_key the inline FOREIGN KEY.
11972                        if (stmt.getColumnList() != null) {
11973                                for (int i = 0; i < stmt.getColumnList().size(); i++) {
11974                                        TColumnDefinition fkColumnDef = stmt.getColumnList().getColumn(i);
11975                                        TConstraintList fkConstraints = fkColumnDef.getConstraints();
11976                                        if (fkConstraints == null) {
11977                                                continue;
11978                                        }
11979                                        for (int c = 0; c < fkConstraints.size(); c++) {
11980                                                TConstraint inlineConstraint = fkConstraints.getConstraint(c);
11981                                                EConstraintType inlineType = inlineConstraint.getConstraint_type();
11982                                                if (inlineType != EConstraintType.foreign_key
11983                                                                && inlineType != EConstraintType.reference) {
11984                                                        continue;
11985                                                }
11986                                                TObjectName referencedTableName = inlineConstraint.getReferencedObject();
11987                                                TObjectNameList referencedTableColumns = inlineConstraint.getReferencedColumnList();
11988                                                if (referencedTableName == null || referencedTableColumns == null) {
11989                                                        continue;
11990                                                }
11991                                                TableColumn fkColumn = modelFactory.createTableColumn(tableModel,
11992                                                                fkColumnDef.getColumnName(), true);
11993                                                if (fkColumn == null) {
11994                                                        continue;
11995                                                }
11996                                                Table referencedTable = modelManager.getTableByName(
11997                                                                DlineageUtil.getTableFullName(referencedTableName.toString()));
11998                                                if (referencedTable == null) {
11999                                                        referencedTable = modelFactory.createTableByName(referencedTableName);
12000                                                }
12001                                                for (int j = 0; j < referencedTableColumns.size(); j++) {
12002                                                        TableColumn referencedColumn = modelFactory.createTableColumn(referencedTable,
12003                                                                        referencedTableColumns.getObjectName(j), false);
12004                                                        if (referencedColumn != null) {
12005                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
12006                                                                relation.addSource(new TableColumnRelationshipElement(referencedColumn));
12007                                                                relation.setTarget(new TableColumnRelationshipElement(fkColumn));
12008                                                                relation.setEffectType(EffectType.foreign_key);
12009                                                                Process process = modelFactory.createProcess(stmt);
12010                                                                relation.setProcess(process);
12011                                                                if (this.option.isShowERDiagram()) {
12012                                                                        ERRelationship erRelation = modelFactory.createERRelation();
12013                                                                        erRelation.addSource(new TableColumnRelationshipElement(referencedColumn));
12014                                                                        erRelation.setTarget(new TableColumnRelationshipElement(fkColumn));
12015                                                                }
12016                                                        }
12017                                                }
12018                                        }
12019                                }
12020                        }
12021
12022                        if (stmt.getTableConstraints() != null && stmt.getTableConstraints().size() > 0) {
12023                                for (int i = 0; i < stmt.getTableConstraints().size(); i++) {
12024                                        TConstraint createTableConstraint = stmt.getTableConstraints().getConstraint(i);
12025                                        TPTNodeList<TColumnWithSortOrder> keyNames = createTableConstraint.getColumnList();
12026                                        if (keyNames != null) {
12027                                                for (int k = 0; k < keyNames.size(); k++) {
12028                                                        TObjectName keyName = keyNames.getElement(k).getColumnName();
12029                                                        // Skip functional indexes (expression-based indexes) where columnName is null
12030                                                        if (keyName == null) {
12031                                                                continue;
12032                                                        }
12033                                                        TObjectName referencedTableName = createTableConstraint.getReferencedObject();
12034                                                        TObjectNameList referencedTableColumns = createTableConstraint.getReferencedColumnList();
12035
12036                                                        TableColumn tableConstraint = modelFactory.createTableColumn(tableModel, keyName, true);
12037                                                        if (createTableConstraint.getConstraint_type() == EConstraintType.primary_key) {
12038                                                                tableConstraint.setPrimaryKey(true);
12039                                                        } else if (createTableConstraint.getConstraint_type() == EConstraintType.table_index) {
12040                                                                tableConstraint.setIndexKey(true);
12041                                                        } else if (createTableConstraint.getConstraint_type() == EConstraintType.unique) {
12042                                                                tableConstraint.setUnqiueKey(true);
12043                                                        } else if (createTableConstraint.getConstraint_type() == EConstraintType.foreign_key) {
12044                                                                tableConstraint.setForeignKey(true);
12045                                                                Table referencedTable = modelManager
12046                                                                                .getTableByName(DlineageUtil.getTableFullName(referencedTableName.toString()));
12047                                                                if (referencedTable == null) {
12048                                                                        referencedTable = modelFactory.createTableByName(referencedTableName);
12049                                                                }
12050                                                                if (k == keyNames.size() - 1) {
12051                                                                        captureCompositeForeignKey(tableModel, keyNames, referencedTable,
12052                                                                                        referencedTableColumns);
12053                                                                }
12054
12055                                                                if (referencedTableColumns != null) {
12056                                                                        for (int j = 0; j < referencedTableColumns.size(); j++) {
12057                                                                                TableColumn tableColumn = modelFactory.createTableColumn(referencedTable,
12058                                                                                                referencedTableColumns.getObjectName(j), false);
12059                                                                                if (tableColumn != null) {
12060                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
12061                                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
12062                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableConstraint));
12063                                                                                        relation.setEffectType(EffectType.foreign_key);
12064                                                                                        Process process = modelFactory.createProcess(stmt);
12065                                                                                        relation.setProcess(process);
12066                                                                                        if (this.option.isShowERDiagram()) {
12067                                                                                                ERRelationship erRelation = modelFactory.createERRelation();
12068                                                                                                erRelation.addSource(new TableColumnRelationshipElement(tableColumn));
12069                                                                                                erRelation
12070                                                                                                                .setTarget(new TableColumnRelationshipElement(tableConstraint));
12071                                                                                        }
12072                                                                                }
12073                                                                        }
12074                                                                }
12075                                                        }
12076                                                }
12077                                        }
12078                                }
12079                        }
12080
12081                        if (stmt.getHiveTablePartition() != null && stmt.getHiveTablePartition().getColumnDefList() != null) {
12082                                for (int i = 0; i < stmt.getHiveTablePartition().getColumnDefList().size(); i++) {
12083                                        TColumnDefinition column = stmt.getHiveTablePartition().getColumnDefList().getColumn(i);
12084                                        modelFactory.createTableColumn(tableModel, column.getColumnName(), true);
12085                                        appendTableColumnToSQLEnv(tableModel, column.getColumnName());
12086                                }
12087                        }
12088
12089                        analyzeDynamicTableRefresh(stmt, tableModel);
12090
12091                } else {
12092                        ErrorInfo errorInfo = new ErrorInfo();
12093                        errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
12094                        errorInfo.setErrorMessage("Can't get target table. CreateTableSqlStatement is " + stmt.toString());
12095                        errorInfo.setStartPosition(new Pair3<Long, Long, String>(stmt.getStartToken().lineNo,
12096                                        stmt.getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
12097                        errorInfo.setEndPosition(new Pair3<Long, Long, String>(stmt.getEndToken().lineNo,
12098                                        stmt.getEndToken().columnNo + stmt.getEndToken().getAstext().length(),
12099                                        ModelBindingManager.getGlobalHash()));
12100                        errorInfos.add(errorInfo);
12101                }
12102        }
12103
12104        private void analyzeDynamicTableRefresh(TCreateTableSqlStatement stmt,
12105                        Table dynamicTable) {
12106                TCustomSqlStatement refreshStatement = stmt.getRefreshUsingStatement();
12107                if (refreshStatement == null || refreshStatement.getTargetTable() == null) {
12108                        return;
12109                }
12110                String targetToken = refreshStatement.getTargetTable().getTableName()
12111                                .toString();
12112                if (!"SELF".equalsIgnoreCase(targetToken)) { // non-identifier-compare: Snowflake pseudo-target keyword
12113                        return;
12114                }
12115
12116                // Snowflake defines SELF as the dynamic table currently being created.
12117                // Bind the lexical SELF target to that existing model before running the
12118                // normal INSERT/MERGE analyzer, so all emitted target edges use the real
12119                // dynamic-table identity and its declared columns.
12120                modelManager.bindModel(refreshStatement.getTargetTable(), dynamicTable);
12121                analyzeCustomSqlStmt(refreshStatement);
12122        }
12123
12124        private void captureCompositeForeignKey(Table foreignKeyTable,
12125                        TPTNodeList<TColumnWithSortOrder> foreignKeyNames, Table referencedTable,
12126                        TObjectNameList referencedNames) {
12127                if (!shouldCollectAuthoritativeLineageEvidence() || foreignKeyTable == null
12128                                || referencedTable == null || foreignKeyNames == null
12129                                || referencedNames == null || foreignKeyNames.size() < 2
12130                                || foreignKeyNames.size() != referencedNames.size()) return;
12131                List<TableColumn> foreignKeyFields = new ArrayList<TableColumn>();
12132                List<TableColumn> referencedFields = new ArrayList<TableColumn>();
12133                for (int ordinal = 0; ordinal < foreignKeyNames.size(); ordinal++) {
12134                        TColumnWithSortOrder foreignKey = foreignKeyNames.getElement(ordinal);
12135                        if (foreignKey == null || foreignKey.getColumnName() == null
12136                                        || referencedNames.getObjectName(ordinal) == null) return;
12137                        TableColumn local = findExistingTableColumn(foreignKeyTable,
12138                                        foreignKey.getColumnName());
12139                        TableColumn referenced = findExistingTableColumn(referencedTable,
12140                                        referencedNames.getObjectName(ordinal));
12141                        if (local == null || referenced == null) return;
12142                        foreignKeyFields.add(local);
12143                        referencedFields.add(referenced);
12144                }
12145                authoritativeEvidenceCollector.addCompositeForeignKey(foreignKeyFields, referencedFields);
12146        }
12147
12148        /** Lookup-only: evidence capture must never create or mutate legacy model nodes. */
12149        private TableColumn findExistingTableColumn(Table table, TObjectName columnName) {
12150                Object exact = modelManager.getModel(new Pair<Table, TObjectName>(table, columnName));
12151                if (exact instanceof TableColumn && ((TableColumn) exact).getTable() == table) {
12152                        return (TableColumn) exact;
12153                }
12154                if (table.getColumns() == null) return null;
12155                for (TableColumn candidate : table.getColumns()) {
12156                        if (candidate != null && DlineageUtil.sameColumnName(candidate.getName(), columnName)) {
12157                                return candidate;
12158                        }
12159                }
12160                return null;
12161        }
12162
12163        protected void flattenStructColumns(boolean hasDefinition, Table tableModel, TColumnDefinition column,
12164                        Stack<TColumnDefinition> columnPaths, int index) {
12165                columnPaths.push(column);
12166                for (int j = 0; j < column.getDatatype().getColumnDefList().size(); j++) {
12167                        TColumnDefinition columnDefinition = column.getDatatype().getColumnDefList().getColumn(j);
12168                        if (columnDefinition.getDatatype().getColumnDefList() != null) {
12169                                flattenStructColumns(hasDefinition, tableModel, columnDefinition, columnPaths, index);
12170                        } else {
12171                                TObjectName columnName = new TObjectName();
12172                                columnName.setString(getColumnName(columnPaths, columnDefinition));
12173                                TableColumn tableColumn = modelFactory.createTableColumn(tableModel, columnName, hasDefinition);
12174                                tableColumn.setColumnIndex(index);
12175                                tableColumn.setStruct(true);
12176                                
12177                                if(option.getAnalyzeMode() == AnalyzeMode.crud) {
12178                                        CrudRelationship crudRelationship = modelFactory.createCrudRelation();
12179                                        crudRelationship.setTarget(new TableColumnRelationshipElement(tableColumn));
12180                                        crudRelationship.setEffectType(EffectType.create_table);
12181                                }
12182                                
12183                                appendTableColumnToSQLEnv(tableModel, tableColumn);
12184                        }
12185                }
12186                columnPaths.pop();
12187        }
12188
12189        private String getColumnName(Stack<TColumnDefinition> columnPaths, TColumnDefinition column) {
12190                StringBuilder buffer = new StringBuilder();
12191                Iterator<TColumnDefinition> iter = columnPaths.iterator();
12192                while(iter.hasNext()) {
12193                        buffer.append(iter.next().getColumnName().getColumnNameOnly()).append(".");
12194                }
12195                buffer.append(column.getColumnName().getColumnNameOnly());
12196                return buffer.toString();
12197        }
12198
12199        private void appendTableColumnToSQLEnv(Table tableModel, TableColumn tableColumn) {
12200                //tableModel如果非determined,请不要添加到sqlenv里
12201                if (sqlenv != null && tableColumn!=null) {
12202                        TSQLSchema schema = sqlenv.getSQLSchema(DlineageUtil.getTableSchema(tableModel), true);
12203                        if (schema != null) {
12204                                TSQLTable tempTable = schema.createTable(DlineageUtil.getSimpleTableName(tableModel.getName()));
12205                                if (tableColumn.hasStarLinkColumn()) {
12206                                        for (String column : tableColumn.getStarLinkColumnNames()) {
12207                                                tempTable.addColumn(DlineageUtil.getColumnNameOnly(column));
12208                                        }
12209                                } else {
12210                                        tempTable.addColumn(DlineageUtil.getColumnNameOnly(tableColumn.getName()));
12211                                }
12212                        }
12213                }
12214        }
12215
12216        private void appendTableColumnToSQLEnv(Table tableModel, TObjectName tableColumn) {
12217                if (sqlenv != null) {
12218                        TSQLSchema schema = sqlenv.getSQLSchema(DlineageUtil.getTableSchema(tableModel), true);
12219                        if (schema != null) {
12220                                TSQLTable tempTable = schema.createTable(DlineageUtil.getSimpleTableName(tableModel.getName()));
12221                                tempTable.addColumn(DlineageUtil.getColumnNameOnly(tableColumn.getColumnNameOnly()));
12222                        }
12223                }
12224        }
12225
12226        private void analyzeCreateStageStmt(TCreateStageStmt stmt) {
12227                TObjectName stageName = stmt.getStageName();
12228
12229                if (stageName != null) {
12230
12231                        Table tableModel = modelFactory.createStage(stageName);
12232                        tableModel.setCreateTable(true);
12233                        tableModel.setStage(true);
12234                        tableModel.setLocation(stmt.getExternalStageURL());
12235
12236                        Process process = modelFactory.createProcess(stmt);
12237                        tableModel.addProcess(process);
12238
12239                        TableColumn locationColumn = null;
12240                        if (stmt.getExternalStageURL() != null) {
12241                                TObjectName location = new TObjectName();
12242                                location.setString(stmt.getExternalStageURL());
12243                                locationColumn = modelFactory.createStageLocation(tableModel, location);
12244
12245                                Table pathModel = modelFactory.createTableByName(stmt.getExternalStageURL(), true);
12246                                pathModel.setPath(true);
12247                                pathModel.setCreateTable(true);
12248                                TObjectName fileUri = new TObjectName();
12249                                fileUri.setString(stmt.getExternalStageURL());
12250                                TableColumn fileUriColumn = modelFactory.createFileUri(pathModel, fileUri);
12251
12252                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
12253                                relation.addSource(new TableColumnRelationshipElement(fileUriColumn));
12254                                relation.setTarget(new TableColumnRelationshipElement(locationColumn));
12255                                relation.setProcess(process);
12256
12257                        } else {
12258                                for (TCustomSqlStatement temp : stmt.getGsqlparser().getSqlstatements()) {
12259                                        if (temp instanceof TPutStmt) {
12260                                                TPutStmt put = (TPutStmt) temp;
12261                                                TStageLocation stageLocation = put.getStageLocation();
12262                                                if (stageLocation != null && stageLocation.getStageName() != null) {
12263                                                        Table stage = modelManager.getTableByName(
12264                                                                        DlineageUtil.getTableFullName(stageLocation.getStageName().toString()));
12265                                                        if (stage == tableModel) {
12266                                                                TObjectName location = new TObjectName();
12267                                                                if (!SQLUtil.isEmpty(put.getFileName())) {
12268                                                                        location.setString(put.getFileName());
12269                                                                        locationColumn = modelFactory.createStageLocation(tableModel, location);
12270                                                                }
12271                                                                break;
12272                                                        }
12273                                                }
12274                                        }
12275                                }
12276                        }
12277
12278                        String fileFormat = stmt.getFileFormatName();
12279                        if (fileFormat != null) {
12280                                for (TCustomSqlStatement temp : stmt.getGsqlparser().getSqlstatements()) {
12281                                        if (temp instanceof TCreateFileFormatStmt) {
12282                                                TCreateFileFormatStmt fileFormatStmt = (TCreateFileFormatStmt) temp;
12283                                                if (fileFormatStmt.getFileFormatName() != null
12284                                                                && SQLUtil.compareIdentifier(option.getVendor(), ESQLDataObjectType.dotTable, fileFormatStmt.getFileFormatName().toString(), fileFormat)) {
12285                                                        tableModel.setFileType(fileFormatStmt.getTypeName());
12286                                                        break;
12287                                                }
12288                                        }
12289                                }
12290                        }
12291
12292                        String procedureParent = getProcedureParentName(stmt);
12293                        if (procedureParent != null) {
12294                                tableModel.setParent(procedureParent);
12295                        }
12296
12297                        if (locationColumn != null) {
12298                                List<Table> tables = modelManager.getTablesByName();
12299                                if (tables != null) {
12300                                        for (Table referTable : tables) {
12301                                                if (referTable.getLocation() != null && referTable.getLocation()
12302                                                                .equals(DlineageUtil.getIdentifierNormalTableName(tableModel.getName()))) {
12303                                                        for (int i = 0; i < referTable.getColumns().size(); i++) {
12304                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
12305                                                                relation.addSource(new TableColumnRelationshipElement(locationColumn));
12306                                                                relation.setTarget(new TableColumnRelationshipElement(referTable.getColumns().get(i)));
12307                                                        }
12308                                                }
12309                                        }
12310                                }
12311                        }
12312                } else {
12313                        ErrorInfo errorInfo = new ErrorInfo();
12314                        errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
12315                        errorInfo.setErrorMessage("Can't get target table. CreateStageStmt is " + stmt.toString());
12316                        errorInfo.setStartPosition(new Pair3<Long, Long, String>(stmt.getStartToken().lineNo,
12317                                        stmt.getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
12318                        errorInfo.setEndPosition(new Pair3<Long, Long, String>(stmt.getEndToken().lineNo,
12319                                        stmt.getEndToken().columnNo + stmt.getEndToken().getAstext().length(),
12320                                        ModelBindingManager.getGlobalHash()));
12321                        errorInfo.fillInfo(this);
12322                        errorInfos.add(errorInfo);
12323                }
12324        }
12325
12326        private void analyzeCreateExternalDataSourceStmt(TCreateExternalDataSourceStmt stmt) {
12327                TObjectName dataSourceName = stmt.getDataSourceName();
12328                String locationUrl = stmt.getOption("LOCATION");
12329                if (dataSourceName != null && locationUrl != null) {
12330                        Table tableModel = modelFactory.createDataSource(dataSourceName);
12331                        tableModel.setCreateTable(true);
12332                        tableModel.setDataSource(true);
12333                        tableModel.setLocation(locationUrl);
12334
12335                        Process process = modelFactory.createProcess(stmt);
12336                        tableModel.addProcess(process);
12337
12338                        TObjectName location = new TObjectName();
12339                        location.setString(locationUrl);
12340                        modelFactory.createTableColumn(tableModel, location, true);
12341
12342                        String procedureParent = getProcedureParentName(stmt);
12343                        if (procedureParent != null) {
12344                                tableModel.setParent(procedureParent);
12345                        }
12346                } else {
12347                        ErrorInfo errorInfo = new ErrorInfo();
12348                        errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
12349                        errorInfo.setErrorMessage("Can't get target table. CreateExternalDataSourceStmt is " + stmt.toString());
12350                        errorInfo.setStartPosition(new Pair3<Long, Long, String>(stmt.getStartToken().lineNo,
12351                                        stmt.getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
12352                        errorInfo.setEndPosition(new Pair3<Long, Long, String>(stmt.getEndToken().lineNo,
12353                                        stmt.getEndToken().columnNo + stmt.getEndToken().getAstext().length(),
12354                                        ModelBindingManager.getGlobalHash()));
12355                        errorInfo.fillInfo(this);
12356                        errorInfos.add(errorInfo);
12357                }
12358        }
12359
12360        private void analyzeCreateStreamStmt(TCreateStreamStmt stmt) {
12361                TObjectName streamName = stmt.getStreamName();
12362
12363                if (streamName != null) {
12364                        Table tableModel = modelFactory.createTableByName(stmt.getTableName());
12365                        tableModel.setCreateTable(true);
12366
12367                        Table streamModel = modelFactory.createStream(streamName);
12368                        streamModel.setCreateTable(true);
12369                        streamModel.setStream(true);
12370
12371                        Process process = modelFactory.createProcess(stmt);
12372                        tableModel.addProcess(process);
12373
12374                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
12375                        relation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(streamModel.getRelationRows()));
12376                        relation.addSource(new RelationRowsRelationshipElement<TableRelationRows>(tableModel.getRelationRows()));
12377                        relation.setProcess(process);
12378                }
12379        }
12380
12381        private String getProcedureParentName(TCustomSqlStatement stmt) {
12382                if (stmt instanceof TStoredProcedureSqlStatement) {
12383                        if (((TStoredProcedureSqlStatement) stmt).getStoredProcedureName() != null) {
12384                                return ((TStoredProcedureSqlStatement) stmt).getStoredProcedureName().toString();
12385                        }
12386                }
12387
12388                stmt = stmt.getParentStmt();
12389                if (stmt == null)
12390                        return null;
12391
12392        if (stmt instanceof TCommonBlock) {
12393                        if(((TCommonBlock) stmt).getBlockBody().getParentObjectName() instanceof TStoredProcedureSqlStatement) {
12394                                stmt = (TStoredProcedureSqlStatement)((TCommonBlock) stmt).getBlockBody().getParentObjectName();
12395                        }
12396                }
12397
12398                if (stmt instanceof TStoredProcedureSqlStatement) {
12399                        if (((TStoredProcedureSqlStatement) stmt).getStoredProcedureName() != null) {
12400                                return ((TStoredProcedureSqlStatement) stmt).getStoredProcedureName().toString();
12401                        }
12402                }
12403                if (stmt instanceof TTeradataCreateProcedure) {
12404                        if (((TTeradataCreateProcedure) stmt).getProcedureName() != null) {
12405                                return ((TTeradataCreateProcedure) stmt).getProcedureName().toString();
12406                        }
12407                }
12408
12409                return getProcedureParentName(stmt);
12410        }
12411
12412        private TStoredProcedureSqlStatement getProcedureParent(TCustomSqlStatement stmt) {
12413                if (stmt instanceof TStoredProcedureSqlStatement) {
12414                        return (TStoredProcedureSqlStatement) stmt;
12415                }
12416
12417                stmt = stmt.getParentStmt();
12418                if (stmt == null)
12419                        return null;
12420
12421                if (stmt instanceof TCommonBlock) {
12422                        if (((TCommonBlock) stmt).getBlockBody().getParentObjectName() instanceof TStoredProcedureSqlStatement) {
12423                                stmt = (TStoredProcedureSqlStatement) ((TCommonBlock) stmt).getBlockBody().getParentObjectName();
12424                        }
12425                }
12426
12427                if (stmt instanceof TStoredProcedureSqlStatement) {
12428                        return ((TStoredProcedureSqlStatement) stmt);
12429                }
12430                if (stmt instanceof TTeradataCreateProcedure) {
12431                        return ((TTeradataCreateProcedure) stmt);
12432                }
12433
12434                return getProcedureParent(stmt);
12435        }
12436
12437        private static final class CteWriteIntent {
12438                private final List<TObjectName> columns = new ArrayList<TObjectName>();
12439                private String unsupportedReason;
12440        }
12441
12442        private static final class CteTargetColumnMapping {
12443                private final String exposedName;
12444                private final String baseColumnName;
12445
12446                private CteTargetColumnMapping(String exposedName, String baseColumnName) {
12447                        this.exposedName = exposedName;
12448                        this.baseColumnName = baseColumnName;
12449                }
12450        }
12451
12452        private static final class CteWriteTargetResolution {
12453                private TTable baseTable;
12454                private final List<CteTargetColumnMapping> columns = new ArrayList<CteTargetColumnMapping>();
12455                private String failureReason;
12456
12457                private boolean isWritable() {
12458                        return baseTable != null;
12459                }
12460        }
12461
12462        private static final class ProjectedColumnResolution {
12463                private TTable baseTable;
12464                private String baseColumnName;
12465                private String failureReason;
12466        }
12467
12468        private static final class CteSelectRelationKey {
12469                private final ResultColumn target;
12470                private final ResultColumn source;
12471
12472                private CteSelectRelationKey(ResultColumn target, ResultColumn source) {
12473                        this.target = target;
12474                        this.source = source;
12475                }
12476
12477                @Override
12478                public int hashCode() {
12479                        return 31 * System.identityHashCode(target) + System.identityHashCode(source);
12480                }
12481
12482                @Override
12483                public boolean equals(Object other) {
12484                        if (!(other instanceof CteSelectRelationKey)) {
12485                                return false;
12486                        }
12487                        CteSelectRelationKey key = (CteSelectRelationKey) other;
12488                        return target == key.target && source == key.source;
12489                }
12490        }
12491
12492        private CteWriteIntent collectMergeCteWriteIntent(TMergeSqlStatement stmt, TTable targetTable) {
12493                CteWriteIntent intent = new CteWriteIntent();
12494                if (stmt.getWhenClauses() == null) {
12495                        intent.unsupportedReason = "the MERGE has no writable target columns";
12496                        return intent;
12497                }
12498                for (int i = 0; i < stmt.getWhenClauses().size(); i++) {
12499                        TMergeWhenClause clause = stmt.getWhenClauses().getElement(i);
12500                        if (clause.getUpdateClause() != null) {
12501                                TResultColumnList assignments = clause.getUpdateClause().getUpdateColumnList();
12502                                if (assignments != null) {
12503                                        for (int j = 0; j < assignments.size(); j++) {
12504                                                TExpression assignment = assignments.getResultColumn(j).getExpr();
12505                                                TExpression target = assignment == null ? null : assignment.getLeftOperand();
12506                                                if (target == null || target.getExpressionType() != EExpressionType.simple_object_name_t) {
12507                                                        addCteWriteIntentWarning(intent,
12508                                                                        "an UPDATE assignment target cannot be mapped to a CTE projection");
12509                                                        continue;
12510                                                }
12511                                                addCteWriteIntentColumn(intent, target.getObjectOperand());
12512                                        }
12513                                }
12514                        }
12515                        if (clause.getInsertClause() != null) {
12516                                if (clause.getInsertClause().getInsertValue() != null) {
12517                                        addCteWriteIntentWarning(intent,
12518                                                        "an object-constructor INSERT target cannot be mapped to CTE projections");
12519                                        continue;
12520                                }
12521                                TObjectNameList columns = clause.getInsertClause().getColumnList();
12522                                if (columns == null || columns.size() == 0) {
12523                                        List<TObjectName> inferredColumns = inferProjectedInsertColumns(targetTable,
12524                                                        clause.getInsertClause().getValuelist());
12525                                        if (inferredColumns == null) {
12526                                                addCteWriteIntentWarning(intent,
12527                                                                "an INSERT without an explicit target column list cannot be mapped safely");
12528                                                continue;
12529                                        }
12530                                        for (TObjectName inferredColumn : inferredColumns) {
12531                                                addCteWriteIntentColumn(intent, inferredColumn);
12532                                        }
12533                                        continue;
12534                                }
12535                                for (int j = 0; j < columns.size(); j++) {
12536                                        addCteWriteIntentColumn(intent, columns.getObjectName(j));
12537                                }
12538                        }
12539                }
12540                if (intent.columns.isEmpty() && intent.unsupportedReason == null) {
12541                        intent.unsupportedReason = "the MERGE has no modified columns from which to resolve a base table";
12542                }
12543                return intent;
12544        }
12545
12546        private void addCteWriteIntentWarning(CteWriteIntent intent, String reason) {
12547                if (intent.unsupportedReason == null) {
12548                        intent.unsupportedReason = reason;
12549                } else if (!intent.unsupportedReason.contains(reason)) {
12550                        intent.unsupportedReason += "; " + reason;
12551                }
12552        }
12553
12554        private List<TObjectName> inferProjectedInsertColumns(TTable targetTable, TResultColumnList values) {
12555                if (targetTable == null || values == null) {
12556                        return null;
12557                }
12558                TSelectSqlStatement subquery = null;
12559                TObjectNameList exposedColumns = null;
12560                if (targetTable.getCTE() != null) {
12561                        subquery = targetTable.getCTE().getSubquery();
12562                        exposedColumns = targetTable.getCTE().getColumnList();
12563                } else if (targetTable.getSubquery() != null) {
12564                        subquery = targetTable.getSubquery();
12565                        exposedColumns = getProjectedTargetColumns(targetTable);
12566                } else if (targetTable.getLinkTable() != null
12567                                && targetTable.getLinkTable().getSubquery() != null) {
12568                        subquery = targetTable.getLinkTable().getSubquery();
12569                        exposedColumns = getProjectedTargetColumns(targetTable.getLinkTable());
12570                }
12571                TResultColumnList projections = subquery == null ? null : subquery.getResultColumnList();
12572                if (projections == null || projections.size() != values.size()) {
12573                        return null;
12574                }
12575                if (exposedColumns != null && exposedColumns.size() > 0
12576                                && exposedColumns.size() != projections.size()) {
12577                        return null;
12578                }
12579                List<TObjectName> inferredColumns = new ArrayList<TObjectName>();
12580                for (int i = 0; i < projections.size(); i++) {
12581                        TResultColumn projection = projections.getResultColumn(i);
12582                        if (isCteStarProjection(projection)) {
12583                                return null;
12584                        }
12585                        String exposedName = getProjectedColumnName(exposedColumns, projection, i);
12586                        if (SQLUtil.isEmpty(exposedName)) {
12587                                return null;
12588                        }
12589                        TObjectName column = new TObjectName();
12590                        TSourceToken nameToken = new TSourceToken(exposedName);
12591                        TResultColumn value = values.getResultColumn(i);
12592                        column.setPartToken(nameToken);
12593                        column.setObjectType(TObjectName.ttobjColumn);
12594                        column.setSourceTable(targetTable);
12595                        column.setStartToken(value.getStartToken() == null
12596                                        ? nameToken : value.getStartToken());
12597                        column.setEndToken(value.getEndToken() == null
12598                                        ? nameToken : value.getEndToken());
12599                        inferredColumns.add(column);
12600                }
12601                return inferredColumns;
12602        }
12603
12604        private CteWriteIntent collectUpdateCteWriteIntent(TUpdateSqlStatement stmt) {
12605                CteWriteIntent intent = new CteWriteIntent();
12606                TResultColumnList assignments = stmt.getResultColumnList();
12607                if (assignments == null) {
12608                        intent.unsupportedReason = "the UPDATE has no writable target columns";
12609                        return intent;
12610                }
12611                for (int i = 0; i < assignments.size(); i++) {
12612                        TExpression assignment = assignments.getResultColumn(i).getExpr();
12613                        if (assignment == null || assignment.getExpressionType() == EExpressionType.function_t) {
12614                                addCteWriteIntentWarning(intent,
12615                                                "an UPDATE assignment target cannot be mapped to a CTE projection");
12616                                continue;
12617                        }
12618                        TExpression target = assignment.getLeftOperand();
12619                        if (target == null) {
12620                                addCteWriteIntentWarning(intent,
12621                                                "an UPDATE assignment target cannot be mapped to a CTE projection");
12622                                continue;
12623                        }
12624                        if (target.getExpressionType() == EExpressionType.simple_object_name_t) {
12625                                addCteWriteIntentColumn(intent, target.getObjectOperand());
12626                        } else if (target.getExpressionType() == EExpressionType.list_t && target.getExprList() != null) {
12627                                for (int j = 0; j < target.getExprList().size(); j++) {
12628                                        TExpression item = target.getExprList().getExpression(j);
12629                                        if (item.getExpressionType() == EExpressionType.simple_object_name_t) {
12630                                                addCteWriteIntentColumn(intent, item.getObjectOperand());
12631                                        } else {
12632                                                addCteWriteIntentWarning(intent,
12633                                                                "an UPDATE assignment target cannot be mapped to a CTE projection");
12634                                        }
12635                                }
12636                        } else {
12637                                addCteWriteIntentWarning(intent,
12638                                                "an UPDATE assignment target cannot be mapped to a CTE projection");
12639                        }
12640                }
12641                if (intent.columns.isEmpty() && intent.unsupportedReason == null) {
12642                        intent.unsupportedReason = "the UPDATE has no modified columns from which to resolve a base table";
12643                }
12644                return intent;
12645        }
12646
12647        private void addCteWriteIntentColumn(CteWriteIntent intent, TObjectName column) {
12648                if (column == null || column.getDbObjectType() == EDbObjectType.variable) {
12649                        return;
12650                }
12651                String name = column.getColumnNameOnly();
12652                if (name == null) {
12653                        return;
12654                }
12655                if (name.startsWith("@") && (option.getVendor() == EDbVendor.dbvmssql
12656                                || option.getVendor() == EDbVendor.dbvazuresql)) {
12657                        return;
12658                }
12659                if (name.startsWith(":") && (option.getVendor() == EDbVendor.dbvhana
12660                                || option.getVendor() == EDbVendor.dbvteradata)) {
12661                        return;
12662                }
12663                intent.columns.add(column);
12664        }
12665
12666        private CteWriteTargetResolution resolveCteWriteTarget(TTable targetTable, CteWriteIntent intent) {
12667                TSelectSqlStatement subquery = targetTable == null || targetTable.getCTE() == null
12668                                ? null : targetTable.getCTE().getSubquery();
12669                TObjectNameList exposedColumns = targetTable == null || targetTable.getCTE() == null
12670                                ? null : targetTable.getCTE().getColumnList();
12671                return resolveProjectedWriteTarget(subquery, exposedColumns, intent);
12672        }
12673
12674        private CteWriteTargetResolution resolveProjectedWriteTarget(TSelectSqlStatement subquery,
12675                        TObjectNameList exposedColumns, CteWriteIntent intent) {
12676                CteWriteTargetResolution resolution = new CteWriteTargetResolution();
12677                if (intent.unsupportedReason != null) {
12678                        appendCteResolutionWarning(resolution, intent.unsupportedReason);
12679                }
12680                if (subquery == null) {
12681                        appendCteResolutionWarning(resolution, "the target CTE has no SELECT body");
12682                        return resolution;
12683                }
12684
12685                if (subquery.isCombinedQuery() || subquery.getSetOperatorType() != ESetOperatorType.none) {
12686                        appendCteResolutionWarning(resolution,
12687                                        "a CTE with a set operator is not a writable single-table projection");
12688                        return resolution;
12689                }
12690                if (subquery.getSelectDistinct() != null && !subquery.getSelectDistinct().isAll()) {
12691                        appendCteResolutionWarning(resolution,
12692                                        "a DISTINCT CTE is not a writable single-table projection");
12693                        return resolution;
12694                }
12695                if (hasCteAggregate(subquery)) {
12696                        appendCteResolutionWarning(resolution,
12697                                        "a grouped or aggregate CTE is not a writable single-table projection");
12698                        return resolution;
12699                }
12700                TResultColumnList projections = subquery.getResultColumnList();
12701                if (projections == null || projections.size() == 0) {
12702                        appendCteResolutionWarning(resolution, "the target CTE has no selectable columns");
12703                        return resolution;
12704                }
12705
12706                for (TObjectName modifiedColumn : intent.columns) {
12707                        int projectionIndex = findCteProjectionIndex(exposedColumns, modifiedColumn, projections);
12708                        if (projectionIndex < 0) {
12709                                appendCteResolutionWarning(resolution, "modified column '" + modifiedColumn.getColumnNameOnly()
12710                                                + "' does not map uniquely to a CTE projection");
12711                                continue;
12712                        }
12713                        TResultColumn projection = projections.getResultColumn(projectionIndex);
12714                        boolean starProjection = isCteStarProjection(projection);
12715                        if (starProjection && exposedColumns != null && exposedColumns.size() > 0) {
12716                                appendCteResolutionWarning(resolution, "modified column '" + modifiedColumn.getColumnNameOnly()
12717                                                + "' cannot be mapped safely through an explicitly named wildcard projection");
12718                                continue;
12719                        }
12720                        TObjectName projectedColumn = projection.getColumnFullname();
12721                        if (starProjection && projectedColumn != null
12722                                        && SQLUtil.isEmpty(projectedColumn.getTableString())
12723                                        && subquery.tables != null && subquery.tables.size() > 1) {
12724                                appendCteResolutionWarning(resolution,
12725                                                "an unqualified wildcard over multiple tables is ambiguous");
12726                                continue;
12727                        }
12728                        if (projectedColumn == null || projectedColumn.getColumnSource() == null) {
12729                                appendCteResolutionWarning(resolution, "modified column '" + modifiedColumn.getColumnNameOnly()
12730                                                + "' is calculated or has no resolvable base column");
12731                                continue;
12732                        }
12733
12734                        gudusoft.gsqlparser.resolver2.model.ColumnSource source = projectedColumn.getColumnSource();
12735                        if (!starProjection && SQLUtil.isEmpty(projectedColumn.getTableString())
12736                                        && isAmbiguousUnqualifiedProjection(subquery, projectedColumn)) {
12737                                appendCteResolutionWarning(resolution,
12738                                                "an unqualified projection over multiple tables is ambiguous");
12739                                continue;
12740                        }
12741                        if (source.isAmbiguous()) {
12742                                appendCteResolutionWarning(resolution, "modified column '" + modifiedColumn.getColumnNameOnly()
12743                                                + "' has more than one possible base table");
12744                                continue;
12745                        }
12746                        String projectedColumnName = starProjection
12747                                        ? modifiedColumn.getColumnNameOnly() : projectedColumn.getColumnNameOnly();
12748                        ProjectedColumnResolution projectedResolution = resolveProjectedColumn(
12749                                        getResolvedProjectedSourceTable(projectedColumn), projectedColumnName,
12750                                        new HashSet<TSelectSqlStatement>());
12751                        if (projectedResolution.failureReason != null) {
12752                                appendCteResolutionWarning(resolution, projectedResolution.failureReason);
12753                                continue;
12754                        }
12755                        TTable baseTable = projectedResolution.baseTable;
12756                        if (!isPhysicalCteWriteTable(baseTable)) {
12757                                appendCteResolutionWarning(resolution, "modified column '" + modifiedColumn.getColumnNameOnly()
12758                                                + "' does not resolve to a physical base table");
12759                                continue;
12760                        }
12761                        if (resolution.baseTable != null && resolution.baseTable != baseTable) {
12762                                appendCteResolutionWarning(resolution,
12763                                                "modified columns resolve to more than one base table");
12764                                resolution.baseTable = null;
12765                                resolution.columns.clear();
12766                                return resolution;
12767                        }
12768
12769                        String baseColumnName = projectedResolution.baseColumnName;
12770                        if (SQLUtil.isEmpty(baseColumnName)) {
12771                                baseColumnName = projectedColumn.getColumnNameOnly();
12772                        }
12773                        if (SQLUtil.isEmpty(baseColumnName)) {
12774                                appendCteResolutionWarning(resolution, "modified column '" + modifiedColumn.getColumnNameOnly()
12775                                                + "' has no resolvable physical column name");
12776                                continue;
12777                        }
12778
12779                        resolution.baseTable = baseTable;
12780                        resolution.columns.add(new CteTargetColumnMapping(modifiedColumn.getColumnNameOnly(), baseColumnName));
12781                }
12782                if (resolution.baseTable == null && resolution.failureReason == null) {
12783                        resolution.failureReason = "no modified column resolves safely to a physical base table";
12784                }
12785                return resolution;
12786        }
12787
12788        private boolean isAmbiguousUnqualifiedProjection(TSelectSqlStatement subquery,
12789                        TObjectName projectedColumn) {
12790                if (subquery.tables == null || subquery.tables.size() <= 1) {
12791                        return false;
12792                }
12793                String columnName = projectedColumn.getColumnNameOnly();
12794                int matches = 0;
12795                int unknowns = 0;
12796                TTable onlyUnknown = null;
12797                for (int i = 0; i < subquery.tables.size(); i++) {
12798                        TTable candidate = subquery.tables.getTable(i);
12799                        Boolean hasColumn = hasAuthoritativeColumn(candidate, columnName);
12800                        if (hasColumn == null) {
12801                                unknowns++;
12802                                onlyUnknown = candidate;
12803                                continue;
12804                        }
12805                        if (hasColumn.booleanValue() && ++matches > 1) {
12806                                return true;
12807                        }
12808                }
12809                if (matches == 1) {
12810                        return unknowns != 0;
12811                }
12812                return unknowns != 1 || getResolvedProjectedSourceTable(projectedColumn) != onlyUnknown;
12813        }
12814
12815        private TTable getResolvedProjectedSourceTable(TObjectName column) {
12816                if (column == null) {
12817                        return null;
12818                }
12819                TTable sourceTable = column.getSourceTable();
12820                if (sourceTable != null || column.getColumnSource() == null
12821                                || column.getColumnSource().getSourceNamespace() == null) {
12822                        return sourceTable;
12823                }
12824                return column.getColumnSource().getSourceNamespace().getSourceTable();
12825        }
12826
12827        private Boolean hasAuthoritativeColumn(TTable table, String columnName) {
12828                if (table == null || SQLUtil.isEmpty(columnName)) {
12829                        return null;
12830                }
12831                if (!isPhysicalCteWriteTable(table)) {
12832                        TSelectSqlStatement subquery = null;
12833                        TObjectNameList exposedColumns = null;
12834                        if (table.getCTE() != null) {
12835                                subquery = table.getCTE().getSubquery();
12836                                exposedColumns = table.getCTE().getColumnList();
12837                        } else if (table.getSubquery() != null) {
12838                                subquery = table.getSubquery();
12839                                exposedColumns = getProjectedTargetColumns(table);
12840                        } else if (table.getLinkTable() != null
12841                                        && table.getLinkTable().getSubquery() != null) {
12842                                subquery = table.getLinkTable().getSubquery();
12843                                exposedColumns = getProjectedTargetColumns(table.getLinkTable());
12844                        }
12845                        TResultColumnList projections = subquery == null
12846                                        ? null : subquery.getResultColumnList();
12847                        if (projections == null || projections.size() == 0
12848                                        || (exposedColumns != null && exposedColumns.size() > 0
12849                                                        && exposedColumns.size() != projections.size())) {
12850                                return null;
12851                        }
12852                        boolean matched = false;
12853                        for (int i = 0; i < projections.size(); i++) {
12854                                TResultColumn projection = projections.getResultColumn(i);
12855                                if (isCteStarProjection(projection)) {
12856                                        return null;
12857                                }
12858                                String exposedName = getProjectedColumnName(exposedColumns, projection, i);
12859                                if (SQLUtil.isEmpty(exposedName)) {
12860                                        return null;
12861                                }
12862                                if (SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotColumn,
12863                                                columnName, exposedName)) {
12864                                        matched = true;
12865                                }
12866                        }
12867                        return Boolean.valueOf(matched);
12868                }
12869                TColumnDefinitionList definitions = table.getColumnDefinitions();
12870                if (definitions != null && definitions.size() > 0) {
12871                        for (int i = 0; i < definitions.size(); i++) {
12872                                TObjectName name = definitions.getColumn(i).getColumnName();
12873                                if (name != null && SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotColumn,
12874                                                columnName, name.getColumnNameOnly())) {
12875                                        return Boolean.TRUE;
12876                                }
12877                        }
12878                        return Boolean.FALSE;
12879                }
12880                if (sqlenv == null || table.getTableName() == null) {
12881                        return null;
12882                }
12883                TSQLTable sqlTable = sqlenv.searchTable(table.getTableName());
12884                if (sqlTable == null || sqlTable.getColumnList() == null
12885                                || sqlTable.getColumnList().isEmpty()) {
12886                        return null;
12887                }
12888                for (TSQLColumn column : sqlTable.getColumnList()) {
12889                        if (SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotColumn,
12890                                        columnName, column.getName())) {
12891                                return Boolean.TRUE;
12892                        }
12893                }
12894                return Boolean.FALSE;
12895        }
12896
12897        private ProjectedColumnResolution resolveProjectedColumn(TTable sourceTable, String columnName,
12898                        Set<TSelectSqlStatement> visited) {
12899                ProjectedColumnResolution resolution = new ProjectedColumnResolution();
12900                if (sourceTable == null) {
12901                        resolution.failureReason = "a projected column has no uniquely resolved source table";
12902                        return resolution;
12903                }
12904                if (isPhysicalCteWriteTable(sourceTable)) {
12905                        resolution.baseTable = sourceTable;
12906                        resolution.baseColumnName = columnName;
12907                        return resolution;
12908                }
12909                TSelectSqlStatement subquery = null;
12910                TObjectNameList exposedColumns = null;
12911                if (sourceTable.getCTE() != null) {
12912                        subquery = sourceTable.getCTE().getSubquery();
12913                        exposedColumns = sourceTable.getCTE().getColumnList();
12914                } else if (sourceTable.getSubquery() != null) {
12915                        subquery = sourceTable.getSubquery();
12916                        exposedColumns = getProjectedTargetColumns(sourceTable);
12917                } else if (sourceTable.getLinkTable() != null
12918                                && sourceTable.getLinkTable().getSubquery() != null) {
12919                        subquery = sourceTable.getLinkTable().getSubquery();
12920                        exposedColumns = getProjectedTargetColumns(sourceTable.getLinkTable());
12921                }
12922                if (subquery == null || !visited.add(subquery)) {
12923                        resolution.failureReason = subquery == null
12924                                        ? "an intermediate projection has no SELECT body"
12925                                        : "the projected target contains a cyclic intermediate query";
12926                        return resolution;
12927                }
12928                if (subquery.isCombinedQuery() || subquery.getSetOperatorType() != ESetOperatorType.none) {
12929                        resolution.failureReason = "an intermediate projection with a set operator is not writable";
12930                        return resolution;
12931                }
12932                if (subquery.getSelectDistinct() != null && !subquery.getSelectDistinct().isAll()) {
12933                        resolution.failureReason = "an intermediate DISTINCT projection is not writable";
12934                        return resolution;
12935                }
12936                if (hasCteAggregate(subquery)) {
12937                        resolution.failureReason = "an intermediate grouped or aggregate projection is not writable";
12938                        return resolution;
12939                }
12940                TResultColumnList projections = subquery.getResultColumnList();
12941                if (projections == null || projections.size() == 0 || SQLUtil.isEmpty(columnName)) {
12942                        resolution.failureReason = "an intermediate projection has no mappable target column";
12943                        return resolution;
12944                }
12945                TObjectName exposedColumn = new TObjectName();
12946                TSourceToken columnToken = new TSourceToken(columnName);
12947                exposedColumn.setPartToken(columnToken);
12948                exposedColumn.setObjectType(TObjectName.ttobjColumn);
12949                int projectionIndex = findCteProjectionIndex(exposedColumns, exposedColumn, projections);
12950                if (projectionIndex < 0) {
12951                        resolution.failureReason = "modified column '" + columnName
12952                                        + "' does not map uniquely through an intermediate projection";
12953                        return resolution;
12954                }
12955                TResultColumn projection = projections.getResultColumn(projectionIndex);
12956                boolean starProjection = isCteStarProjection(projection);
12957                if (starProjection && exposedColumns != null && exposedColumns.size() > 0) {
12958                        resolution.failureReason = "modified column '" + columnName
12959                                        + "' cannot be mapped safely through an explicitly named intermediate wildcard projection";
12960                        return resolution;
12961                }
12962                TObjectName innerColumn = projection.getColumnFullname();
12963                if (innerColumn == null || innerColumn.getColumnSource() == null) {
12964                        resolution.failureReason = "modified column '" + columnName
12965                                        + "' has no resolvable intermediate base column";
12966                        return resolution;
12967                }
12968                if (starProjection && SQLUtil.isEmpty(innerColumn.getTableString())
12969                                && subquery.tables != null && subquery.tables.size() > 1) {
12970                        resolution.failureReason = "an unqualified intermediate wildcard over multiple tables is ambiguous";
12971                        return resolution;
12972                }
12973                gudusoft.gsqlparser.resolver2.model.ColumnSource source = innerColumn.getColumnSource();
12974                if (!starProjection && SQLUtil.isEmpty(innerColumn.getTableString())
12975                                && isAmbiguousUnqualifiedProjection(subquery, innerColumn)) {
12976                        resolution.failureReason = "an unqualified intermediate projection over multiple tables is ambiguous";
12977                        return resolution;
12978                }
12979                if (source.isAmbiguous()) {
12980                        resolution.failureReason = "modified column '" + columnName
12981                                        + "' has more than one possible intermediate base table";
12982                        return resolution;
12983                }
12984                String innerName = starProjection ? columnName : innerColumn.getColumnNameOnly();
12985                return resolveProjectedColumn(getResolvedProjectedSourceTable(innerColumn), innerName, visited);
12986        }
12987
12988        private void appendCteResolutionWarning(CteWriteTargetResolution resolution, String reason) {
12989                if (resolution.failureReason == null) {
12990                        resolution.failureReason = reason;
12991                } else if (!resolution.failureReason.contains(reason)) {
12992                        resolution.failureReason += "; " + reason;
12993                }
12994        }
12995
12996        private boolean hasCteAggregate(TSelectSqlStatement subquery) {
12997                if (subquery.getGroupByClause() != null) {
12998                        return true;
12999                }
13000                TResultColumnList projections = subquery.getResultColumnList();
13001                if (projections == null) {
13002                        return false;
13003                }
13004                for (int i = 0; i < projections.size(); i++) {
13005                        TExpression expression = projections.getResultColumn(i).getExpr();
13006                        if (expression == null) {
13007                                continue;
13008                        }
13009                        columnsInExpr visitor = new columnsInExpr();
13010                        expression.inOrderTraverse(visitor);
13011                        List<TParseTreeNode> functions = visitor.getFunctions();
13012                        if (functions == null) {
13013                                continue;
13014                        }
13015                        for (TParseTreeNode function : functions) {
13016                                if (function instanceof TFunctionCall
13017                                                && ((TFunctionCall) function).getWindowDef() == null
13018                                                && isAggregateFunction((TFunctionCall) function)) {
13019                                        return true;
13020                                }
13021                        }
13022                }
13023                return false;
13024        }
13025
13026        private int findCteProjectionIndex(TObjectNameList exposedColumns, TObjectName modifiedColumn,
13027                        TResultColumnList projections) {
13028                int match = -1;
13029                int starMatch = -1;
13030                for (int i = 0; i < projections.size(); i++) {
13031                        TResultColumn projection = projections.getResultColumn(i);
13032                        String exposedName;
13033                        if (exposedColumns != null && exposedColumns.size() > 0) {
13034                                if (i >= exposedColumns.size()) {
13035                                        continue;
13036                                }
13037                                exposedName = getProjectedColumnName(exposedColumns, projection, i);
13038                        } else {
13039                                exposedName = SQLUtil.isEmpty(projection.getColumnAlias())
13040                                                ? projection.getDisplayName() : projection.getColumnAlias();
13041                                if (isCteStarProjection(projection)) {
13042                                        if (starMatch == -1) {
13043                                                starMatch = i;
13044                                        } else {
13045                                                starMatch = -2;
13046                                        }
13047                                }
13048                        }
13049                        if (SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotColumn,
13050                                        modifiedColumn.getColumnNameOnly(), exposedName)) {
13051                                if (match >= 0) {
13052                                        return -1;
13053                                }
13054                                match = i;
13055                        }
13056                }
13057                return match >= 0 ? match : starMatch;
13058        }
13059
13060        private String getProjectedColumnName(TObjectNameList exposedColumns,
13061                        TResultColumn projection, int index) {
13062                if (exposedColumns != null && exposedColumns.size() > 0) {
13063                        if (index >= exposedColumns.size()) {
13064                                return null;
13065                        }
13066                        TObjectName exposedColumn = exposedColumns.getObjectName(index);
13067                        String exposedName = exposedColumn.getColumnNameOnly();
13068                        return SQLUtil.isEmpty(exposedName) ? exposedColumn.toString() : exposedName;
13069                }
13070                return SQLUtil.isEmpty(projection.getColumnAlias())
13071                                ? projection.getDisplayName() : projection.getColumnAlias();
13072        }
13073
13074        private boolean isCteStarProjection(TResultColumn projection) {
13075                if (projection == null || projection.getExpr() == null
13076                                || projection.getExpr().getExpressionType() != EExpressionType.simple_object_name_t
13077                                || projection.getExpr().getObjectOperand() == null) {
13078                        return false;
13079                }
13080                return "*".equals(projection.getExpr().getObjectOperand().getColumnNameOnly());
13081        }
13082
13083        private boolean isPhysicalCteWriteTable(TTable table) {
13084                return table != null && table.getTableName() != null && table.getCTE() == null
13085                                && table.getSubquery() == null
13086                                && (table.getLinkTable() == null || table.getLinkTable().getSubquery() == null);
13087        }
13088
13089        private CteWriteTargetResolution resolveCteDeleteTarget(TSelectSqlStatement subquery) {
13090                return resolveCteDeleteTarget(subquery, new HashSet<TSelectSqlStatement>());
13091        }
13092
13093        private CteWriteTargetResolution resolveCteDeleteTarget(TSelectSqlStatement subquery,
13094                        Set<TSelectSqlStatement> visited) {
13095                CteWriteTargetResolution resolution = new CteWriteTargetResolution();
13096                if (subquery == null || !visited.add(subquery)) {
13097                        resolution.failureReason = "the target CTE has no acyclic SELECT body";
13098                        return resolution;
13099                }
13100                if (subquery.isCombinedQuery() || subquery.getSetOperatorType() != ESetOperatorType.none) {
13101                        resolution.failureReason = "a CTE with a set operator is not a writable single-table projection";
13102                        return resolution;
13103                }
13104                if (subquery.getSelectDistinct() != null && !subquery.getSelectDistinct().isAll()) {
13105                        resolution.failureReason = "a DISTINCT CTE is not a writable single-table projection";
13106                        return resolution;
13107                }
13108                if (hasCteAggregate(subquery)) {
13109                        resolution.failureReason = "a grouped or aggregate CTE is not a writable single-table projection";
13110                        return resolution;
13111                }
13112                if (subquery.tables == null || subquery.tables.size() != 1) {
13113                        resolution.failureReason = "a DELETE target CTE resolves to more than one base table";
13114                        return resolution;
13115                }
13116                TTable table = subquery.tables.getTable(0);
13117                if (isPhysicalCteWriteTable(table)) {
13118                        resolution.baseTable = table;
13119                        return resolution;
13120                }
13121                if (table.getCTE() != null) {
13122                        return resolveCteDeleteTarget(table.getCTE().getSubquery(), visited);
13123                }
13124                if (table.getSubquery() != null) {
13125                        return resolveCteDeleteTarget(table.getSubquery(), visited);
13126                }
13127                if (table.getLinkTable() != null && table.getLinkTable().getSubquery() != null) {
13128                        return resolveCteDeleteTarget(table.getLinkTable().getSubquery(), visited);
13129                }
13130                resolution.failureReason = "the DELETE target CTE does not resolve to a physical base table";
13131                return resolution;
13132        }
13133
13134        private TObjectNameList getProjectedTargetColumns(TTable targetTable) {
13135                return targetTable.getAliasClause() == null
13136                                ? null : targetTable.getAliasClause().getColumns();
13137        }
13138
13139        private TObjectName getCteBaseColumn(CteWriteTargetResolution resolution, TObjectName targetColumn) {
13140                if (resolution == null) {
13141                        return targetColumn;
13142                }
13143                if (!resolution.isWritable() || targetColumn == null) {
13144                        return null;
13145                }
13146                for (CteTargetColumnMapping mapping : resolution.columns) {
13147                        if (SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotColumn,
13148                                        targetColumn.getColumnNameOnly(), mapping.exposedName)) {
13149                                TObjectName baseColumn = new TObjectName();
13150                                TSourceToken nameToken = new TSourceToken(mapping.baseColumnName);
13151                                baseColumn.setPartToken(nameToken);
13152                                baseColumn.setObjectType(TObjectName.ttobjColumn);
13153                                baseColumn.setSourceTable(resolution.baseTable);
13154                                baseColumn.setStartToken(targetColumn.getStartToken() == null
13155                                                ? nameToken : targetColumn.getStartToken());
13156                                baseColumn.setEndToken(targetColumn.getEndToken() == null
13157                                                ? nameToken : targetColumn.getEndToken());
13158                                return baseColumn;
13159                        }
13160                }
13161                return null;
13162        }
13163
13164        private QueryTable analyzeCteWriteTarget(TTable targetTable, Process process) {
13165                return analyzeProjectedWriteTarget(targetTable, targetTable.getCTE().getSubquery(),
13166                                targetTable.getCTE().getColumnList(), process);
13167        }
13168
13169        private QueryTable analyzeProjectedWriteTarget(TTable targetTable, TSelectSqlStatement subquery,
13170                        TObjectNameList exposedColumns, Process process) {
13171                QueryTable queryTable = modelFactory.createQueryTable(targetTable);
13172                if (exposedColumns != null) {
13173                        for (int i = 0; i < exposedColumns.size(); i++) {
13174                                modelFactory.createResultColumn(queryTable, exposedColumns.getObjectName(i));
13175                        }
13176                }
13177
13178                if (subquery == null || stmtStack.contains(subquery)) {
13179                        return queryTable;
13180                }
13181                analyzeSelectStmt(subquery);
13182                ResultSet resultSetModel = (ResultSet) modelManager.getModel(subquery);
13183                if (resultSetModel == null || resultSetModel == queryTable) {
13184                        return queryTable;
13185                }
13186
13187                if (!resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
13188                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
13189                        impactRelation.setEffectType(EffectType.select);
13190                        impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
13191                                        resultSetModel.getRelationRows()));
13192                        impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
13193                                        queryTable.getRelationRows()));
13194                }
13195
13196                if (subquery.getSetOperatorType() != ESetOperatorType.none) {
13197                        SelectSetResultSet selectSetResultSetModel = (SelectSetResultSet) resultSetModel;
13198                        for (int i = 0; i < selectSetResultSetModel.getColumns().size(); i++) {
13199                                ResultColumn sourceColumn = selectSetResultSetModel.getColumns().get(i);
13200                                ResultColumn targetColumn = exposedColumns != null && i < queryTable.getColumns().size()
13201                                                ? queryTable.getColumns().get(i)
13202                                                : modelFactory.createSelectSetResultColumn(queryTable, sourceColumn);
13203                                for (Set<TObjectName> starLinkColumns : sourceColumn.getStarLinkColumns().values()) {
13204                                        for (TObjectName starLinkColumn : starLinkColumns) {
13205                                                targetColumn.bindStarLinkColumn(starLinkColumn);
13206                                        }
13207                                }
13208                                createCteSelectRelation(process, targetColumn, sourceColumn);
13209                        }
13210                } else {
13211                        for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
13212                                ResultColumn sourceColumn = resultSetModel.getColumns().get(i);
13213                                ResultColumn targetColumn = exposedColumns != null && i < queryTable.getColumns().size()
13214                                                ? queryTable.getColumns().get(i)
13215                                                : modelFactory.createSelectSetResultColumn(queryTable, sourceColumn);
13216                                for (TObjectName starLinkColumn : sourceColumn.getStarLinkColumnList()) {
13217                                        targetColumn.bindStarLinkColumn(starLinkColumn);
13218                                }
13219                                createCteSelectRelation(process, targetColumn, sourceColumn);
13220                        }
13221                }
13222                return queryTable;
13223        }
13224
13225        private void createProjectedWriteImpact(TSelectSqlStatement subquery, Table tableModel,
13226                        EffectType effectType, Process process) {
13227                if (subquery == null || tableModel == null) {
13228                        return;
13229                }
13230                Object model = modelManager.getModel(subquery);
13231                if (!(model instanceof ResultSet)) {
13232                        return;
13233                }
13234                ResultSet resultSet = (ResultSet) model;
13235                if (resultSet.getRelationRows().getHoldRelations().isEmpty()) {
13236                        return;
13237                }
13238                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
13239                impactRelation.setEffectType(effectType);
13240                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
13241                                resultSet.getRelationRows()));
13242                impactRelation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(
13243                                tableModel.getRelationRows()));
13244                impactRelation.setProcess(process);
13245        }
13246
13247        private void createCteSelectRelation(Process process, ResultColumn targetColumn, ResultColumn sourceColumn) {
13248                if (!cteSelectRelationKeys.add(new CteSelectRelationKey(targetColumn, sourceColumn))) {
13249                        return;
13250                }
13251                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
13252                relation.setEffectType(EffectType.select);
13253                relation.setTarget(new ResultColumnRelationshipElement(targetColumn));
13254                relation.addSource(new ResultColumnRelationshipElement(sourceColumn));
13255                relation.setProcess(process);
13256        }
13257
13258        private void addCteWriteTargetHint(TTable targetTable, String reason) {
13259                addCteWriteTargetHint(targetTable, reason, false);
13260        }
13261
13262        private void addCteWriteTargetHint(TTable targetTable, String reason, boolean partiallyResolved) {
13263                ErrorInfo errorInfo = new ErrorInfo();
13264                errorInfo.setErrorType(ErrorInfo.SYNTAX_HINT);
13265                String targetKind = targetTable.getCTE() == null ? "Projected write target" : "CTE write target";
13266                String targetName = targetTable.getTableName() == null
13267                                ? targetTable.getAliasName() : targetTable.getTableName().toString();
13268                errorInfo.setErrorMessage(targetKind + " '" + targetName + "' "
13269                                + (partiallyResolved ? "was only partially resolved: " : "was kept as a result set: ")
13270                                + reason + ".");
13271                if (targetTable.getStartToken() != null && targetTable.getEndToken() != null) {
13272                        errorInfo.setStartPosition(new Pair3<Long, Long, String>(targetTable.getStartToken().lineNo,
13273                                        targetTable.getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
13274                        errorInfo.setEndPosition(new Pair3<Long, Long, String>(targetTable.getEndToken().lineNo,
13275                                        targetTable.getEndToken().columnNo
13276                                                        + targetTable.getEndToken().getAstext().length(), ModelBindingManager.getGlobalHash()));
13277                }
13278                errorInfo.fillInfo(this);
13279                errorInfos.add(errorInfo);
13280        }
13281
13282        private void analyzeMergeStmt(TMergeSqlStatement stmt) {
13283                Object tableModel;
13284                CteWriteTargetResolution cteTargetResolution = null;
13285                if (stmt.getUsingTable() != null) {
13286                        TTable table = stmt.getTargetTable();
13287                        Process process = modelFactory.createProcess(stmt);
13288                        if (table.getCTE() != null) {
13289                                QueryTable cteTargetModel = analyzeCteWriteTarget(table, process);
13290                                cteTargetResolution = resolveCteWriteTarget(table, collectMergeCteWriteIntent(stmt, table));
13291                                if (cteTargetResolution.isWritable()) {
13292                                        tableModel = modelFactory.createTable(cteTargetResolution.baseTable);
13293                                        ((Table) tableModel).addProcess(process);
13294                                        createProjectedWriteImpact(table.getCTE().getSubquery(), (Table) tableModel,
13295                                                        EffectType.merge, process);
13296                                } else {
13297                                        tableModel = cteTargetModel;
13298                                }
13299                                if (cteTargetResolution.failureReason != null) {
13300                                        addCteWriteTargetHint(table, cteTargetResolution.failureReason,
13301                                                        cteTargetResolution.isWritable());
13302                                }
13303                        } else if(table.getSubquery()!=null) {
13304                                TObjectNameList exposedColumns = getProjectedTargetColumns(table);
13305                                QueryTable projectedTargetModel = analyzeProjectedWriteTarget(table,
13306                                                table.getSubquery(), exposedColumns, process);
13307                                cteTargetResolution = resolveProjectedWriteTarget(table.getSubquery(), exposedColumns,
13308                                                collectMergeCteWriteIntent(stmt, table));
13309                                if (cteTargetResolution.isWritable()) {
13310                                        tableModel = modelFactory.createTable(cteTargetResolution.baseTable);
13311                                        ((Table) tableModel).addProcess(process);
13312                                        createProjectedWriteImpact(table.getSubquery(), (Table) tableModel,
13313                                                        EffectType.merge, process);
13314                                } else {
13315                                        tableModel = projectedTargetModel;
13316                                }
13317                                if (cteTargetResolution.failureReason != null) {
13318                                        addCteWriteTargetHint(table, cteTargetResolution.failureReason,
13319                                                        cteTargetResolution.isWritable());
13320                                }
13321                        }
13322                        else {
13323                                tableModel = modelFactory.createTable(table);
13324                                ((Table)tableModel).addProcess(process);
13325                        }
13326                        
13327                        for(TTable item: stmt.tables) {
13328                                // Skip subqueries and CTE references: a CTE feeding USING is resolved
13329                                // through its subquery (see the getCTE() branch below), so materializing
13330                                // it here as a physical base table leaves a spurious table named after
13331                                // the CTE in the lineage output. (MantisBT #4493)
13332                                if(item.getSubquery()!=null || item.getCTE()!=null) {
13333                                        continue;
13334                                }
13335                                Table tableItemModel = modelFactory.createTable(item);                          
13336                                if (tableItemModel.getColumns() == null || tableItemModel.getColumns().isEmpty()) {
13337                                        tableItemModel.addColumnsFromSQLEnv();
13338                                }
13339                        }
13340
13341                        if (stmt.getUsingTable().getSubquery() != null) {
13342                                QueryTable queryTable = modelFactory.createQueryTable(stmt.getUsingTable());
13343                                analyzeSelectStmt(stmt.getUsingTable().getSubquery());
13344
13345                                ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt.getUsingTable().getSubquery());
13346                                
13347                                if (queryTable != null && resultSetModel != null && queryTable != resultSetModel) {
13348                                        if (queryTable.getColumns().size() == resultSetModel.getColumns().size()) {
13349                                                for (int i = 0; i < queryTable.getColumns().size(); i++) {
13350                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
13351                                                        relation.setEffectType(EffectType.select);
13352                                                        relation.setTarget(new ResultColumnRelationshipElement(queryTable.getColumns().get(i)));
13353                                                        relation.addSource(new ResultColumnRelationshipElement(resultSetModel.getColumns().get(i)));
13354                                                        relation.setProcess(process);
13355                                                }
13356                                        }
13357                                }
13358                                
13359                                if (resultSetModel != null && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
13360                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
13361                                        impactRelation.setEffectType(EffectType.merge);
13362                                        impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
13363                                                        resultSetModel.getRelationRows()));
13364                                        if (tableModel instanceof Table) {
13365                                                impactRelation.setTarget(
13366                                                                new RelationRowsRelationshipElement<TableRelationRows>(((Table)tableModel).getRelationRows()));
13367                                        }
13368                                        else {
13369                                                impactRelation.setTarget(
13370                                                                new RelationRowsRelationshipElement<ResultSetRelationRows>(((ResultSet)tableModel).getRelationRows()));
13371                                        }
13372                                }
13373                        } else if (stmt.getUsingTable().getCTE() != null) {
13374                                // USING references a CTE defined in the WITH clause attached to the
13375                                // MERGE. Mirror the SELECT-side CTE handling so the CTE subquery (and
13376                                // the base tables inside it) are analyzed and linked through to the
13377                                // merge source; otherwise the CTE is treated as an opaque base table
13378                                // and the tables feeding it are dropped from the lineage. (MantisBT #4493)
13379                                QueryTable queryTable = modelFactory.createQueryTable(stmt.getUsingTable());
13380
13381                                TObjectNameList cteColumns = stmt.getUsingTable().getCTE().getColumnList();
13382                                if (cteColumns != null) {
13383                                        for (int j = 0; j < cteColumns.size(); j++) {
13384                                                modelFactory.createResultColumn(queryTable, cteColumns.getObjectName(j));
13385                                        }
13386                                }
13387
13388                                TSelectSqlStatement subquery = stmt.getUsingTable().getCTE().getSubquery();
13389                                if (subquery != null && !stmtStack.contains(subquery)) {
13390                                        analyzeSelectStmt(subquery);
13391
13392                                        ResultSet resultSetModel = (ResultSet) modelManager.getModel(subquery);
13393
13394                                        if (resultSetModel != null && resultSetModel != queryTable) {
13395                                                if (subquery.getSetOperatorType() != ESetOperatorType.none) {
13396                                                        SelectSetResultSet selectSetResultSetModel = (SelectSetResultSet) modelManager
13397                                                                        .getModel(subquery);
13398                                                        for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) {
13399                                                                ResultColumn sourceColumn = selectSetResultSetModel.getColumns().get(j);
13400                                                                ResultColumn targetColumn = null;
13401                                                                if (cteColumns != null && j < queryTable.getColumns().size()) {
13402                                                                        targetColumn = queryTable.getColumns().get(j);
13403                                                                } else {
13404                                                                        targetColumn = modelFactory.createSelectSetResultColumn(queryTable, sourceColumn);
13405                                                                }
13406                                                                for (Set<TObjectName> starLinkColumns : sourceColumn.getStarLinkColumns().values()) {
13407                                                                        for (TObjectName starLinkColumn : starLinkColumns) {
13408                                                                                targetColumn.bindStarLinkColumn(starLinkColumn);
13409                                                                        }
13410                                                                }
13411                                                                createCteSelectRelation(process, targetColumn, sourceColumn);
13412                                                        }
13413                                                } else {
13414                                                        for (int j = 0; j < resultSetModel.getColumns().size(); j++) {
13415                                                                ResultColumn sourceColumn = resultSetModel.getColumns().get(j);
13416                                                                ResultColumn targetColumn = null;
13417                                                                if (cteColumns != null && j < queryTable.getColumns().size()) {
13418                                                                        targetColumn = queryTable.getColumns().get(j);
13419                                                                } else {
13420                                                                        targetColumn = modelFactory.createSelectSetResultColumn(queryTable, sourceColumn);
13421                                                                }
13422                                                                for (TObjectName starLinkColumn : sourceColumn.getStarLinkColumnList()) {
13423                                                                        targetColumn.bindStarLinkColumn(starLinkColumn);
13424                                                                }
13425                                                                createCteSelectRelation(process, targetColumn, sourceColumn);
13426                                                        }
13427                                                }
13428                                        }
13429
13430                                        if (resultSetModel != null && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
13431                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
13432                                                impactRelation.setEffectType(EffectType.merge);
13433                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
13434                                                                resultSetModel.getRelationRows()));
13435                                                if (tableModel instanceof Table) {
13436                                                        impactRelation.setTarget(
13437                                                                        new RelationRowsRelationshipElement<TableRelationRows>(((Table) tableModel).getRelationRows()));
13438                                                } else {
13439                                                        impactRelation.setTarget(
13440                                                                        new RelationRowsRelationshipElement<ResultSetRelationRows>(((ResultSet) tableModel).getRelationRows()));
13441                                                }
13442                                        }
13443                                }
13444                        } else {
13445                                if (stmt.getUsingTable().getAliasClause() != null && stmt.getUsingTable().getAliasClause().getColumns() != null && stmt.getUsingTable().getValueClause().getRows()!=null) {
13446                                        Table usingTable = modelFactory.createTableFromCreateDDL(stmt.getUsingTable(), false, stmt.getUsingTable().getAliasName() + stmt.getUsingTable().getTableName());
13447                                        usingTable.setCreateTable(true);
13448                                        usingTable.setSubType(SubType.function);
13449                                        for (int z=0;z<stmt.getUsingTable().getAliasClause().getColumns().size();z++) {
13450                                                TObjectName columnName = stmt.getUsingTable().getAliasClause().getColumns().getObjectName(z);
13451                                                TableColumn tableColumn = modelFactory.createTableColumn(usingTable, columnName, true);
13452                                                TResultColumn resultColumn = stmt.getUsingTable().getValueClause().getRows().get(0).getResultColumn(z);
13453                                                modelManager.bindModel(resultColumn, tableColumn);
13454                                                analyzeResultColumnExpressionRelation(tableColumn, resultColumn.getExpr());                                             }
13455                                }
13456                                else {
13457                                        modelFactory.createTable(stmt.getUsingTable());
13458                                }
13459                        }
13460                
13461
13462                        if (stmt.getWhenClauses() != null && stmt.getWhenClauses().size() > 0) {
13463                                for (int i = 0; i < stmt.getWhenClauses().size(); i++) {
13464                                        TMergeWhenClause clause = stmt.getWhenClauses().getElement(i);
13465                                        if (clause.getCondition() != null) {
13466                                                analyzeFilterCondition(null, clause.getCondition(), null, null, EffectType.merge_when);
13467                                        }
13468                                        if (clause.getUpdateClause() != null) {
13469                                                TResultColumnList columns = clause.getUpdateClause().getUpdateColumnList();
13470                                                if (columns == null || columns.size() == 0)
13471                                                        continue;
13472
13473                                                ResultSet resultSet = modelFactory.createResultSet(clause.getUpdateClause(), false);
13474                                                createPseudoImpactRelation(stmt, resultSet, EffectType.merge_update);
13475
13476                                                for (int j = 0; j < columns.size(); j++) {
13477                                                        TResultColumn resultColumn = columns.getResultColumn(j);
13478                                                        if (resultColumn.getExpr().getLeftOperand()
13479                                                                        .getExpressionType() == EExpressionType.simple_object_name_t) {
13480                                                                TObjectName columnObject = resultColumn.getExpr().getLeftOperand().getObjectOperand();
13481
13482                                                                if (columnObject.getDbObjectType() == EDbObjectType.variable) {
13483                                                                        continue;
13484                                                                }
13485
13486                                                                if (columnObject.getColumnNameOnly().startsWith("@")
13487                                                                                && (option.getVendor() == EDbVendor.dbvmssql
13488                                                                                                || option.getVendor() == EDbVendor.dbvazuresql)) {
13489                                                                        continue;
13490                                                                }
13491
13492                                                                if (columnObject.getColumnNameOnly().startsWith(":")
13493                                                                                && (option.getVendor() == EDbVendor.dbvhana
13494                                                                                                || option.getVendor() == EDbVendor.dbvteradata)) {
13495                                                                        continue;
13496                                                                }
13497
13498                                                                ResultColumn updateColumn = modelFactory.createMergeResultColumn(resultSet,
13499                                                                                columnObject);
13500
13501                                                                TExpression valueExpression = resultColumn.getExpr().getRightOperand();
13502                                                                if (valueExpression == null)
13503                                                                        continue;
13504
13505                                                                columnsInExpr visitor = new columnsInExpr();
13506                                                                valueExpression.inOrderTraverse(visitor);
13507                                                                List<TObjectName> objectNames = visitor.getObjectNames();
13508                                                                List<TParseTreeNode> functions = visitor.getFunctions();
13509
13510                                                                if (functions != null && !functions.isEmpty()) {
13511                                                                        analyzeFunctionDataFlowRelation(updateColumn, functions, EffectType.merge_update);
13512                                                                }
13513
13514                                                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
13515                                                                if (subquerys != null && !subquerys.isEmpty()) {
13516                                                                        analyzeSubqueryDataFlowRelation(updateColumn, subquerys, EffectType.merge_update);
13517                                                                }
13518
13519                                                                analyzeDataFlowRelation(updateColumn, objectNames, EffectType.merge_update, functions);
13520
13521                                                                List<TParseTreeNode> constants = visitor.getConstants();
13522                                                                analyzeConstantDataFlowRelation(updateColumn, constants, EffectType.merge_update,
13523                                                                                functions);
13524
13525                                                                if (tableModel instanceof Table) {
13526                                                                        TObjectName targetColumn = getCteBaseColumn(cteTargetResolution, columnObject);
13527                                                                        TableColumn tableColumn = targetColumn == null ? null
13528                                                                                        : modelFactory.createTableColumn((Table)tableModel, targetColumn, false);
13529
13530                                                                        if (tableColumn != null) {
13531                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
13532                                                                                relation.setEffectType(EffectType.merge_update);
13533                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
13534                                                                                relation.addSource(new ResultColumnRelationshipElement(updateColumn));
13535                                                                                relation.setProcess(process);
13536                                                                        }
13537                                                                }
13538                                                        }
13539                                                }
13540                                        }
13541                                        if (clause.getInsertClause() != null) {
13542                                                TExpression insertValue = clause.getInsertClause().getInsertValue();
13543                                                if (insertValue != null
13544                                                                && insertValue.getExpressionType() == EExpressionType.objectConstruct_t) {
13545                                                        ResultSet resultSet = modelFactory.createResultSet(clause.getInsertClause(), false);
13546
13547                                                        createPseudoImpactRelation(stmt, resultSet, EffectType.merge_insert);
13548
13549                                                        TObjectConstruct objectConstruct = insertValue.getObjectConstruct();
13550                                                        for (int z = 0; z < objectConstruct.getPairs().size(); z++) {
13551                                                                TPair pair = objectConstruct.getPairs().getElement(z);
13552
13553                                                                if (pair.getKeyName().getExpressionType() == EExpressionType.simple_constant_t) {
13554                                                                        TObjectName columnObject = new TObjectName();
13555                                                                        TConstant constant = pair.getKeyName().getConstantOperand();
13556                                                                        TSourceToken newSt = new TSourceToken(
13557                                                                                        constant.getValueToken().getTextWithoutQuoted());
13558                                                                        columnObject.setPartToken(newSt);
13559                                                                        columnObject.setSourceTable(stmt.getTargetTable());
13560                                                                        columnObject.setStartToken(constant.getStartToken());
13561                                                                        columnObject.setEndToken(constant.getEndToken());
13562
13563                                                                        ResultColumn insertColumn = modelFactory.createMergeResultColumn(resultSet,
13564                                                                                        columnObject);
13565
13566                                                                        TExpression valueExpression = pair.getKeyValue();
13567                                                                        if (valueExpression == null)
13568                                                                                continue;
13569
13570                                                                        columnsInExpr visitor = new columnsInExpr();
13571                                                                        valueExpression.inOrderTraverse(visitor);
13572                                                                        List<TObjectName> objectNames = visitor.getObjectNames();
13573                                                                        List<TParseTreeNode> functions = visitor.getFunctions();
13574
13575                                                                        if (functions != null && !functions.isEmpty()) {
13576                                                                                analyzeFunctionDataFlowRelation(insertColumn, functions,
13577                                                                                                EffectType.merge_insert);
13578                                                                        }
13579
13580                                                                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
13581                                                                        if (subquerys != null && !subquerys.isEmpty()) {
13582                                                                                analyzeSubqueryDataFlowRelation(insertColumn, subquerys,
13583                                                                                                EffectType.merge_insert);
13584                                                                        }
13585
13586                                                                        analyzeDataFlowRelation(insertColumn, objectNames, EffectType.merge_insert,
13587                                                                                        functions);
13588
13589                                                                        List<TParseTreeNode> constants = visitor.getConstants();
13590                                                                        analyzeConstantDataFlowRelation(insertColumn, constants, EffectType.merge_insert,
13591                                                                                        functions);
13592
13593                                                                        TObjectName targetColumn = getCteBaseColumn(cteTargetResolution, columnObject);
13594                                                                        TableColumn tableColumn = !(tableModel instanceof Table) || targetColumn == null ? null
13595                                                                                        : modelFactory.createTableColumn((Table)tableModel, targetColumn, false);
13596
13597                                                                        if (tableColumn != null) {
13598                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
13599                                                                                relation.setEffectType(EffectType.merge_insert);
13600                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
13601                                                                                relation.addSource(new ResultColumnRelationshipElement(insertColumn));
13602                                                                                relation.setProcess(process);
13603                                                                        }
13604                                                                }
13605                                                        }
13606                                                } else {
13607                                                        TObjectNameList columns = clause.getInsertClause().getColumnList();
13608                                                        TResultColumnList values = clause.getInsertClause().getValuelist();
13609                                                        if (values == null || values.size() == 0) {
13610                                                                if (clause.getInsertClause().toString().toLowerCase().indexOf("row") != -1) {
13611                                                                        if (stmt.getUsingTable().getSubquery() != null && tableModel instanceof Table) {
13612                                                                                ResultSet sourceResultSet = modelFactory.createQueryTable(stmt.getUsingTable());
13613                                                                                TObjectName targetStarColumn = new TObjectName();
13614                                                                                targetStarColumn.setString("*");
13615                                                                                TableColumn targetTableColumn = modelFactory.createTableColumn((Table)tableModel,
13616                                                                                                targetStarColumn, true);
13617                                                                                if (sourceResultSet.getColumns() == null
13618                                                                                                || sourceResultSet.getColumns().isEmpty()) {
13619                                                                                        TObjectName sourceStarColumn = new TObjectName();
13620                                                                                        sourceStarColumn.setString("*");
13621                                                                                        modelFactory.createResultColumn(sourceResultSet, sourceStarColumn);
13622                                                                                }
13623                                                                                for (ResultColumn sourceColumn : sourceResultSet.getColumns()) {
13624                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
13625                                                                                        relation.setEffectType(EffectType.merge_insert);
13626                                                                                        relation.setTarget(new TableColumnRelationshipElement(targetTableColumn));
13627                                                                                        relation.addSource(new ResultColumnRelationshipElement(sourceColumn));
13628                                                                                        relation.setProcess(process);
13629                                                                                }
13630                                                                        } else if (tableModel instanceof Table) {
13631                                                                                Table sourceTable = modelFactory.createTable(stmt.getUsingTable());
13632                                                                                TObjectName sourceStarColumn = new TObjectName();
13633                                                                                sourceStarColumn.setString("*");
13634                                                                                TableColumn sourceTableColumn = modelFactory.createTableColumn(sourceTable,
13635                                                                                                sourceStarColumn, true);
13636                                                                                TObjectName targetStarColumn = new TObjectName();
13637                                                                                targetStarColumn.setString("*");
13638                                                                                TableColumn targetTableColumn = modelFactory.createTableColumn(((Table)tableModel),
13639                                                                                                targetStarColumn, true);
13640                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
13641                                                                                relation.setEffectType(EffectType.merge_insert);
13642                                                                                relation.setTarget(new TableColumnRelationshipElement(targetTableColumn));
13643                                                                                relation.addSource(new TableColumnRelationshipElement(sourceTableColumn));
13644                                                                                relation.setProcess(process);
13645                                                                        }
13646
13647                                                                }
13648                                                                continue;
13649                                                        }
13650
13651                                                        List<TObjectName> tableColumns = new ArrayList<TObjectName>();
13652                                                        boolean unresolvedProjectedColumns = false;
13653                                                        if (columns == null || columns.size() == 0) {
13654                                                                List<TObjectName> inferredColumns = inferProjectedInsertColumns(
13655                                                                                stmt.getTargetTable(), values);
13656                                                                unresolvedProjectedColumns = inferredColumns == null
13657                                                                                && cteTargetResolution != null;
13658                                                                if (inferredColumns != null) {
13659                                                                        tableColumns.addAll(inferredColumns);
13660                                                                } else {
13661//                                                              if (!((Table)tableModel).getColumns().isEmpty()) {
13662//                                                                      for (int j = 0; j < ((Table)tableModel).getColumns().size(); j++) {
13663//                                                                              if (((Table)tableModel).getColumns().get(j).getColumnObject() == null) {
13664//                                                                                      continue;
13665//                                                                              }
13666//                                                                              tableColumns.add(((Table)tableModel).getColumns().get(j).getColumnObject());
13667//                                                                      }
13668//                                                              } else {
13669                                                                        for (int j = 0; j < values.size(); j++) {
13670                                                                                TResultColumn column = values.getResultColumn(j);
13671                                                                                if (column.getAliasClause() != null) {
13672                                                                                        tableColumns.add(column.getAliasClause().getAliasName());
13673                                                                                } else if (column.getFieldAttr() != null) {
13674                                                                                        tableColumns.add(column.getFieldAttr());
13675                                                                                } else {
13676                                                                                        TObjectName columnName = new TObjectName();
13677                                                                                        columnName.setString(column.toString());
13678                                                                                        tableColumns.add(columnName);
13679                                                                                }
13680//                                                                      }
13681                                                                        }
13682                                                                }
13683                                                        } else {
13684                                                                for (int j = 0; j < columns.size(); j++) {
13685                                                                        tableColumns.add(columns.getObjectName(j));
13686                                                                }
13687                                                        }
13688
13689                                                        ResultSet resultSet = modelFactory.createResultSet(clause.getInsertClause(), false);
13690
13691                                                        createPseudoImpactRelation(stmt, resultSet, EffectType.merge_insert);
13692
13693                                                        for (int j = 0; j < tableColumns.size() && j < values.size(); j++) {
13694                                                                TObjectName columnObject = tableColumns.get(j);
13695
13696                                                                ResultColumn insertColumn = modelFactory.createMergeResultColumn(resultSet,
13697                                                                                columnObject);
13698
13699                                                                TExpression valueExpression = values.getResultColumn(j).getExpr();
13700                                                                if (valueExpression == null)
13701                                                                        continue;
13702
13703                                                                columnsInExpr visitor = new columnsInExpr();
13704                                                                valueExpression.inOrderTraverse(visitor);
13705                                                                List<TObjectName> objectNames = visitor.getObjectNames();
13706                                                                List<TParseTreeNode> functions = visitor.getFunctions();
13707
13708                                                                if (functions != null && !functions.isEmpty()) {
13709                                                                        analyzeFunctionDataFlowRelation(insertColumn, functions, EffectType.merge_insert);
13710                                                                }
13711
13712                                                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
13713                                                                if (subquerys != null && !subquerys.isEmpty()) {
13714                                                                        analyzeSubqueryDataFlowRelation(insertColumn, subquerys, EffectType.merge_insert);
13715                                                                }
13716
13717                                                                analyzeDataFlowRelation(insertColumn, objectNames, EffectType.merge_insert, functions);
13718
13719                                                                List<TParseTreeNode> constants = visitor.getConstants();
13720                                                                analyzeConstantDataFlowRelation(insertColumn, constants, EffectType.merge_insert,
13721                                                                                functions);
13722
13723                                                                TObjectName targetColumn = unresolvedProjectedColumns ? null
13724                                                                                : getCteBaseColumn(cteTargetResolution, columnObject);
13725                                                                TableColumn tableColumn = !(tableModel instanceof Table) || targetColumn == null ? null
13726                                                                                : modelFactory.createTableColumn(((Table)tableModel), targetColumn, false);
13727                                                                if(tableColumn == null) {
13728                                                                        if (cteTargetResolution == null && tableModel instanceof Table
13729                                                                                        && ((Table) tableModel).isCreateTable()) {
13730                                                                                tableColumn = ((Table) tableModel).getColumns().get(j);
13731                                                                        }
13732                                                                        else {
13733                                                                                continue;
13734                                                                        }
13735                                                                }
13736
13737                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
13738                                                                relation.setEffectType(EffectType.merge_insert);
13739                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
13740                                                                relation.addSource(new ResultColumnRelationshipElement(insertColumn));
13741                                                                relation.setProcess(process);
13742                                                        }
13743                                                }
13744                                        }
13745                                }
13746
13747                        }
13748
13749                        if (stmt.getCondition() != null) {
13750                                analyzeFilterCondition(null, stmt.getCondition(), null, JoinClauseType.on, EffectType.merge);
13751                        }
13752                }
13753        }
13754
13755        private List<TableColumn> bindInsertTableColumn(Table tableModel, TInsertIntoValue value, List<TObjectName> keyMap,
13756                        List<TResultColumn> valueMap) {
13757                List<TableColumn> tableColumns = new ArrayList<TableColumn>();
13758                if (value.getColumnList() != null) {
13759                        for (int z = 0; z < value.getColumnList().size(); z++) {
13760                                TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
13761                                                value.getColumnList().getObjectName(z));
13762                                tableColumns.add(tableColumn);
13763                                keyMap.add(tableColumn.getColumnObject());
13764                        }
13765                }
13766
13767                if (value.getTargetList() != null) {
13768                        for (int z = 0; z < value.getTargetList().size(); z++) {
13769                                TMultiTarget target = value.getTargetList().getMultiTarget(z);
13770                                TResultColumnList columns = target.getColumnList();
13771                                for (int i = 0; i < columns.size(); i++) {
13772                                        if (value.getColumnList() == null) {
13773                                                TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
13774                                                                columns.getResultColumn(i).getFieldAttr());
13775                                                tableColumns.add(tableColumn);
13776                                        }
13777                                        valueMap.add(columns.getResultColumn(i));
13778                                }
13779                        }
13780                }
13781
13782                return tableColumns;
13783        }
13784
13785        private TableColumn matchColumn(List<TableColumn> tableColumns, TableColumn targetColumn) {
13786                String columnName = targetColumn.getName();
13787                if (tableColumns == null) {
13788                        return null;
13789                }
13790                for (int i = 0; i < tableColumns.size(); i++) {
13791                        TableColumn column = tableColumns.get(i);
13792                        if (column.getColumnObject() == null) {
13793                                continue;
13794                        }
13795                        if(column.isStruct() && targetColumn.isStruct()) {
13796                                List<String> names = SQLUtil.parseNames(column.getName());
13797                                List<String> targetNames = SQLUtil
13798                                                .parseNames(targetColumn.getName());
13799                                if (!getColumnName(targetNames.get(0))
13800                                                .equals(getColumnName(names.get(0)))) {
13801                                        continue;
13802                                }
13803                        }
13804                        if (getColumnName(column.getColumnObject().toString()).equals(getColumnName(columnName))) {
13805                                return column;
13806                        }
13807                }
13808                return null;
13809        }
13810        
13811        private TableColumn matchColumn(List<TableColumn> tableColumns, TObjectName columnName) {
13812                if (tableColumns == null) {
13813                        return null;
13814                }
13815                for (int i = 0; i < tableColumns.size(); i++) {
13816                        TableColumn column = tableColumns.get(i);
13817                        if (column.getColumnObject() == null) {
13818                                continue;
13819                        }
13820                        if (DlineageUtil.sameColumnName(column.getColumnObject(), columnName))
13821                                return column;
13822                }
13823                return null;
13824        }
13825
13826        private ResultColumn matchResultColumn(List<ResultColumn> resultColumns, ResultColumn resultColumn) {
13827                if (resultColumns == null) {
13828                        return null;
13829                }
13830
13831                TObjectName columnName = getObjectName(resultColumn);
13832                if (columnName == null) {
13833                        return null;
13834                }
13835
13836                for (int i = 0; i < resultColumns.size(); i++) {
13837                        ResultColumn column = resultColumns.get(i);
13838                        if (column.getAlias() != null
13839                                        && DlineageUtil.sameColumnName(column.getAlias(), columnName))
13840                                return column;
13841                        if (column.getName() != null && DlineageUtil.sameColumnName(column.getName(), columnName))
13842                                return column;
13843                        if (column.getName() != null && column.getName().endsWith("*")) {
13844                                if ("*".equals(column.getColumnObject().toString())) {
13845                                        return column;
13846                                } else {
13847                                        TObjectName columnObjectName = getObjectName(column);
13848                                        if (columnObjectName.getTableString() != null
13849                                                        && columnObjectName.getTableString().equals(getResultSetAlias(resultColumn))) {
13850                                                return column;
13851                                        }
13852                                }
13853                        }
13854                }
13855                return null;
13856        }
13857
13858        private String getResultSetAlias(ResultColumn resultColumn) {
13859                ResultSet resultSet = resultColumn.getResultSet();
13860                if (resultSet instanceof QueryTable) {
13861                        return ((QueryTable) resultSet).getAlias();
13862                }
13863                return null;
13864        }
13865
13866        private ResultColumn matchResultColumn(List<ResultColumn> resultColumns, TObjectName columnName) {
13867                if (resultColumns == null) {
13868                        return null;
13869                }
13870                for (int i = 0; i < resultColumns.size(); i++) {
13871                        ResultColumn column = resultColumns.get(i);
13872                        if (column.getAlias() != null
13873                                        && DlineageUtil.sameColumnName(column.getAlias(), columnName))
13874                                return column;
13875                        if (column.getName() != null && DlineageUtil.sameColumnName(column.getName(), columnName))
13876                                return column;
13877                        if (column.getName() != null && column.getName().endsWith("*")) {
13878                                if ("*".equals(column.getColumnObject().toString())) {
13879                                        return column;
13880                                } else {
13881                                        TObjectName columnObjectName = getObjectName(column);
13882                                        if (columnObjectName.getTableString() != null
13883                                                        && columnObjectName.getTableString().equals(columnName.getTableString())) {
13884                                                return column;
13885                                        }
13886                                }
13887                        }
13888                }
13889                return null;
13890        }
13891
13892        private void analyzeInsertStmt(TInsertSqlStatement stmt) {
13893                Map<Table, List<TObjectName>> insertTableKeyMap = new LinkedHashMap<Table, List<TObjectName>>();
13894                Map<Table, List<TResultColumn>> insertTableValueMap = new LinkedHashMap<Table, List<TResultColumn>>();
13895                Map<String, List<TableColumn>> tableColumnMap = new LinkedHashMap<String, List<TableColumn>>();
13896                List<Table> inserTables = new ArrayList<Table>();
13897                List<TExpression> expressions = new ArrayList<TExpression>();
13898                boolean hasInsertColumns = false;
13899                
13900                EffectType effectType = EffectType.insert;
13901                if(stmt.getInsertToken()!=null && stmt.getInsertToken().toString().toLowerCase().startsWith("replace")) {
13902                        effectType = EffectType.replace;
13903                }
13904                
13905                if (stmt.getInsertConditions() != null && stmt.getInsertConditions().size() > 0) {
13906                        for (int i = 0; i < stmt.getInsertConditions().size(); i++) {
13907                                TInsertCondition condition = stmt.getInsertConditions().getElement(i);
13908                                if (condition.getCondition() != null) {
13909                                        expressions.add(condition.getCondition());
13910                                }
13911                                for (int j = 0; j < condition.getInsertIntoValues().size(); j++) {
13912                                        TInsertIntoValue value = condition.getInsertIntoValues().getElement(j);
13913                                        TTable table = value.getTable();
13914                                        Table tableModel = modelFactory.createTable(table);
13915
13916                                        inserTables.add(tableModel);
13917                                        List<TObjectName> keyMap = new ArrayList<TObjectName>();
13918                                        List<TResultColumn> valueMap = new ArrayList<TResultColumn>();
13919                                        insertTableKeyMap.put(tableModel, keyMap);
13920                                        insertTableValueMap.put(tableModel, valueMap);
13921
13922                                        List<TableColumn> tableColumns = bindInsertTableColumn(tableModel, value, keyMap, valueMap);
13923                                        if (tableColumnMap.get(DlineageUtil.getIdentifierNormalTableName(table.getFullName())) == null
13924                                                        && !tableColumns.isEmpty()) {
13925                                                tableColumnMap.put(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName()),
13926                                                                tableColumns);
13927                                        }
13928
13929                                        // if (stmt.getSubQuery() != null)
13930                                        {
13931                                                Process process = modelFactory.createProcess(stmt);
13932                                                tableModel.addProcess(process);
13933                                        }
13934                                }
13935                        }
13936                        hasInsertColumns = true;
13937                } else if (stmt.getInsertIntoValues() != null && stmt.getInsertIntoValues().size() > 0) {
13938                        for (int i = 0; i < stmt.getInsertIntoValues().size(); i++) {
13939                                TInsertIntoValue value = stmt.getInsertIntoValues().getElement(i);
13940                                TTable table = value.getTable();
13941                                Table tableModel = modelFactory.createTable(table);
13942
13943                                inserTables.add(tableModel);
13944                                List<TObjectName> keyMap = new ArrayList<TObjectName>();
13945                                List<TResultColumn> valueMap = new ArrayList<TResultColumn>();
13946                                insertTableKeyMap.put(tableModel, keyMap);
13947                                insertTableValueMap.put(tableModel, valueMap);
13948
13949                                List<TableColumn> tableColumns = bindInsertTableColumn(tableModel, value, keyMap, valueMap);
13950                                if (tableColumnMap.get(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName())) == null && !tableColumns.isEmpty()) {
13951                                        tableColumnMap.put(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName()), tableColumns);
13952                                }
13953
13954                                // if (stmt.getSubQuery() != null)
13955                                {
13956                                        Process process = modelFactory.createProcess(stmt);
13957                                        tableModel.addProcess(process);
13958                                }
13959                        }
13960                        hasInsertColumns = true;
13961                } else if (stmt.getColumnList() != null && stmt.getColumnList().size() > 0) {
13962                        TTable table = stmt.getTargetTable();
13963                        Table tableModel = modelFactory.createTable(table);
13964
13965                        inserTables.add(tableModel);
13966                        List<TObjectName> keyMap = new ArrayList<TObjectName>();
13967                        insertTableKeyMap.put(tableModel, keyMap);
13968                        List<TableColumn> tableColumns = new ArrayList<TableColumn>();
13969                        for (int i = 0; i < stmt.getColumnList().size(); i++) {
13970                                TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
13971                                                stmt.getColumnList().getObjectName(i));
13972                                tableColumns.add(tableColumn);
13973                        }
13974                        if (tableColumnMap.get(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName())) == null && !tableColumns.isEmpty()) {
13975                                tableColumnMap.put(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName()), tableColumns);
13976                        }
13977
13978                        // if (stmt.getSubQuery() != null)
13979                        {
13980                                Process process = modelFactory.createProcess(stmt);
13981                                tableModel.addProcess(process);
13982                        }
13983                        hasInsertColumns = true;
13984                } else if (stmt.getOutputClause() != null && stmt.getOutputClause().getSelectItemList().size() > 0) {
13985                        TTable table = stmt.getTargetTable();
13986                        Table tableModel = modelFactory.createTable(table);
13987
13988                        inserTables.add(tableModel);
13989                        List<TObjectName> keyMap = new ArrayList<TObjectName>();
13990                        insertTableKeyMap.put(tableModel, keyMap);
13991                        List<TableColumn> tableColumns = new ArrayList<TableColumn>();
13992                        for (int i = 0; i < stmt.getOutputClause().getSelectItemList().size(); i++) {
13993                                TObjectName columnName = stmt.getOutputClause().getSelectItemList().getResultColumn(i).getFieldAttr();
13994                                if (columnName.getPseudoTableType() != EPseudoTableType.none) {
13995                                        // Phase 1 already swapped tokens; getColumnNameOnly() returns actual column name
13996                                        String column = columnName.getColumnNameOnly();
13997                                        columnName = new TObjectName();
13998                                        columnName.setString(column);
13999                                } else {
14000                                        String column = columnName.toString().toLowerCase();
14001                                        if ((column.startsWith("inserted.") || column.startsWith("deleted."))
14002                                                        && columnName.getPropertyToken() != null) {
14003                                                column = columnName.getPropertyToken().getAstext();
14004                                                columnName = new TObjectName();
14005                                                columnName.setString(column);
14006                                        }
14007                                }
14008                                TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel, columnName);
14009                                tableColumns.add(tableColumn);
14010                        }
14011                        if (tableColumnMap.get(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName())) == null && !tableColumns.isEmpty()) {
14012                                tableColumnMap.put(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName()), tableColumns);
14013                        }
14014
14015                        // if (stmt.getSubQuery() != null)
14016                        {
14017                                Process process = modelFactory.createProcess(stmt);
14018                                tableModel.addProcess(process);
14019                        }
14020                        hasInsertColumns = true;
14021                } else {
14022                        TTable table = stmt.getTargetTable();
14023                        Table tableModel;
14024                        if (table != null) {
14025                                tableModel = modelFactory.createTable(table);
14026                                // if (stmt.getSubQuery() != null)
14027                                {
14028                                        Process process = modelFactory.createProcess(stmt);
14029                                        tableModel.addProcess(process);
14030                                }
14031                                if (tableModel.getColumns() == null || tableModel.getColumns().isEmpty()) {
14032                                        tableModel.addColumnsFromSQLEnv();
14033                                }
14034                        } else if (stmt.getDirectoryName() != null) {
14035                                tableModel = modelFactory.createTableByName(stmt.getDirectoryName(), true);
14036                                tableModel.setPath(true);
14037                                tableModel.setCreateTable(true);
14038                                TObjectName fileUri = new TObjectName();
14039                                fileUri.setString("uri=" + stmt.getDirectoryName());
14040                                TableColumn tableColumn = modelFactory.createFileUri(tableModel, fileUri);
14041                                // if (stmt.getSubQuery() != null)
14042                                {
14043                                        Process process = modelFactory.createProcess(stmt);
14044                                        tableModel.addProcess(process);
14045                                }
14046                        } else {
14047                                ErrorInfo errorInfo = new ErrorInfo();
14048                                errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
14049                                errorInfo.setErrorMessage("Can't get target table. InsertSqlStatement is " + stmt.toString());
14050                                errorInfo.setStartPosition(new Pair3<Long, Long, String>(stmt.getStartToken().lineNo,
14051                                                stmt.getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
14052                                errorInfo.setEndPosition(new Pair3<Long, Long, String>(stmt.getEndToken().lineNo,
14053                                                stmt.getEndToken().columnNo + stmt.getEndToken().getAstext().length(),
14054                                                ModelBindingManager.getGlobalHash()));
14055                                errorInfo.fillInfo(this);
14056                                errorInfos.add(errorInfo);
14057                                return;
14058                        }
14059                        inserTables.add(tableModel);
14060                        if (table != null
14061                                        && tableColumnMap.get(DlineageUtil.getIdentifierNormalTableName(table.getFullName())) == null) {
14062                                if (tableModel.getColumns() != null && !tableModel.getColumns().isEmpty()) {
14063                                        tableColumnMap.put(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName()),
14064                                                        tableModel.getColumns());
14065                                } else {
14066                                        tableColumnMap.put(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName()), null);
14067                                }
14068                        }
14069                }
14070
14071                if (stmt.getSubQuery() != null) {
14072                        analyzeSelectStmt(stmt.getSubQuery());
14073                }
14074
14075                Iterator<Table> tableIter = inserTables.iterator();
14076                while (tableIter.hasNext()) {
14077                        Table tableModel = tableIter.next();
14078                        List<TableColumn> tableColumns = tableColumnMap.get(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName()));
14079                        List<TObjectName> keyMap = insertTableKeyMap.get(tableModel);
14080                        List<TResultColumn> valueMap = insertTableValueMap.get(tableModel);
14081                        boolean initColumn = (hasInsertColumns && tableColumns != null && !containStarColumn(tableColumns));
14082
14083                        if (stmt.getSubQuery() != null) {
14084
14085                                List<TSelectSqlStatement> subquerys = new ArrayList<TSelectSqlStatement>();
14086                                if (stmt.getSubQuery().getResultColumnList() != null || stmt.getSubQuery().getTransformClause() != null) {
14087                                        subquerys.add(stmt.getSubQuery());
14088                                } else if (stmt.getSubQuery().getValueClause() != null
14089                                                && stmt.getSubQuery().getValueClause().getRows() != null) {
14090                                        for (TResultColumnList resultColumnList : stmt.getSubQuery().getValueClause().getRows()) {
14091                                                for(TResultColumn resultColumn: resultColumnList) {
14092                                                        if(resultColumn.getExpr()!=null && resultColumn.getExpr().getSubQuery()!=null) {
14093                                                                analyzeSelectStmt(resultColumn.getExpr().getSubQuery());
14094                                                                subquerys.add(resultColumn.getExpr().getSubQuery());
14095                                                        }
14096                                                }
14097                                        }
14098                                }
14099
14100                                for(TSelectSqlStatement subquery: subquerys) {
14101                                        if ((tableModel.isCreateTable() && tableModel.getColumns() != null)
14102                                                        || (subquery.getSetOperatorType() == ESetOperatorType.none
14103                                                                        && stmt.getColumnList() != null && stmt.getColumnList().size() > 0)) {
14104
14105                                                ResultSet resultSetModel = null;
14106
14107                                                if (subquery != null) {
14108                                                        resultSetModel = (ResultSet) modelManager.getModel(subquery);
14109                                                }
14110
14111                                                TResultColumnList resultset = subquery.getResultColumnList();
14112                                                if (resultSetModel == null && resultset != null) {
14113                                                        resultSetModel = (ResultSet) modelManager.getModel(resultset);
14114                                                }
14115
14116                                                if (resultSetModel == null) {
14117                                                        ErrorInfo errorInfo = new ErrorInfo();
14118                                                        errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
14119                                                        errorInfo.setErrorMessage("Can't get resultset model");
14120                                                        errorInfo.setStartPosition(new Pair3<Long, Long, String>(resultset.getStartToken().lineNo,
14121                                                                        resultset.getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
14122                                                        errorInfo.setEndPosition(new Pair3<Long, Long, String>(resultset.getEndToken().lineNo,
14123                                                                        resultset.getEndToken().columnNo + resultset.getEndToken().getAstext().length(),
14124                                                                        ModelBindingManager.getGlobalHash()));
14125                                                        errorInfos.add(errorInfo);
14126                                                }
14127
14128                                                if (!resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
14129                                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
14130                                                        impactRelation.setEffectType(effectType);
14131                                                        impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
14132                                                                        resultSetModel.getRelationRows()));
14133                                                        impactRelation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(
14134                                                                        tableModel.getRelationRows()));
14135                                                }
14136
14137                                                int resultSetSize = resultSetModel.getColumns().size();
14138                                                int j = 0;
14139                                                int starIndex = 0;
14140                                                TObjectNameList items = stmt.getColumnList();
14141                                                List<String> itemNames = new ArrayList<String>();
14142                                                int starColumnCount = 0;
14143                                                for (ResultColumn item : resultSetModel.getColumns()) {
14144                                                        if (item.getName().endsWith("*")) {
14145                                                                starColumnCount += 1;
14146                                                        }
14147                                                }
14148                                                if (items != null) {
14149                                                        for (int i = 0; i < items.size() && j < resultSetSize; i++) {
14150                                                                TObjectName column = items.getObjectName(i);
14151
14152                                                                if (column.getDbObjectType() == EDbObjectType.variable) {
14153                                                                        continue;
14154                                                                }
14155
14156                                                                if (column.getColumnNameOnly().startsWith("@")
14157                                                                                && (option.getVendor() == EDbVendor.dbvmssql
14158                                                                                                || option.getVendor() == EDbVendor.dbvazuresql)) {
14159                                                                        continue;
14160                                                                }
14161
14162                                                                if (column.getColumnNameOnly().startsWith(":")
14163                                                                                && (option.getVendor() == EDbVendor.dbvhana
14164                                                                                                || option.getVendor() == EDbVendor.dbvteradata)) {
14165                                                                        continue;
14166                                                                }
14167
14168                                                                ResultColumn resultColumn = resultSetModel.getColumns().get(j);
14169                                                                if (!resultSetModel.getColumns().get(j).getName().contains("*")) {
14170                                                                        j++;
14171                                                                } else {
14172                                                                        starIndex++;
14173                                                                        if (resultSetSize - j == items.size() - i) {
14174                                                                                j++;
14175
14176                                                                        }
14177                                                                }
14178                                                                if (column != null) {
14179                                                                        TableColumn tableColumn;
14180                                                                        // if (!initColumn) {
14181                                                                        tableColumn = matchColumn(tableModel.getColumns(), column);
14182                                                                        if (tableColumn == null) {
14183                                                                                if (tableModel.isCreateTable() && !containStarColumn(tableModel.getColumns())) {
14184                                                                                        if (tableModel.getColumns().size() <= i) {
14185                                                                                                continue;
14186                                                                                        }
14187                                                                                        tableColumn = tableModel.getColumns().get(i);
14188                                                                                } else {
14189                                                                                        tableColumn = modelFactory.createTableColumn(tableModel, column, false);
14190                                                                                        if(tableColumn == null) {
14191                                                                                                continue;
14192                                                                                        }
14193                                                                                }
14194                                                                        }
14195//                                                      } else {
14196//                                                              tableColumn = matchColumn(tableColumns, column);
14197//                                                              if (tableColumn == null) {
14198//                                                                      continue;
14199//                                                              }
14200//                                                      }
14201                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
14202                                                                        relation.setEffectType(effectType);
14203                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
14204                                                                        if (resultColumn.hasStarLinkColumn()
14205                                                                                        && resultColumn.getStarLinkColumnNames().size() > starIndex - 1 && starColumnCount<=1) {
14206                                                                                boolean find = false;
14207                                                                                while (resultColumn.getStarLinkColumnNames().size() > starIndex - 1) {
14208                                                                                        TObjectName name = resultColumn.getStarLinkColumnName(starIndex - 1);
14209                                                                                        if (itemNames.contains(name.toString())) {
14210                                                                                                starIndex++;
14211                                                                                                continue;
14212                                                                                        }
14213                                                                                        ResultColumn expandStarColumn = modelFactory
14214                                                                                                        .createResultColumn(resultSetModel, name, false);
14215                                                                                        relation.addSource(new ResultColumnRelationshipElement(expandStarColumn));
14216                                                                                        itemNames.add(resultColumn.getName());
14217                                                                                        find = true;
14218                                                                                        break;
14219                                                                                }
14220                                                                                if (!find) {
14221                                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
14222                                                                                        itemNames.add(resultColumn.getName());
14223                                                                                }
14224                                                                        } else {
14225                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
14226                                                                                itemNames.add(resultColumn.getName());
14227                                                                        }
14228                                                                        Process process = modelFactory.createProcess(stmt);
14229                                                                        relation.setProcess(process);
14230                                                                }
14231                                                        }
14232                                                } else {
14233                                                        List<TableColumn> columns = tableModel.getColumns();
14234                                                        if (columns.size() == 1 && tableModel.isPath()) {
14235                                                                for(int i=0;i<resultSetSize;i++) {
14236                                                                        ResultColumn resultColumn = resultSetModel.getColumns().get(i);
14237                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
14238                                                                        relation.setEffectType(effectType);
14239                                                                        relation.setTarget(new TableColumnRelationshipElement(tableModel.getColumns().get(0)));
14240                                                                        if (resultColumn.hasStarLinkColumn()
14241                                                                                        && resultColumn.getStarLinkColumnNames().size() > starIndex - 1) {
14242                                                                                ResultColumn expandStarColumn = modelFactory.createResultColumn(
14243                                                                                                resultSetModel, resultColumn.getStarLinkColumnName(starIndex - 1),
14244                                                                                                false);
14245                                                                                relation.addSource(new ResultColumnRelationshipElement(expandStarColumn));
14246                                                                        } else {
14247                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
14248                                                                        }
14249                                                                        Process process = modelFactory.createProcess(stmt);
14250                                                                        relation.setProcess(process);
14251                                                                }
14252                                                        } else {
14253                                                                boolean fromStruct = false;
14254                                                                for (int i = 0; i < columns.size() && j < resultSetSize; i++) {
14255                                                                        String column = columns.get(i).getName();
14256                                                                        ResultColumn resultColumn = resultSetModel.getColumns().get(j);
14257                                                                        if (!resultColumn.getName().contains("*")) {
14258                                                                                if (resultColumn.getName().equals(resultColumn.getRefColumnName())
14259                                                                                                && resultColumn.getColumnObject().toString().endsWith("*")
14260                                                                                                && resultSetSize == 1) {
14261                                                                                        starIndex++;
14262                                                                                        if (resultSetSize - j == columns.size() - i) {
14263                                                                                                j++;
14264
14265                                                                                        }
14266                                                                                }
14267                                                                                else {
14268                                                                                        j++;
14269                                                                                }
14270                                                                        } else {
14271                                                                                starIndex++;
14272                                                                                if (resultSetSize - j == columns.size() - i) {
14273                                                                                        j++;
14274
14275                                                                                }
14276                                                                        }
14277                                                                        if (column != null) {
14278                                                                                TableColumn tableColumn;
14279                                                                                // if (!initColumn) {
14280                                                                                tableColumn = matchColumn(tableModel.getColumns(), columns.get(i));
14281                                                                                if (tableColumn == null) {
14282                                                                                        if (tableModel.isCreateTable()
14283                                                                                                        && !containStarColumn(tableModel.getColumns())) {
14284                                                                                                if (tableModel.getColumns().size() <= i) {
14285                                                                                                        continue;
14286                                                                                                }
14287                                                                                                tableColumn = tableModel.getColumns().get(i);
14288                                                                                        } else {
14289                                                                                                TObjectName columnName = new TObjectName();
14290                                                                                                columnName.setString(column);
14291                                                                                                tableColumn = modelFactory.createTableColumn(tableModel, columnName,
14292                                                                                                                false);
14293                                                                                        }
14294                                                                                }
14295                                                                                else if (!resultColumn.isStruct() && tableColumn.isStruct() && columns.size() != resultSetSize) {
14296                                                                                        j--;
14297                                                                                        fromStruct = true;
14298                                                                                }
14299                                                                                if(fromStruct && !tableColumn.isStruct()) {
14300                                                                                        fromStruct = false;
14301                                                                                        resultColumn = resultSetModel.getColumns().get(j);
14302                                                                                        j++;
14303                                                                                }
14304                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
14305                                                                                relation.setEffectType(effectType);
14306                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
14307                                                                                if (resultColumn.hasStarLinkColumn()
14308                                                                                                && resultColumn.getStarLinkColumnNames().size() > starIndex - 1) {
14309                                                                                        ResultColumn expandStarColumn = modelFactory.createResultColumn(
14310                                                                                                        resultSetModel, resultColumn.getStarLinkColumnName(starIndex - 1),
14311                                                                                                        false);
14312                                                                                        relation.addSource(new ResultColumnRelationshipElement(expandStarColumn));
14313                                                                                } else {
14314                                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn, starIndex - 1));
14315                                                                                }
14316                                                                                Process process = modelFactory.createProcess(stmt);
14317                                                                                relation.setProcess(process);
14318                                                                        }
14319                                                                }
14320                                                        }
14321                                                }
14322                                        } else if (!subquery.isCombinedQuery()) {
14323                                                SelectResultSet resultSetModel = (SelectResultSet) modelManager
14324                                                                .getModel(subquery.getResultColumnList() != null ? subquery.getResultColumnList()
14325                                                                                : subquery.getTransformClause());
14326
14327                                                if (resultSetModel != null && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
14328                                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
14329                                                        impactRelation.setEffectType(effectType);
14330                                                        impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
14331                                                                        resultSetModel.getRelationRows()));
14332                                                        impactRelation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(
14333                                                                        tableModel.getRelationRows()));
14334                                                }
14335
14336                                                List<ResultColumn> columnsSnapshot = new ArrayList<ResultColumn>();
14337                                                columnsSnapshot.addAll(resultSetModel.getColumns());
14338                                                if(resultSetModel.isDetermined() && stmt.getColumnList() == null) {
14339                                                        tableModel.setDetermined(true);
14340                                                }
14341                                                for (int i = 0; i < columnsSnapshot.size(); i++) {
14342                                                        ResultColumn resultColumn = columnsSnapshot.get(i);
14343                                                        if (resultColumn.getColumnObject() instanceof TObjectName) {
14344                                                                TableColumn tableColumn;
14345                                                                if (!initColumn) {
14346                                                                        if (tableModel.isCreateTable() && !containStarColumn(tableModel.getColumns())) {
14347                                                                                if (tableModel.getColumns().size() <= i) {
14348                                                                                        continue;
14349                                                                                }
14350                                                                                tableColumn = tableModel.getColumns().get(i);
14351                                                                        } else {
14352                                                                                tableColumn = modelFactory.createInsertTableColumn(tableModel,
14353                                                                                                (TObjectName) resultColumn.getColumnObject());
14354                                                                        }
14355                                                                        if (containStarColumn(tableColumns)) {
14356                                                                                getStarColumn(tableColumns)
14357                                                                                                .bindStarLinkColumn((TObjectName) resultColumn.getColumnObject());
14358                                                                        }
14359                                                                } else {
14360                                                                        TObjectName matchedColumnName = (TObjectName) resultColumn.getColumnObject();
14361                                                                        tableColumn = matchColumn(tableColumns, matchedColumnName);
14362                                                                        if (tableColumn == null) {
14363                                                                                if (!isEmptyCollection(valueMap)) {
14364                                                                                        int index = indexOfColumn(valueMap, matchedColumnName);
14365                                                                                        if (index != -1) {
14366                                                                                                if (!isEmptyCollection(keyMap) && index < keyMap.size()) {
14367                                                                                                        tableColumn = matchColumn(tableColumns, keyMap.get(index));
14368                                                                                                } else if (isEmptyCollection(keyMap) && index < tableColumns.size()) {
14369                                                                                                        tableColumn = tableColumns.get(index);
14370                                                                                                } else {
14371                                                                                                        continue;
14372                                                                                                }
14373                                                                                        } else {
14374                                                                                                continue;
14375                                                                                        }
14376                                                                                } else if (!isEmptyCollection(keyMap) && i < keyMap.size()) {
14377                                                                                        tableColumn = matchColumn(tableColumns, keyMap.get(i));
14378                                                                                } else if (isEmptyCollection(keyMap) && isEmptyCollection(valueMap)
14379                                                                                                && i < tableColumns.size()) {
14380                                                                                        tableColumn = tableColumns.get(i);
14381                                                                                } else {
14382                                                                                        continue;
14383                                                                                }
14384                                                                        }
14385                                                                }
14386
14387                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
14388                                                                relation.setEffectType(effectType);
14389                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
14390                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
14391                                                                Process process = modelFactory.createProcess(stmt);
14392                                                                relation.setProcess(process);
14393                                                        } else {
14394                                                                TAliasClause alias = ((TResultColumn) resultColumn.getColumnObject()).getAliasClause();
14395                                                                if (alias != null && alias.getAliasName() != null) {
14396                                                                        TableColumn tableColumn;
14397                                                                        if (!initColumn) {
14398                                                                                if (tableModel.isCreateTable() && !containStarColumn(tableModel.getColumns())) {
14399                                                                                        if (tableModel.getColumns().size() <= i) {
14400                                                                                                if (tableModel.isPath()) {
14401                                                                                                        tableColumn = tableModel.getColumns().get(0);
14402                                                                                                } else {
14403                                                                                                        continue;
14404                                                                                                }
14405                                                                                        } else {
14406                                                                                                tableColumn = tableModel.getColumns().get(i);
14407                                                                                        }
14408                                                                                } else {
14409                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
14410                                                                                                        alias.getAliasName());
14411                                                                                        if (containStarColumn(resultSetModel)) {
14412                                                                                                tableColumn.notBindStarLinkColumn(true);
14413                                                                                        }
14414                                                                                }
14415                                                                                if (containStarColumn(tableColumns)) {
14416                                                                                        getStarColumn(tableColumns).bindStarLinkColumn(alias.getAliasName());
14417                                                                                }
14418                                                                        } else {
14419                                                                                TObjectName matchedColumnName = alias.getAliasName();
14420                                                                                tableColumn = matchColumn(tableColumns, matchedColumnName);
14421                                                                                if (tableColumn == null) {
14422                                                                                        if (!isEmptyCollection(valueMap)) {
14423                                                                                                int index = indexOfColumn(valueMap, matchedColumnName);
14424                                                                                                if (index != -1) {
14425                                                                                                        if (!isEmptyCollection(keyMap) && index < keyMap.size()) {
14426                                                                                                                tableColumn = matchColumn(tableColumns, keyMap.get(index));
14427                                                                                                        } else if (isEmptyCollection(keyMap)
14428                                                                                                                        && index < tableColumns.size()) {
14429                                                                                                                tableColumn = tableColumns.get(index);
14430                                                                                                        } else {
14431                                                                                                                continue;
14432                                                                                                        }
14433                                                                                                } else {
14434                                                                                                        continue;
14435                                                                                                }
14436                                                                                        } else if (!isEmptyCollection(keyMap) && i < keyMap.size()) {
14437                                                                                                tableColumn = matchColumn(tableColumns, keyMap.get(i));
14438                                                                                        } else {
14439                                                                                                tableColumn = modelFactory.createInsertTableColumn(tableModel,
14440                                                                                                                alias.getAliasName());
14441                                                                                        }
14442                                                                                }
14443
14444                                                                        }
14445
14446                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
14447                                                                        relation.setEffectType(effectType);
14448                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
14449                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
14450                                                                        Process process = modelFactory.createProcess(stmt);
14451                                                                        relation.setProcess(process);
14452
14453                                                                } else if (((TResultColumn) resultColumn.getColumnObject()).getFieldAttr() != null) {
14454                                                                        TObjectName fieldAttr = ((TResultColumn) resultColumn.getColumnObject())
14455                                                                                        .getFieldAttr();
14456
14457                                                                        Object model = modelManager.getModel( resultColumn.getColumnObject());
14458                                                                        
14459                                                                        TableColumn tableColumn;
14460                                                                        if (!initColumn) {
14461                                                                                if (tableModel.isCreateTable() && !containStarColumn(tableModel.getColumns())) {
14462                                                                                        if (fieldAttr.toString().endsWith("*")) {
14463                                                                                                int starIndex = 0;
14464                                                                                                for (TableColumn column : tableModel.getColumns()) {
14465                                                                                                        starIndex++;
14466                                                                                                        DataFlowRelationship relation = modelFactory
14467                                                                                                                        .createDataFlowRelation();
14468                                                                                                        relation.setEffectType(effectType);
14469                                                                                                        relation.setTarget(new TableColumnRelationshipElement(column));
14470                                                                                                        if (resultColumn.getStarLinkColumnList().size() == tableModel
14471                                                                                                                        .getColumns().size()) {
14472                                                                                                                ResultColumn expandStarColumn = modelFactory.createResultColumn(
14473                                                                                                                                resultSetModel,
14474                                                                                                                                resultColumn.getStarLinkColumnList().get(starIndex - 1),
14475                                                                                                                                false);
14476                                                                                                                relation.addSource(
14477                                                                                                                                new ResultColumnRelationshipElement(expandStarColumn));
14478                                                                                                        } else {
14479                                                                                                                relation.addSource(
14480                                                                                                                                new ResultColumnRelationshipElement(resultColumn));
14481                                                                                                        }
14482                                                                                                        Process process = modelFactory.createProcess(stmt);
14483                                                                                                        relation.setProcess(process);
14484                                                                                                }
14485                                                                                                continue;
14486                                                                                        }
14487                                                                                        if (tableModel.getColumns().size() <= i) {
14488                                                                                                continue;
14489                                                                                        }
14490                                                                                        tableColumn = tableModel.getColumns().get(i);
14491                                                                                } else {
14492                                                                                        if(model instanceof LinkedHashMap) {
14493                                                                                                LinkedHashMap<String, ResultColumn> resultColumns = (LinkedHashMap<String, ResultColumn>)model;
14494                                                                                                for(String key: resultColumns.keySet()) {
14495                                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel, resultColumns.get(key).getName());
14496                                                                                                        DataFlowRelationship relation = modelFactory
14497                                                                                                                        .createDataFlowRelation();
14498                                                                                                        relation.setEffectType(effectType);
14499                                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
14500                                                                                                        relation.addSource(
14501                                                                                                                        new ResultColumnRelationshipElement(resultColumns.get(key)));
14502                                                                                                        Process process = modelFactory.createProcess(stmt);
14503                                                                                                        relation.setProcess(process);
14504                                                                                                }
14505                                                                                                continue;
14506                                                                                        }
14507                                                                                        else {
14508                                                                                                tableColumn = modelFactory.createInsertTableColumn(tableModel, fieldAttr);
14509                                                                                        }
14510                                                                                }
14511                                                                        } else if (tableModel.isDetermined() && i < tableModel.getColumns().size()) {
14512                                                                                if (!isEmptyCollection(valueMap)) {
14513                                                                                        TObjectName matchedColumnName = fieldAttr;
14514                                                                                        int index = indexOfColumn(valueMap, matchedColumnName);
14515                                                                                        if (index != -1) {
14516                                                                                                if (!isEmptyCollection(keyMap) && index < keyMap.size()) {
14517                                                                                                        tableColumn = matchColumn(tableColumns, keyMap.get(index));
14518                                                                                                } else if (isEmptyCollection(keyMap)
14519                                                                                                                && index < tableColumns.size()) {
14520                                                                                                        tableColumn = tableColumns.get(index);
14521                                                                                                } else {
14522                                                                                                        continue;
14523                                                                                                }
14524                                                                                        } else {
14525                                                                                                continue;
14526                                                                                        }
14527                                                                                } else if (!isEmptyCollection(keyMap) && i < keyMap.size()) {
14528                                                                                        tableColumn = matchColumn(tableColumns, keyMap.get(i));
14529                                                                                } else {
14530                                                                                        tableColumn = tableModel.getColumns().get(i);
14531                                                                                }
14532                                                                        } else {
14533                                                                                TObjectName matchedColumnName = fieldAttr;
14534                                                                                tableColumn = matchColumn(tableColumns, matchedColumnName);
14535                                                                                if (tableColumn == null) {
14536                                                                                        if (!isEmptyCollection(valueMap)) {
14537                                                                                                int index = indexOfColumn(valueMap, matchedColumnName);
14538                                                                                                if (index != -1) {
14539                                                                                                        if (!isEmptyCollection(keyMap) && index < keyMap.size()) {
14540                                                                                                                tableColumn = matchColumn(tableColumns, keyMap.get(index));
14541                                                                                                        } else if (isEmptyCollection(keyMap)
14542                                                                                                                        && index < tableColumns.size()) {
14543                                                                                                                tableColumn = tableColumns.get(index);
14544                                                                                                        } else {
14545                                                                                                                continue;
14546                                                                                                        }
14547                                                                                                } else {
14548                                                                                                        continue;
14549                                                                                                }
14550                                                                                        } else if (!isEmptyCollection(keyMap) && i < keyMap.size()) {
14551                                                                                                tableColumn = matchColumn(tableColumns, keyMap.get(i));
14552                                                                                        } else {
14553                                                                                                tableColumn = modelFactory.createInsertTableColumn(tableModel,
14554                                                                                                                fieldAttr);
14555                                                                                        }
14556                                                                                }
14557                                                                        }
14558
14559                                                                        if (!"*".equals(getColumnName(tableColumn.getColumnObject()))
14560                                                                                        && "*".equals(getColumnName(fieldAttr))) {
14561                                                                                TObjectName columnObject = fieldAttr;
14562                                                                                TTable sourceTable = columnObject.getSourceTable();
14563                                                                                if (columnObject.getTableToken() != null && sourceTable != null) {
14564                                                                                        TObjectName[] columns = modelManager.getTableColumns(sourceTable);
14565                                                                                        for (int j = 0; j < columns.length; j++) {
14566                                                                                                TObjectName columnName = columns[j];
14567                                                                                                if (columnName == null || "*".equals(getColumnName(columnName))) {
14568                                                                                                        continue;
14569                                                                                                }
14570                                                                                                resultColumn.bindStarLinkColumn(columnName);
14571                                                                                        }
14572                                                                                } else {
14573                                                                                        TTableList tables = stmt.getTables();
14574                                                                                        for (int k = 0; k < tables.size(); k++) {
14575                                                                                                TTable tableElement = tables.getTable(k);
14576                                                                                                TObjectName[] columns = modelManager.getTableColumns(tableElement);
14577                                                                                                for (int j = 0; j < columns.length; j++) {
14578                                                                                                        TObjectName columnName = columns[j];
14579                                                                                                        if (columnName == null || "*".equals(getColumnName(columnName))) {
14580                                                                                                                continue;
14581                                                                                                        }
14582                                                                                                        resultColumn.bindStarLinkColumn(columnName);
14583                                                                                                }
14584                                                                                        }
14585                                                                                }
14586                                                                        }
14587
14588                                                                        if ("*".equals(getColumnName(tableColumn.getColumnObject())) && resultColumn != null
14589                                                                                        && !resultColumn.getStarLinkColumns().isEmpty()) {
14590                                                                                tableColumn.bindStarLinkColumns(resultColumn.getStarLinkColumns());
14591                                                                        }
14592
14593                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
14594                                                                        relation.setEffectType(effectType);
14595                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
14596                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
14597
14598                                                                        if (tableColumn.getName().endsWith("*") && resultColumn.getName().endsWith("*")) {
14599                                                                                tableColumn.getTable().setStarStmt("insert");
14600                                                                        }
14601
14602                                                                        Process process = modelFactory.createProcess(stmt);
14603                                                                        relation.setProcess(process);
14604                                                                } else if (((TResultColumn) resultColumn.getColumnObject()).getExpr()
14605                                                                                .getExpressionType() == EExpressionType.simple_constant_t) {
14606                                                                        if (!initColumn) {
14607                                                                                TableColumn tableColumn;
14608                                                                                if (tableModel.isCreateTable() && !containStarColumn(tableModel.getColumns())) {
14609                                                                                        if (tableModel.getColumns().size() <= i) {
14610                                                                                                continue;
14611                                                                                        }
14612                                                                                        tableColumn = tableModel.getColumns().get(i);
14613                                                                                } else {
14614                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
14615                                                                                                        ((TResultColumn) resultColumn.getColumnObject()).getExpr()
14616                                                                                                                        .getConstantOperand(),
14617                                                                                                        i);
14618                                                                                }
14619
14620                                                                                if (DlineageUtil.isTempTable(tableModel, option.getVendor()) && sqlenv != null
14621                                                                                                && tableModel.getDatabase() != null && tableModel.getSchema() != null) {
14622                                                                                        TSQLSchema schema = sqlenv.getSQLSchema(
14623                                                                                                        tableModel.getDatabase() + "." + tableModel.getSchema(), true);
14624                                                                                        if (schema != null) {
14625                                                                                                TSQLTable tempTable = schema.createTable(
14626                                                                                                                DlineageUtil.getSimpleTableName(tableModel.getName()));
14627                                                                                                tempTable.addColumn(tableColumn.getName());
14628                                                                                        }
14629                                                                                }
14630                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
14631                                                                                relation.setEffectType(effectType);
14632                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
14633                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
14634                                                                                Process process = modelFactory.createProcess(stmt);
14635                                                                                relation.setProcess(process);
14636                                                                        } else {
14637                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
14638                                                                                relation.setEffectType(effectType);
14639                                                                                relation.setTarget(
14640                                                                                                new TableColumnRelationshipElement(tableModel.getColumns().get(i)));
14641                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
14642                                                                                Process process = modelFactory.createProcess(stmt);
14643                                                                                relation.setProcess(process);
14644                                                                        }
14645                                                                } else {
14646                                                                        if (!initColumn) {
14647                                                                                TableColumn tableColumn;
14648                                                                                if (tableModel.isCreateTable() && !containStarColumn(tableModel.getColumns())) {
14649                                                                                        if (tableModel.getColumns().size() <= i) {
14650                                                                                                continue;
14651                                                                                        }
14652                                                                                        tableColumn = tableModel.getColumns().get(i);
14653                                                                                } else {
14654                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
14655                                                                                                        ((TResultColumn) resultColumn.getColumnObject()).getExpr(), i);
14656                                                                                }
14657                                                                                if (DlineageUtil.isTempTable(tableModel, option.getVendor()) && sqlenv != null
14658                                                                                                && tableModel.getDatabase() != null && tableModel.getSchema() != null) {
14659                                                                                        TSQLSchema schema = sqlenv.getSQLSchema(
14660                                                                                                        tableModel.getDatabase() + "." + tableModel.getSchema(), true);
14661                                                                                        if (schema != null) {
14662                                                                                                TSQLTable tempTable = schema.createTable(
14663                                                                                                                DlineageUtil.getSimpleTableName(tableModel.getName()));
14664                                                                                                tempTable.addColumn(tableColumn.getName());
14665                                                                                        }
14666                                                                                }
14667                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
14668                                                                                relation.setEffectType(effectType);
14669                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
14670                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
14671                                                                                Process process = modelFactory.createProcess(stmt);
14672                                                                                relation.setProcess(process);
14673                                                                        } else {
14674                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
14675                                                                                relation.setEffectType(effectType);
14676                                                                                relation.setTarget(
14677                                                                                                new TableColumnRelationshipElement(tableModel.getColumns().get(i)));
14678                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
14679                                                                                Process process = modelFactory.createProcess(stmt);
14680                                                                                relation.setProcess(process);
14681                                                                        }
14682                                                                }
14683                                                        }
14684                                                }
14685                                        } else if (stmt.getSubQuery() != null) {
14686                                                ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt.getSubQuery());
14687                                                if (resultSetModel != null) {
14688
14689                                                        if (!resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
14690                                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
14691                                                                impactRelation.setEffectType(effectType);
14692                                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
14693                                                                                resultSetModel.getRelationRows()));
14694                                                                impactRelation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(
14695                                                                                tableModel.getRelationRows()));
14696                                                        }
14697
14698                                                        if(stmt.getColumnList()!=null && stmt.getColumnList().size()>0) {
14699                                                                for(int i=0;i<stmt.getColumnList().size();i++) {
14700                                                                        TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
14701                                                                                        stmt.getColumnList().getObjectName(i));
14702                                                                }
14703                                                                
14704                                                                List<TableColumn> columns = tableModel.getColumns();
14705                                                                int resultSetSize = resultSetModel.getColumns().size();
14706                                                                int starIndex = 0; 
14707                                                                int j = 0;
14708                                                                boolean fromStruct = false;
14709                                                                for (int i = 0; i < columns.size() && j < resultSetSize; i++) {
14710                                                                        String column = columns.get(i).getName();
14711                                                                        ResultColumn resultColumn = resultSetModel.getColumns().get(j);
14712                                                                        if (!resultColumn.getName().contains("*")) {
14713                                                                                if (resultColumn.getName().equals(resultColumn.getRefColumnName())
14714                                                                                                && resultColumn.getColumnObject().toString().endsWith("*")
14715                                                                                                && resultSetSize == 1) {
14716                                                                                        starIndex++;
14717                                                                                        if (resultSetSize - j == columns.size() - i) {
14718                                                                                                j++;
14719
14720                                                                                        }
14721                                                                                }
14722                                                                                else {
14723                                                                                        j++;
14724                                                                                }
14725                                                                        } else {
14726                                                                                starIndex++;
14727                                                                                if (resultSetSize - j == columns.size() - i) {
14728                                                                                        j++;
14729
14730                                                                                }
14731                                                                        }
14732                                                                        if (column != null) {
14733                                                                                TableColumn tableColumn;
14734                                                                                // if (!initColumn) {
14735                                                                                tableColumn = matchColumn(tableModel.getColumns(), columns.get(i));
14736                                                                                if (tableColumn == null) {
14737                                                                                        if (tableModel.isCreateTable()
14738                                                                                                        && !containStarColumn(tableModel.getColumns())) {
14739                                                                                                if (tableModel.getColumns().size() <= i) {
14740                                                                                                        continue;
14741                                                                                                }
14742                                                                                                tableColumn = tableModel.getColumns().get(i);
14743                                                                                        } else {
14744                                                                                                TObjectName columnName = new TObjectName();
14745                                                                                                columnName.setString(column);
14746                                                                                                tableColumn = modelFactory.createTableColumn(tableModel, columnName,
14747                                                                                                                false);
14748                                                                                        }
14749                                                                                }
14750                                                                                else if (!resultColumn.isStruct() && tableColumn.isStruct() && columns.size() != resultSetSize) {
14751                                                                                        j--;
14752                                                                                        fromStruct = true;
14753                                                                                }
14754                                                                                if(fromStruct && !tableColumn.isStruct()) {
14755                                                                                        fromStruct = false;
14756                                                                                        resultColumn = resultSetModel.getColumns().get(j);
14757                                                                                        j++;
14758                                                                                }
14759                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
14760                                                                                relation.setEffectType(effectType);
14761                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
14762                                                                                if (resultColumn.hasStarLinkColumn()
14763                                                                                                && resultColumn.getStarLinkColumnNames().size() > starIndex - 1) {
14764                                                                                        ResultColumn expandStarColumn = modelFactory.createResultColumn(
14765                                                                                                        resultSetModel, resultColumn.getStarLinkColumnName(starIndex - 1),
14766                                                                                                        false);
14767                                                                                        relation.addSource(new ResultColumnRelationshipElement(expandStarColumn));
14768                                                                                } else {
14769                                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn, starIndex - 1));
14770                                                                                }
14771                                                                                Process process = modelFactory.createProcess(stmt);
14772                                                                                relation.setProcess(process);
14773                                                                        }
14774                                                                }
14775                                                        }
14776                                                        else {
14777                                                                for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
14778                                                                        ResultColumn resultColumn = resultSetModel.getColumns().get(i);
14779                                                                        TAliasClause alias = null;
14780                                                                        if(resultColumn.getColumnObject() instanceof TResultColumn) {
14781                                                                                alias = ((TResultColumn) resultColumn.getColumnObject()).getAliasClause();
14782                                                                        }
14783                                                                        if (stmt.getColumnList() != null) {
14784                                                                                if (i < stmt.getColumnList().size()) {
14785                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
14786                                                                                        relation.setEffectType(effectType);
14787                                                                                        TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
14788                                                                                                        stmt.getColumnList().getObjectName(i));
14789                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
14790                                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
14791                                                                                        Process process = modelFactory.createProcess(stmt);
14792                                                                                        relation.setProcess(process);
14793                                                                                }
14794                                                                        } else {
14795                                                                                if (alias != null && alias.getAliasName() != null) {
14796                                                                                        TableColumn tableColumn;
14797                                                                                        if (!initColumn) {
14798                                                                                                if (tableModel.isCreateTable()
14799                                                                                                                && !containStarColumn(tableModel.getColumns())) {
14800                                                                                                        if (tableModel.getColumns().size() <= i) {
14801                                                                                                                continue;
14802                                                                                                        }
14803                                                                                                        tableColumn = tableModel.getColumns().get(i);
14804                                                                                                } else {
14805                                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
14806                                                                                                                        alias.getAliasName());
14807                                                                                                }
14808                                                                                        } else {
14809                                                                                                tableColumn = matchColumn(tableColumns, alias.getAliasName());
14810                                                                                                if (tableColumn == null) {
14811                                                                                                        continue;
14812                                                                                                }
14813                                                                                        }
14814        
14815                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
14816                                                                                        relation.setEffectType(effectType);
14817                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
14818                                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
14819                                                                                        Process process = modelFactory.createProcess(stmt);
14820                                                                                        relation.setProcess(process);
14821                                                                                } else if (resultColumn.getColumnObject() instanceof TObjectName 
14822                                                                                                || ( resultColumn.getColumnObject() instanceof TResultColumn && ((TResultColumn) resultColumn.getColumnObject())
14823                                                                                                .getFieldAttr() != null)) {
14824                                                                                        TObjectName fieldAttr = null;
14825                                                                                        if (resultColumn.getColumnObject() instanceof TObjectName) {
14826                                                                                                fieldAttr = (TObjectName)resultColumn.getColumnObject();
14827                                                                                        }
14828                                                                                        else if (resultColumn.getColumnObject() instanceof TResultColumn) {
14829                                                                                                fieldAttr = ((TResultColumn) resultColumn.getColumnObject())
14830                                                                                                                .getFieldAttr();
14831                                                                                        }
14832                                
14833                                                                                        TableColumn tableColumn;
14834                                                                                        if (!initColumn) {
14835                                                                                                if (tableModel.isCreateTable()
14836                                                                                                                && !containStarColumn(tableModel.getColumns())) {
14837                                                                                                        if (tableModel.getColumns().size() <= i) {
14838                                                                                                                continue;
14839                                                                                                        }
14840                                                                                                        tableColumn = tableModel.getColumns().get(i);
14841                                                                                                } else {
14842                                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
14843                                                                                                                        fieldAttr);
14844                                                                                                }
14845                                                                                        } else {
14846                                                                                                tableColumn = matchColumn(tableColumns, fieldAttr);
14847                                                                                                if (tableColumn == null) {
14848                                                                                                        continue;
14849                                                                                                }
14850                                                                                        }
14851        
14852                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
14853                                                                                        relation.setEffectType(effectType);
14854                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
14855                                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
14856                                                                                        Process process = modelFactory.createProcess(stmt);
14857                                                                                        relation.setProcess(process);
14858                                                                                } else if (((TResultColumn) resultColumn.getColumnObject()).getExpr()
14859                                                                                                .getExpressionType() == EExpressionType.simple_constant_t) {
14860                                                                                        if (!initColumn) {
14861                                                                                                TableColumn tableColumn;
14862                                                                                                if (tableModel.isCreateTable()
14863                                                                                                                && !containStarColumn(tableModel.getColumns())) {
14864                                                                                                        if (tableModel.getColumns().size() <= i) {
14865                                                                                                                continue;
14866                                                                                                        }
14867                                                                                                        tableColumn = tableModel.getColumns().get(i);
14868                                                                                                } else {
14869                                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
14870                                                                                                                        ((TResultColumn) resultColumn.getColumnObject()).getExpr()
14871                                                                                                                                        .getConstantOperand(),
14872                                                                                                                        i);
14873                                                                                                }
14874                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
14875                                                                                                relation.setEffectType(effectType);
14876                                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
14877                                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
14878                                                                                                Process process = modelFactory.createProcess(stmt);
14879                                                                                                relation.setProcess(process);
14880                                                                                        }
14881                                                                                } else {
14882                                                                                        if (!initColumn) {
14883                                                                                                TableColumn tableColumn;
14884                                                                                                if (tableModel.isCreateTable()
14885                                                                                                                && !containStarColumn(tableModel.getColumns())) {
14886                                                                                                        if (tableModel.getColumns().size() <= i) {
14887                                                                                                                continue;
14888                                                                                                        }
14889                                                                                                        tableColumn = tableModel.getColumns().get(i);
14890                                                                                                } else {
14891                                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
14892                                                                                                                        ((TResultColumn) resultColumn.getColumnObject()).getExpr(), i);
14893                                                                                                }
14894                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
14895                                                                                                relation.setEffectType(effectType);
14896                                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
14897                                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
14898                                                                                                Process process = modelFactory.createProcess(stmt);
14899                                                                                                relation.setProcess(process);
14900                                                                                        }
14901                                                                                }
14902                                                                        }
14903                                                                }
14904                                                        }
14905                                                }
14906                                        }
14907                                }
14908                        } else if (stmt.getColumnList() != null && stmt.getColumnList().size() > 0) {
14909                                TObjectNameList items = stmt.getColumnList();
14910                                TMultiTargetList values = stmt.getValues();
14911                                if (values != null) {
14912                                        for (int k = 0; values != null && k < values.size(); k++) {
14913                                                int j = 0;
14914                                                for (int i = 0; i < items.size(); i++) {
14915                                                        TObjectName column = items.getObjectName(i);
14916                                                        TableColumn tableColumn;
14917                                                        if (!initColumn) {
14918                                                                if (tableModel.isCreateTable() && !containStarColumn(tableModel.getColumns())) {
14919                                                                        if (tableModel.getColumns().size() <= i) {
14920                                                                                continue;
14921                                                                        }
14922                                                                        tableColumn = tableModel.getColumns().get(i);
14923                                                                } else {
14924                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel, column);
14925                                                                }
14926                                                        } else {
14927                                                                tableColumn = matchColumn(tableColumns, column);
14928                                                                if (tableColumn == null) {
14929                                                                        continue;
14930                                                                }
14931                                                        }
14932                                                        TResultColumn columnObject = values.getMultiTarget(k).getColumnList().getResultColumn(j);
14933                                                        if (columnObject == null) {
14934                                                                continue;
14935                                                        }
14936                                                        TExpression valueExpr = columnObject.getExpr();
14937                                                        columnsInExpr visitor = new columnsInExpr();
14938                                                        valueExpr.inOrderTraverse(visitor);
14939                                                        List<TObjectName> objectNames = visitor.getObjectNames();
14940                                                        List<TParseTreeNode> constants = visitor.getConstants();
14941                                                        List<TParseTreeNode> functions = visitor.getFunctions();
14942                                                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
14943
14944                                                        Process process = modelFactory.createProcess(stmt);
14945
14946
14947
14948                                                        if (functions != null && !functions.isEmpty()) {
14949                                                                analyzeFunctionDataFlowRelation(tableColumn, functions, effectType, process);
14950                                                        }
14951
14952                                                        if (subquerys != null && !subquerys.isEmpty()) {
14953                                                                analyzeSubqueryDataFlowRelation(tableColumn, subquerys, effectType, process);
14954                                                        }
14955                                                        if (objectNames != null && !objectNames.isEmpty()) {
14956                                                                analyzeDataFlowRelation(tableColumn, objectNames, null, effectType, functions,
14957                                                                                process, i);
14958                                                        }
14959                                                        //insert into values generate too many constant relations, ignore constant relations.
14960                                                        if (constants != null && !constants.isEmpty()) {
14961                                                                if (!option.isIgnoreInsertIntoValues() || stmt.getParentStmt() != null) {
14962                                                                        analyzeConstantDataFlowRelation(tableColumn, constants, effectType,
14963                                                                                        functions, process);
14964                                                                }
14965                                                        }
14966                                                        j++;
14967                                                }
14968                                        }
14969                                } else if (stmt.getExecuteStmt() != null && stmt.getExecuteStmt().getModuleName() != null) {
14970                                        analyzeCustomSqlStmt(stmt.getExecuteStmt());
14971                                        Procedure procedure = modelManager.getProcedureByName(DlineageUtil
14972                                                        .getIdentifierNormalTableName(stmt.getExecuteStmt().getModuleName().toString()));
14973                                        if ((procedure != null && procedure.getProcedureObject() instanceof TStoredProcedureSqlStatement)
14974                                                        || isSpExecuteExternalScript(stmt.getExecuteStmt().getModuleName())) {
14975                                                List<ResultSet> resultSetModels = calleeResultSetsForInsertExec(stmt.getExecuteStmt(), procedure);
14976                                                if (resultSetModels != null) {
14977                                                        for (ResultSet resultSetModel : resultSetModels) {
14978                                                                if (resultSetModel != null) {
14979                                                                        for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
14980                                                                                ResultColumn resultColumn = resultSetModel.getColumns().get(i);
14981                                                                                Transform transform = new Transform();
14982                                                                                transform.setType(Transform.FUNCTION);
14983                                                                                transform.setCode(stmt.getExecuteStmt().getModuleName());
14984                                                                                resultColumn.setTransform(transform);
14985                                                                                
14986                                                                                if (stmt.getColumnList() != null) {
14987                                                                                        if (i < stmt.getColumnList().size()) {
14988                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
14989                                                                                                relation.setEffectType(effectType);
14990                                                                                                TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
14991                                                                                                                stmt.getColumnList().getObjectName(i));                                                                                         
14992                                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
14993                                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
14994                                                                                                Process process = modelFactory.createProcess(stmt);
14995                                                                                                relation.setProcess(process);                                   
14996                                                                                        }
14997                                                                                } else {
14998                                                                                        if (resultColumn.getColumnObject() instanceof TObjectName) {
14999                                                                                                TObjectName fieldAttr = ((TObjectName) resultColumn.getColumnObject());
15000                                                                                                TableColumn tableColumn;
15001                                                                                                if (!initColumn) {
15002                                                                                                        if (tableModel.isCreateTable()
15003                                                                                                                        && !containStarColumn(tableModel.getColumns())) {
15004                                                                                                                if (tableModel.getColumns().size() <= i) {
15005                                                                                                                        continue;
15006                                                                                                                }
15007                                                                                                                tableColumn = tableModel.getColumns().get(i);
15008                                                                                                        } else {
15009                                                                                                                tableColumn = modelFactory.createInsertTableColumn(tableModel,
15010                                                                                                                                fieldAttr);
15011                                                                                                        }
15012                                                                                                } else {
15013                                                                                                        tableColumn = matchColumn(tableColumns, fieldAttr);
15014                                                                                                        if (tableColumn == null) {
15015                                                                                                                continue;
15016                                                                                                        }
15017                                                                                                }
15018        
15019                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
15020                                                                                                relation.setEffectType(effectType);
15021                                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
15022                                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
15023                                                                                                Process process = modelFactory.createProcess(stmt);
15024                                                                                                relation.setProcess(process);
15025
15026                                                                                                
15027                                                                                        } else {
15028                                                                                                TAliasClause alias = ((TResultColumn) resultColumn.getColumnObject())
15029                                                                                                                .getAliasClause();
15030                                                                                                if (alias != null && alias.getAliasName() != null) {
15031                                                                                                        TableColumn tableColumn;
15032                                                                                                        if (!initColumn) {
15033                                                                                                                if (tableModel.isCreateTable()
15034                                                                                                                                && !containStarColumn(tableModel.getColumns())) {
15035                                                                                                                        if (tableModel.getColumns().size() <= i) {
15036                                                                                                                                continue;
15037                                                                                                                        }
15038                                                                                                                        tableColumn = tableModel.getColumns().get(i);
15039                                                                                                                } else {
15040                                                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
15041                                                                                                                                        alias.getAliasName());
15042                                                                                                                }
15043                                                                                                        } else {
15044                                                                                                                tableColumn = matchColumn(tableColumns, alias.getAliasName());
15045                                                                                                                if (tableColumn == null) {
15046                                                                                                                        continue;
15047                                                                                                                }
15048                                                                                                        }
15049                                                                                                        
15050                                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
15051                                                                                                        relation.setEffectType(effectType);
15052                                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
15053                                                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
15054                                                                                                        Process process = modelFactory.createProcess(stmt);
15055                                                                                                        relation.setProcess(process);
15056                                                                                                } else if (((TResultColumn) resultColumn.getColumnObject())
15057                                                                                                                .getFieldAttr() != null) {
15058                                                                                                        TObjectName fieldAttr = ((TResultColumn) resultColumn.getColumnObject())
15059                                                                                                                        .getFieldAttr();
15060                                                                                                        TableColumn tableColumn;
15061                                                                                                        if (!initColumn) {
15062                                                                                                                if (tableModel.isCreateTable()
15063                                                                                                                                && !containStarColumn(tableModel.getColumns())) {
15064                                                                                                                        if (tableModel.getColumns().size() <= i) {
15065                                                                                                                                continue;
15066                                                                                                                        }
15067                                                                                                                        tableColumn = tableModel.getColumns().get(i);
15068                                                                                                                } else {
15069                                                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
15070                                                                                                                                        fieldAttr);
15071                                                                                                                }
15072                                                                                                        } else {
15073                                                                                                                tableColumn = matchColumn(tableColumns, fieldAttr);
15074                                                                                                                if (tableColumn == null) {
15075                                                                                                                        continue;
15076                                                                                                                }
15077                                                                                                        }
15078                                                                                                        
15079                                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
15080                                                                                                        relation.setEffectType(effectType);
15081                                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
15082                                                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
15083                                                                                                        Process process = modelFactory.createProcess(stmt);
15084                                                                                                        relation.setProcess(process);                                                                                                   
15085                                                                                                } else if (((TResultColumn) resultColumn.getColumnObject()).getExpr()
15086                                                                                                                .getExpressionType() == EExpressionType.simple_constant_t) {
15087                                                                                                        if (!initColumn) {
15088                                                                                                                TableColumn tableColumn;
15089                                                                                                                if (tableModel.isCreateTable()
15090                                                                                                                                && !containStarColumn(tableModel.getColumns())) {
15091                                                                                                                        if (tableModel.getColumns().size() <= i) {
15092                                                                                                                                continue;
15093                                                                                                                        }
15094                                                                                                                        tableColumn = tableModel.getColumns().get(i);
15095                                                                                                                } else {
15096                                                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
15097                                                                                                                                        ((TResultColumn) resultColumn.getColumnObject()).getExpr()
15098                                                                                                                                                        .getConstantOperand(),
15099                                                                                                                                        i);
15100                                                                                                                }
15101                                                                                                                
15102                                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
15103                                                                                                                relation.setEffectType(effectType);
15104                                                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
15105                                                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
15106                                                                                                                Process process = modelFactory.createProcess(stmt);
15107                                                                                                                relation.setProcess(process);
15108                                                                                                        }
15109                                                                                                } else {
15110                                                                                                        if (!initColumn) {
15111                                                                                                                TableColumn tableColumn;
15112                                                                                                                if (tableModel.isCreateTable()
15113                                                                                                                                && !containStarColumn(tableModel.getColumns())) {
15114                                                                                                                        if (tableModel.getColumns().size() <= i) {
15115                                                                                                                                continue;
15116                                                                                                                        }
15117                                                                                                                        tableColumn = tableModel.getColumns().get(i);
15118                                                                                                                } else {
15119                                                                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
15120                                                                                                                                        ((TResultColumn) resultColumn.getColumnObject()).getExpr(),
15121                                                                                                                                        i);
15122                                                                                                                }
15123                                                                                                                
15124                                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
15125                                                                                                                relation.setEffectType(effectType);
15126                                                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
15127                                                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
15128                                                                                                                Process process = modelFactory.createProcess(stmt);
15129                                                                                                                relation.setProcess(process);
15130                                                                                                        }
15131                                                                                                }
15132                                                                                        }
15133                                                                                }
15134                                                                        }
15135                                                                }
15136                                                        }
15137                                                }
15138                                        }
15139                                        else if (procedure!=null && procedure.getProcedureObject() instanceof TObjectName) {
15140                                                TObjectName functionName = new TObjectName();
15141                                                functionName.setString(procedure.getName());
15142                                                Function function = (Function)createFunction(functionName);
15143                                                if (stmt.getColumnList() != null) {
15144                                                        for (int i = 0; i < stmt.getColumnList().size(); i++) {
15145                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
15146                                                                relation.setEffectType(effectType);
15147                                                                TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
15148                                                                                stmt.getColumnList().getObjectName(i));
15149                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
15150                                                                relation.addSource(new ResultColumnRelationshipElement(function.getColumns().get(0)));
15151                                                                Process process = modelFactory.createProcess(stmt);
15152                                                                relation.setProcess(process);
15153                                                        }
15154                                                }
15155                                        }
15156                                } else if (stmt.getExecuteStmt() != null) {
15157                                        // INSERT INTO t EXEC(@sql) / EXEC('...'): the string form has no module name,
15158                                        // so the branch above never saw it - no dynamic site was recorded and the
15159                                        // inner statement was never analyzed. Analyze the EXEC like its top-level
15160                                        // twin (site + inner lineage); the INSERT target itself stays unlinked.
15161                                        analyzeCustomSqlStmt(stmt.getExecuteStmt());
15162                                }
15163                        } else if (stmt.getValues() != null && stmt.getValues().size() > 0 && tableModel.isCreateTable() && !tableModel.getColumns().isEmpty()) {
15164                                for (int k = 0; stmt.getValues() != null && k < stmt.getValues().size(); k++) {
15165                                        TResultColumnList columns = stmt.getValues().getMultiTarget(k).getColumnList();
15166                                        boolean allConstant = true;
15167                                        Process process = modelFactory.createProcess(stmt);
15168                                        int columnSize = tableModel.getColumns().size();
15169                                        for (int x = 0; x < columns.size(); x++) {
15170                                                if (x >= columnSize) {
15171                                                        break;
15172                                                }
15173                                                TableColumn tableColumn = tableModel.getColumns().get(x);
15174                                                TResultColumn columnObject = columns.getResultColumn(x);
15175                                                if (columnObject == null) {
15176                                                        continue;
15177                                                }
15178                                                TExpression valueExpr = columnObject.getExpr();
15179                                                columnsInExpr visitor = new columnsInExpr();
15180                                                valueExpr.inOrderTraverse(visitor);
15181                                                List<TObjectName> objectNames = visitor.getObjectNames();
15182                                                List<TParseTreeNode> constants = visitor.getConstants();
15183                                                List<TParseTreeNode> functions = visitor.getFunctions();
15184                                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
15185
15186                                                if (functions != null && !functions.isEmpty()) {
15187                                                        analyzeFunctionDataFlowRelation(tableColumn, functions, effectType, process);
15188                                                        allConstant = false;
15189                                                }
15190
15191                                                if (subquerys != null && !subquerys.isEmpty()) {
15192                                                        analyzeSubqueryDataFlowRelation(tableColumn, subquerys, effectType, process);
15193                                                        allConstant = false;
15194                                                }
15195                                                if (objectNames != null && !objectNames.isEmpty()) {
15196                                                        analyzeDataFlowRelation(tableColumn, objectNames, null, effectType, functions,
15197                                                                        process);
15198                                                        allConstant = false;
15199                                                }
15200                                                //insert into values generate too many constant relations, ignore constant relations.
15201                                                if (constants != null && !constants.isEmpty() && stmt.getParentStmt() != null) {
15202                                                        analyzeConstantDataFlowRelation(tableColumn, constants, effectType, functions,
15203                                                                        process);
15204                                                        allConstant = false;
15205                                                }
15206                                        }
15207                                        
15208                                        if(allConstant) {
15209                                                modelManager.unbindProcessModel(stmt);
15210                                                tableModel.removeProcess(process);
15211                                        }
15212                                }
15213                        } else if (stmt.getRecordName() != null) {
15214                                String procedureName = DlineageUtil.getProcedureParentName(stmt);
15215                                String variableString = stmt.getRecordName().toString();
15216                                if (variableString.startsWith(":")) {
15217                                        variableString = variableString.substring(variableString.indexOf(":") + 1);
15218                                }
15219                                if (!SQLUtil.isEmpty(procedureName)) {
15220                                        variableString = procedureName + "." + SQLUtil.getIdentifierNormalTableName(variableString);
15221                                }
15222                                
15223                                Table recordTable = modelManager
15224                                                .getTableByName(DlineageUtil.getTableFullName(variableString));
15225                                if (recordTable != null) {
15226                                        for (int i = 0; i < recordTable.getColumns().size(); i++) {
15227                                                TableColumn sourceTableColumn = recordTable.getColumns().get(i);
15228                                                TableColumn targetTableColumn = modelFactory.createTableColumn(tableModel,
15229                                                                sourceTableColumn.getColumnObject(), false);
15230                                                if (targetTableColumn != null) {
15231                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
15232                                                        relation.setEffectType(effectType);
15233                                                        relation.setTarget(new TableColumnRelationshipElement(targetTableColumn));
15234                                                        relation.addSource(new TableColumnRelationshipElement(sourceTableColumn));
15235                                                        Process process = modelFactory.createProcess(stmt);
15236                                                        relation.setProcess(process);
15237                                                } else if (sourceTableColumn.getName().endsWith("*") && tableModel.isCreateTable()) {
15238                                                        for (TableColumn column : tableModel.getColumns()) {
15239                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
15240                                                                relation.setEffectType(effectType);
15241                                                                relation.setTarget(new TableColumnRelationshipElement(column));
15242                                                                relation.addSource(new TableColumnRelationshipElement(sourceTableColumn));
15243                                                                Process process = modelFactory.createProcess(stmt);
15244                                                                relation.setProcess(process);
15245                                                        }
15246                                                }
15247                                        }
15248                                }
15249                        } else if (stmt.getInsertSource() == EInsertSource.values_function && stmt.getFunctionCall() != null) {
15250                                Table cursor = modelManager.getTableByName(
15251                                                DlineageUtil.getTableFullName(stmt.getFunctionCall().getFunctionName().toString()));
15252                                if (cursor != null) {
15253                                        TObjectName starColumn = new TObjectName();
15254                                        starColumn.setString("*");
15255                                        TableColumn insertColumn = modelFactory.createTableColumn(tableModel, starColumn, true);
15256                                        insertColumn.setShowStar(false);
15257                                        insertColumn.setExpandStar(true);
15258                                        for (int j = 0; j < cursor.getColumns().size(); j++) {
15259                                                DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation();
15260                                                dataflowRelation.setEffectType(effectType);
15261                                                dataflowRelation.addSource(new TableColumnRelationshipElement(cursor.getColumns().get(j)));
15262                                                dataflowRelation.setTarget(new TableColumnRelationshipElement(insertColumn));
15263                                                Process process = modelFactory.createProcess(stmt);
15264                                                dataflowRelation.setProcess(process);
15265                                        }
15266                                }
15267
15268                        } else if (stmt.getInsertSource() == EInsertSource.values && stmt.getValues() != null) {
15269                                TObjectName starColumn = new TObjectName();
15270                                starColumn.setString("*");
15271                                TableColumn insertColumn = modelFactory.createTableColumn(tableModel, starColumn, true);
15272                                insertColumn.setShowStar(false);
15273                                insertColumn.setExpandStar(true);
15274                                for (int k = 0; stmt.getValues() != null && k < stmt.getValues().size(); k++) {
15275                                        TResultColumnList columns = stmt.getValues().getMultiTarget(k).getColumnList();
15276                                        for (int x = 0; x < columns.size(); x++) {
15277                                                TResultColumn columnObject = columns.getResultColumn(x);
15278                                                if (columnObject == null) {
15279                                                        continue;
15280                                                }
15281                                                TExpression valueExpr = columnObject.getExpr();
15282                                                columnsInExpr visitor = new columnsInExpr();
15283                                                valueExpr.inOrderTraverse(visitor);
15284                                                List<TObjectName> objectNames = visitor.getObjectNames();
15285                                                List<TParseTreeNode> constants = visitor.getConstants();
15286                                                List<TParseTreeNode> functions = visitor.getFunctions();
15287                                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
15288
15289                                                Process process = modelFactory.createProcess(stmt);
15290                                                if (functions != null && !functions.isEmpty()) {
15291                                                        analyzeFunctionDataFlowRelation(insertColumn, functions, effectType, process);
15292                                                }
15293
15294                                                if (subquerys != null && !subquerys.isEmpty()) {
15295                                                        analyzeSubqueryDataFlowRelation(insertColumn, subquerys, effectType, process);
15296                                                }
15297                                                if (objectNames != null && !objectNames.isEmpty()) {
15298                                                        analyzeDataFlowRelation(insertColumn, objectNames, null, effectType, functions,
15299                                                                        process);
15300                                                }
15301                                                //insert into values generate too many constant relations, ignore constant relations.
15302                                                if (constants != null && !constants.isEmpty() && stmt.getParentStmt() != null) {
15303                                                        analyzeConstantDataFlowRelation(insertColumn, constants, effectType, functions,
15304                                                                        process);
15305                                                }
15306                                        }
15307                                }
15308                        }else if (stmt.getExecuteStmt() != null && stmt.getExecuteStmt().getModuleName() != null) {
15309                                analyzeCustomSqlStmt(stmt.getExecuteStmt());
15310                                Procedure procedure = modelManager.getProcedureByName(DlineageUtil
15311                                                .getIdentifierNormalTableName(stmt.getExecuteStmt().getModuleName().toString()));
15312                                if ((procedure != null && procedure.getProcedureObject() instanceof TStoredProcedureSqlStatement)
15313                                                || isSpExecuteExternalScript(stmt.getExecuteStmt().getModuleName())) {
15314                                        List<ResultSet> resultSetModels = calleeResultSetsForInsertExec(stmt.getExecuteStmt(), procedure);
15315                                        if (resultSetModels != null) {
15316                                                for (ResultSet resultSetModel : resultSetModels) {
15317                                                        if (resultSetModel != null) {
15318                                                                for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
15319                                                                        ResultColumn resultColumn = resultSetModel.getColumns().get(i);
15320                                                                        
15321                                                                        Transform transform = new Transform();
15322                                                                        transform.setType(Transform.FUNCTION);
15323                                                                        transform.setCode(stmt.getExecuteStmt().getModuleName());
15324                                                                        resultColumn.setTransform(transform);
15325                                                                        
15326                                                                        TAliasClause alias = null;
15327
15328                                                                        if (resultColumn.getColumnObject() instanceof TResultColumn) {
15329                                                                                alias = ((TResultColumn) resultColumn.getColumnObject())
15330                                                                                                .getAliasClause();
15331                                                                        }
15332                                                                        
15333                                                                        if (alias != null && alias.getAliasName() != null) {
15334                                                                                TableColumn tableColumn;
15335                                                                                if (!initColumn) {
15336                                                                                        if (tableModel.isCreateTable()
15337                                                                                                        && !containStarColumn(tableModel.getColumns())) {
15338                                                                                                if(resultColumn.getName().endsWith("*")) {
15339                                                                                                        for(TableColumn tableColumnItem: tableModel.getColumns()) {
15340                                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
15341                                                                                                                relation.setEffectType(effectType);
15342                                                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumnItem));
15343                                                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
15344                                                                                                                Process process = modelFactory.createProcess(stmt);
15345                                                                                                                relation.setProcess(process);
15346                                                                                                        }
15347                                                                                                        continue;
15348                                                                                                }
15349                                                                                                else {
15350                                                                                                        if (tableModel.getColumns().size() <= i) {
15351                                                                                                                continue;
15352                                                                                                        }
15353                                                                                                        tableColumn = tableModel.getColumns().get(i);
15354                                                                                                }
15355                                                                                        } else {
15356                                                                                                tableColumn = modelFactory.createInsertTableColumn(tableModel,
15357                                                                                                                alias.getAliasName());
15358                                                                                        }
15359                                                                                } else {
15360                                                                                        tableColumn = matchColumn(tableColumns, alias.getAliasName());
15361                                                                                        if (tableColumn == null) {
15362                                                                                                continue;
15363                                                                                        }
15364                                                                                }
15365                                        
15366                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
15367                                                                                relation.setEffectType(effectType);
15368                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
15369                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
15370                                                                                Process process = modelFactory.createProcess(stmt);
15371                                                                                relation.setProcess(process);
15372                                                                        } else if (resultColumn.getColumnObject() instanceof TObjectName
15373                                                                                        || (resultColumn.getColumnObject() instanceof TResultColumn
15374                                                                                                        && ((TResultColumn) resultColumn.getColumnObject())
15375                                                                                                                        .getFieldAttr() != null)) {
15376                                                                                TObjectName fieldAttr = null;
15377                                                                                if (resultColumn.getColumnObject() instanceof TObjectName) {
15378                                                                                        fieldAttr = (TObjectName) resultColumn.getColumnObject();
15379                                                                                } else {
15380                                                                                        fieldAttr = ((TResultColumn) resultColumn.getColumnObject()).getFieldAttr();
15381                                                                                }
15382                                                                                TableColumn tableColumn;
15383                                                                                if (!initColumn) {
15384                                                                                        if (tableModel.isCreateTable()
15385                                                                                                        && !containStarColumn(tableModel.getColumns())) {
15386                                                                                                if(resultColumn.getName().endsWith("*")) {
15387                                                                                                        for(TableColumn tableColumnItem: tableModel.getColumns()) {
15388                                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
15389                                                                                                                relation.setEffectType(effectType);
15390                                                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumnItem));
15391                                                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
15392                                                                                                                Process process = modelFactory.createProcess(stmt);
15393                                                                                                                relation.setProcess(process);
15394                                                                                                        }
15395                                                                                                        continue;
15396                                                                                                }
15397                                                                                                else {
15398                                                                                                        if (tableModel.getColumns().size() <= i) {
15399                                                                                                                continue;
15400                                                                                                        }
15401                                                                                                        tableColumn = tableModel.getColumns().get(i);
15402                                                                                                }
15403                                                                                        } else {
15404                                                                                                tableColumn = modelFactory.createInsertTableColumn(tableModel,
15405                                                                                                                fieldAttr);
15406                                                                                        }
15407                                                                                } else {
15408                                                                                        tableColumn = matchColumn(tableColumns, fieldAttr);
15409                                                                                        if (tableColumn == null) {
15410                                                                                                continue;
15411                                                                                        }
15412                                                                                }
15413                                                                                
15414                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
15415                                                                                relation.setEffectType(effectType);
15416                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
15417                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
15418                                                                                Process process = modelFactory.createProcess(stmt);
15419                                                                                relation.setProcess(process);
15420                                                                        } else if (((TResultColumn) resultColumn.getColumnObject()).getExpr()
15421                                                                                        .getExpressionType() == EExpressionType.simple_constant_t) {
15422                                                                                if (!initColumn) {
15423                                                                                        TableColumn tableColumn;
15424                                                                                        if (tableModel.isCreateTable()
15425                                                                                                        && !containStarColumn(tableModel.getColumns())) {
15426                                                                                                if(resultColumn.getName().endsWith("*")) {
15427                                                                                                        for(TableColumn tableColumnItem: tableModel.getColumns()) {
15428                                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
15429                                                                                                                relation.setEffectType(effectType);
15430                                                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumnItem));
15431                                                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
15432                                                                                                                Process process = modelFactory.createProcess(stmt);
15433                                                                                                                relation.setProcess(process);
15434                                                                                                        }
15435                                                                                                        continue;
15436                                                                                                }
15437                                                                                                else {
15438                                                                                                        if (tableModel.getColumns().size() <= i) {
15439                                                                                                                continue;
15440                                                                                                        }
15441                                                                                                        tableColumn = tableModel.getColumns().get(i);
15442                                                                                                }
15443                                                                                        } else {
15444                                                                                                tableColumn = modelFactory.createInsertTableColumn(tableModel,
15445                                                                                                                ((TResultColumn) resultColumn.getColumnObject()).getExpr()
15446                                                                                                                                .getConstantOperand(),
15447                                                                                                                i);
15448                                                                                        }
15449                                                                                        
15450                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
15451                                                                                        relation.setEffectType(effectType);
15452                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
15453                                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
15454                                                                                        Process process = modelFactory.createProcess(stmt);
15455                                                                                        relation.setProcess(process);
15456                                                                                }
15457                                                                        } else {
15458                                                                                if (!initColumn) {
15459                                                                                        TableColumn tableColumn;
15460                                                                                        if (tableModel.isCreateTable()
15461                                                                                                        && !containStarColumn(tableModel.getColumns())) {
15462                                                                                                if(resultColumn.getName().endsWith("*")) {
15463                                                                                                        for(TableColumn tableColumnItem: tableModel.getColumns()) {
15464                                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
15465                                                                                                                relation.setEffectType(effectType);
15466                                                                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumnItem));
15467                                                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
15468                                                                                                                Process process = modelFactory.createProcess(stmt);
15469                                                                                                                relation.setProcess(process);
15470                                                                                                        }
15471                                                                                                        continue;
15472                                                                                                }
15473                                                                                                else {
15474                                                                                                        if (tableModel.getColumns().size() <= i) {
15475                                                                                                                continue;
15476                                                                                                        }
15477                                                                                                        tableColumn = tableModel.getColumns().get(i);
15478                                                                                                }
15479                                                                                        } else {
15480                                                                                                tableColumn = modelFactory.createInsertTableColumn(tableModel,
15481                                                                                                                ((TResultColumn) resultColumn.getColumnObject()).getExpr(),
15482                                                                                                                i);
15483                                                                                        }
15484                                                                                        
15485                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
15486                                                                                        relation.setEffectType(effectType);
15487                                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
15488                                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
15489                                                                                        Process process = modelFactory.createProcess(stmt);
15490                                                                                        relation.setProcess(process);
15491                                                                                }
15492                                                                        }
15493                                                                }
15494                                                        }
15495                                                }
15496                                        }
15497                                }
15498                                else if (procedure!=null && procedure.getProcedureObject() instanceof TObjectName) {
15499                                        TObjectName functionName = new TObjectName();
15500                                        functionName.setString(procedure.getName());
15501                                        Function function = (Function)createFunction(functionName);
15502                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
15503                                        relation.setEffectType(effectType);
15504                                        TObjectName starColumn = new TObjectName();
15505                                        starColumn.setString("*");
15506                                        TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
15507                                                        starColumn);
15508                                        tableColumn.setExpandStar(false);
15509                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
15510                                        relation.addSource(new ResultColumnRelationshipElement(function.getColumns().get(0)));
15511                                        Process process = modelFactory.createProcess(stmt);
15512                                        relation.setProcess(process);
15513                                }
15514                        } else if (stmt.getExecuteStmt() != null) {
15515                                // INSERT INTO t EXEC(@sql) / EXEC('...'): the string form has no module name,
15516                                // so the branch above never saw it - no dynamic site was recorded and the
15517                                // inner statement was never analyzed. Analyze the EXEC like its top-level
15518                                // twin (site + inner lineage); the INSERT target itself stays unlinked.
15519                                analyzeCustomSqlStmt(stmt.getExecuteStmt());
15520                        }
15521                }
15522                
15523                if(stmt.getOnDuplicateKeyUpdate()!=null) {
15524                        TTable table = stmt.getTargetTable();
15525                        Table tableModel = modelFactory.createTable(table);
15526                        for(TResultColumn column: stmt.getOnDuplicateKeyUpdate()) {
15527                                if(column.getExpr()==null || column.getExpr().getExpressionType() != EExpressionType.assignment_t) {
15528                                        continue;
15529                                }
15530                                TExpression left = column.getExpr().getLeftOperand();
15531                                TExpression right = column.getExpr().getRightOperand();
15532                                TObjectName columnObject = left.getObjectOperand();
15533                                if (columnObject != null) {
15534                                        TableColumn tableColumn = modelFactory.createTableColumn(tableModel, columnObject, false);
15535                                        if (tableColumn != null) {
15536                                                columnsInExpr visitor = new columnsInExpr();
15537                                                right.inOrderTraverse(visitor);
15538                                                List<TObjectName> objectNames = visitor.getObjectNames();
15539                                                List<TParseTreeNode> functions = visitor.getFunctions();
15540                                                List<TParseTreeNode> constants = visitor.getConstants();
15541                                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
15542
15543                                                if (functions != null && !functions.isEmpty()) {
15544                                                        analyzeFunctionDataFlowRelation(tableColumn, functions, EffectType.update);
15545                                                }
15546                                                if (subquerys != null && !subquerys.isEmpty()) {
15547                                                        analyzeSubqueryDataFlowRelation(tableColumn, subquerys, EffectType.update);
15548                                                }
15549                                                if (objectNames != null && !objectNames.isEmpty()) {
15550                                                        analyzeDataFlowRelation(tableColumn, objectNames, EffectType.update, functions);
15551                                                }
15552                                                if (constants != null && !constants.isEmpty()) {
15553                                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.update, functions);
15554                                                }
15555                                        }
15556                                }
15557                        }
15558                }
15559
15560                if (!expressions.isEmpty() && stmt.getSubQuery() != null) {
15561                        analyzeInsertImpactRelation(stmt.getSubQuery(), tableColumnMap, expressions, effectType);
15562                }
15563        }
15564
15565        /**
15566         * Check whether a SELECT statement actually returns a result set to the client.
15567         * A SELECT does NOT return a result set when:
15568         * <ul>
15569         *   <li>It has an INTO clause ({@code SELECT ... INTO @var})</li>
15570         *   <li>It has an INTO TABLE clause ({@code SELECT ... INTO TABLE t})</li>
15571         *   <li>All result columns are variable assignments ({@code SELECT @var = expr}) —
15572         *       MSSQL-only, detected via {@code EExpressionType.sqlserver_proprietary_column_alias_t}</li>
15573         * </ul>
15574         */
15575        private static boolean isResultSetReturningSelect(TSelectSqlStatement select) {
15576                if (select.getIntoClause() != null || select.getIntoTableClause() != null) {
15577                        return false;
15578                }
15579                if (select.getResultColumnList() != null && select.getResultColumnList().size() > 0) {
15580                        boolean allAssignments = true;
15581                        for (int i = 0; i < select.getResultColumnList().size(); i++) {
15582                                TResultColumn col = select.getResultColumnList().getResultColumn(i);
15583                                if (col.getExpr() == null
15584                                                || col.getExpr().getExpressionType() != EExpressionType.sqlserver_proprietary_column_alias_t) {
15585                                        allAssignments = false;
15586                                        break;
15587                                }
15588                        }
15589                        if (allAssignments) {
15590                                return false;
15591                        }
15592                }
15593                return true;
15594        }
15595
15596        private List<TSelectSqlStatement> getLastSelectStmt(TStoredProcedureSqlStatement procedureStmt) {
15597                List<TSelectSqlStatement> stmts = new ArrayList<TSelectSqlStatement>();
15598                if (procedureStmt.getBodyStatements().size() > 0) {
15599                        for (int j = procedureStmt.getBodyStatements().size() - 1; j >= 0; j--) {
15600                                TCustomSqlStatement stmtItem = procedureStmt.getBodyStatements().get(j);
15601                                if (stmtItem instanceof TReturnStmt || stmtItem instanceof TMssqlReturn) {
15602                                        if (stmtItem.getStatements() != null) {
15603                                                List<TSelectSqlStatement> item = getLastSelectStmt(stmtItem);
15604                                                if (item != null && !item.isEmpty()) {
15605                                                        stmts.addAll(item);
15606                                                        if(option.getVendor()!=EDbVendor.dbvmssql && option.getVendor()!=EDbVendor.dbvazuresql) {
15607                                                                break;
15608                                                        }
15609                                                }
15610                                        }
15611                                        break;
15612                                }
15613                                if (stmtItem instanceof TSelectSqlStatement && !((TSelectSqlStatement) stmtItem).isQueryOfCTE()) {
15614                                        if (isResultSetReturningSelect((TSelectSqlStatement) stmtItem)) {
15615                                                if(!(DlineageUtil.getTopBasicStmt(stmtItem) instanceof TSelectSqlStatement)) {
15616                                                        continue;
15617                                                }
15618                                                stmts.add((TSelectSqlStatement) stmtItem);
15619                                                if(option.getVendor()!=EDbVendor.dbvmssql && option.getVendor()!=EDbVendor.dbvazuresql) {
15620                                                        break;
15621                                                }
15622                                        }
15623                                } else if (stmtItem.getStatements() != null) {
15624                                        List<TSelectSqlStatement> item = getLastSelectStmt(stmtItem);
15625                                        if (item != null && !item.isEmpty()) {
15626                                                stmts.addAll(item);
15627                                                if(option.getVendor()!=EDbVendor.dbvmssql && option.getVendor()!=EDbVendor.dbvazuresql) {
15628                                                        break;
15629                                                }
15630                                        }
15631                                }
15632                        }
15633                }
15634                return stmts;
15635        }
15636
15637        private List<TSelectSqlStatement> getLastSelectStmt(TCustomSqlStatement stmt) {
15638                List<TSelectSqlStatement> stmts = new ArrayList<TSelectSqlStatement>();
15639                for (int j = stmt.getStatements().size() - 1; j >= 0; j--) {
15640                        TCustomSqlStatement stmtItem = stmt.getStatements().get(j);
15641                        if (stmtItem instanceof TReturnStmt || stmtItem instanceof TMssqlReturn) {
15642                                if (stmtItem.getStatements() != null) {
15643                                        List<TSelectSqlStatement> item = getLastSelectStmt(stmtItem);
15644                                        if (item != null && !item.isEmpty()) {
15645                                                stmts.addAll(item);
15646                                                if(option.getVendor()!=EDbVendor.dbvmssql && option.getVendor()!=EDbVendor.dbvazuresql) {
15647                                                        break;
15648                                                }
15649                                        }
15650                                }
15651                                break;
15652                        }
15653                        if (stmtItem instanceof TSelectSqlStatement && !((TSelectSqlStatement) stmtItem).isQueryOfCTE()) {
15654                                if (isResultSetReturningSelect((TSelectSqlStatement) stmtItem)) {
15655                                        if(!(DlineageUtil.getTopBasicStmt(stmtItem) instanceof TSelectSqlStatement)) {
15656                                                continue;
15657                                        }
15658                                        stmts.add((TSelectSqlStatement) stmtItem);
15659                                        if(option.getVendor()!=EDbVendor.dbvmssql && option.getVendor()!=EDbVendor.dbvazuresql) {
15660                                                break;
15661                                        }
15662                                }
15663                        } else if (stmtItem.getStatements() != null) {
15664                                List<TSelectSqlStatement> item = getLastSelectStmt(stmtItem);
15665                                if (item != null && !item.isEmpty()) {
15666                                        stmts.addAll(item);
15667                                        if(option.getVendor()!=EDbVendor.dbvmssql && option.getVendor()!=EDbVendor.dbvazuresql) {
15668                                                break;
15669                                        }
15670                                }
15671                        }
15672                }
15673                return stmts;
15674        }
15675
15676        private TableColumn getStarColumn(List<TableColumn> columns) {
15677                for (TableColumn column : columns) {
15678                        if (column.getName().endsWith("*")) {
15679                                return column;
15680                        }
15681                }
15682                return null;
15683        }
15684
15685        private boolean containStarColumn(List<TableColumn> columns) {
15686                if (columns == null)
15687                        return false;
15688                for (TableColumn column : columns) {
15689                        if (column.getName().endsWith("*")) {
15690                                return true;
15691                        }
15692                }
15693                return false;
15694        }
15695
15696        private boolean containStarColumn(ResultSet resultSet) {
15697                if (resultSet == null || resultSet.getColumns() == null)
15698                        return false;
15699                for (ResultColumn column : resultSet.getColumns()) {
15700                        if (column.getName().endsWith("*")) {
15701                                return true;
15702                        }
15703                }
15704                return false;
15705        }
15706
15707        private int indexOfColumn(List<TResultColumn> columns, TObjectName objectName) {
15708                for (int i = 0; i < columns.size(); i++) {
15709                        if (columns.get(i).toString().trim().equalsIgnoreCase(objectName.toString().trim())) {
15710                                return i;
15711                        }
15712                }
15713                return -1;
15714        }
15715
15716        private boolean isEmptyCollection(Collection<?> keyMap) {
15717                return keyMap == null || keyMap.isEmpty();
15718        }
15719
15720        private void analyzeInsertImpactRelation(TSelectSqlStatement stmt, Map<String, List<TableColumn>> insertMap,
15721                        List<TExpression> expressions, EffectType effectType) {
15722                List<TObjectName> objectNames = new ArrayList<TObjectName>();
15723                for (int i = 0; i < expressions.size(); i++) {
15724                        TExpression condition = expressions.get(i);
15725                        columnsInExpr visitor = new columnsInExpr();
15726                        condition.inOrderTraverse(visitor);
15727                        objectNames.addAll(visitor.getObjectNames());
15728                }
15729
15730                Iterator<String> iter = insertMap.keySet().iterator();
15731                while (iter.hasNext()) {
15732                        String table = iter.next();
15733                        List<TableColumn> tableColumns = insertMap.get(table);
15734                        for (int i = 0; i < tableColumns.size(); i++) {
15735
15736                                TableColumn column = tableColumns.get(i);
15737                                ImpactRelationship relation = modelFactory.createImpactRelation();
15738                                relation.setEffectType(effectType);
15739                                relation.setTarget(new TableColumnRelationshipElement(column));
15740
15741                                for (int j = 0; j < objectNames.size(); j++) {
15742                                        TObjectName columnName = objectNames.get(j);
15743                                        Object model = modelManager.getModel(stmt);
15744                                        if (model instanceof SelectResultSet) {
15745                                                SelectResultSet queryTable = (SelectResultSet) model;
15746                                                List<ResultColumn> columns = queryTable.getColumns();
15747                                                for (int k = 0; k < columns.size(); k++) {
15748                                                        ResultColumn resultColumn = columns.get(k);
15749                                                        if (resultColumn.getAlias() != null
15750                                                                        && SQLUtil.compareIdentifier(option.getVendor(), ESQLDataObjectType.dotColumn, columnName.toString(), resultColumn.getAlias())) {
15751                                                                relation.addSource(
15752                                                                                new ResultColumnRelationshipElement(resultColumn, columnName.getLocation(), columnName));
15753                                                        } else if (resultColumn.getName() != null
15754                                                                        && SQLUtil.compareIdentifier(option.getVendor(), ESQLDataObjectType.dotColumn, columnName.toString(), resultColumn.getName())) {
15755                                                                relation.addSource(
15756                                                                                new ResultColumnRelationshipElement(resultColumn, columnName.getLocation(), columnName));
15757                                                        }
15758                                                }
15759                                        }
15760                                }
15761                        }
15762                }
15763        }
15764
15765        private void analyzeUpdateStmt(TUpdateSqlStatement stmt) {
15766                if (stmt.getResultColumnList() == null)
15767                        return;
15768
15769                TTable targetTable = stmt.getTargetTable();
15770                Process process = modelFactory.createProcess(stmt);
15771                Table tableModel = null;
15772                QueryTable cteTargetModel = null;
15773                CteWriteTargetResolution cteTargetResolution = null;
15774                if (targetTable.getCTE() != null) {
15775                        cteTargetModel = analyzeCteWriteTarget(targetTable, process);
15776                        cteTargetResolution = resolveCteWriteTarget(targetTable, collectUpdateCteWriteIntent(stmt));
15777                        if (cteTargetResolution.isWritable()) {
15778                                tableModel = modelFactory.createTable(cteTargetResolution.baseTable);
15779                                tableModel.addProcess(process);
15780                                createProjectedWriteImpact(targetTable.getCTE().getSubquery(), tableModel,
15781                                                EffectType.update, process);
15782                        }
15783                        if (cteTargetResolution.failureReason != null) {
15784                                addCteWriteTargetHint(targetTable, cteTargetResolution.failureReason,
15785                                                cteTargetResolution.isWritable());
15786                        }
15787                } else if (targetTable.getSubquery() != null
15788                                || (targetTable.getLinkTable() != null
15789                                                && targetTable.getLinkTable().getSubquery() != null)) {
15790                        TTable projectedTarget = targetTable.getSubquery() != null
15791                                        ? targetTable : targetTable.getLinkTable();
15792                        TSelectSqlStatement projectedSubquery = projectedTarget.getSubquery();
15793                        TObjectNameList exposedColumns = getProjectedTargetColumns(projectedTarget);
15794                        cteTargetModel = analyzeProjectedWriteTarget(targetTable, projectedSubquery,
15795                                        exposedColumns, process);
15796                        cteTargetResolution = resolveProjectedWriteTarget(projectedSubquery, exposedColumns,
15797                                        collectUpdateCteWriteIntent(stmt));
15798                        if (cteTargetResolution.isWritable()) {
15799                                tableModel = modelFactory.createTable(cteTargetResolution.baseTable);
15800                                tableModel.addProcess(process);
15801                                createProjectedWriteImpact(projectedSubquery, tableModel, EffectType.update, process);
15802                        }
15803                        if (cteTargetResolution.failureReason != null) {
15804                                addCteWriteTargetHint(targetTable, cteTargetResolution.failureReason,
15805                                                cteTargetResolution.isWritable());
15806                        }
15807                } else {
15808                        tableModel = modelFactory.createTable(targetTable);
15809                        tableModel.addProcess(process);
15810                }
15811
15812                for (int i = 0; i < stmt.tables.size(); i++) {
15813                        TTable tableElement = stmt.tables.getTable(i);
15814                        if (tableElement == targetTable && cteTargetModel != null) {
15815                                continue;
15816                        }
15817                        if (tableElement.getSubquery() != null) {
15818                                QueryTable queryTable = modelFactory.createQueryTable(tableElement);
15819                                TSelectSqlStatement subquery = tableElement.getSubquery();
15820                                analyzeSelectStmt(subquery);
15821
15822                                if (subquery.getSetOperatorType() != ESetOperatorType.none) {
15823                                        SelectSetResultSet selectSetResultSetModel = (SelectSetResultSet) modelManager.getModel(subquery);
15824                                        for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) {
15825                                                ResultColumn sourceColumn = selectSetResultSetModel.getColumns().get(j);
15826                                                ResultColumn targetColumn = modelFactory.createSelectSetResultColumn(queryTable, sourceColumn);
15827                                                DataFlowRelationship selectSetRalation = modelFactory.createDataFlowRelation();
15828                                                selectSetRalation.setEffectType(EffectType.select);
15829                                                selectSetRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
15830                                                selectSetRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
15831                                                selectSetRalation.setProcess(process);
15832                                        }
15833                                }
15834
15835                                ResultSet resultSetModel = (ResultSet) modelManager.getModel(tableElement.getSubquery());
15836                                if (resultSetModel != null && resultSetModel != queryTable
15837                                                && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
15838                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
15839                                        impactRelation.setEffectType(EffectType.update);
15840                                        impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
15841                                                        resultSetModel.getRelationRows()));
15842                                        impactRelation.setTarget(
15843                                                        new RelationRowsRelationshipElement<ResultSetRelationRows>(queryTable.getRelationRows()));
15844                                }
15845
15846                        } else if (tableElement.getCTE() != null) {
15847                                QueryTable queryTable = modelFactory.createQueryTable(tableElement);
15848
15849                                TObjectNameList cteColumns = tableElement.getCTE().getColumnList();
15850                                if (cteColumns != null) {
15851                                        for (int j = 0; j < cteColumns.size(); j++) {
15852                                                modelFactory.createResultColumn(queryTable, cteColumns.getObjectName(j));
15853                                        }
15854                                }
15855                                TSelectSqlStatement subquery = tableElement.getCTE().getSubquery();
15856                                if (subquery != null && !stmtStack.contains(subquery)) {
15857                                        analyzeSelectStmt(subquery);
15858
15859                                        ResultSet resultSetModel = (ResultSet) modelManager.getModel(subquery);
15860                                        if (resultSetModel != null && resultSetModel != queryTable
15861                                                        && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
15862                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
15863                                                impactRelation.setEffectType(EffectType.select);
15864                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
15865                                                                resultSetModel.getRelationRows()));
15866                                                impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
15867                                                                queryTable.getRelationRows()));
15868                                        }
15869
15870                                        if (subquery.getSetOperatorType() != ESetOperatorType.none) {
15871                                                SelectSetResultSet selectSetResultSetModel = (SelectSetResultSet) modelManager
15872                                                                .getModel(subquery);
15873                                                for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) {
15874                                                        ResultColumn sourceColumn = selectSetResultSetModel.getColumns().get(j);
15875                                                        ResultColumn targetColumn = null;
15876                                                        if (cteColumns != null) {
15877                                                                targetColumn = queryTable.getColumns().get(j);
15878                                                        } else {
15879                                                                targetColumn = modelFactory.createSelectSetResultColumn(queryTable, sourceColumn);
15880                                                        }
15881                                                        for (Set<TObjectName> starLinkColumns : sourceColumn.getStarLinkColumns().values()) {
15882                                                                for (TObjectName starLinkColumn : starLinkColumns) {
15883                                                                        targetColumn.bindStarLinkColumn(starLinkColumn);
15884                                                                }
15885                                                        }
15886                                                        createCteSelectRelation(process, targetColumn, sourceColumn);
15887                                                }
15888                                        } else {
15889                                                for (int j = 0; j < resultSetModel.getColumns().size(); j++) {
15890                                                        ResultColumn sourceColumn = resultSetModel.getColumns().get(j);
15891                                                        ResultColumn targetColumn = null;
15892                                                        if (cteColumns != null) {
15893                                                                targetColumn = queryTable.getColumns().get(j);
15894                                                        } else {
15895                                                                targetColumn = modelFactory.createSelectSetResultColumn(queryTable, sourceColumn);
15896                                                        }
15897                                                        for (TObjectName starLinkColumn : sourceColumn.getStarLinkColumnList()) {
15898                                                                targetColumn.bindStarLinkColumn(starLinkColumn);
15899                                                        }
15900                                                        createCteSelectRelation(process, targetColumn, sourceColumn);
15901                                                }
15902                                        }
15903                                } else if (tableElement.getCTE().getUpdateStmt() != null) {
15904                                        analyzeCustomSqlStmt(tableElement.getCTE().getUpdateStmt());
15905                                } else if (tableElement.getCTE().getInsertStmt() != null) {
15906                                        analyzeCustomSqlStmt(tableElement.getCTE().getInsertStmt());
15907                                } else if (tableElement.getCTE().getDeleteStmt() != null) {
15908                                        analyzeCustomSqlStmt(tableElement.getCTE().getDeleteStmt());
15909                                }
15910                        } else {
15911                                modelFactory.createTable(stmt.tables.getTable(i));
15912                        }
15913                }
15914
15915                for (int i = 0; i < stmt.getResultColumnList().size(); i++) {
15916                        TResultColumn field = stmt.getResultColumnList().getResultColumn(i);
15917
15918                        if (field.getExpr().getExpressionType() == EExpressionType.function_t) {
15919                                // Handle SQL Server XML modify() method for data lineage
15920                                TFunctionCall funcCall = field.getExpr().getFunctionCall();
15921                                if (tableModel != null && funcCall != null
15922                                                && funcCall.getFunctionType() == EFunctionType.xmlmodify_t) {
15923                                        analyzeXmlModifyFunction(stmt, tableModel, process, funcCall);
15924                                }
15925                                continue;
15926                        }
15927
15928                        TExpression expression = field.getExpr().getLeftOperand();
15929                        if (expression == null) {
15930                                ErrorInfo errorInfo = new ErrorInfo();
15931                                errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
15932                                errorInfo.setErrorMessage(
15933                                                "Can't get result column expression. Expression is " + field.getExpr().toString());
15934                                errorInfo.setStartPosition(new Pair3<Long, Long, String>(field.getExpr().getStartToken().lineNo,
15935                                                field.getExpr().getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
15936                                errorInfo.setEndPosition(new Pair3<Long, Long, String>(field.getExpr().getEndToken().lineNo,
15937                                                field.getExpr().getEndToken().columnNo + field.getExpr().getEndToken().getAstext().length(),
15938                                                ModelBindingManager.getGlobalHash()));
15939                                errorInfo.fillInfo(this);
15940                                errorInfos.add(errorInfo);
15941                                continue;
15942                        }
15943                        if (expression.getExpressionType() == EExpressionType.list_t) {
15944                                TExpression setExpression = field.getExpr().getRightOperand();
15945                                if (setExpression != null && setExpression.getSubQuery() != null) {
15946                                        TSelectSqlStatement query = setExpression.getSubQuery();
15947                                        analyzeSelectStmt(query);
15948
15949                                        SelectResultSet resultSetModel = (SelectResultSet) modelManager
15950                                                        .getModel(query.getResultColumnList());
15951
15952                                        if (!resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
15953                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
15954                                                impactRelation.setEffectType(EffectType.update);
15955                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
15956                                                                resultSetModel.getRelationRows()));
15957                                                if (tableModel != null) {
15958                                                        impactRelation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(
15959                                                                        tableModel.getRelationRows()));
15960                                                } else if (cteTargetModel != null) {
15961                                                        impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
15962                                                                        cteTargetModel.getRelationRows()));
15963                                                }
15964                                        }
15965
15966                                        TExpressionList columnList = expression.getExprList();
15967                                        for (int j = 0; j < columnList.size(); j++) {
15968                                                TObjectName column = columnList.getExpression(j).getObjectOperand();
15969
15970                                                if (column.getDbObjectType() == EDbObjectType.variable) {
15971                                                        continue;
15972                                                }
15973
15974                                                if (column.getColumnNameOnly().startsWith("@") && (option.getVendor() == EDbVendor.dbvmssql
15975                                                                || option.getVendor() == EDbVendor.dbvazuresql)) {
15976                                                        continue;
15977                                                }
15978
15979                                                if (column.getColumnNameOnly().startsWith(":") && (option.getVendor() == EDbVendor.dbvhana
15980                                                                || option.getVendor() == EDbVendor.dbvteradata)) {
15981                                                        continue;
15982                                                }
15983
15984                                                ResultColumn resultColumn = resultSetModel.getColumns().get(j);
15985                                                TObjectName targetColumn = getCteBaseColumn(cteTargetResolution, column);
15986                                                TableColumn tableColumn = tableModel == null || targetColumn == null ? null
15987                                                                : modelFactory.createTableColumn(tableModel, targetColumn, false);
15988                                                if (tableColumn != null) {
15989                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
15990                                                        relation.setEffectType(EffectType.update);
15991                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
15992                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
15993                                                        relation.setProcess(process);
15994                                                }
15995
15996                                        }
15997                                }
15998                        } else if (expression.getExpressionType() == EExpressionType.simple_object_name_t) {
15999                                TExpression setExpression = field.getExpr().getRightOperand();
16000                                if (setExpression != null && setExpression.getSubQuery() != null) {
16001                                        TSelectSqlStatement query = setExpression.getSubQuery();
16002                                        analyzeSelectStmt(query);
16003
16004                                        SelectResultSet resultSetModel = (SelectResultSet) modelManager
16005                                                        .getModel(query.getResultColumnList());
16006
16007                                        if (!resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
16008                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
16009                                                impactRelation.setEffectType(EffectType.update);
16010                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
16011                                                                resultSetModel.getRelationRows()));
16012                                                if (tableModel != null) {
16013                                                        impactRelation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(
16014                                                                        tableModel.getRelationRows()));
16015                                                } else if (cteTargetModel != null) {
16016                                                        impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
16017                                                                        cteTargetModel.getRelationRows()));
16018                                                }
16019                                        }
16020
16021                                        TObjectName column = expression.getObjectOperand();
16022                                        ResultColumn resultColumn = resultSetModel.getColumns().get(0);
16023                                        TObjectName targetColumn = getCteBaseColumn(cteTargetResolution, column);
16024                                        TableColumn tableColumn = tableModel == null || targetColumn == null ? null
16025                                                        : modelFactory.createTableColumn(tableModel, targetColumn, false);
16026                                        if (tableColumn != null) {
16027                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16028                                                relation.setEffectType(EffectType.update);
16029                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
16030                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
16031                                                relation.setProcess(process);
16032                                        }
16033                                } else if (setExpression != null) {
16034                                        // ResultSet resultSet = modelFactory.createResultSet(stmt,
16035                                        // true);
16036
16037                                        ResultSet resultSet = modelFactory.createResultSet(stmt, false);
16038
16039                                        createPseudoImpactRelation(stmt, resultSet, EffectType.update);
16040
16041                                        TObjectName columnObject = expression.getObjectOperand();
16042
16043                                        ResultColumn updateColumn = modelFactory.createUpdateResultColumn(resultSet, columnObject);
16044
16045                                        columnsInExpr visitor = new columnsInExpr();
16046                                        field.getExpr().getRightOperand().inOrderTraverse(visitor);
16047
16048                                        List<TObjectName> objectNames = visitor.getObjectNames();
16049                                        List<TParseTreeNode> functions = visitor.getFunctions();
16050                                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
16051
16052                                        if (functions != null && !functions.isEmpty()) {
16053                                                analyzeFunctionDataFlowRelation(updateColumn, functions, EffectType.update);
16054                                        }
16055
16056                                        if (subquerys != null && !subquerys.isEmpty()) {
16057                                                analyzeSubqueryDataFlowRelation(updateColumn, subquerys, EffectType.update);
16058                                        }
16059
16060                                        Transform transform = new Transform();
16061                                        transform.setType(Transform.EXPRESSION);
16062                                        transform.setCode(setExpression);
16063                                        updateColumn.setTransform(transform);
16064                                        analyzeDataFlowRelation(updateColumn, objectNames, EffectType.update, functions);
16065
16066                                        List<TParseTreeNode> constants = visitor.getConstants();
16067                                        analyzeConstantDataFlowRelation(updateColumn, constants, EffectType.update, functions);
16068
16069                                        TObjectName targetColumn = getCteBaseColumn(cteTargetResolution, columnObject);
16070                                        TableColumn tableColumn = tableModel == null || targetColumn == null ? null
16071                                                        : modelFactory.createTableColumn(tableModel, targetColumn, false);
16072                                        if(tableColumn!=null) {
16073                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16074                                                relation.setEffectType(EffectType.update);
16075                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
16076                                                relation.addSource(new ResultColumnRelationshipElement(updateColumn));
16077                                                relation.setProcess(process);
16078                                        }
16079                                        
16080                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
16081                                        impactRelation.setEffectType(EffectType.update);
16082                                        impactRelation.addSource(
16083                                                        new RelationRowsRelationshipElement<ResultSetRelationRows>(resultSet.getRelationRows()));
16084                                        if (tableModel != null) {
16085                                                impactRelation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(
16086                                                                tableModel.getRelationRows()));
16087                                        } else if (cteTargetModel != null) {
16088                                                impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
16089                                                                cteTargetModel.getRelationRows()));
16090                                        }
16091                                }
16092                        }
16093                }
16094
16095                if (stmt.getJoins() != null && stmt.getJoins().size() > 0) {
16096                        for (int i = 0; i < stmt.getJoins().size(); i++) {
16097                                TJoin join = stmt.getJoins().getJoin(i);
16098                                if (join.getJoinItems() != null) {
16099                                        for (int j = 0; j < join.getJoinItems().size(); j++) {
16100                                                TJoinItem joinItem = join.getJoinItems().getJoinItem(j);
16101                                                TExpression expr = joinItem.getOnCondition();
16102                                                analyzeFilterCondition(null, expr, joinItem.getJoinType(), JoinClauseType.on,
16103                                                                EffectType.update);
16104                                        }
16105                                }
16106                        }
16107                }
16108
16109                if (stmt.getWhereClause() != null && stmt.getWhereClause().getCondition() != null) {
16110                        analyzeFilterCondition(null, stmt.getWhereClause().getCondition(), null, JoinClauseType.where,
16111                                        EffectType.update);
16112                }
16113
16114                if (stmt.getOutputClause() != null) {
16115                        TOutputClause outputClause = stmt.getOutputClause();
16116                        if (outputClause.getSelectItemList() != null) {
16117                                ResultSet resultSet = modelFactory.createResultSet(outputClause, false);
16118                                for (int j = 0; j < outputClause.getSelectItemList().size(); j++) {
16119                                        TResultColumn sourceColumn = outputClause.getSelectItemList().getResultColumn(j);
16120                                        ResultColumn sourceColumnModel = modelFactory.createResultColumn(resultSet, sourceColumn);
16121                                        analyzeResultColumn(sourceColumn, EffectType.select);
16122
16123                                        if (outputClause.getIntoTable() != null) {
16124                                                Table intoTableModel = modelFactory.createTableByName(outputClause.getIntoTable());
16125                                                intoTableModel.addProcess(process);
16126                                                if (outputClause.getIntoColumnList() != null) {
16127                                                        TableColumn intoTableColumn = modelFactory.createInsertTableColumn(intoTableModel,
16128                                                                        outputClause.getIntoColumnList().getObjectName(j));
16129
16130                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16131                                                        relation.setEffectType(EffectType.insert);
16132                                                        relation.setTarget(new TableColumnRelationshipElement(intoTableColumn));
16133                                                        relation.addSource(new ResultColumnRelationshipElement(sourceColumnModel));
16134                                                } else if (sourceColumn.getAliasClause() != null
16135                                                                || sourceColumn.getExpr().getObjectOperand() != null) {
16136                                                        TObjectName tableColumnObject = null;
16137                                                        if (sourceColumn.getAliasClause() != null) {
16138                                                                tableColumnObject = sourceColumn.getAliasClause().getAliasName();
16139                                                        } else {
16140                                                                tableColumnObject = sourceColumn.getExpr().getObjectOperand();
16141                                                        }
16142
16143                                                        TableColumn intoTableColumn = modelFactory.createInsertTableColumn(intoTableModel,
16144                                                                        tableColumnObject);
16145                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16146                                                        relation.setEffectType(EffectType.insert);
16147                                                        relation.setTarget(new TableColumnRelationshipElement(intoTableColumn));
16148                                                        relation.addSource(new ResultColumnRelationshipElement(sourceColumnModel));
16149                                                }
16150                                        }
16151                                }
16152                        }
16153                }
16154        }
16155
16156        /**
16157         * Analyzes SQL Server XML modify() function to extract data lineage from sql:column() references
16158         * in XQuery expressions.
16159         *
16160         * Example SQL:
16161         * SET [Demographics].modify('... sql:column("deleted.LineTotal") ...')
16162         *
16163         * This extracts:
16164         * - Target column: Demographics (from the XML column being modified)
16165         * - Source column: deleted.LineTotal (from sql:column() reference in XQuery)
16166         */
16167        private void analyzeXmlModifyFunction(TUpdateSqlStatement stmt, Table tableModel, Process process, TFunctionCall funcCall) {
16168                TObjectName funcName = funcCall.getFunctionName();
16169                if (funcName == null || funcName.getPartToken() == null) {
16170                        return;
16171                }
16172
16173                // Get the XML column being modified (e.g., [Demographics] from "[Demographics].modify")
16174                String xmlColumnName = funcName.getPartToken().astext;
16175
16176                // Get the XQuery argument
16177                if (funcCall.getArgs() == null || funcCall.getArgs().size() == 0) {
16178                        return;
16179                }
16180
16181                String xqueryString = funcCall.getArgs().getExpression(0).toString();
16182
16183                // Extract sql:column() references from the XQuery string
16184                List<String> sqlColumnRefs = extractSqlColumnReferences(xqueryString);
16185                if (sqlColumnRefs.isEmpty()) {
16186                        return;
16187                }
16188
16189                // Extract XPath target from XQuery (e.g., /IndividualSurvey/TotalPurchaseYTD)
16190                String xpathTarget = extractXPathTarget(xqueryString);
16191
16192                // Build full target column name including XML path
16193                String fullTargetColumnName = xmlColumnName;
16194                if (xpathTarget != null && !xpathTarget.isEmpty()) {
16195                        fullTargetColumnName = xmlColumnName + "." + xpathTarget;
16196                }
16197
16198                // Create the target table column for the XML column being modified
16199                TableColumn targetTableColumn = modelFactory.createInsertTableColumn(tableModel, fullTargetColumnName);
16200
16201                // Create source-to-target relationships for each sql:column reference
16202                for (String sqlColRef : sqlColumnRefs) {
16203                        // Parse the column reference (e.g., "deleted.LineTotal" -> table="deleted", column="LineTotal")
16204                        String[] parts = sqlColRef.split("\\.", 2);
16205                        String sourceTableName = parts.length > 1 ? parts[0] : null;
16206                        String sourceColumnName = parts.length > 1 ? parts[1] : parts[0];
16207
16208                        // Find the source table in the statement's tables
16209                        TTable sourceTable = null;
16210                        if (sourceTableName != null && stmt.tables != null) {
16211                                for (int j = 0; j < stmt.tables.size(); j++) {
16212                                        TTable t = stmt.tables.getTable(j);
16213                                        String tableName = t.getTableName().toString();
16214                                        String alias = t.getAliasName();
16215                                        if (SQLUtil.compareIdentifier(option.getVendor(), ESQLDataObjectType.dotTable, tableName, sourceTableName) ||
16216                                                (alias != null && SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotTable, alias, sourceTableName))) {
16217                                                sourceTable = t;
16218                                                break;
16219                                        }
16220                                }
16221                        }
16222
16223                        if (sourceTable != null && targetTableColumn != null) {
16224                                // Create a table model for the source if needed
16225                                Table sourceTableModel = modelFactory.createTable(sourceTable);
16226                                TableColumn sourceColumn = modelFactory.createInsertTableColumn(sourceTableModel, sourceColumnName);
16227
16228                                if (sourceColumn != null) {
16229                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16230                                        relation.setEffectType(EffectType.update);
16231                                        relation.setTarget(new TableColumnRelationshipElement(targetTableColumn));
16232                                        relation.addSource(new TableColumnRelationshipElement(sourceColumn));
16233                                        relation.setProcess(process);
16234                                        relation.setFunction("modify");
16235                                }
16236                        }
16237                }
16238        }
16239
16240        /**
16241         * Extracts sql:column() references from an XQuery string.
16242         * Example: 'sql:column("deleted.LineTotal")' -> ["deleted.LineTotal"]
16243         */
16244        private List<String> extractSqlColumnReferences(String xquery) {
16245                List<String> refs = new ArrayList<>();
16246                if (xquery == null) {
16247                        return refs;
16248                }
16249
16250                int startIdx = 0;
16251                while ((startIdx = xquery.indexOf("sql:column(", startIdx)) >= 0) {
16252                        int parenStart = startIdx + "sql:column(".length();
16253                        int parenEnd = xquery.indexOf(")", parenStart);
16254                        if (parenEnd < 0) {
16255                                break;
16256                        }
16257
16258                        String arg = xquery.substring(parenStart, parenEnd).trim();
16259                        // Remove quotes (single or double)
16260                        if ((arg.startsWith("\"") && arg.endsWith("\"")) ||
16261                                (arg.startsWith("'") && arg.endsWith("'"))) {
16262                                arg = arg.substring(1, arg.length() - 1);
16263                        }
16264
16265                        if (!arg.isEmpty()) {
16266                                refs.add(arg);
16267                        }
16268
16269                        startIdx = parenEnd + 1;
16270                }
16271
16272                return refs;
16273        }
16274
16275        /**
16276         * Extracts the XPath target from an XQuery modify expression.
16277         * Example: 'replace value of (/IndividualSurvey/TotalPurchaseYTD)[1]' -> "IndividualSurvey.TotalPurchaseYTD"
16278         */
16279        private String extractXPathTarget(String xquery) {
16280                if (xquery == null) {
16281                        return null;
16282                }
16283
16284                // Look for patterns like "(/path/to/element)" or "(/path/to/element)[1]"
16285                int replaceIdx = xquery.indexOf("replace value of");
16286                if (replaceIdx < 0) {
16287                        return null;
16288                }
16289
16290                int parenStart = xquery.indexOf("(/", replaceIdx);
16291                if (parenStart < 0) {
16292                        return null;
16293                }
16294
16295                int parenEnd = xquery.indexOf(")", parenStart);
16296                if (parenEnd < 0) {
16297                        return null;
16298                }
16299
16300                String xpath = xquery.substring(parenStart + 1, parenEnd);
16301                // Remove any predicates like [1]
16302                int bracketIdx = xpath.indexOf("[");
16303                if (bracketIdx > 0) {
16304                        xpath = xpath.substring(0, bracketIdx);
16305                }
16306
16307                // Convert XPath to dot notation (e.g., /IndividualSurvey/TotalPurchaseYTD -> IndividualSurvey.TotalPurchaseYTD)
16308                if (xpath.startsWith("/")) {
16309                        xpath = xpath.substring(1);
16310                }
16311                xpath = xpath.replace("/", ".");
16312
16313                return xpath;
16314        }
16315
16316        private void analyzeConstantDataFlowRelation(Object modelObject, List<TParseTreeNode> constants,
16317                        EffectType effectType, List<TParseTreeNode> functions) {
16318                analyzeConstantDataFlowRelation(modelObject, constants, effectType, functions, null);
16319        }
16320
16321        private void analyzeConstantDataFlowRelation(Object modelObject, List<TParseTreeNode> constants,
16322                        EffectType effectType, List<TParseTreeNode> functions, Process process) {
16323                if (constants == null || constants.size() == 0)
16324                        return;
16325
16326                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16327                relation.setEffectType(effectType);
16328                relation.setProcess(process);
16329
16330                if (functions != null && !functions.isEmpty()) {
16331                        relation.setFunction(getFunctionName(functions.get(0)));
16332                }
16333
16334                if (modelObject instanceof ResultColumn) {
16335                        relation.setTarget(new ResultColumnRelationshipElement((ResultColumn) modelObject));
16336
16337                } else if (modelObject instanceof TableColumn) {
16338                        relation.setTarget(new TableColumnRelationshipElement((TableColumn) modelObject));
16339
16340                } else {
16341                        throw new UnsupportedOperationException();
16342                }
16343
16344                if (option.isShowConstantTable()) {
16345                        Table constantTable = null;
16346                        if(modelObject instanceof FunctionResultColumn && ((FunctionResultColumn)modelObject).getFunction() instanceof TFunctionCall) {
16347                                TFunctionCall function = (TFunctionCall)((FunctionResultColumn)modelObject).getFunction();
16348                                if(function.getFunctionType() == EFunctionType.struct_t) {
16349                                        constantTable = modelFactory.createConstantsTable(String.valueOf(function.toString().hashCode()));
16350                                }
16351                        }
16352                        if(constantTable == null) {
16353                                constantTable = modelFactory.createConstantsTable(stmtStack.peek());
16354                        }
16355                        for (int i = 0; i < constants.size(); i++) {
16356                                TParseTreeNode constant = constants.get(i);
16357                                if (constant instanceof TConstant) {
16358                                        TableColumn constantColumn = modelFactory.createTableColumn(constantTable, (TConstant) constant);
16359                                        relation.addSource(new ConstantRelationshipElement(constantColumn));
16360                                } else if (constant instanceof TObjectName) {
16361                                        TableColumn constantColumn = modelFactory.createTableColumn(constantTable, (TObjectName) constant,
16362                                                        false);
16363                                        if(constantColumn == null) {
16364                                                continue;
16365                                        }
16366                                        relation.addSource(new ConstantRelationshipElement(constantColumn));
16367                                }
16368                        }
16369                }
16370
16371        }
16372
16373        private String getFunctionName(TParseTreeNode parseTreeNode) {
16374                if (parseTreeNode instanceof TFunctionCall) {
16375                        return ((TFunctionCall) parseTreeNode).getFunctionName().toString();
16376                }
16377                if (parseTreeNode instanceof TCaseExpression) {
16378                        return "case-when";
16379                }
16380                return null;
16381        }
16382
16383        private void analyzeCreateViewStmt(TCustomSqlStatement stmt, TSelectSqlStatement subquery,
16384                        TViewAliasClause viewAlias, TObjectName viewName) {
16385
16386                if (subquery != null) {
16387                        TTableList tables = subquery.getTables();
16388                        if (tables != null) {
16389                                for (int i = 0; i < tables.size(); i++) {
16390                                        TTable table = tables.getTable(i);
16391                                        TCustomSqlStatement createView = viewDDLMap
16392                                                        .get(DlineageUtil.getTableFullName(table.getTableName().toString()));
16393                                        if (createView != null) {
16394                                                analyzeCustomSqlStmt(createView);
16395                                        }
16396                                }
16397                        }
16398                        analyzeSelectStmt(subquery);
16399                }
16400
16401        
16402                if (viewAlias != null && viewAlias.getViewAliasItemList() != null) {
16403                        TViewAliasItemList viewItems = viewAlias.getViewAliasItemList();
16404                        Table viewModel = modelFactory.createView(stmt, viewName, true);
16405                        viewModel.setFromDDL(true);
16406                        viewModel.setDetermined(true);
16407                        Process process = modelFactory.createProcess(stmt);
16408                        viewModel.addProcess(process);
16409                        ResultSet resultSetModel = (ResultSet) modelManager.getModel(subquery);
16410                        if (resultSetModel != null) {
16411                                int resultSetSize = resultSetModel.getColumns().size();
16412                                int viewItemSize = viewItems.size();
16413                                int j = 0;
16414                                int viewColumnSize = viewItemSize;
16415                                if (resultSetModel.isDetermined() && resultSetSize > viewColumnSize) {
16416                                        viewColumnSize = resultSetSize;
16417                                }
16418                                for (int i = 0; i < viewColumnSize && j < resultSetSize; i++) {
16419                                        ResultColumn resultColumn = resultSetModel.getColumns().get(j);                                 
16420                                        if (i < viewItemSize) {
16421                                                TObjectName alias = viewItems.getViewAliasItem(i).getAlias();
16422
16423                                                if (!resultSetModel.getColumns().get(j).getName().contains("*")) {
16424                                                        j++;
16425                                                } else {
16426                                                        if (resultSetSize - j == viewItems.size() - i) {
16427                                                                j++;
16428                                                        }
16429                                                }
16430
16431                                                if (alias != null) {
16432                                                        TableColumn viewColumn = modelFactory.createViewColumn(viewModel, alias, i, true);
16433                                                        appendTableColumnToSQLEnv(viewModel, viewColumn);
16434                                                        if (resultColumn != null) {
16435                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16436                                                                relation.setEffectType(EffectType.create_view);
16437                                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
16438                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
16439                                                                relation.setProcess(process);
16440                                                        }
16441                                                } else if (resultColumn.getColumnObject() instanceof TObjectName) {
16442                                                        TableColumn viewColumn = modelFactory.createViewColumn(viewModel,
16443                                                                        (TObjectName) resultColumn.getColumnObject(), i, true);
16444                                                        appendTableColumnToSQLEnv(viewModel, viewColumn);
16445                                                        if (resultColumn != null) {
16446                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16447                                                                relation.setEffectType(EffectType.create_view);
16448                                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
16449                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
16450                                                                relation.setProcess(process);
16451                                                        }
16452                                                } else if (resultColumn.getColumnObject() instanceof TResultColumn) {
16453                                                        TableColumn viewColumn = modelFactory.createViewColumn(viewModel,
16454                                                                        ((TResultColumn) resultColumn.getColumnObject()).getFieldAttr(), i, true);
16455                                                        appendTableColumnToSQLEnv(viewModel, viewColumn);
16456                                                        ResultColumn column = (ResultColumn) modelManager.getModel(resultColumn.getColumnObject());
16457                                                        if (column != null && !column.getStarLinkColumns().isEmpty()) {
16458                                                                viewColumn.bindStarLinkColumns(column.getStarLinkColumns());
16459                                                        }
16460                                                        if (resultColumn != null) {
16461                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16462                                                                relation.setEffectType(EffectType.create_view);
16463                                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
16464                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
16465                                                                relation.setProcess(process);
16466                                                        }
16467                                                }
16468                                        }
16469                                        else if(resultSetModel.isDetermined()){
16470                                                TObjectName viewColumnName = new TObjectName();
16471                                                viewColumnName.setString(resultColumn.getName());
16472                                                TableColumn viewColumn = modelFactory.createViewColumn(viewModel, viewColumnName, viewModel.getColumns().size(), true);
16473                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16474                                                relation.setEffectType(EffectType.create_view);
16475                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
16476                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
16477                                                relation.setProcess(process);
16478                                                j++;
16479                                        }
16480                                }
16481                                if (resultSetModel != null && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
16482                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
16483                                        impactRelation.setEffectType(EffectType.create_view);
16484                                        impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
16485                                                        resultSetModel.getRelationRows()));
16486                                        impactRelation.setTarget(
16487                                                        new RelationRowsRelationshipElement<TableRelationRows>(viewModel.getRelationRows()));
16488                                }
16489                        }
16490
16491                        if (subquery.getResultColumnList() == null && subquery.getValueClause() != null
16492                                        && subquery.getValueClause().getValueRows().size() == viewItems.size()) {
16493                                for (int i = 0; i < viewItems.size(); i++) {
16494                                        TObjectName alias = viewItems.getViewAliasItem(i).getAlias();
16495
16496                                        if (alias != null) {
16497                                                TableColumn viewColumn = modelFactory.createViewColumn(viewModel, alias, i, true);
16498                                                appendTableColumnToSQLEnv(viewModel, viewColumn);
16499                                                TExpression expression = subquery.getValueClause().getValueRows().getValueRowItem(i).getExpr();
16500
16501                                                columnsInExpr visitor = new columnsInExpr();
16502                                                expression.inOrderTraverse(visitor);
16503                                                List<TObjectName> objectNames = visitor.getObjectNames();
16504                                                List<TParseTreeNode> functions = visitor.getFunctions();
16505
16506                                                if (functions != null && !functions.isEmpty()) {
16507                                                        analyzeFunctionDataFlowRelation(viewColumn, functions, EffectType.select);
16508
16509                                                }
16510
16511                                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
16512                                                if (subquerys != null && !subquerys.isEmpty()) {
16513                                                        analyzeSubqueryDataFlowRelation(viewColumn, subquerys, EffectType.select);
16514                                                }
16515
16516                                                analyzeDataFlowRelation(viewColumn, objectNames, EffectType.select, functions);
16517                                                List<TParseTreeNode> constants = visitor.getConstants();
16518                                                analyzeConstantDataFlowRelation(viewColumn, constants, EffectType.select, functions);
16519                                        }
16520                                }
16521                        }
16522
16523                } else {
16524                        if (viewName == null) {
16525                                ErrorInfo errorInfo = new ErrorInfo();
16526                                errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
16527                                errorInfo.setErrorMessage("Can't get view name. CreateView is " + stmt.toString());
16528                                errorInfo.setStartPosition(new Pair3<Long, Long, String>(stmt.getStartToken().lineNo,
16529                                                stmt.getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
16530                                errorInfo.setEndPosition(new Pair3<Long, Long, String>(stmt.getEndToken().lineNo,
16531                                                stmt.getEndToken().columnNo + stmt.getEndToken().getAstext().length(),
16532                                                ModelBindingManager.getGlobalHash()));
16533                                errorInfo.fillInfo(this);
16534                                errorInfos.add(errorInfo);
16535                                return;
16536                        }
16537                        Table viewModel = modelFactory.createView(stmt, viewName);
16538                        Process process = modelFactory.createProcess(stmt);
16539                        viewModel.addProcess(process);
16540                        if (subquery != null && !subquery.isCombinedQuery()) {
16541                                SelectResultSet resultSetModel = (SelectResultSet) modelManager
16542                                                .getModel(subquery.getResultColumnList());
16543
16544                                boolean determined = false;
16545                                if (!containStarColumn(resultSetModel)) {
16546                                        viewModel.setCreateTable(true);
16547                                        determined = true;
16548                                        viewModel.setFromDDL(true);
16549                                }
16550                                for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
16551                                        ResultColumn resultColumn = resultSetModel.getColumns().get(i);
16552                                        if (resultColumn.getColumnObject() instanceof TResultColumn) {
16553                                                TResultColumn columnObject = ((TResultColumn) resultColumn.getColumnObject());
16554
16555                                                TAliasClause alias = ((TResultColumn) resultColumn.getColumnObject()).getAliasClause();
16556                                                if (alias != null && alias.getAliasName() != null) {
16557                                                        TableColumn viewColumn = modelFactory.createViewColumn(viewModel, alias.getAliasName(), i,
16558                                                                        determined);
16559                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16560                                                        relation.setEffectType(EffectType.create_view);
16561                                                        relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
16562                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
16563                                                        relation.setProcess(process);
16564                                                        if (determined) {
16565                                                                appendTableColumnToSQLEnv(viewModel, viewColumn);
16566                                                        }
16567                                                } else if (columnObject.getFieldAttr() != null) {
16568                                                        TObjectName viewColumnObject = columnObject.getFieldAttr();
16569                                                        TableColumn viewColumn;
16570                                                        Object model = modelManager.getModel(resultColumn.getColumnObject());
16571                                                        if ("*".equals(viewColumnObject.getColumnNameOnly())) {
16572                                                                if (model instanceof LinkedHashMap) {
16573                                                                        String columnName = getColumnNameOnly(resultColumn.getName());
16574                                                                        LinkedHashMap<String, ResultColumn> resultColumns = (LinkedHashMap<String, ResultColumn>) model;
16575                                                                        if ("*".equals(columnName)) {
16576                                                                                for (String key : resultColumns.keySet()) {
16577                                                                                        ResultColumn column = resultColumns.get(key);
16578                                                                                        viewColumn = modelFactory.createInsertTableColumn(viewModel, column.getName());
16579                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16580                                                                                        relation.setEffectType(EffectType.create_view);
16581                                                                                        relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
16582                                                                                        relation.addSource(new ResultColumnRelationshipElement(column));
16583                                                                                        relation.setProcess(process);
16584                                                                                }
16585                                                                                continue;
16586                                                                        } else if (resultColumns.containsKey(columnName)) {
16587                                                                                ResultColumn column = resultColumns.get(columnName);
16588                                                                                viewColumn = modelFactory.createInsertTableColumn(viewModel, column.getName());
16589                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16590                                                                                relation.setEffectType(EffectType.create_view);
16591                                                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
16592                                                                                relation.addSource(new ResultColumnRelationshipElement(column));
16593                                                                                relation.setProcess(process);
16594                                                                                continue;
16595                                                                        }
16596                                                                }
16597                                                        }
16598                                                        
16599                                                        if (!SQLUtil.isEmpty(viewColumnObject.getColumnNameOnly())
16600                                                                        && viewColumnObject.getPropertyToken() != null) {
16601                                                                TObjectName object = new TObjectName();
16602                                                                // object.setString(viewColumnObject.getPropertyToken().astext);
16603                                                                object.setPartToken(viewColumnObject.getPropertyToken());
16604                                                                object.setStartToken(viewColumnObject.getPropertyToken());
16605                                                                object.setEndToken(viewColumnObject.getPropertyToken());
16606                                                                viewColumn = modelFactory.createViewColumn(viewModel, object, i, determined);
16607                                                        } else {
16608                                                                viewColumn = modelFactory.createViewColumn(viewModel, columnObject.getFieldAttr(),
16609                                                                                i, determined);
16610                                                        }
16611                                                        
16612                                                        if(determined) {
16613                                                                appendTableColumnToSQLEnv(viewModel, viewColumn);
16614                                                        }
16615                                                        
16616                                                        if (model instanceof ResultColumn) {
16617                                                                ResultColumn column = (ResultColumn) modelManager
16618                                                                                .getModel(resultColumn.getColumnObject());
16619                                                                if (column != null && !column.getStarLinkColumns().isEmpty()) {
16620                                                                        viewColumn.bindStarLinkColumns(column.getStarLinkColumns());
16621                                                                }
16622                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16623                                                                relation.setEffectType(EffectType.create_view);
16624                                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
16625                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
16626                                                                if (sqlenv == null) {
16627                                                                        if (resultColumn.isShowStar()) {
16628                                                                                relation.setShowStarRelation(true);
16629                                                                                viewColumn.setShowStar(true);
16630                                                                                resultColumn.setShowStar(true);
16631                                                                                setSourceShowStar(resultColumn);
16632                                                                        }
16633                                                                }
16634                                                                if (viewColumn.getName().endsWith("*") && resultColumn.getName().endsWith("*")) {
16635                                                                        viewModel.setStarStmt("create_view");
16636                                                                }
16637                                                                relation.setProcess(process);
16638                                                        }
16639                                                } else if (resultColumn.getAlias() != null && columnObject.getExpr()
16640                                                                .getExpressionType() == EExpressionType.sqlserver_proprietary_column_alias_t) {
16641                                                        TableColumn viewColumn = modelFactory.createViewColumn(viewModel,
16642                                                                        columnObject.getExpr().getLeftOperand().getObjectOperand(), i, determined);
16643                                                        if(determined) {
16644                                                                appendTableColumnToSQLEnv(viewModel, viewColumn);
16645                                                        }
16646                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16647                                                        relation.setEffectType(EffectType.create_view);
16648                                                        relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
16649                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
16650                                                        relation.setProcess(process);
16651                                                } else {
16652                                                        TGSqlParser parser = columnObject.getGsqlparser();
16653                                                        TObjectName viewColumnName = parser
16654                                                                        .parseObjectName(generateQuotedName(parser, columnObject.toString()));
16655                                                        if (viewColumnName == null) {
16656                                                                ErrorInfo errorInfo = new ErrorInfo();
16657                                                                errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
16658                                                                errorInfo.setErrorMessage(
16659                                                                                "Can't parse view column. Column is " + columnObject.toString());
16660                                                                errorInfo.setStartPosition(new Pair3<Long, Long, String>(
16661                                                                                columnObject.getStartToken().lineNo, columnObject.getStartToken().columnNo,
16662                                                                                ModelBindingManager.getGlobalHash()));
16663                                                                errorInfo
16664                                                                                .setEndPosition(new Pair3<Long, Long, String>(columnObject.getEndToken().lineNo,
16665                                                                                                columnObject.getEndToken().columnNo
16666                                                                                                                + columnObject.getEndToken().getAstext().length(),
16667                                                                                                ModelBindingManager.getGlobalHash()));
16668                                                                errorInfo.fillInfo(this);
16669                                                                errorInfos.add(errorInfo);
16670                                                        } else {
16671                                                                TableColumn viewColumn = modelFactory.createViewColumn(viewModel, viewColumnName, i,
16672                                                                                determined);
16673                                                                if(determined) {
16674                                                                        appendTableColumnToSQLEnv(viewModel, viewColumn);
16675                                                                }
16676                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16677                                                                relation.setEffectType(EffectType.create_view);
16678                                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
16679                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
16680                                                                relation.setProcess(process);
16681                                                        }
16682                                                }
16683                                        } else if (resultColumn.getColumnObject() instanceof TObjectName) {
16684                                                TableColumn viewColumn = modelFactory.createViewColumn(viewModel,
16685                                                                (TObjectName) resultColumn.getColumnObject(), i, determined);
16686                                                if(determined) {
16687                                                        appendTableColumnToSQLEnv(viewModel, viewColumn);
16688                                                }
16689                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16690                                                relation.setEffectType(EffectType.create_view);
16691                                                relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
16692                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
16693                                                relation.setProcess(process);
16694                                        }
16695                                }
16696                                if (!resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
16697                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
16698                                        impactRelation.setEffectType(EffectType.create_view);
16699                                        impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
16700                                                        resultSetModel.getRelationRows()));
16701                                        impactRelation.setTarget(
16702                                                        new RelationRowsRelationshipElement<TableRelationRows>(viewModel.getRelationRows()));
16703                                }
16704                        } else if (subquery != null && subquery.isCombinedQuery()) {
16705                                SelectSetResultSet resultSetModel = (SelectSetResultSet) modelManager.getModel(subquery);
16706
16707                                boolean determined = false;
16708                                if (!containStarColumn(resultSetModel)) {
16709                                        viewModel.setCreateTable(true);
16710                                        viewModel.setFromDDL(true);
16711                                        determined = true;
16712                                }
16713
16714                                for (int i = 0; i < resultSetModel.getColumns().size(); i++) {
16715                                        ResultColumn resultColumn = resultSetModel.getColumns().get(i);
16716
16717                                        if (resultColumn.getColumnObject() instanceof TResultColumn) {
16718                                                TResultColumn columnObject = ((TResultColumn) resultColumn.getColumnObject());
16719
16720                                                TAliasClause alias = columnObject.getAliasClause();
16721                                                if (alias != null && alias.getAliasName() != null) {
16722                                                        TableColumn viewColumn = modelFactory.createViewColumn(viewModel, alias.getAliasName(), i,
16723                                                                        determined);
16724                                                        if(determined) {
16725                                                                appendTableColumnToSQLEnv(viewModel, viewColumn);
16726                                                        }
16727                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16728                                                        relation.setEffectType(EffectType.create_view);
16729                                                        relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
16730                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
16731                                                        relation.setProcess(process);
16732                                                } else if (columnObject.getFieldAttr() != null) {
16733                                                        TableColumn viewColumn = modelFactory.createViewColumn(viewModel,
16734                                                                        columnObject.getFieldAttr(), i, determined);
16735                                                        if(determined) {
16736                                                                appendTableColumnToSQLEnv(viewModel, viewColumn);
16737                                                        }
16738                                                        ResultColumn column = (ResultColumn) modelManager.getModel(resultColumn.getColumnObject());
16739                                                        if (column != null && !column.getStarLinkColumns().isEmpty()) {
16740                                                                viewColumn.bindStarLinkColumns(column.getStarLinkColumns());
16741                                                        }
16742                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16743                                                        relation.setEffectType(EffectType.create_view);
16744                                                        relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
16745                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
16746                                                        relation.setProcess(process);
16747                                                } else if (resultColumn.getAlias() != null && columnObject.getExpr()
16748                                                                .getExpressionType() == EExpressionType.sqlserver_proprietary_column_alias_t) {
16749                                                        TableColumn viewColumn = modelFactory.createViewColumn(viewModel,
16750                                                                        columnObject.getExpr().getLeftOperand().getObjectOperand(), i, determined);
16751                                                        if(determined) {
16752                                                                appendTableColumnToSQLEnv(viewModel, viewColumn);
16753                                                        }
16754                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16755                                                        relation.setEffectType(EffectType.create_view);
16756                                                        relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
16757                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
16758                                                        relation.setProcess(process);
16759                                                } else {
16760                                                        TGSqlParser parser = columnObject.getGsqlparser();
16761                                                        TObjectName viewColumnName = parser
16762                                                                        .parseObjectName(generateQuotedName(parser, columnObject.toString()));
16763                                                        TableColumn viewColumn = modelFactory.createViewColumn(viewModel, viewColumnName, i,
16764                                                                        determined);
16765                                                        if(determined) {
16766                                                                appendTableColumnToSQLEnv(viewModel, viewColumn);
16767                                                        }
16768                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16769                                                        relation.setEffectType(EffectType.create_view);
16770                                                        relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
16771                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
16772                                                        relation.setProcess(process);
16773                                                }
16774                                        } else if (resultColumn.getColumnObject() instanceof TObjectName) {
16775                                                TableColumn viewColumn = modelFactory.createViewColumn(viewModel,
16776                                                                (TObjectName) resultColumn.getColumnObject(), i, determined);
16777                                                if(viewColumn!=null) {
16778                                                        if(determined) {
16779                                                                appendTableColumnToSQLEnv(viewModel, viewColumn);
16780                                                        }
16781                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
16782                                                        relation.setEffectType(EffectType.create_view);
16783                                                        relation.setTarget(new ViewColumnRelationshipElement(viewColumn));
16784                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
16785                                                        relation.setProcess(process);
16786                                                }
16787                                        }
16788                                }
16789                                if (!resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
16790                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
16791                                        impactRelation.setEffectType(EffectType.create_view);
16792                                        impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
16793                                                        resultSetModel.getRelationRows()));
16794                                        impactRelation.setTarget(
16795                                                        new RelationRowsRelationshipElement<TableRelationRows>(viewModel.getRelationRows()));
16796                                }
16797                        }
16798                }
16799        }
16800
16801        private void setSourceShowStar(Object resultColumn) {
16802                for (Relationship relation : modelManager.getRelations()) {
16803                        Collection<RelationshipElement<?>> sources = (Collection<RelationshipElement<?>>)relation.getSources();
16804                        if (relation.getTarget().getElement() == resultColumn && sources != null) {
16805
16806                                ((AbstractRelationship) relation).setShowStarRelation(true);
16807                                for (RelationshipElement<?> source : sources) {
16808                                        Object column = source.getElement();
16809                                        if (column instanceof TableColumn) {
16810                                                if (((TableColumn) column).isShowStar())
16811                                                        continue;
16812                                                ((TableColumn) column).setShowStar(true);
16813                                        }
16814                                        if (column instanceof ResultColumn) {
16815                                                if (((ResultColumn) column).isShowStar())
16816                                                        continue;
16817                                                ((ResultColumn) column).setShowStar(true);
16818                                        }
16819                                        setSourceShowStar(column);
16820                                }
16821                        }
16822                }
16823        }
16824
16825        private String generateQuotedName(TGSqlParser parser, String name) {
16826                return "\"" + name + "\"";
16827        }
16828
16829        private void appendRelations(dataflow dataflow) {
16830                Relationship[] relations = modelManager.getRelations();
16831
16832                appendRelation(dataflow, relations, DataFlowRelationship.class);
16833                appendRelation(dataflow, relations, IndirectImpactRelationship.class);
16834                appendRecordSetRelation(dataflow, relations);
16835                appendCallRelation(dataflow, relations);
16836                appendRelation(dataflow, relations, ImpactRelationship.class);
16837                appendRelation(dataflow, relations, JoinRelationship.class);
16838                appendRelation(dataflow, relations, ERRelationship.class);
16839                appendRelation(dataflow, relations, CrudRelationship.class);
16840                reconcileCitedColumns(dataflow);
16841        }
16842
16843        /**
16844         * Invariant enforcement (codex-696 F5): a relationship must never cite a
16845         * column its parent element does not declare. Star/variant columns of
16846         * record variables can be cited raw (a whole-record source in a filter,
16847         * an unexpanded fill) while the model writer only emitted their
16848         * star-linked expansions — declare exactly the cited leftovers, nothing
16849         * more.
16850         */
16851        private void reconcileCitedColumns(dataflow dataflow) {
16852                // VARIABLES only: the undeclared-endpoint family is a record variable's
16853                // star/variant column cited raw. Declaring cited leftovers on tables or
16854                // resultsets changes what downstream consumers (JSON conversion,
16855                // normalized comparison) derive from the model — churn far beyond the
16856                // invariant this pass exists to keep.
16857                Map<String, table> parents = new HashMap<String, table>();
16858                if (dataflow.getVariables() != null) {
16859                        for (table t : dataflow.getVariables()) {
16860                                if (t.getId() != null) {
16861                                        parents.put(t.getId(), t);
16862                                }
16863                        }
16864                }
16865                if (dataflow.getRelationships() == null) {
16866                        return;
16867                }
16868                Set<String> declared = new HashSet<String>();
16869                for (table t : parents.values()) {
16870                        if (t.getColumns() == null) {
16871                                continue;
16872                        }
16873                        for (column c : t.getColumns()) {
16874                                declared.add(t.getId() + ":" + c.getId());
16875                        }
16876                }
16877                for (relationship relation : dataflow.getRelationships()) {
16878                        if (relation.getSources() != null) {
16879                                for (sourceColumn source : relation.getSources()) {
16880                                        declareCitedColumn(parents, declared, source.getParent_id(), source.getId(),
16881                                                        source.getColumn(), source.getCoordinate());
16882                                }
16883                        }
16884                        targetColumn target = relation.getTarget();
16885                        if (target != null) {
16886                                declareCitedColumn(parents, declared, target.getParent_id(), target.getId(),
16887                                                target.getColumn(), target.getCoordinate());
16888                        }
16889                }
16890        }
16891
16892        private void declareCitedColumn(Map<String, table> parents, Set<String> declared,
16893                        String parentId, String columnId, String columnName, String coordinate) {
16894                if (parentId == null || columnId == null || columnName == null) {
16895                        return;
16896                }
16897                String key = parentId + ":" + columnId;
16898                if (declared.contains(key)) {
16899                        return;
16900                }
16901                table parent = parents.get(parentId);
16902                if (parent == null) {
16903                        return;
16904                }
16905                column columnElement = new column();
16906                columnElement.setId(columnId);
16907                columnElement.setName(columnName);
16908                if (coordinate != null) {
16909                        columnElement.setCoordinate(coordinate);
16910                }
16911                parent.getColumns().add(columnElement);
16912                declared.add(key);
16913        }
16914        
16915        
16916        private relationship cloneRelationshipWithSingleSource(relationship orig, sourceColumn src) {
16917                relationship rel = new relationship();
16918                rel.setType(orig.getType());
16919                rel.setEffectType(orig.getEffectType());
16920                rel.setId(orig.getId() + "_" + src.getColumn());
16921                rel.setSqlHash(orig.getSqlHash());
16922                rel.setSqlComment(orig.getSqlComment());
16923                rel.setProcessId(orig.getProcessId());
16924                rel.setProcessType(orig.getProcessType());
16925                rel.setFunction(orig.getFunction());
16926                rel.setProcedureId(orig.getProcedureId());
16927
16928                targetColumn t = new targetColumn();
16929                t.setId(orig.getTarget().getId());
16930                t.setColumn(orig.getTarget().getColumn());
16931                t.setParent_id(orig.getTarget().getParent_id());
16932                t.setParent_name(orig.getTarget().getParent_name());
16933                t.setParent_alias(orig.getTarget().getParent_alias());
16934                t.setCoordinate(orig.getTarget().getCoordinate());
16935                rel.setTarget(t);
16936
16937                sourceColumn s = new sourceColumn();
16938                s.setId(src.getId());
16939                s.setColumn(src.getColumn());
16940                s.setParent_id(src.getParent_id());
16941                s.setParent_name(src.getParent_name());
16942                s.setCoordinate(src.getCoordinate());
16943                rel.addSource(s);
16944                return rel;
16945        }
16946
16947        private Set<String> appendStarColumns = new HashSet<String>();
16948        private void appendRelation(dataflow dataflow, Relationship[] relations, Class<? extends Relationship> clazz) {
16949                for (int i = 0; i < relations.length; i++) {
16950                        AbstractRelationship relation = (AbstractRelationship) relations[i];
16951                        if (relation.getClass() == clazz) {
16952                                if (relation.getSources() == null || relation.getTarget() == null) {
16953                                        continue;
16954                                }
16955                                Object targetElement = relation.getTarget().getElement();
16956                                TObjectName targetColumnName = null;
16957                                if(relation.getTarget() instanceof ResultColumnRelationshipElement) {
16958                                        targetColumnName = ((ResultColumnRelationshipElement) relation.getTarget()).getColumnName();
16959                                }
16960                                if (targetElement instanceof ResultColumn) {
16961                                        ResultColumn targetColumn = (ResultColumn) targetElement;
16962                                        if (!targetColumn.isPseduo()) 
16963                                        {
16964                                                if (targetColumnName == null) 
16965                                                {
16966                                                        if ("*".equals(targetColumn.getName())) {
16967                                                                updateResultColumnStarLinks(dataflow, relation, -1);
16968                                                        }
16969
16970                                                        if (targetColumn.hasStarLinkColumn()) {
16971                                                                for (int j = 0; j < targetColumn.getStarLinkColumnNames().size(); j++) {
16972                                                                        appendStarRelation(dataflow, relation, j);
16973                                                                }
16974
16975                                                                if (!containsStar(relation.getSources())) {
16976                                                                        continue;
16977                                                                }
16978                                                        }
16979                                                }
16980                                                else {
16981                                                        String columnName = DlineageUtil.getColumnName(targetColumnName);
16982                                                        int index = targetColumn.getStarLinkColumnNames().indexOf(columnName);
16983                                                        if (index != -1) {
16984                                                                int size = targetColumn.getStarLinkColumnNames().size();
16985                                                                if ("*".equals(targetColumn.getName())) {
16986                                                                        if (appendStarColumns.contains(columnName)) {
16987                                                                                updateResultColumnStarLinks(dataflow, relation, index);
16988                                                                        }
16989                                                                        else {
16990                                                                                updateResultColumnStarLinks(dataflow, relation, -1);
16991                                                                                appendStarColumns.add(columnName);
16992                                                                        }
16993                                                                }
16994                                                                appendStarRelation(dataflow, relation, index);
16995                                                                for(int j= size; j < targetColumn.getStarLinkColumnNames().size(); j++) {
16996                                                                        appendStarRelation(dataflow, relation, j);
16997                                                                }
16998                                                                if (!containsStar(relation.getSources())) {
16999                                                                        continue;
17000                                                                }
17001                                                        }
17002                                                }
17003                                        }
17004                                } else if (targetElement instanceof TableColumn) {
17005                                        TableColumn targetColumn = (TableColumn) targetElement;
17006                                        if (!targetColumn.isPseduo()) {
17007                                                if ("*".equals(targetColumn.getName())) {
17008                                                        updateTableColumnStarLinks(dataflow, relation);
17009                                                }
17010
17011                                                // Star expansion selected by HOLDER type (#695): a
17012                                                // VARIABLE's variant column is serialized as its
17013                                                // star-linked fields (appendVariableModel), so a
17014                                                // relation targeting it must expand onto those fields
17015                                                // or it cites a column the model never declares. A
17016                                                // physical table's variant column keeps the historical
17017                                                // refusal (appendTableModel refuses variant).
17018                                                boolean variantOfVariable = targetColumn.isVariant()
17019                                                                && targetColumn.getTable() instanceof Variable;
17020                                                if (targetColumn.hasStarLinkColumn()
17021                                                                && ((!targetColumn.isVariant() && targetColumn.isExpandStar())
17022                                                                                || variantOfVariable)) {
17023                                                        for (int j = 0; j < targetColumn.getStarLinkColumnNames().size(); j++) {
17024                                                                appendStarRelation(dataflow, relation, j);
17025                                                        }
17026                                                        if (!containsStar(relation.getSources())) {
17027                                                                continue;
17028                                                        }
17029                                                }
17030                                        }
17031                                }
17032
17033                                relationship relationElement = new relationship();
17034                                relationElement.setType(relation.getRelationshipType().name());
17035                                if (relation.getEffectType() != null) {
17036                                        relationElement.setEffectType(relation.getEffectType().name());
17037                                }
17038                                if (relation.getFunction() != null) {
17039                                        relationElement.setFunction(relation.getFunction());
17040                                }
17041                                relationElement.setSqlHash(relation.getSqlHash());
17042                                relationElement.setSqlComment(relation.getSqlComment());
17043
17044                                if (relation.getProcedureId() != null) {
17045                                        relationElement.setProcedureId(String.valueOf(relation.getProcedureId()));
17046                                }
17047                                relationElement.setId(String.valueOf(relation.getId()));
17048                                if (relation.getProcess() != null) {
17049                                        relationElement.setProcessId(String.valueOf(relation.getProcess().getId()));
17050                                        if (relation.getProcess().getGspObject() != null) {
17051                                                relationElement.setProcessType(relation.getProcess().getGspObject().sqlstatementtype.name());
17052                                        }
17053                                }
17054
17055                                if (relation.getPartition() != null) {
17056                                        relationElement.setPartition(relation.getPartition());
17057                                }
17058                                relationElement.setSqlHash(relation.getSqlHash());
17059                                relationElement.setSqlComment(relation.getSqlComment());
17060                                
17061                                if (relation.getProcedureId() != null) {
17062                                        relationElement.setProcedureId(String.valueOf(relation.getProcedureId()));
17063                                }
17064                                
17065                                if (relation instanceof JoinRelationship) {
17066                                        relationElement.setCondition(((JoinRelationship) relation).getJoinCondition());
17067                                        relationElement.setJoinType(((JoinRelationship) relation).getJoinType().name());
17068                                        relationElement.setClause(((JoinRelationship) relation).getJoinClauseType().name());
17069                                }
17070
17071                                if(relation instanceof ImpactRelationship){
17072                                        ImpactRelationship impactRelationship = (ImpactRelationship)relation;
17073                                        if(impactRelationship.getJoinClauseType()!=null){
17074                                                relationElement.setClause(impactRelationship.getJoinClauseType().name());
17075                                        }
17076                                }
17077
17078                                String targetName = null;
17079                                Object columnObject = null;
17080                                List<TObjectName> targetObjectNames = null;
17081
17082                                if (targetElement instanceof ResultSetRelationRows) {
17083                                        ResultSetRelationRows targetColumn = (ResultSetRelationRows) targetElement;
17084                                        targetColumn target = new targetColumn();
17085                                        target.setId(String.valueOf(targetColumn.getId()));
17086                                        target.setColumn(targetColumn.getName());
17087                                        target.setParent_id(String.valueOf(targetColumn.getHolder().getId()));
17088                                        target.setParent_name(getResultSetName(targetColumn.getHolder()));
17089                                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
17090                                                target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
17091                                                                + convertCoordinate(targetColumn.getEndPosition()));
17092                                        }
17093                                        if (relation instanceof RecordSetRelationship) {
17094                                                target.setFunction(((RecordSetRelationship) relation).getAggregateFunction());
17095                                        }
17096                                        target.setSource("system");
17097                                        targetName = targetColumn.getName();
17098                                        relationElement.setTarget(target);
17099                                } else if (targetElement instanceof TableRelationRows) {
17100                                        TableRelationRows targetColumn = (TableRelationRows) targetElement;
17101                                        targetColumn target = new targetColumn();
17102                                        target.setId(String.valueOf(targetColumn.getId()));
17103                                        target.setColumn(targetColumn.getName());
17104                                        target.setParent_id(String.valueOf(targetColumn.getHolder().getId()));
17105                                        target.setParent_name(getTableName(targetColumn.getHolder()));
17106                                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
17107                                                target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
17108                                                                + convertCoordinate(targetColumn.getEndPosition()));
17109                                        }
17110                                        if (relation instanceof RecordSetRelationship) {
17111                                                target.setFunction(((RecordSetRelationship) relation).getAggregateFunction());
17112                                        }
17113                                        target.setSource("system");
17114                                        targetName = targetColumn.getName();
17115                                        relationElement.setTarget(target);
17116                                } else if (targetElement instanceof ResultColumn) {
17117                                        ResultColumn targetColumn = (ResultColumn) targetElement;
17118                                        targetColumn target = new targetColumn();
17119                                        target.setId(String.valueOf(targetColumn.getId()));
17120                                        target.setColumn(targetColumn.getName());
17121                                        target.setStruct(targetColumn.isStruct());
17122                                        target.setParent_id(String.valueOf(targetColumn.getResultSet().getId()));
17123                                        target.setParent_name(getResultSetName(targetColumn.getResultSet()));
17124                                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
17125                                                target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
17126                                                                + convertCoordinate(targetColumn.getEndPosition()));
17127                                        }
17128                                        if (relation instanceof RecordSetRelationship) {
17129                                                target.setFunction(((RecordSetRelationship) relation).getAggregateFunction());
17130                                        }
17131                                        targetName = targetColumn.getName();
17132                                        if (((ResultColumn) targetColumn).getColumnObject() instanceof TResultColumn) {
17133                                                columnsInExpr visitor = new columnsInExpr();
17134                                                ((TResultColumn) ((ResultColumn) targetColumn).getColumnObject()).getExpr()
17135                                                                .inOrderTraverse(visitor);
17136                                                targetObjectNames = visitor.getObjectNames();
17137                                        }
17138
17139                                        if (targetElement instanceof FunctionResultColumn) {
17140                                                columnObject = ((FunctionResultColumn) targetElement).getColumnObject();
17141                                        }
17142                                        if(targetColumn.isPseduo()) {
17143                                                target.setSource("system");
17144                                        }
17145                                        relationElement.setTarget(target);
17146                                } else if (targetElement instanceof TableColumn) {
17147                                        TableColumn targetColumn = (TableColumn) targetElement;
17148                                        targetColumn target = new targetColumn();
17149                                        target.setId(String.valueOf(targetColumn.getId()));
17150                                        target.setColumn(targetColumn.getName());
17151                                        target.setStruct(targetColumn.isStruct());
17152                                        target.setParent_id(String.valueOf(targetColumn.getTable().getId()));
17153                                        target.setParent_name(getTableName(targetColumn.getTable()));
17154                                        if (relation.getTarget() instanceof TableColumnRelationshipElement) {
17155                                                target.setParent_alias(((TableColumnRelationshipElement) relation.getTarget()).getTableAlias());
17156                                        }
17157                                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
17158                                                target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
17159                                                                + convertCoordinate(targetColumn.getEndPosition()));
17160                                        }
17161                                        if (relation instanceof RecordSetRelationship) {
17162                                                target.setFunction(((RecordSetRelationship) relation).getAggregateFunction());
17163                                        }
17164                                        if(targetColumn.isPseduo()) {
17165                                                target.setSource("system");
17166                                        }
17167                                        targetName = targetColumn.getName();
17168                                        relationElement.setTarget(target);
17169                                } else if (targetElement instanceof Argument) {
17170                                        Argument targetColumn = (Argument) targetElement;
17171                                        targetColumn target = new targetColumn();
17172                                        target.setId(String.valueOf(targetColumn.getId()));
17173                                        target.setColumn(targetColumn.getName());
17174                                        target.setParent_id(String.valueOf(targetColumn.getProcedure().getId()));
17175                                        target.setParent_name(targetColumn.getProcedure().getName());
17176                                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
17177                                                target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
17178                                                                + convertCoordinate(targetColumn.getEndPosition()));
17179                                        }
17180                                        if (relation instanceof RecordSetRelationship) {
17181                                                target.setFunction(((RecordSetRelationship) relation).getAggregateFunction());
17182                                        }
17183                                        targetName = targetColumn.getName();
17184                                        relationElement.setTarget(target);
17185                                } else if (targetElement instanceof Table) {
17186                                        Table table = (Table) targetElement;
17187                                        targetColumn target = new targetColumn();
17188                                        target.setTarget_id(String.valueOf(table.getId()));
17189                                        target.setTarget_name(getTableName(table));
17190                                        if (table.getStartPosition() != null && table.getEndPosition() != null) {
17191                                                target.setCoordinate(convertCoordinate(table.getStartPosition()) + ","
17192                                                                + convertCoordinate(table.getEndPosition()));
17193                                        }
17194                                        relationElement.setTarget(target);
17195                                } else {
17196                                        continue;
17197                                }
17198
17199                                Collection<RelationshipElement<?>> sourceElements = relation.getSources();
17200                                if (sourceElements.size() == 0) {
17201                                        if(clazz == CrudRelationship.class && option.getAnalyzeMode() == AnalyzeMode.crud) {
17202                                                dataflow.getRelationships().add(relationElement);
17203                                        }
17204                                        continue;
17205                                }
17206
17207                                boolean append = false;
17208                                for (RelationshipElement<?> sourceItem: relation.getSources()) {
17209                                        Object sourceElement = sourceItem.getElement();
17210                                        TObjectName sourceColumnName = null;
17211                                        if (sourceItem instanceof ResultColumnRelationshipElement) {
17212                                                sourceColumnName = ((ResultColumnRelationshipElement) sourceItem).getColumnName();
17213                                        }
17214//                                      if (sourceItem instanceof TableColumnRelationElement
17215//                                                      && (((TableColumnRelationElement) sourceItem).getColumnIndex() != null)) {
17216//                                              TableColumnRelationElement tableColumnRelationElement = (TableColumnRelationElement) sourceItem;
17217//                                              sourceElement = tableColumnRelationElement.getElement().getTable().getColumns()
17218//                                                              .get(tableColumnRelationElement.getColumnIndex() + 1);
17219//                                      }
17220                                        if (sourceElement instanceof ResultColumn) {
17221                                                ResultColumn sourceColumn = (ResultColumn) sourceElement;
17222                                                if (sourceColumn.hasStarLinkColumn() && !relation.isShowStarRelation() && !sourceColumn.isPseduo()) {
17223                                                        List<String> sourceStarColumnNames = sourceColumn.getStarLinkColumnNames();
17224                                                        sourceColumn source = new sourceColumn();
17225                                                        if (targetObjectNames != null && !targetObjectNames.isEmpty()) {
17226
17227                                                                for (int k = 0; k < targetObjectNames.size(); k++) {
17228                                                                        String targetObjectName = getColumnName(targetObjectNames.get(k));
17229                                                                        if(sourceColumn.getResultSet()!=null && sourceColumn.getResultSet().getColumns()!=null) {
17230                                                                                boolean find = false;
17231                                                                                for(ResultColumn column: sourceColumn.getResultSet().getColumns()) {
17232                                                                                        if(SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotColumn, column.getName(), targetObjectName)) {
17233                                                                                                source = new sourceColumn();
17234                                                                                                source.setId(String.valueOf(column.getId()));
17235                                                                                                source.setColumn(targetObjectName);
17236                                                                                                source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
17237                                                                                                source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
17238                                                                                                setSourceCoordinate(source, sourceItem, sourceColumn);
17239                                                                                                append = true;
17240                                                                                                relationElement.addSource(source);
17241                                                                                                find = true;
17242                                                                                                break;
17243                                                                                        }
17244                                                                                }
17245                                                                                if(find) {
17246                                                                                        continue;
17247                                                                                }
17248                                                                        }
17249                                                                        if (sourceColumn.getStarLinkColumns().containsKey(targetObjectName)) {
17250                                                                                source = new sourceColumn();
17251                                                                                source.setId(String.valueOf(sourceColumn.getId()) + "_"
17252                                                                                                + sourceStarColumnNames.indexOf(targetObjectName));
17253                                                                                source.setColumn(targetObjectName);
17254                                                                                source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
17255                                                                                source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
17256                                                                                setSourceCoordinate(source, sourceItem, sourceColumn);
17257                                                                                append = true;
17258                                                                                relationElement.addSource(source);
17259                                                                        } else {
17260                                                                                source = new sourceColumn();
17261                                                                                source.setId(String.valueOf(sourceColumn.getId()));
17262                                                                                source.setColumn(relationElement.getTarget().getColumn());
17263                                                                                source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
17264                                                                                source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
17265                                                                                setSourceCoordinate(source, sourceItem, sourceColumn);
17266                                                                                append = true;
17267                                                                                relationElement.addSource(source);
17268                                                                        }
17269                                                                }
17270                                                        } else {
17271                                                                if (columnObject instanceof TWhenClauseItemList) {
17272                                                                        TCaseExpression expr = (TCaseExpression)((FunctionResultColumn) targetElement).getResultSet().getGspObject();
17273                                                                        List<TExpression> directExpressions = new ArrayList<TExpression>();
17274                                                                        
17275//                                                                      TExpression inputExpr = expr.getInput_expr();
17276//                                                                      if (inputExpr != null) {
17277//                                                                              directExpressions.add(inputExpr);
17278//                                                                      }
17279                                                                        TExpression defaultExpr = expr.getElse_expr();
17280                                                                        if (defaultExpr != null) {
17281                                                                                directExpressions.add(defaultExpr);
17282                                                                        }
17283                                                                        TWhenClauseItemList list = expr.getWhenClauseItemList();
17284                                                                        for (int k = 0; k < list.size(); k++) {
17285                                                                                TWhenClauseItem element = list.getWhenClauseItem(k);
17286                                                                                directExpressions.add(element.getReturn_expr());
17287                                                                        }
17288                                                                        
17289                                                                        for (int k = 0; k < directExpressions.size(); k++) {
17290                                                                                columnsInExpr visitor = new columnsInExpr();
17291                                                                                directExpressions.get(k).inOrderTraverse(visitor);
17292                                                                                List<TObjectName> objectNames = visitor.getObjectNames();
17293                                                                                if (objectNames == null) {
17294                                                                                        continue;
17295                                                                                }
17296                                                                                for (int x = 0; x < objectNames.size(); x++) {
17297                                                                                        String objectName = getColumnName(objectNames.get(x));
17298                                                                                        if (sourceColumn.getStarLinkColumns().containsKey(objectName)) {
17299
17300                                                                                                if(sourceColumn.getResultSet()!=null && sourceColumn.getResultSet().getColumns()!=null) {
17301                                                                                                        boolean find = false;
17302                                                                                                        for(ResultColumn column: sourceColumn.getResultSet().getColumns()) {
17303                                                                                                                if(SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotColumn, column.getName(), objectName)) {
17304                                                                                                                        source = new sourceColumn();
17305                                                                                                                        source.setId(String.valueOf(column.getId()));
17306                                                                                                                        source.setColumn(objectName);
17307                                                                                                                        source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
17308                                                                                                                        source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
17309                                                                                                                        setSourceCoordinate(source, sourceItem, sourceColumn);
17310                                                                                                                        append = true;
17311                                                                                                                        relationElement.addSource(source);
17312                                                                                                                        find = true;
17313                                                                                                                        break;
17314                                                                                                                }
17315                                                                                                        }
17316                                                                                                        if(find) {
17317                                                                                                                continue;
17318                                                                                                        }
17319                                                                                                }
17320                                                                                                
17321                                                                                                source.setId(String.valueOf(sourceColumn.getId()) + "_"
17322                                                                                                                + sourceStarColumnNames.indexOf(objectName));
17323                                                                                                source.setColumn(objectName);
17324                                                                                        } else {
17325                                                                                                source.setId(String.valueOf(sourceColumn.getId()));
17326                                                                                                source.setColumn(sourceColumn.getName());
17327                                                                                        }
17328                                                                                        source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
17329                                                                                        source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
17330                                                                                        setSourceCoordinate(source, sourceItem, sourceColumn);
17331                                                                                        append = true;
17332                                                                                        relationElement.addSource(source);
17333                                                                                }
17334                                                                        }
17335                                                                } else {
17336                                                                        String objectName = getColumnName(targetName);
17337                                                                        boolean find = false;
17338                                                                        
17339                                                                        if (!find && sourceColumn.getStarLinkColumns().containsKey(objectName)) {
17340                                                                                source.setId(String.valueOf(sourceColumn.getId()) + "_"
17341                                                                                                + sourceStarColumnNames.indexOf(objectName));
17342                                                                                source.setColumn(objectName);
17343                                                                                find = true;
17344                                                                        }
17345
17346                                                                        if (!find && sourceColumnName != null) {
17347                                                                                objectName = getColumnName(sourceColumnName);
17348                                                                                
17349                                                                                if(sourceColumn.getResultSet()!=null && sourceColumn.getResultSet().getColumns()!=null) {
17350                                                                                        for(ResultColumn column: sourceColumn.getResultSet().getColumns()) {
17351                                                                                                if(SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotColumn, column.getName(), objectName)) {
17352                                                                                                        source = new sourceColumn();
17353                                                                                                        source.setId(String.valueOf(column.getId()));
17354                                                                                                        source.setColumn(objectName);
17355                                                                                                        source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
17356                                                                                                        source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
17357                                                                                                        setSourceCoordinate(source, sourceItem, sourceColumn);
17358                                                                                                        append = true;
17359                                                                                                        relationElement.addSource(source);
17360                                                                                                        find = true;
17361                                                                                                        break;
17362                                                                                                }
17363                                                                                        }
17364                                                                                        if(find) {
17365                                                                                                continue;
17366                                                                                        }
17367                                                                                }
17368                                                                                
17369                                                                                if (sourceColumn.getStarLinkColumns().containsKey(objectName)) {
17370                                                                                        source.setId(String.valueOf(sourceColumn.getId()) + "_"
17371                                                                                                        + sourceStarColumnNames.indexOf(objectName));
17372                                                                                        source.setColumn(objectName);
17373                                                                                        find = true;
17374                                                                                }
17375                                                                        }
17376                                                                        
17377                                                                        if (!find && sourceItem instanceof ResultColumnRelationshipElement) {
17378                                                                                int starIndex = ((ResultColumnRelationshipElement) sourceItem)
17379                                                                                                .getStarIndex();
17380                                                                                if (starIndex > -1 && sourceColumn.getStarLinkColumnList().size() > starIndex) {
17381                                                                                        objectName = getColumnName(sourceColumn.getStarLinkColumnList().get(starIndex));
17382                                                                                        source.setId(String.valueOf(sourceColumn.getId()) + "_"
17383                                                                                                        + starIndex);
17384                                                                                        source.setColumn(objectName);
17385                                                                                        find = true;
17386                                                                                }
17387                                                                        }
17388
17389                                                                        if (!find) {
17390                                                                                source.setId(String.valueOf(sourceColumn.getId()));
17391                                                                                source.setColumn(sourceColumn.getName());
17392                                                                        }
17393
17394                                                                        source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
17395                                                                        source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
17396                                                                        setSourceCoordinate(source, sourceItem, sourceColumn);
17397                                                                        append = true;
17398                                                                        relationElement.addSource(source);
17399                                                                }
17400                                                        }
17401
17402                                                } else {
17403                                                        sourceColumn source = new sourceColumn();
17404                                                        source.setId(String.valueOf(sourceColumn.getId()));
17405                                                        source.setColumn(sourceColumn.getName());
17406                                                        source.setStruct(sourceColumn.isStruct());
17407                                                        source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
17408                                                        source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
17409                                                        setSourceCoordinate(source, sourceItem, sourceColumn);
17410                                                        append = true;
17411                                                        if (sourceItem.getTransforms() != null) {
17412                                                                for (Transform transform : sourceItem.getTransforms()) {
17413                                                                        source.addTransform(transform);
17414                                                                }
17415                                                        }
17416                                                        if(sourceColumn.isPseduo()) {
17417                                                                source.setSource("system");
17418                                                        }
17419                                                        relationElement.addSource(source);
17420                                                }
17421                                        } else if (sourceElement instanceof TableColumn) {
17422                                                TableColumn sourceColumn = (TableColumn) sourceElement;
17423                                                if (!sourceColumn.isPseduo() && sourceColumn.hasStarLinkColumn()
17424                                                                && sourceItem instanceof TableColumnRelationshipElement) {
17425                                                        sourceColumn source = new sourceColumn();
17426                                                        boolean find = false;
17427                                                        if (((TableColumnRelationshipElement) sourceItem).getColumnIndex() != null) {
17428                                                                int columnIndex = ((TableColumnRelationshipElement) sourceItem).getColumnIndex();
17429                                                                if (sourceColumn.getStarLinkColumns().size() > columnIndex) {
17430                                                                        source = new sourceColumn();
17431                                                                        source.setId(String.valueOf(sourceColumn.getId()) + "_" + columnIndex);
17432                                                                        String targetObjectName = getColumnName(
17433                                                                                        sourceColumn.getStarLinkColumnList().get(columnIndex));
17434                                                                        source.setColumn(targetObjectName);
17435                                                                        source.setParent_id(String.valueOf(sourceColumn.getTable().getId()));
17436                                                                        source.setParent_name(sourceColumn.getTable().getName());
17437                                                                        if (sourceItem instanceof TableColumnRelationshipElement) {
17438                                                                                source.setParent_alias(
17439                                                                                                ((TableColumnRelationshipElement) sourceItem).getTableAlias());
17440                                                                        }
17441                                                                        setSourceCoordinate(source, sourceItem, sourceColumn);
17442                                                                        append = true;
17443                                                                        relationElement.addSource(source);
17444                                                                        find = true;
17445                                                                }
17446                                                        }
17447
17448                                                        if (!find) {
17449                                                                String objectName = getColumnName(targetName);
17450
17451                                                                table tableElement = null;
17452                                                                if (dataflow.getTables() != null) {
17453                                                                        for (table t : dataflow.getTables()) {
17454                                                                                if (t.getId().equals(String.valueOf(sourceColumn.getTable().getId()))) {
17455                                                                                        tableElement = t;
17456                                                                                        break;
17457                                                                                }
17458                                                                        }
17459                                                                }
17460                                                                if (tableElement == null && dataflow.getViews() != null) {
17461                                                                        for (table t : dataflow.getViews()) {
17462                                                                                if (t.getId().equals(String.valueOf(sourceColumn.getTable().getId()))) {
17463                                                                                        tableElement = t;
17464                                                                                        break;
17465                                                                                }
17466                                                                        }
17467                                                                }
17468                                                                if (tableElement == null && dataflow.getVariables() != null) {
17469                                                                        for (table t : dataflow.getVariables()) {
17470                                                                                if (t.getId().equals(String.valueOf(sourceColumn.getTable().getId()))) {
17471                                                                                        tableElement = t;
17472                                                                                        break;
17473                                                                                }
17474                                                                        }
17475                                                                }
17476
17477                                                                if(tableElement!=null) {
17478                                                                        for (column column : tableElement.getColumns()) {
17479                                                                                if (column.getName() != null && SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotColumn, column.getName(), objectName)) {
17480                                                                                        source = new sourceColumn();
17481                                                                                        source.setId(String.valueOf(column.getId()));
17482                                                                                        source.setColumn(objectName);
17483                                                                                        source.setParent_id(String.valueOf(sourceColumn.getTable().getId()));
17484                                                                                        source.setParent_name(sourceColumn.getTable().getName());
17485                                                                                        if (sourceItem instanceof TableColumnRelationshipElement) {
17486                                                                                                source.setParent_alias(
17487                                                                                                                ((TableColumnRelationshipElement) sourceItem).getTableAlias());
17488                                                                                        }
17489                                                                                        setSourceCoordinate(source, sourceItem, sourceColumn);
17490                                                                                        if (sourceItem.getTransforms() != null) {
17491                                                                                                for (Transform transform : sourceItem.getTransforms()) {
17492                                                                                                        source.addTransform(transform);
17493                                                                                                }
17494                                                                                        }
17495                                                                                        if (sourceColumn.getCandidateParents() != null) {
17496                                                                                                for(Object item: sourceColumn.getCandidateParents()) {
17497                                                                                                        candidateTable candidateParent = new candidateTable();
17498                                                                                                        if(item instanceof Table) {
17499                                                                                                                candidateParent.setId(String.valueOf(((Table)item).getId()));
17500                                                                                                                candidateParent.setName(getTableName((Table)item));
17501                                                                                                                source.addCandidateParent(candidateParent);
17502                                                                                                        }
17503                                                                                                        else if(item instanceof ResultSet) {
17504                                                                                                                candidateParent.setId(String.valueOf(((ResultSet)item).getId()));
17505                                                                                                                candidateParent.setName(getResultSetName((ResultSet)item));
17506                                                                                                                source.addCandidateParent(candidateParent);
17507                                                                                                        }
17508                                                                                                }
17509                                                                                        }
17510                                                                                        append = true;
17511                                                                                        relationElement.addSource(source);
17512                                                                                        find = true;
17513                                                                                        break;
17514                                                                                }
17515                                                                        }
17516                                                                }
17517                                                        }
17518
17519                                                        if(!find) {
17520                                                                source = new sourceColumn();
17521                                                                source.setId(String.valueOf(sourceColumn.getId()));
17522                                                                source.setColumn(sourceColumn.getName());
17523                                                                source.setParent_id(String.valueOf(sourceColumn.getTable().getId()));
17524                                                                source.setParent_name(sourceColumn.getTable().getName());
17525                                                                if (sourceItem instanceof TableColumnRelationshipElement) {
17526                                                                        source.setParent_alias(
17527                                                                                        ((TableColumnRelationshipElement) sourceItem).getTableAlias());
17528                                                                }
17529                                                                setSourceCoordinate(source, sourceItem, sourceColumn);
17530                                                                if (sourceItem.getTransforms() != null) {
17531                                                                        for (Transform transform : sourceItem.getTransforms()) {
17532                                                                                source.addTransform(transform);
17533                                                                        }
17534                                                                }
17535                                                                if (sourceColumn.getCandidateParents() != null) {
17536                                                                        for(Object item: sourceColumn.getCandidateParents()) {
17537                                                                                candidateTable candidateParent = new candidateTable();
17538                                                                                if(item instanceof Table) {
17539                                                                                        candidateParent.setId(String.valueOf(((Table)item).getId()));
17540                                                                                        candidateParent.setName(getTableName((Table)item));
17541                                                                                        source.addCandidateParent(candidateParent);
17542                                                                                }
17543                                                                                else if(item instanceof ResultSet) {
17544                                                                                        candidateParent.setId(String.valueOf(((ResultSet)item).getId()));
17545                                                                                        candidateParent.setName(getResultSetName((ResultSet)item));
17546                                                                                        source.addCandidateParent(candidateParent);
17547                                                                                }
17548                                                                        }
17549                                                                }
17550                                                                append = true;
17551                                                                relationElement.addSource(source);
17552                                                        }
17553
17554                                                } else {
17555                                                        sourceColumn source = new sourceColumn();
17556                                                        source.setId(String.valueOf(sourceColumn.getId()));
17557                                                        source.setColumn(sourceColumn.getName());
17558                                                        source.setStruct(sourceColumn.isStruct());
17559                                                        source.setParent_id(String.valueOf(sourceColumn.getTable().getId()));
17560                                                        source.setParent_name(getTableName(sourceColumn.getTable()));
17561                                                        if (sourceItem instanceof TableColumnRelationshipElement) {
17562                                                                source.setParent_alias(
17563                                                                                ((TableColumnRelationshipElement) sourceItem).getTableAlias());
17564                                                        }
17565                                                        setSourceCoordinate(source, sourceItem, sourceColumn);
17566                                                        if (sourceItem.getTransforms() != null) {
17567                                                                for (Transform transform : sourceItem.getTransforms()) {
17568                                                                        source.addTransform(transform);
17569                                                                }
17570                                                        }
17571                                                        if(sourceColumn.isPseduo()) {
17572                                                                source.setSource("system");
17573                                                        }
17574                                                        if (sourceColumn.getCandidateParents() != null) {
17575                                                                for(Object item: sourceColumn.getCandidateParents()) {
17576                                                                        candidateTable candidateParent = new candidateTable();
17577                                                                        if(item instanceof Table) {
17578                                                                                candidateParent.setId(String.valueOf(((Table)item).getId()));
17579                                                                                candidateParent.setName(getTableName((Table)item));
17580                                                                                source.addCandidateParent(candidateParent);
17581                                                                        }
17582                                                                        else if(item instanceof ResultSet) {
17583                                                                                candidateParent.setId(String.valueOf(((ResultSet)item).getId()));
17584                                                                                candidateParent.setName(getResultSetName((ResultSet)item));
17585                                                                                source.addCandidateParent(candidateParent);
17586                                                                        }
17587                                                                }
17588                                                        }
17589                                                        append = true;
17590                                                        relationElement.addSource(source);
17591                                                }
17592                                        } else if (sourceElement instanceof Argument) {
17593                                                Argument sourceColumn = (Argument) sourceElement;
17594                                                sourceColumn source = new sourceColumn();
17595                                                source.setId(String.valueOf(sourceColumn.getId()));
17596                                                source.setColumn(sourceColumn.getName());
17597                                                source.setParent_id(String.valueOf(sourceColumn.getProcedure().getId()));
17598                                                source.setParent_name(sourceColumn.getProcedure().getName());
17599                                                if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
17600                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
17601                                                                        + convertCoordinate(sourceColumn.getEndPosition()));
17602                                                }
17603                                                append = true;
17604                                                relationElement.addSource(source);
17605                                        } else if (sourceElement instanceof TableRelationRows) {
17606                                                TableRelationRows sourceColumn = (TableRelationRows) sourceElement;
17607                                                sourceColumn source = new sourceColumn();
17608                                                source.setId(String.valueOf(sourceColumn.getId()));
17609                                                source.setColumn(sourceColumn.getName());
17610                                                source.setParent_id(String.valueOf(sourceColumn.getHolder().getId()));
17611                                                source.setParent_name(getTableName(sourceColumn.getHolder()));
17612                                                if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
17613                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
17614                                                                        + convertCoordinate(sourceColumn.getEndPosition()));
17615                                                }
17616                                                source.setSource("system");
17617                                                append = true;
17618                                                relationElement.addSource(source);
17619                                        } else if (sourceElement instanceof ResultSetRelationRows) {
17620                                                ResultSetRelationRows sourceColumn = (ResultSetRelationRows) sourceElement;
17621                                                sourceColumn source = new sourceColumn();
17622                                                source.setId(String.valueOf(sourceColumn.getId()));
17623                                                source.setColumn(sourceColumn.getName());
17624                                                source.setParent_id(String.valueOf(sourceColumn.getHolder().getId()));
17625                                                source.setParent_name(getResultSetName(sourceColumn.getHolder()));
17626                                                if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
17627                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
17628                                                                        + convertCoordinate(sourceColumn.getEndPosition()));
17629                                                }
17630                                                source.setSource("system");
17631                                                append = true;
17632                                                relationElement.addSource(source);
17633                                        } else if (sourceElement instanceof Constant) {
17634                                                Constant sourceColumn = (Constant) sourceElement;
17635                                                sourceColumn source = new sourceColumn();
17636                                                source.setId(String.valueOf(sourceColumn.getId()));
17637                                                source.setColumn(sourceColumn.getName());
17638                                                source.setColumn_type("constant");
17639                                                if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
17640                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
17641                                                                        + convertCoordinate(sourceColumn.getEndPosition()));
17642                                                }
17643                                                append = true;
17644                                                relationElement.addSource(source);
17645                                        } else if (sourceElement instanceof Table) {
17646                                                Table table = (Table) sourceElement;
17647                                                sourceColumn source = new sourceColumn();
17648                                                source.setSource_id(String.valueOf(table.getId()));
17649                                                source.setSource_name(getTableName(table));
17650                                                if (table.getStartPosition() != null && table.getEndPosition() != null) {
17651                                                        source.setCoordinate(convertCoordinate(table.getStartPosition()) + ","
17652                                                                        + convertCoordinate(table.getEndPosition()));
17653                                                }
17654                                                append = true;
17655                                                relationElement.addSource(source);
17656                                        }
17657
17658                                        if (relation instanceof ImpactRelationship) {
17659                                                ESqlClause clause = getSqlClause(sourceItem);
17660                                                if (clause != null
17661                                                                && (relationElement.getSources() != null && !relationElement.getSources().isEmpty())) {
17662                                                        relationElement.getSources().get(relationElement.getSources().size() - 1)
17663                                                                        .setClauseType(clause.name());
17664                                                }
17665                                        }
17666                                }
17667                                if (append)
17668                                        dataflow.getRelationships().add(relationElement);
17669                        }
17670                }
17671        }
17672
17673        private boolean containsStar(Collection<RelationshipElement<?>> elements) {
17674                if (elements == null || elements.size() == 0)
17675                        return false;
17676
17677                for (RelationshipElement<?> element : elements) {
17678                        if (element.getElement() instanceof ResultColumn) {
17679                                ResultColumn object = (ResultColumn) element.getElement();
17680                                if (object.getName().endsWith("*") && object.isShowStar())
17681                                        return true;
17682                        } else if (element.getElement() instanceof TableColumn) {
17683                                TableColumn object = (TableColumn) element.getElement();
17684                                if (object.getName().endsWith("*") && object.isShowStar())
17685                                        return true;
17686                        }
17687                }
17688                return false;
17689        }
17690
17691        private Map<String, Set<String>> appendTableStarColumns = new HashMap<String, Set<String>>();
17692
17693        // Tracks column names that have explicit (non-star) sources per target star column.
17694        // Used across relationships to prevent phantom star expansions.
17695        // Key: target star column ID, Value: set of column names with explicit sources.
17696        private Map<Long, Set<String>> explicitStarTargetColumns = new HashMap<Long, Set<String>>();
17697        private void updateResultColumnStarLinks(dataflow dataflow, AbstractRelationship relation, int index) {
17698        try {
17699            if (option.getAnalyzeMode() == AnalyzeMode.crud) {
17700                return;
17701            }
17702
17703            ResultColumn targetColumn = (ResultColumn) relation.getTarget().getElement();
17704
17705            Collection<RelationshipElement<?>> sourceElements = (Collection<RelationshipElement<?>>) relation.getSources();
17706            if (sourceElements == null || sourceElements.size() == 0)
17707                return;
17708
17709            for (RelationshipElement<?> sourceItem : sourceElements) {
17710                Object sourceElement = sourceItem.getElement();
17711                if (sourceElement instanceof ResultColumn) {
17712                    ResultColumn source = (ResultColumn) sourceElement;
17713                    if (source.hasStarLinkColumn()) {
17714                        for (Map.Entry<String, Set<TObjectName>> item : source.getStarLinkColumns().entrySet()) {
17715                            if (!targetColumn.getStarLinkColumns().containsKey(item.getKey())) {
17716                                targetColumn.getStarLinkColumns().put(item.getKey(), new LinkedHashSet<TObjectName>());
17717                            }
17718                            targetColumn.getStarLinkColumns().get(item.getKey()).addAll(item.getValue());
17719                        }
17720                        if (!source.isShowStar()) {
17721                            targetColumn.setShowStar(false);
17722                            relation.setShowStarRelation(false);
17723                        }
17724                    } else if (!"*".equals(source.getName())) {
17725                        if (source.getColumnObject() instanceof TObjectName) {
17726                            if (source instanceof FunctionResultColumn) {
17727
17728                            } else {
17729                                targetColumn.bindStarLinkColumn((TObjectName) source.getColumnObject());
17730                            }
17731                        } else if (source.getColumnObject() instanceof TResultColumn) {
17732                            TResultColumn sourceColumn = (TResultColumn) source.getColumnObject();
17733                            if (sourceColumn.getAliasClause() != null) {
17734                                targetColumn.bindStarLinkColumn(sourceColumn.getAliasClause().getAliasName());
17735                            } else if (sourceColumn.getFieldAttr() != null) {
17736                                targetColumn.bindStarLinkColumn(sourceColumn.getFieldAttr());
17737                            } else {
17738                                TObjectName column = new TObjectName();
17739                                if (sourceColumn.getExpr().getExpressionType() == EExpressionType.typecast_t) {
17740                                    column.setString(sourceColumn.getExpr().getLeftOperand().toString());
17741                                } else {
17742                                    column.setString(sourceColumn.toString());
17743                                }
17744                                targetColumn.bindStarLinkColumn(column);
17745                            }
17746                        }
17747                    }
17748                } else if (sourceElement instanceof TableColumn && !targetColumn.isStruct()) {
17749                    TableColumn source = (TableColumn) sourceElement;
17750                    if (!source.isPseduo() && source.hasStarLinkColumn()) {
17751                                                for (Map.Entry<String, Set<TObjectName>> item : source.getStarLinkColumns().entrySet()) {
17752                                                        if (!targetColumn.getStarLinkColumns().containsKey(item.getKey())) {
17753                                                                targetColumn.getStarLinkColumns().put(item.getKey(),
17754                                                                                new LinkedHashSet<TObjectName>());
17755                                                        }
17756                                                        targetColumn.getStarLinkColumns().get(item.getKey()).addAll(item.getValue());
17757                                                }
17758                                                if ((source.getTable().isCreateTable() || source.getTable().hasSQLEnv())
17759                                                                && !source.isShowStar()) {
17760                                                        targetColumn.setShowStar(false);
17761                                                        relation.setShowStarRelation(false);
17762                                                }
17763                    } else if (!"*".equals(source.getName())) {
17764                        targetColumn.bindStarLinkColumn(source.getColumnObject());
17765                    }
17766                }
17767            }
17768
17769            if (targetColumn.hasStarLinkColumn()) {
17770                table resultSetElement = null;
17771                for (table t : dataflow.getResultsets()) {
17772                    if (t.getId().equals(String.valueOf(targetColumn.getResultSet().getId()))) {
17773                        resultSetElement = t;
17774                        break;
17775                    }
17776                }
17777
17778                int starColumnCount = 0;
17779                for (column item : resultSetElement.getColumns()) {
17780                    if (item.getName() != null && item.getName().endsWith("*")) {
17781                        starColumnCount += 1;
17782                    }
17783                }
17784
17785                if (resultSetElement != null && starColumnCount <= 1) {
17786                    List<String> columns = targetColumn.getStarLinkColumnNames();
17787                    if (index == -1) {
17788                        for (int k = 0; k < columns.size(); k++) {
17789                            String columnName = columns.get(k);
17790                            String id = String.valueOf(targetColumn.getId()) + "_" + k;
17791                            if (appendTableStarColumns.containsKey(resultSetElement.getId()) && appendTableStarColumns.get(resultSetElement.getId()).contains(id)) {
17792                                continue;
17793                            }
17794                            column columnElement = new column();
17795                            columnElement.setId(id);
17796                            columnElement.setName(columnName);
17797                            if (targetColumn.isFunction()) {
17798                                columnElement.setIsFunction(String.valueOf(targetColumn.isFunction()));
17799                            }
17800                            if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
17801                                columnElement.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
17802                                        + convertCoordinate(targetColumn.getEndPosition()));
17803                            }
17804
17805                            if (targetColumn.getResultSet() != null && targetColumn.getResultSet().getColumns() != null) {
17806                                boolean find = false;
17807                                for (int i = 0; i < targetColumn.getResultSet().getColumns().size(); i++) {
17808                                    ResultColumn columnModel = targetColumn.getResultSet().getColumns().get(i);
17809                                    if (DlineageUtil.getIdentifierNormalColumnName(columnModel.getName())
17810                                            .equals(columnName)) {
17811                                        find = true;
17812                                        break;
17813                                    }
17814                                }
17815                                if (find) {
17816                                    continue;
17817                                }
17818                            }
17819
17820                            if (!resultSetElement.getColumns().contains(columnElement)) {
17821                                resultSetElement.getColumns().add(columnElement);
17822                                appendTableStarColumns.putIfAbsent(resultSetElement.getId(), new HashSet<String>());
17823                                appendTableStarColumns.get(resultSetElement.getId()).add(id);
17824                            }
17825                        }
17826                    } else {
17827                        int k = index;
17828                        String columnName = columns.get(k);
17829                        column columnElement = new column();
17830                        columnElement.setId(String.valueOf(targetColumn.getId()) + "_" + k);
17831                        columnElement.setName(columnName);
17832                        if (targetColumn.isFunction()) {
17833                            columnElement.setIsFunction(String.valueOf(targetColumn.isFunction()));
17834                        }
17835                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
17836                            columnElement.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
17837                                    + convertCoordinate(targetColumn.getEndPosition()));
17838                        }
17839
17840                        if (targetColumn.getResultSet() != null && targetColumn.getResultSet().getColumns() != null) {
17841                            boolean find = false;
17842                            for (int i = 0; i < targetColumn.getResultSet().getColumns().size(); i++) {
17843                                ResultColumn columnModel = targetColumn.getResultSet().getColumns().get(i);
17844                                if (DlineageUtil.getIdentifierNormalColumnName(columnModel.getName()).equals(columnName)) {
17845                                    find = true;
17846                                    break;
17847                                }
17848                            }
17849                            if (find) {
17850                                return;
17851                            }
17852                        }
17853
17854                        if (!resultSetElement.getColumns().contains(columnElement)) {
17855                            resultSetElement.getColumns().add(columnElement);
17856                        }
17857                    }
17858                }
17859            }
17860        } catch (Exception e) {
17861            logger.error("updateResultColumnStarLinks occurs unknown exceptions.", e);
17862        }
17863        }
17864
17865        private void updateTableColumnStarLinks(dataflow dataflow, AbstractRelationship relation) {
17866                TableColumn targetColumn = (TableColumn) relation.getTarget().getElement();             
17867                Collection<RelationshipElement<?>> sourceElements = (Collection<RelationshipElement<?>>) relation.getSources();
17868                if (sourceElements == null || sourceElements.size() == 0)
17869                        return;
17870
17871                TableColumn sourceStarColumn = null;
17872
17873                for (RelationshipElement<?> sourceItem: sourceElements) {
17874                        Object sourceElement = sourceItem.getElement();
17875                        if (sourceElement instanceof ResultColumn) {
17876                                ResultColumn source = (ResultColumn) sourceElement;
17877                                if(source.getResultSet() instanceof Function) {
17878                                        continue;
17879                                }
17880                                if (source.hasStarLinkColumn()) {
17881                                        for (Map.Entry<String, Set<TObjectName>> item : source.getStarLinkColumns().entrySet()) {
17882                                                if (!targetColumn.getStarLinkColumns().containsKey(item.getKey())) {
17883                                                        targetColumn.getStarLinkColumns().put(item.getKey(), new LinkedHashSet<TObjectName>());
17884                                                }
17885                                                targetColumn.getStarLinkColumns().get(item.getKey()).addAll(item.getValue());
17886                                        }
17887                                        if (!source.isShowStar()) {
17888                                                targetColumn.setShowStar(false);
17889                                                relation.setShowStarRelation(false);
17890                                        }
17891                                } else if (!"*".equals(source.getName())) {
17892                                        if (source.getColumnObject() instanceof TObjectName) {
17893                                                targetColumn.bindStarLinkColumn((TObjectName) source.getColumnObject());
17894                                        } else if (source.getColumnObject() instanceof TResultColumn) {
17895                                                if (((TResultColumn) source.getColumnObject()).getAliasClause() != null) {
17896                                                        TObjectName field = ((TResultColumn) source.getColumnObject()).getAliasClause()
17897                                                                        .getAliasName();
17898                                                        if (field != null) {
17899                                                                targetColumn.bindStarLinkColumn(field);
17900                                                        }
17901                                                } else {
17902                                                        TObjectName field = ((TResultColumn) source.getColumnObject()).getFieldAttr();
17903                                                        if (field != null) {
17904                                                                targetColumn.bindStarLinkColumn(field);
17905                                                        } else {
17906                                                                TObjectName column = new TObjectName();
17907                                                                if (((TResultColumn) source.getColumnObject()).getExpr()
17908                                                                                .getExpressionType() == EExpressionType.typecast_t) {
17909                                                                        column.setString(((TResultColumn) source.getColumnObject()).getExpr()
17910                                                                                        .getLeftOperand().toString());
17911                                                                } else {
17912                                                                        column.setString(((TResultColumn) source.getColumnObject()).toString());
17913                                                                }
17914                                                                targetColumn.bindStarLinkColumn(column);
17915                                                        }
17916                                                }
17917                                        }
17918                                }
17919                        } else if (sourceElement instanceof TableColumn) {
17920                                TableColumn source = (TableColumn) sourceElement;
17921                                if (source.hasStarLinkColumn()) {
17922                                        for (Map.Entry<String, Set<TObjectName>> item : source.getStarLinkColumns().entrySet()) {
17923                                                if (!targetColumn.getStarLinkColumns().containsKey(item.getKey())) {
17924                                                        targetColumn.getStarLinkColumns().put(item.getKey(), new LinkedHashSet<TObjectName>());
17925                                                }
17926                                                targetColumn.getStarLinkColumns().get(item.getKey()).addAll(item.getValue());
17927                                        }
17928                                        for (Map.Entry<String, Set<TObjectName>> item : targetColumn.getStarLinkColumns().entrySet()) {
17929                                                if (!source.getStarLinkColumns().containsKey(item.getKey())) {
17930                                                        source.getStarLinkColumns().put(item.getKey(), new LinkedHashSet<TObjectName>());
17931                                                }
17932                                                source.getStarLinkColumns().get(item.getKey()).addAll(item.getValue());
17933                                        }
17934
17935                                        sourceStarColumn = source;
17936
17937                                        if (source.getTable().isCreateTable() && !source.isShowStar()) {
17938                                                targetColumn.setShowStar(false);
17939                                                relation.setShowStarRelation(false);
17940                                        }
17941                                } else if (!"*".equals(source.getName())) {
17942                                        if (source.isStruct()) {
17943                                                targetColumn.bindStarLinkColumn(source.getColumnObject());
17944                                        }
17945                                        else {
17946                                                TObjectName objectName = new TObjectName();
17947                                                objectName.setString(DlineageUtil.getColumnNameOnly(source.getName()));
17948                                                targetColumn.bindStarLinkColumn(objectName);
17949                                        }
17950                                }
17951                        }
17952                }
17953
17954                if (targetColumn.hasStarLinkColumn()) {
17955                        table tableElement = null;
17956                        if (dataflow.getTables() != null) {
17957                                for (table t : dataflow.getTables()) {
17958                                        if (t.getId().equals(String.valueOf(targetColumn.getTable().getId()))) {
17959                                                tableElement = t;
17960                                                break;
17961                                        }
17962                                }
17963                        }
17964                        if (tableElement == null && dataflow.getViews() != null) {
17965                                for (table t : dataflow.getViews()) {
17966                                        if (t.getId().equals(String.valueOf(targetColumn.getTable().getId()))) {
17967                                                tableElement = t;
17968                                                break;
17969                                        }
17970                                }
17971                        }
17972                        if (tableElement == null && dataflow.getVariables() != null) {
17973                                for (table t : dataflow.getVariables()) {
17974                                        if (t.getId().equals(String.valueOf(targetColumn.getTable().getId()))) {
17975                                                tableElement = t;
17976                                                break;
17977                                        }
17978                                }
17979                        }
17980
17981                        if (tableElement != null) {
17982                                List<String> columns = new ArrayList<String>(targetColumn.getStarLinkColumns().keySet());
17983                                for (int k = 0; k < columns.size(); k++) {
17984                                        String columnName = columns.get(k);
17985                                        if (containStarColumn(targetColumn.getTable().getColumns(), columnName)) {
17986                                                continue;
17987                                        }
17988                                        column columnElement = new column();
17989                                        columnElement.setId(String.valueOf(targetColumn.getId()) + "_" + k);
17990                                        columnElement.setName(columnName);
17991                                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
17992                                                columnElement.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
17993                                                                + convertCoordinate(targetColumn.getEndPosition()));
17994                                        }
17995                                        if (!tableElement.getColumns().contains(columnElement)) {
17996                                                tableElement.getColumns().add(columnElement);
17997                                        }
17998                                }
17999                        }
18000                }
18001
18002                if (sourceStarColumn != null) {
18003                        table tableElement = null;
18004                        if (dataflow.getTables() != null) {
18005                                for (table t : dataflow.getTables()) {
18006                                        if (t.getId().equals(String.valueOf(sourceStarColumn.getTable().getId()))) {
18007                                                tableElement = t;
18008                                                break;
18009                                        }
18010                                }
18011                        }
18012                        if (tableElement == null && dataflow.getViews() != null) {
18013                                for (table t : dataflow.getViews()) {
18014                                        if (t.getId().equals(String.valueOf(sourceStarColumn.getTable().getId()))) {
18015                                                tableElement = t;
18016                                                break;
18017                                        }
18018                                }
18019                        }
18020                        if (tableElement == null && dataflow.getVariables() != null) {
18021                                for (table t : dataflow.getVariables()) {
18022                                        if (t.getId().equals(String.valueOf(sourceStarColumn.getTable().getId()))) {
18023                                                tableElement = t;
18024                                                break;
18025                                        }
18026                                }
18027                        }
18028
18029                        if (tableElement != null) {
18030                                List<String> columns = new ArrayList<String>(sourceStarColumn.getStarLinkColumns().keySet());
18031                                for (int k = 0; k < columns.size(); k++) {
18032                                        String columnName = columns.get(k);
18033                                        if (containStarColumn(sourceStarColumn.getTable().getColumns(), columnName)) {
18034                                                continue;
18035                                        }
18036                                        column columnElement = new column();
18037                                        columnElement.setId(sourceStarColumn.getId() + "_" + k);
18038                                        columnElement.setName(columnName);
18039                                        if (sourceStarColumn.getStartPosition() != null && sourceStarColumn.getEndPosition() != null) {
18040                                                columnElement.setCoordinate(convertCoordinate(sourceStarColumn.getStartPosition()) + ","
18041                                                                + convertCoordinate(sourceStarColumn.getEndPosition()));
18042                                        }
18043                                        if (!tableElement.getColumns().contains(columnElement)) {
18044                                                tableElement.getColumns().add(columnElement);
18045                                        }
18046                                }
18047                        }
18048                }
18049        }
18050
18051        private ESqlClause getSqlClause(RelationshipElement<?> relationshipElement) {
18052                if (relationshipElement instanceof TableColumnRelationshipElement) {
18053                        return ((TableColumnRelationshipElement) relationshipElement).getRelationLocation();
18054                } else if (relationshipElement instanceof ResultColumnRelationshipElement) {
18055                        return ((ResultColumnRelationshipElement) relationshipElement).getRelationLocation();
18056                }
18057                return null;
18058        }
18059
18060        /**
18061         * Emits a source coordinate, preferring the relationship element's own
18062         * reference site (where THIS relation referenced the column — e.g. the
18063         * ON/WHERE operand) over the shared column model's position. A created
18064         * (e.g. #temp) table reuses one TableColumn model for the definition and
18065         * every reference, so the model's position alone would report the
18066         * definition site for every operand.
18067         */
18068        private void setSourceCoordinate(sourceColumn source, RelationshipElement<?> sourceItem,
18069                        TableColumn sourceTableColumn) {
18070                Pair3<Long, Long, String> modelStart = sourceTableColumn.getStartPosition();
18071                Pair3<Long, Long, String> modelEnd = sourceTableColumn.getEndPosition();
18072                Pair3<Long, Long, String> start = modelStart;
18073                Pair3<Long, Long, String> end = modelEnd;
18074                if (sourceItem instanceof TableColumnRelationshipElement) {
18075                        TableColumnRelationshipElement element = (TableColumnRelationshipElement) sourceItem;
18076                        if (element.getReferenceStartPosition() != null && element.getReferenceEndPosition() != null) {
18077                                start = element.getReferenceStartPosition();
18078                                end = element.getReferenceEndPosition();
18079                                if (modelStart != null && modelEnd != null) {
18080                                        // Keep the definition span for the merge pass's dedup steps.
18081                                        source.setDedupCoordinate(convertCoordinate(modelStart) + "," + convertCoordinate(modelEnd));
18082                                }
18083                        }
18084                }
18085                if (start != null && end != null) {
18086                        source.setCoordinate(convertCoordinate(start) + "," + convertCoordinate(end));
18087                }
18088        }
18089
18090        /**
18091         * ResultColumn flavor of {@link #setSourceCoordinate}: a derived table's /
18092         * CTE's result column model carries its select-list definition span, so the
18093         * relationship element's reference site (the ON/WHERE operand) wins when it
18094         * was captured.
18095         */
18096        private void setSourceCoordinate(sourceColumn source, RelationshipElement<?> sourceItem,
18097                        ResultColumn sourceResultColumn) {
18098                Pair3<Long, Long, String> modelStart = sourceResultColumn.getStartPosition();
18099                Pair3<Long, Long, String> modelEnd = sourceResultColumn.getEndPosition();
18100                Pair3<Long, Long, String> start = modelStart;
18101                Pair3<Long, Long, String> end = modelEnd;
18102                if (sourceItem instanceof ResultColumnRelationshipElement) {
18103                        ResultColumnRelationshipElement element = (ResultColumnRelationshipElement) sourceItem;
18104                        if (element.getReferenceStartPosition() != null && element.getReferenceEndPosition() != null) {
18105                                start = element.getReferenceStartPosition();
18106                                end = element.getReferenceEndPosition();
18107                                if (modelStart != null && modelEnd != null) {
18108                                        // Keep the definition span for the merge pass's dedup steps.
18109                                        source.setDedupCoordinate(convertCoordinate(modelStart) + "," + convertCoordinate(modelEnd));
18110                                }
18111                        }
18112                }
18113                if (start != null && end != null) {
18114                        source.setCoordinate(convertCoordinate(start) + "," + convertCoordinate(end));
18115                }
18116        }
18117
18118        private void appendStarRelation(dataflow dataflow, AbstractRelationship relation, int index) {
18119                if(option.getAnalyzeMode() == AnalyzeMode.crud) {
18120                        return;
18121                }
18122                
18123                Object targetElement = relation.getTarget().getElement();
18124
18125                relationship relationElement = new relationship();
18126                relationElement.setType(relation.getRelationshipType().name());
18127                if (relation.getEffectType() != null) {
18128                        relationElement.setEffectType(relation.getEffectType().name());
18129                }
18130                relationElement.setSqlHash(relation.getSqlHash());
18131                relationElement.setSqlComment(relation.getSqlComment());
18132
18133                if (relation.getProcedureId() != null) {
18134                        relationElement.setProcedureId(String.valueOf(relation.getProcedureId()));
18135                }
18136                relationElement.setId(String.valueOf(relation.getId()) + "_" + index);
18137                if (relation.getProcess() != null) {
18138                        relationElement.setProcessId(String.valueOf(relation.getProcess().getId()));
18139                        if (relation.getProcess().getGspObject() != null) {
18140                                relationElement.setProcessType(relation.getProcess().getGspObject().sqlstatementtype.name());
18141                        }
18142                }
18143                if (relation instanceof DataFlowRelationship) {
18144                        relationElement.setSqlHash(((DataFlowRelationship) relation).getSqlHash());
18145                        relationElement.setSqlComment(((DataFlowRelationship) relation).getSqlComment());
18146                
18147                        if (relation.getProcedureId() != null) {
18148                                relationElement.setProcedureId(String.valueOf(relation.getProcedureId()));
18149                        }
18150                }
18151                String targetName = "";
18152
18153                if (targetElement instanceof ResultColumn) {
18154                        ResultColumn targetColumn = (ResultColumn) targetElement;
18155
18156                        targetName = targetColumn.getStarLinkColumnNames().get(index);
18157
18158                        
18159                        targetColumn target = new targetColumn();
18160                        target.setId(String.valueOf(targetColumn.getId()) + "_" + index);
18161                        if(targetColumn.getResultSet()!=null && targetColumn.getResultSet().getColumns()!=null) {
18162                                Map<String, ResultColumn> lookup = getResultSetColumnLookup(targetColumn.getResultSet().getColumns());
18163                                ResultColumn matched = lookup.get(targetName);
18164                                if (matched != null) {
18165                                        target.setId(String.valueOf(matched.getId()));
18166                                }
18167                        }
18168                        target.setColumn(targetName);
18169                        target.setStruct(targetColumn.isStruct());
18170                        target.setParent_id(String.valueOf(targetColumn.getResultSet().getId()));
18171                        target.setParent_name(getResultSetName(targetColumn.getResultSet()));
18172                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
18173                                target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
18174                                                + convertCoordinate(targetColumn.getEndPosition()));
18175                        }
18176                        relationElement.setTarget(target);
18177                } else if (targetElement instanceof TableColumn) {
18178                        TableColumn targetColumn = (TableColumn) targetElement;
18179
18180                        targetName = targetColumn.getStarLinkColumnNames().get(index);
18181
18182                        Map<String, TableColumn> tableLookup = getTableColumnLookup(targetColumn.getTable().getColumns());
18183                        TableColumn tableColumn = tableLookup.get(targetName);
18184
18185                        targetColumn target = new targetColumn();
18186                        if (tableColumn == null) {
18187                                target.setId(targetColumn.getId() + "_" + index);
18188                        } else {
18189                                target.setId(String.valueOf(tableColumn.getId()));
18190                        }
18191                        target.setStruct(targetColumn.isStruct());
18192                        target.setColumn(targetName);
18193                        target.setParent_id(String.valueOf(targetColumn.getTable().getId()));
18194                        target.setParent_name(targetColumn.getTable().getName());
18195                        if (relation.getTarget() instanceof TableColumnRelationshipElement) {
18196                                target.setParent_alias(((TableColumnRelationshipElement) relation.getTarget()).getTableAlias());
18197                        }
18198                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
18199                                target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
18200                                                + convertCoordinate(targetColumn.getEndPosition()));
18201                        }
18202                        relationElement.setTarget(target);
18203                } else {
18204                        return;
18205                }
18206
18207                Collection<RelationshipElement<?>> sourceElements = (Collection<RelationshipElement<?>>) relation.getSources();
18208                if (sourceElements.size() == 0) {
18209                        return;
18210                }
18211
18212                String targetIdentifierName = DlineageUtil.getIdentifierNormalColumnName(targetName);
18213                if (targetIdentifierName == null) {
18214                        return;
18215                }
18216
18217                // Record-fill expansion (#695): when the star being expanded is a
18218                // record VARIABLE's own * (holder type decided at analysis, not
18219                // inferred here), every field is filled by the SAME producer, so the
18220                // name filters below — written for SELECT-* expansion, where source
18221                // and target columns pair by name — must not drop it.
18222                // Record-fill keep-all applies to a SINGLE-source fill only: one opaque
18223                // producer genuinely feeds every field. A multi-source relation fanned
18224                // across the fields would multiply unproven field-level claims.
18225                boolean recordFillTarget = targetElement instanceof TableColumn
18226                                && ((TableColumn) targetElement).getTable() instanceof Variable
18227                                && relation.getSources() != null && relation.getSources().size() == 1;
18228
18229                // Track explicit (non-star) source columns per target star column to prevent
18230                // phantom star expansions. Skip for UNION targets where branches contribute independently.
18231                long targetColumnId = -1;
18232                boolean isUnionTarget = false;
18233                if (targetElement instanceof ResultColumn) {
18234                        ResultColumn tc = (ResultColumn) targetElement;
18235                        targetColumnId = tc.getId();
18236                        isUnionTarget = (tc.getResultSet() instanceof SelectSetResultSet);
18237                } else if (targetElement instanceof TableColumn) {
18238                        targetColumnId = ((TableColumn) targetElement).getId();
18239                }
18240
18241                if (!isUnionTarget && targetColumnId != -1) {
18242                        // Check if any source in this relationship is a non-star source matching the target
18243                        // column name, AND whose parent does NOT also contribute a star source.
18244                        // This distinguishes:
18245                        //   - "b.col_a" (definitive: b only provides explicit cols, not *) => suppress star expansion
18246                        //   - "aTab.id" (not definitive: aTab also provides *) => don't suppress
18247                        boolean hasDefinitiveExplicitSource = hasDefinitiveNonStarSource(sourceElements, targetIdentifierName);
18248                        if (hasDefinitiveExplicitSource) {
18249                                if (!explicitStarTargetColumns.containsKey(targetColumnId)) {
18250                                        explicitStarTargetColumns.put(targetColumnId, new HashSet<String>());
18251                                }
18252                                explicitStarTargetColumns.get(targetColumnId).add(targetIdentifierName);
18253                        }
18254                }
18255
18256                // Column names that have explicit (non-star) sources for this target, across all relationships
18257                Set<String> targetExplicitNames = isUnionTarget ? null : explicitStarTargetColumns.get(targetColumnId);
18258
18259                for (RelationshipElement<?> sourceItem: sourceElements) {
18260                        Object sourceElement = sourceItem.getElement();
18261                        if (sourceElement instanceof ResultColumn) {
18262                                ResultColumn sourceColumn = (ResultColumn) sourceElement;
18263                                if (sourceColumn.hasStarLinkColumn()) {
18264                                        List<String> linkColumnNames = sourceColumn.getStarLinkColumnNames();
18265                                        int linkColumnNameSize = linkColumnNames.size();
18266                                        for (int k = 0; k < linkColumnNameSize; k++) {
18267                                                String sourceName = linkColumnNames.get(k);
18268                                                if (relation.getRelationshipType() == RelationshipType.fdd) {
18269                                                        if (!targetIdentifierName.equalsIgnoreCase(sourceName) && !"*".equals(sourceName))
18270                                                                continue;
18271                                                }
18272                                                // Skip star-expanded source when an explicit (non-star) source provides
18273                                                // the same column, either in this relationship or a prior one.
18274                                                if (targetExplicitNames != null && targetExplicitNames.contains(sourceName)) {
18275                                                        continue;
18276                                                }
18277                                                sourceColumn source = new sourceColumn();                                               
18278                                                
18279                                                boolean find = false;
18280                                                if(sourceColumn.getResultSet()!=null && sourceColumn.getResultSet().getColumns()!=null) {
18281                                                        Map<String, ResultColumn> srcLookup = getResultSetColumnLookup(sourceColumn.getResultSet().getColumns());
18282                                                        ResultColumn matched = srcLookup.get(sourceName);
18283                                                        if (matched != null) {
18284                                                                source.setId(String.valueOf(matched.getId()));
18285                                                                find = true;
18286                                                        }
18287                                                }
18288                                                if(!find) {
18289                                                        source.setId(String.valueOf(sourceColumn.getId()) + "_" + k);
18290                                                }
18291                                                source.setColumn(sourceName);
18292                                                source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
18293                                                source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
18294                                                if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
18295                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
18296                                                                        + convertCoordinate(sourceColumn.getEndPosition()));
18297                                                }
18298                                                if (sourceItem.getTransforms() != null) {
18299                                                        for (Transform transform : sourceItem.getTransforms()) {
18300                                                                source.addTransform(transform);
18301                                                        }
18302                                                }
18303                                                relationElement.addSource(source);
18304                                        }
18305                                        if(relationElement.getSources().isEmpty()
18306                                                && !(targetExplicitNames != null && targetExplicitNames.contains(targetIdentifierName))
18307                                                && sourceColumn.getResultSet()!=null && sourceColumn.getResultSet().getColumns()!=null) {
18308                                                Map<String, ResultColumn> srcLookup = getResultSetColumnLookup(sourceColumn.getResultSet().getColumns());
18309                                                for (Map.Entry<String, ResultColumn> entry : srcLookup.entrySet()) {
18310                                                        if (entry.getKey().equalsIgnoreCase(targetIdentifierName)) {
18311                                                                ResultColumn columnModel = entry.getValue();
18312                                                                sourceColumn source = new sourceColumn();
18313                                                                source.setId(String.valueOf(columnModel.getId()));
18314                                                                source.setColumn(columnModel.getName());
18315                                                                source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
18316                                                                source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
18317                                                                if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
18318                                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
18319                                                                                        + convertCoordinate(sourceColumn.getEndPosition()));
18320                                                                }
18321                                                                if (sourceItem.getTransforms() != null) {
18322                                                                        for (Transform transform : sourceItem.getTransforms()) {
18323                                                                                source.addTransform(transform);
18324                                                                        }
18325                                                                }
18326                                                                relationElement.addSource(source);
18327                                                                break;
18328                                                        }
18329                                                }
18330                                        }
18331                                        if (relationElement.getSources().isEmpty()
18332                                                && !(targetExplicitNames != null && targetExplicitNames.contains(targetIdentifierName))) {
18333                                                TObjectName sourceStarLinkColumn = new TObjectName();
18334                                                sourceStarLinkColumn.setString(targetName);
18335                                                boolean newBinding = sourceColumn.bindStarLinkColumn(sourceStarLinkColumn);
18336                                                sourceColumn source = new sourceColumn();
18337                                                String sourceName = DlineageUtil.getColumnName(sourceStarLinkColumn);
18338                                                if (!newBinding) {
18339                                                        source.setId(String.valueOf(sourceColumn.getId()) + "_"
18340                                                                        + sourceColumn.indexOfStarLinkColumn(sourceStarLinkColumn));
18341                                                } else {
18342                                                        source.setId(String.valueOf(sourceColumn.getId()) + "_" + linkColumnNameSize);
18343                                                }
18344                                                source.setColumn(sourceName);
18345                                                source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
18346                                                source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
18347                                                if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
18348                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
18349                                                                        + convertCoordinate(sourceColumn.getEndPosition()));
18350                                                }
18351                                                if (sourceItem.getTransforms() != null) {
18352                                                        for (Transform transform : sourceItem.getTransforms()) {
18353                                                                source.addTransform(transform);
18354                                                        }
18355                                                }
18356                                                relationElement.addSource(source);
18357
18358                                                if (newBinding) {
18359                                                        table resultSetElement = null;
18360                                                        for (table t : dataflow.getResultsets()) {
18361                                                                if (t.getId().equals(String.valueOf(sourceColumn.getResultSet().getId()))) {
18362                                                                        resultSetElement = t;
18363                                                                        break;
18364                                                                }
18365                                                        }
18366                                                        if (resultSetElement != null) {
18367                                                                column columnElement = new column();
18368                                                                columnElement.setId(String.valueOf(sourceColumn.getId()) + "_" + linkColumnNameSize);
18369                                                                columnElement.setName(sourceName);
18370                                                                if (sourceColumn.isFunction()) {
18371                                                                        columnElement.setIsFunction(String.valueOf(sourceColumn.isFunction()));
18372                                                                }
18373                                                                if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
18374                                                                        columnElement.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
18375                                                                                        + convertCoordinate(sourceColumn.getEndPosition()));
18376                                                                }
18377                                                                resultSetElement.getColumns().add(columnElement);
18378                                                        }
18379                                                }
18380                                        }
18381                                } else {
18382                                        sourceColumn source = new sourceColumn();
18383                                        source.setId(String.valueOf(sourceColumn.getId()));
18384                                        source.setColumn(sourceColumn.getName());
18385                                        source.setStruct(sourceColumn.isStruct());
18386                                        source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
18387                                        source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
18388                                        if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
18389                                                source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
18390                                                                + convertCoordinate(sourceColumn.getEndPosition()));
18391                                        }
18392                                        if (relation.getRelationshipType() == RelationshipType.fdd) {
18393                                                // Model display-name compares (can be constants/expressions) — see
18394                                                // the P0d.3c2 reclassification note above.
18395                                                if (!targetIdentifierName
18396                                                                .equalsIgnoreCase(DlineageUtil.getIdentifierNormalColumnName(sourceColumn.getName()))) {
18397                                                        if (!"*".equals(sourceColumn.getName())) {
18398                                                                // Record-fill (#695): keep a producer only when it
18399                                                                // is OPAQUE — a single-column result (function
18400                                                                // value, collection element) genuinely feeds every
18401                                                                // field. A source from a multi-column result shape
18402                                                                // pairs its own fields by name; fanning it onto
18403                                                                // fields of OTHER shapes asserts dependencies the
18404                                                                // SQL never made.
18405                                                                if (!recordFillTarget) {
18406                                                                        continue;
18407                                                                }
18408                                                                boolean opaqueProducer = sourceColumn.getResultSet() == null
18409                                                                                || sourceColumn.getResultSet().getColumns() == null
18410                                                                                || sourceColumn.getResultSet().getColumns().size() <= 1;
18411                                                                if (!opaqueProducer) {
18412                                                                        continue;
18413                                                                }
18414                                                        }
18415                                                        else {
18416                                                                Map<String, ResultColumn> srcLookup = getResultSetColumnLookup(sourceColumn.getResultSet().getColumns());
18417                                                                boolean flag = false;
18418                                                                for (String key : srcLookup.keySet()) {
18419                                                                        if (targetIdentifierName.equalsIgnoreCase(key)) {
18420                                                                                flag = true;
18421                                                                                break;
18422                                                                        }
18423                                                                }
18424                                                                if(flag) {
18425                                                                        continue;
18426                                                                }
18427                                                        }
18428                                                }
18429                                        }
18430                                        if (sourceItem.getTransforms() != null) {
18431                                                for (Transform transform : sourceItem.getTransforms()) {
18432                                                        source.addTransform(transform);
18433                                                }
18434                                        }
18435                                        relationElement.addSource(source);
18436                                }
18437                        } else if (sourceElement instanceof TableColumn) {
18438                                TableColumn sourceColumn = (TableColumn) sourceElement;
18439                                if (!sourceColumn.isPseduo() && sourceColumn.hasStarLinkColumn()) {
18440                                        List<String> linkColumnNames = sourceColumn.getStarLinkColumnNames();
18441                                        int linkColumnNameSize = linkColumnNames.size();
18442                                        for (int k = 0; k < linkColumnNameSize; k++) {
18443                                                String sourceName = linkColumnNames.get(k);
18444                                                if (relation.getRelationshipType() == RelationshipType.fdd) {
18445                                                        if (!targetIdentifierName.equalsIgnoreCase(sourceName) && !"*".equals(sourceName))
18446                                                                continue;
18447                                                }
18448
18449                                                Map<String, TableColumn> srcTableLookup = getTableColumnLookup(sourceColumn.getTable().getColumns());
18450                                                TableColumn tableColumn = srcTableLookup.get(sourceName);
18451
18452                                                sourceColumn source = new sourceColumn();
18453                                                if (tableColumn == null) {
18454                                                        source.setId(sourceColumn.getId() + "_" + k);
18455                                                } else {
18456                                                        source.setId(String.valueOf(tableColumn.getId()));
18457                                                }
18458                                                source.setColumn(sourceName);
18459                                                source.setStruct(sourceColumn.isStruct());
18460                                                if (containStarColumn(sourceElements, sourceName)) {
18461                                                        continue;
18462                                                }
18463                                                // Cross-relationship check: skip if explicit source was found in prior relationship
18464                                                if (targetExplicitNames != null && targetExplicitNames.contains(sourceName)) {
18465                                                        continue;
18466                                                }
18467                                                if (sourceColumn.getTable().getColumns().size() > 1) {
18468                                                        for (int y = 0; y < sourceColumn.getTable().getColumns().size(); y++) {
18469                                                                if (SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotColumn,
18470                                                                                sourceColumn.getTable().getColumns().get(y).getName(), sourceName)) {
18471                                                                        source.setId(String.valueOf(sourceColumn.getTable().getColumns().get(y).getId()));
18472                                                                        break;
18473                                                                }
18474                                                        }
18475                                                }
18476                                                source.setParent_id(String.valueOf(sourceColumn.getTable().getId()));
18477                                                source.setParent_name(getTableName(sourceColumn.getTable()));
18478                                                if (sourceItem instanceof TableColumnRelationshipElement) {
18479                                                        source.setParent_alias(
18480                                                                        ((TableColumnRelationshipElement) sourceItem).getTableAlias());
18481                                                }
18482                                                if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
18483                                                        source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
18484                                                                        + convertCoordinate(sourceColumn.getEndPosition()));
18485                                                }
18486                                                if (sourceItem.getTransforms() != null) {
18487                                                        for (Transform transform : sourceItem.getTransforms()) {
18488                                                                source.addTransform(transform);
18489                                                        }
18490                                                }
18491                                                relationElement.addSource(source);
18492                                        }
18493                                } else {
18494                                        sourceColumn source = new sourceColumn();
18495                                        source.setId(String.valueOf(sourceColumn.getId()));
18496                                        source.setColumn(sourceColumn.getName());
18497                                        source.setStruct(sourceColumn.isStruct());
18498                                        source.setParent_id(String.valueOf(sourceColumn.getTable().getId()));
18499                                        source.setParent_name(getTableName(sourceColumn.getTable()));
18500                                        if (sourceItem instanceof TableColumnRelationshipElement) {
18501                                                source.setParent_alias(((TableColumnRelationshipElement) sourceItem).getTableAlias());
18502                                        }
18503                                        if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) {
18504                                                source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + ","
18505                                                                + convertCoordinate(sourceColumn.getEndPosition()));
18506                                        }
18507                                        if (relation.getRelationshipType() == RelationshipType.fdd) {
18508                                                // Model display-name compare — see the P0d.3c2 reclassification note.
18509                                                if (!targetIdentifierName
18510                                                                .equalsIgnoreCase(DlineageUtil.getIdentifierNormalColumnName(sourceColumn.getName()))
18511                                                                && !"*".equals(sourceColumn.getName())) {
18512                                                        // Record-fill (#695): same keep-only-if-opaque rule as
18513                                                        // the resultset-source branch above.
18514                                                        if (!recordFillTarget) {
18515                                                                continue;
18516                                                        }
18517                                                        boolean opaqueProducer = sourceColumn.getTable() == null
18518                                                                        || sourceColumn.getTable().getColumns() == null
18519                                                                        || sourceColumn.getTable().getColumns().size() <= 1;
18520                                                        if (!opaqueProducer) {
18521                                                                continue;
18522                                                        }
18523                                                }
18524                                        }
18525                                        if (sourceItem.getTransforms() != null) {
18526                                                for (Transform transform : sourceItem.getTransforms()) {
18527                                                        source.addTransform(transform);
18528                                                }
18529                                        }
18530                                        relationElement.addSource(source);
18531                                }
18532                        }
18533                }
18534
18535                if (relationElement.getTarget() != null && relationElement.getSources() != null
18536                                && !relationElement.getSources().isEmpty()) {
18537                        dataflow.getRelationships().add(relationElement);
18538                }
18539        }
18540
18541        private String getColumnName(TObjectName column) {
18542                if (column == null) {
18543                        return null;
18544                }
18545                String name = column.getColumnNameOnly();
18546                if (name == null || "".equals(name.trim())) {
18547                        return DlineageUtil.getIdentifierNormalColumnName(column.toString().trim());
18548                } else
18549                        return DlineageUtil.getIdentifierNormalColumnName(name.trim());
18550        }
18551
18552        /**
18553         * For BigQuery/Redshift struct field access, returns the full struct path
18554         * (e.g., "customer.name" from ColumnSource with exposedName="customer", fieldPath=["name"]).
18555         * Checks StructFieldHint first (for 3+ part no-alias), then ColumnSource (for 2-part/alias).
18556         * Returns null if this is not a struct field access.
18557         */
18558        private String getStructFieldFullName(TObjectName column) {
18559                if (column == null) return null;
18560                if (getOption().getVendor() != EDbVendor.dbvbigquery
18561                        && getOption().getVendor() != EDbVendor.dbvredshift) return null;
18562                // Priority 1: StructFieldHint (side-channel, for 3+ part no-alias deep struct access)
18563                gudusoft.gsqlparser.resolver2.model.StructFieldHint hint = column.getStructFieldHint();
18564                if (hint != null && hint.getFieldPath() != null && !hint.getFieldPath().isEmpty()) {
18565                        return hint.toFullReference();
18566                }
18567                // Priority 2: ColumnSource (main resolution, for 2-part and alias cases)
18568                gudusoft.gsqlparser.resolver2.model.ColumnSource source = column.getColumnSource();
18569                if (source != null && source.isStructFieldAccess() && source.hasFieldPath()) {
18570                        return source.getFieldPath().toFullReference(source.getExposedName());
18571                }
18572                return null;
18573        }
18574
18575        /**
18576         * Get the base column name for a struct field access column.
18577         * Checks StructFieldHint first (3+ part no-alias), then ColumnSource (2-part/alias).
18578         * Returns null if not a struct field access.
18579         */
18580        private String getStructFieldBaseName(TObjectName column) {
18581                if (column == null) return null;
18582                gudusoft.gsqlparser.resolver2.model.StructFieldHint hint = column.getStructFieldHint();
18583                if (hint != null && hint.getBaseColumn() != null) {
18584                        return hint.getBaseColumn();
18585                }
18586                gudusoft.gsqlparser.resolver2.model.ColumnSource source = column.getColumnSource();
18587                if (source != null && source.isStructFieldAccess()) {
18588                        return source.getExposedName();
18589                }
18590                return null;
18591        }
18592
18593        private String getColumnName(String column) {
18594                if (column == null) {
18595                        return null;
18596                }
18597                String name = column.substring(column.lastIndexOf(".") + 1).trim();
18598                if (name.length() == 0) {
18599                        name = column.trim();
18600                }
18601                // Memoize the normalization. DlineageUtil.getIdentifierNormalColumnName(name)
18602                // resolves the vendor from ModelBindingManager.getGlobalVendor() internally,
18603                // so the result is a pure function of (vendor, name) — NOT of name alone.
18604                // The cache key therefore includes the vendor; passing it explicitly to the
18605                // 2-arg overload is identical to the original call for that same vendor.
18606                EDbVendor vendor = ModelBindingManager.getGlobalVendor();
18607                String cacheKey = (vendor == null ? "" : vendor.name()) + ":" + name;
18608                String normalized = normalizedColumnNameCache.get(cacheKey);
18609                if (normalized == null) {
18610                        normalized = DlineageUtil.getIdentifierNormalColumnName(name, vendor);
18611                        if (normalized != null) {
18612                                normalizedColumnNameCache.put(cacheKey, normalized);
18613                        }
18614                }
18615                return normalized;
18616        }
18617        
18618        private String getColumnNameOnly(String column) {
18619                if (column == null) {
18620                        return null;
18621                }
18622                return DlineageUtil.getColumnNameOnly(column);
18623        }
18624
18625        private void appendRecordSetRelation(dataflow dataflow, Relationship[] relations) {
18626                for (int i = 0; i < relations.length; i++) {
18627                        AbstractRelationship relation = (AbstractRelationship) relations[i];
18628                        relationship relationElement = new relationship();
18629                        relationElement.setType(relation.getRelationshipType().name());
18630                        if (relation.getFunction() != null) {
18631                                relationElement.setFunction(relation.getFunction());
18632                        }
18633                        if (relation.getEffectType() != null) {
18634                                relationElement.setEffectType(relation.getEffectType().name());
18635                        }
18636                        relationElement.setSqlHash(relation.getSqlHash());
18637                        relationElement.setSqlComment(relation.getSqlComment());
18638
18639                        if (relation.getProcedureId() != null) {
18640                                relationElement.setProcedureId(String.valueOf(relation.getProcedureId()));
18641                        }
18642                        relationElement.setId(String.valueOf(relation.getId()));
18643
18644                        if (relation instanceof RecordSetRelationship) {
18645                                RecordSetRelationship recordCountRelation = (RecordSetRelationship) relation;
18646
18647                                Object targetElement = recordCountRelation.getTarget().getElement();
18648                                if (targetElement instanceof ResultColumn) {
18649                                        ResultColumn targetColumn = (ResultColumn) targetElement;
18650                                        targetColumn target = new targetColumn();
18651                                        target.setId(String.valueOf(targetColumn.getId()));
18652                                        target.setColumn(targetColumn.getName());
18653                                        target.setStruct(targetColumn.isStruct());
18654                                        target.setFunction(recordCountRelation.getAggregateFunction());
18655                                        target.setParent_id(String.valueOf(targetColumn.getResultSet().getId()));
18656                                        target.setParent_name(getResultSetName(targetColumn.getResultSet()));
18657                                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
18658                                                target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
18659                                                                + convertCoordinate(targetColumn.getEndPosition()));
18660                                        }
18661                                        relationElement.setTarget(target);
18662                                } else if (targetElement instanceof TableColumn) {
18663                                        TableColumn targetColumn = (TableColumn) targetElement;
18664                                        targetColumn target = new targetColumn();
18665                                        target.setId(String.valueOf(targetColumn.getId()));
18666                                        target.setColumn(targetColumn.getName());
18667                                        target.setStruct(targetColumn.isStruct());
18668                                        target.setFunction(recordCountRelation.getAggregateFunction());
18669                                        target.setParent_id(String.valueOf(targetColumn.getTable().getId()));
18670                                        target.setParent_name(getTableName(targetColumn.getTable()));
18671                                        if (recordCountRelation.getTarget() instanceof TableColumnRelationshipElement) {
18672                                                target.setParent_alias(
18673                                                                ((TableColumnRelationshipElement) recordCountRelation.getTarget()).getTableAlias());
18674                                        }
18675                                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
18676                                                target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
18677                                                                + convertCoordinate(targetColumn.getEndPosition()));
18678                                        }
18679                                        relationElement.setTarget(target);
18680                                } else if (targetElement instanceof ResultSetRelationRows) {
18681                                        ResultSetRelationRows targetColumn = (ResultSetRelationRows) targetElement;
18682                                        targetColumn target = new targetColumn();
18683                                        target.setId(String.valueOf(targetColumn.getId()));
18684                                        target.setColumn(targetColumn.getName());
18685                                        target.setParent_id(String.valueOf(targetColumn.getHolder().getId()));
18686                                        target.setParent_name(getResultSetName(targetColumn.getHolder()));
18687                                        if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) {
18688                                                target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + ","
18689                                                                + convertCoordinate(targetColumn.getEndPosition()));
18690                                        }
18691                                        target.setSource("system");
18692                                        relationElement.setTarget(target);
18693                                } else {
18694                                        continue;
18695                                }
18696
18697                                Collection<RelationshipElement<?>> sourceElements = (Collection<RelationshipElement<?>>)recordCountRelation.getSources();
18698                                if (sourceElements.size() == 0) {
18699                                        continue;
18700                                }
18701
18702                                boolean append = false;
18703                                for (RelationshipElement<?> sourceItem: sourceElements) {
18704                                        Object sourceElement = sourceItem.getElement();
18705                                        if (sourceElement instanceof Table) {
18706                                                Table table = (Table) sourceElement;
18707                                                sourceColumn source = new sourceColumn();
18708                                                source.setSource_id(String.valueOf(table.getId()));
18709                                                source.setSource_name(getTableName(table));
18710                                                if (table.getStartPosition() != null && table.getEndPosition() != null) {
18711                                                        source.setCoordinate(convertCoordinate(table.getStartPosition()) + ","
18712                                                                        + convertCoordinate(table.getEndPosition()));
18713                                                }
18714                                                append = true;
18715                                                relationElement.addSource(source);
18716                                        } else if (sourceElement instanceof QueryTable) {
18717                                                QueryTable table = (QueryTable) sourceElement;
18718                                                sourceColumn source = new sourceColumn();
18719                                                source.setSource_id(String.valueOf(table.getId()));
18720                                                source.setSource_name(getResultSetName(table));
18721                                                if (table.getStartPosition() != null && table.getEndPosition() != null) {
18722                                                        source.setCoordinate(convertCoordinate(table.getStartPosition()) + ","
18723                                                                        + convertCoordinate(table.getEndPosition()));
18724                                                }
18725                                                append = true;
18726                                                relationElement.addSource(source);
18727                                        } else if (sourceElement instanceof TableRelationRows) {
18728                                                TableRelationRows relationRows = (TableRelationRows) sourceElement;
18729                                                sourceColumn source = new sourceColumn();
18730                                                source.setId(String.valueOf(relationRows.getId()));
18731                                                source.setColumn(relationRows.getName());
18732                                                source.setParent_id(String.valueOf(relationRows.getHolder().getId()));
18733                                                source.setParent_name(getTableName(relationRows.getHolder()));
18734                                                if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
18735                                                        source.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
18736                                                                        + convertCoordinate(relationRows.getEndPosition()));
18737                                                }
18738                                                source.setSource("system");
18739                                                append = true;
18740                                                relationElement.addSource(source);
18741                                        } else if (sourceElement instanceof ResultSetRelationRows) {
18742                                                ResultSetRelationRows relationRows = (ResultSetRelationRows) sourceElement;
18743                                                sourceColumn source = new sourceColumn();
18744                                                source.setId(String.valueOf(relationRows.getId()));
18745                                                source.setColumn(relationRows.getName());
18746                                                source.setParent_id(String.valueOf(relationRows.getHolder().getId()));
18747                                                source.setParent_name(getResultSetName(relationRows.getHolder()));
18748                                                if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
18749                                                        source.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
18750                                                                        + convertCoordinate(relationRows.getEndPosition()));
18751                                                }
18752                                                source.setSource("system");
18753                                                append = true;
18754                                                relationElement.addSource(source);
18755                                        } else if (sourceElement instanceof TableColumn) {
18756                                                TableColumn sourceColumn = (TableColumn) sourceElement;
18757                                                sourceColumn source = new sourceColumn();
18758                                                source.setId(String.valueOf(sourceColumn.getId()));
18759                                                source.setColumn(sourceColumn.getName());
18760                                                source.setStruct(sourceColumn.isStruct());
18761                                                source.setParent_id(String.valueOf(sourceColumn.getTable().getId()));
18762                                                source.setParent_name(getTableName(sourceColumn.getTable()));
18763                                                if (sourceItem instanceof TableColumnRelationshipElement) {
18764                                                        source.setParent_alias(
18765                                                                        ((TableColumnRelationshipElement) sourceItem).getTableAlias());
18766                                                }
18767                                                setSourceCoordinate(source, sourceItem, sourceColumn);
18768                                                append = true;
18769                                                relationElement.addSource(source);
18770                                        }
18771                                        if (sourceElement instanceof ResultColumn) {
18772                                                ResultColumn sourceColumn = (ResultColumn) sourceElement;
18773                                                sourceColumn source = new sourceColumn();
18774                                                source.setId(String.valueOf(sourceColumn.getId()));
18775                                                source.setColumn(sourceColumn.getName());
18776                                                source.setStruct(sourceColumn.isStruct());
18777                                                source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId()));
18778                                                source.setParent_name(getResultSetName(sourceColumn.getResultSet()));
18779                                                setSourceCoordinate(source, sourceItem, sourceColumn);
18780                                                append = true;
18781                                                relationElement.addSource(source);
18782                                        }
18783                                }
18784
18785                                if (append)
18786                                        dataflow.getRelationships().add(relationElement);
18787                        }
18788                }
18789        }
18790
18791        private void appendCallRelation(dataflow dataflow, Relationship[] relations) {
18792                for (int i = 0; i < relations.length; i++) {
18793                        AbstractRelationship relation = (AbstractRelationship) relations[i];
18794                        relationship relationElement = new relationship();
18795                        relationElement.setType(relation.getRelationshipType().name());
18796                        if (relation.getFunction() != null) {
18797                                relationElement.setFunction(relation.getFunction());
18798                        }
18799                        if (relation.getEffectType() != null) {
18800                                relationElement.setEffectType(relation.getEffectType().name());
18801                        }
18802                        relationElement.setSqlHash(relation.getSqlHash());
18803                        relationElement.setSqlComment(relation.getSqlComment());
18804                        
18805                        if (relation.getProcedureId() != null) {
18806                                relationElement.setProcedureId(String.valueOf(relation.getProcedureId()));
18807                        }
18808                        relationElement.setId(String.valueOf(relation.getId()));
18809
18810                        if (relation instanceof CallRelationship) {
18811                                CallRelationship callRelation = (CallRelationship) relation;
18812                                
18813                                if (callRelation.getCallObject() != null) {
18814                                        relationElement.setCallStmt(callRelation.getCallObject().toString());
18815                                        if (callRelation.getStartPosition() != null && callRelation.getEndPosition() != null) {
18816                                                relationElement.setCallCoordinate(convertCoordinate(callRelation.getStartPosition()) + ","
18817                                                                + convertCoordinate(callRelation.getEndPosition()));
18818                                        }
18819                                }
18820                                
18821                                if (Boolean.TRUE.equals(callRelation.getBuiltIn())) {
18822                                        relationElement.setBuiltIn(true);
18823                                }
18824                                Object targetElement = callRelation.getTarget().getElement();
18825                                if (targetElement instanceof Procedure) {
18826                                        Procedure procedure = (Procedure) targetElement;
18827                                        targetColumn target = new targetColumn();
18828                                        target.setId(String.valueOf(procedure.getId()));
18829                                        target.setName(getProcedureName(procedure));
18830                                        if (procedure.getStartPosition() != null && procedure.getEndPosition() != null) {
18831                                                target.setCoordinate(convertCoordinate(procedure.getStartPosition()) + ","
18832                                                                + convertCoordinate(procedure.getEndPosition()));
18833                                        }
18834                                        String clazz = procedure.getProcedureObject().getClass().getSimpleName().toLowerCase();
18835                                        if (clazz.indexOf("function") != -1) {
18836                                                target.setType("function");
18837                                        } else if (clazz.indexOf("trigger") != -1) {
18838                                                target.setType("trigger");
18839                                        } else if (clazz.indexOf("macro") != -1) {
18840                                                target.setType("macro");
18841                                        } else {
18842                                                target.setType("procedure");
18843                                        }
18844                                        relationElement.setCaller(target);
18845                                } else {
18846                                        continue;
18847                                }
18848
18849                                Collection<RelationshipElement<?>> sourceElements = (Collection<RelationshipElement<?>>)callRelation.getSources();
18850                                if (sourceElements.size() == 0) {
18851                                        continue;
18852                                }
18853
18854                                boolean append = false;
18855                                for (RelationshipElement<?> sourceItem: sourceElements) {
18856                                        Object sourceElement = sourceItem.getElement();
18857                                        if (sourceElement instanceof Procedure) {
18858                                                Procedure procedure = (Procedure) sourceElement;
18859                                                sourceColumn source = new sourceColumn();
18860                                                source.setId(String.valueOf(procedure.getId()));
18861                                                source.setName(getProcedureName(procedure));
18862                                                if (procedure.getStartPosition() != null && procedure.getEndPosition() != null) {
18863                                                        source.setCoordinate(convertCoordinate(procedure.getStartPosition()) + ","
18864                                                                        + convertCoordinate(procedure.getEndPosition()));
18865                                                }
18866                                                String clazz = procedure.getProcedureObject().getClass().getSimpleName().toLowerCase();
18867                                                if (clazz.indexOf("function") != -1) {
18868                                                        source.setType("function");
18869                                                } else if (clazz.indexOf("trigger") != -1) {
18870                                                        source.setType("trigger");
18871                                                } else if (clazz.indexOf("macro") != -1) {
18872                                                        source.setType("macro");
18873                                                } else {
18874                                                        source.setType("procedure");
18875                                                }
18876                                                append = true;
18877                                                relationElement.getCallees().add(source);
18878                                        } else if (sourceElement instanceof Function) {
18879                                                Function function = (Function) sourceElement;
18880                                                sourceColumn source = new sourceColumn();
18881                                                source.setId(String.valueOf(function.getId()));
18882                                                source.setName(getFunctionName(function.getFunctionObject()));
18883                                                if (function.getStartPosition() != null && function.getEndPosition() != null) {
18884                                                        source.setCoordinate(convertCoordinate(function.getStartPosition()) + ","
18885                                                                        + convertCoordinate(function.getEndPosition()));
18886                                                }
18887                                                source.setType("function");
18888                                                append = true;
18889                                                relationElement.getCallees().add(source);
18890                                        }
18891                                }
18892
18893                                if (append)
18894                                        dataflow.getRelationships().add(relationElement);
18895                        }
18896                }
18897        }
18898
18899        private void appendResultSets(dataflow dataflow) {
18900                Set<ResultSet> resultSets = modelManager.getResultSets(); 
18901                for (ResultSet resultSet: resultSets) {
18902                        appendResultSet(dataflow, resultSet);
18903                }
18904        }
18905
18906        private void appendResultSet(dataflow dataflow, ResultSet resultSetModel) {
18907                if (!appendResultSets.contains(resultSetModel)) {
18908                        appendResultSets.add(resultSetModel);
18909                } else {
18910                        return;
18911                }
18912
18913                table resultSetElement = new table();
18914                resultSetElement.setId(String.valueOf(resultSetModel.getId()));
18915                resultSetElement.setServer(resultSetModel.getServer());
18916                if (!SQLUtil.isEmpty(resultSetModel.getDatabase())) {
18917                        resultSetElement.setDatabase(resultSetModel.getDatabase());
18918                }
18919                if (!SQLUtil.isEmpty(resultSetModel.getSchema())) {
18920                        resultSetElement.setSchema(resultSetModel.getSchema());
18921                }
18922                resultSetElement.setName(getResultSetName(resultSetModel));
18923                resultSetElement.setType(getResultSetType(resultSetModel));
18924                // if ((ignoreRecordSet || simpleOutput) && resultSetModel.isTarget()) {
18925                resultSetElement.setIsTarget(String.valueOf(resultSetModel.isTarget()));
18926                // }
18927                resultSetElement.setIsDetermined(String.valueOf(resultSetModel.isDetermined()));
18928                if (resultSetModel.getProcedureId() != null) {
18929                        resultSetElement.setProcedureId(resultSetModel.getProcedureId());
18930                }
18931                if (resultSetModel.getStartPosition() != null && resultSetModel.getEndPosition() != null) {
18932                        resultSetElement.setCoordinate(convertCoordinate(resultSetModel.getStartPosition()) + ","
18933                                        + convertCoordinate(resultSetModel.getEndPosition()));
18934                }
18935                dataflow.getResultsets().add(resultSetElement);
18936
18937                List<ResultColumn> columns = resultSetModel.getColumns();
18938
18939                Map<String, Integer> columnCounts = new HashMap<String, Integer>();
18940                for (ResultColumn column : columns) {
18941                        String columnName = DlineageUtil.getIdentifierNormalColumnName(column.getName());
18942                        if (!columnCounts.containsKey(columnName)) {
18943                                columnCounts.put(columnName, 0);
18944                        }
18945                        columnCounts.put(columnName, columnCounts.get(columnName) + 1);
18946                        // if (column.hasStarLinkColumn()) {
18947                        // List<String> starLinkColumns = column.getStarLinkColumnNames();
18948                        // for (int k = 0; k < starLinkColumns.size(); k++) {
18949                        // columnName = starLinkColumns.get(k);
18950                        // if (!columnCounts.containsKey(columnName)) {
18951                        // columnCounts.put(columnName, 0);
18952                        // }
18953                        // columnCounts.put(columnName, columnCounts.get(columnName) + 1);
18954                        // }
18955                        // }
18956                }
18957
18958                for (int j = 0; j < columns.size(); j++) {
18959                        ResultColumn columnModel = columns.get(j);
18960                        if (columnModel.hasStarLinkColumn()) {
18961                                // List<String> starLinkColumns =
18962                                // columnModel.getStarLinkColumnNames();
18963                                // for (int k = 0; k < starLinkColumns.size(); k++) {
18964                                // column columnElement = new column();
18965                                // columnElement.setId( String.valueOf(columnModel.getId()) +
18966                                // "_" + k);
18967                                // String columnName = starLinkColumns.get(k);
18968                                // columnElement.setName(columnName);
18969                                // if(columnModel.isFunction()){
18970                                // columnElement.setIsFunction(String.valueOf(columnModel.isFunction()));
18971                                // }
18972                                // if (columnModel.getStartPosition() != null &&
18973                                // columnModel.getEndPosition() != null) {
18974                                // columnElement.setCoordinate(
18975                                // columnModel.getStartPosition() + "," +
18976                                // columnModel.getEndPosition());
18977                                // }
18978                                // String identifier = columnName;
18979                                // if(columnCounts.containsKey(identifier) &&
18980                                // columnCounts.get(identifier)>1){
18981                                // TObjectName column =
18982                                // columnModel.getStarLinkColumns().get(columnName).iterator().next();
18983                                // if(!SQLUtil.isEmpty(getQualifiedTable(column))){
18984                                // columnElement.setQualifiedTable(getQualifiedTable(column));
18985                                // }
18986                                // }
18987                                // resultSetElement.getColumns().add(columnElement);
18988                                // }
18989                                if (columnModel.isShowStar()) {
18990                                        column columnElement = new column();
18991                                        columnElement.setId(String.valueOf(columnModel.getId()));
18992                                        columnElement.setName(columnModel.getName());
18993                                        if (columnModel.isFunction()) {
18994                                                columnElement.setIsFunction(String.valueOf(columnModel.isFunction()));
18995                                        }
18996                                        if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
18997                                                columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
18998                                                                + convertCoordinate(columnModel.getEndPosition()));
18999                                        }
19000
19001                                        String identifier = DlineageUtil.getIdentifierNormalColumnName(columnModel.getName());
19002                                        if (columnCounts.containsKey(identifier) && columnCounts.get(identifier) > 1) {
19003                                                String qualifiedTable = getQualifiedTable(columnModel);
19004                                                if (!SQLUtil.isEmpty(qualifiedTable)) {
19005                                                        columnElement.setQualifiedTable(qualifiedTable);
19006                                                }
19007                                        }
19008                                        resultSetElement.getColumns().add(columnElement);
19009                                }
19010                        } else {
19011                                column columnElement = new column();
19012                                columnElement.setId(String.valueOf(columnModel.getId()));
19013                                columnElement.setName(columnModel.getName());
19014                                if (columnModel.isFunction()) {
19015                                        columnElement.setIsFunction(String.valueOf(columnModel.isFunction()));
19016                                }
19017                                if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
19018                                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
19019                                                        + convertCoordinate(columnModel.getEndPosition()));
19020                                }
19021
19022                                String identifier = DlineageUtil.getIdentifierNormalColumnName(columnModel.getName());
19023                                if (columnCounts.containsKey(identifier) && columnCounts.get(identifier) > 1) {
19024                                        String qualifiedTable = getQualifiedTable(columnModel);
19025                                        if (!SQLUtil.isEmpty(qualifiedTable)) {
19026                                                columnElement.setQualifiedTable(qualifiedTable);
19027                                        }
19028                                }
19029                                resultSetElement.getColumns().add(columnElement);
19030                        }
19031                }
19032
19033                ResultSetRelationRows relationRows = resultSetModel.getRelationRows();
19034                if (relationRows.hasRelation()) {
19035                        column relationRowsElement = new column();
19036                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
19037                        relationRowsElement.setName(relationRows.getName());
19038                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
19039                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
19040                                                + convertCoordinate(relationRows.getEndPosition()));
19041                        }
19042                        relationRowsElement.setSource("system");
19043                        resultSetElement.getColumns().add(relationRowsElement);
19044                }
19045        }
19046
19047        private String getQualifiedTable(ResultColumn columnModel) {
19048                if (columnModel.getColumnObject() instanceof TObjectName) {
19049                        return getQualifiedTable((TObjectName) columnModel.getColumnObject());
19050                }
19051                if (columnModel.getColumnObject() instanceof TResultColumn) {
19052                        TObjectName field = ((TResultColumn) columnModel.getColumnObject()).getFieldAttr();
19053                        if (field != null) {
19054                                return getQualifiedTable(field);
19055                        }
19056                }
19057                return null;
19058        }
19059
19060        private String getQualifiedTable(TObjectName column) {
19061                if (column == null)
19062                        return null;
19063                String[] splits = column.toString().split("\\.");
19064                if (splits.length > 1) {
19065                        return splits[splits.length - 2];
19066                }
19067                return null;
19068        }
19069
19070        /**
19071         * Get the qualified prefix (schema.table) from a column name for 3-part names.
19072         * For example, for "sch.pk_constv2.c_cdsl", returns "sch.pk_constv2".
19073         * Returns null if the column doesn't have both schema and table parts.
19074         */
19075        private String getQualifiedPrefixFromColumn(TObjectName column) {
19076                if (column == null) return null;
19077
19078                // Check if both schema and table tokens are present (3-part name)
19079                String schemaStr = column.getSchemaString();
19080                String tableStr = column.getTableString();
19081
19082                if (schemaStr != null && !schemaStr.isEmpty() &&
19083                        tableStr != null && !tableStr.isEmpty()) {
19084                        return schemaStr + "." + tableStr;
19085                }
19086
19087                // Fallback: parse from toString() for complex cases
19088                String[] splits = column.toString().split("\\.");
19089                if (splits.length >= 3) {
19090                        // Return all parts except the last one (column name)
19091                        StringBuilder prefix = new StringBuilder();
19092                        for (int i = 0; i < splits.length - 1; i++) {
19093                                if (i > 0) prefix.append(".");
19094                                prefix.append(splits[i]);
19095                        }
19096                        return prefix.toString();
19097                }
19098
19099                return null;
19100        }
19101
19102        private String getResultSetType(ResultSet resultSetModel) {
19103                if (resultSetModel instanceof QueryTable) {
19104                        QueryTable table = (QueryTable) resultSetModel;
19105                        if (table.getTableObject().getCTE() != null) {
19106                                return "with_cte";
19107                        }
19108                }
19109
19110                if (resultSetModel instanceof SelectSetResultSet) {
19111                        ESetOperatorType type = ((SelectSetResultSet) resultSetModel).getSetOperatorType();
19112                        return "select_" + type.name();
19113                }
19114
19115                if (resultSetModel instanceof SelectResultSet) {
19116                        if (((SelectResultSet) resultSetModel).getSelectStmt().getParentStmt() instanceof TInsertSqlStatement) {
19117                                return "insert-select";
19118                        }
19119                        if (((SelectResultSet) resultSetModel).getSelectStmt().getParentStmt() instanceof TUpdateSqlStatement) {
19120                                return "update-select";
19121                        }
19122                }
19123
19124                if (resultSetModel.getGspObject() instanceof TMergeUpdateClause) {
19125                        return "merge-update";
19126                }
19127
19128                if (resultSetModel.getGspObject() instanceof TOutputClause) {
19129                        return ResultSetType.output.name();
19130                }
19131
19132                if (resultSetModel.getGspObject() instanceof TMergeInsertClause) {
19133                        return "merge-insert";
19134                }
19135
19136                if (resultSetModel.getGspObject() instanceof TUpdateSqlStatement) {
19137                        return "update-set";
19138                }
19139
19140                if (resultSetModel.getGspObject() instanceof TFunctionCall && ((TFunctionCall)resultSetModel.getGspObject()).getFunctionType() == EFunctionType.array_t) {
19141                        return ResultSetType.array.name();
19142                }
19143
19144                if (resultSetModel.getGspObject() instanceof TFunctionCall && ((TFunctionCall)resultSetModel.getGspObject()).getFunctionType() == EFunctionType.struct_t) {
19145                        return ResultSetType.struct.name();
19146                }
19147
19148                if (resultSetModel.getGspObject() instanceof TFunctionCall || resultSetModel instanceof Function) {
19149                        return ResultSetType.function.name();
19150                }
19151
19152                if (resultSetModel.getGspObject() instanceof TAliasClause) {
19153                        return ResultSetType.alias.name();
19154                }
19155
19156                if (resultSetModel.getGspObject() instanceof TCursorDeclStmt) {
19157                        return ResultSetType.cursor.name();
19158                }
19159
19160                if (resultSetModel instanceof PivotedTable) {
19161                        if (((PivotedTable) resultSetModel).isUnpivoted()) {
19162                                return ResultSetType.unpivot_table.name();
19163                        }
19164                        return ResultSetType.pivot_table.name();
19165                }
19166
19167                return "select_list";
19168        }
19169
19170        private String getTableName(Table tableModel) {
19171                if (modelManager.DISPLAY_NAME.containsKey(tableModel.getId())) {
19172                        return modelManager.DISPLAY_NAME.get(tableModel.getId());
19173                }
19174
19175                String tableName;
19176                if (tableModel.getFullName() != null && tableModel.getFullName().trim().length() > 0) {
19177                        return tableModel.getFullName();
19178                }
19179                if (tableModel.getAlias() != null && tableModel.getAlias().trim().length() > 0) {
19180                        tableName = getResultSetWithId("RESULT_OF_" + tableModel.getAlias());
19181
19182                } else {
19183                        tableName = getResultSetDisplayId("RS");
19184                }
19185                modelManager.DISPLAY_NAME.put(tableModel.getId(), tableName);
19186                return tableName;
19187        }
19188
19189        private String getProcedureName(Procedure procedureModel) {
19190                if (modelManager.DISPLAY_NAME.containsKey(procedureModel.getId())) {
19191                        return modelManager.DISPLAY_NAME.get(procedureModel.getId());
19192                }
19193
19194                String procedureName = procedureModel.getFullName();
19195
19196                modelManager.DISPLAY_NAME.put(procedureModel.getId(), procedureName);
19197                return procedureName;
19198        }
19199
19200        private String getProcessName(Process processModel) {
19201                if (modelManager.DISPLAY_NAME.containsKey(processModel.getId())) {
19202                        return modelManager.DISPLAY_NAME.get(processModel.getId());
19203                } else {
19204                        if (processModel.getCustomType() != null) {
19205                                String name = processModel.getCustomType();
19206                                modelManager.DISPLAY_NAME.put(processModel.getId(), name);
19207                                return name;
19208                        }
19209                        String name = processModel.getType();
19210                        String procedureName = getProcedureParentName(processModel.getGspObject());
19211                        if (procedureName != null) {
19212                                name = getResultSetDisplayId(procedureName + " " + name);
19213                        } else {
19214                                name = getResultSetDisplayId("Query " + name);
19215                        }
19216                        modelManager.DISPLAY_NAME.put(processModel.getId(), name);
19217                        return name;
19218                }
19219        }
19220
19221        private String getDisplayIdByType(String type) {
19222                if (!modelManager.DISPLAY_ID.containsKey(type)) {
19223                        modelManager.DISPLAY_ID.put(type, option.getStartId() + 1);
19224                        return type + "-" + (option.getStartId() + 1);
19225                } else {
19226                        long id = modelManager.DISPLAY_ID.get(type);
19227                        modelManager.DISPLAY_ID.put(type, id + 1);
19228                        return type + "-" + (id + 1);
19229                }
19230        }
19231        
19232        private String getDisplayIdByTypeFromZero(String type) {
19233                if (!modelManager.DISPLAY_ID.containsKey(type)) {
19234                        modelManager.DISPLAY_ID.put(type, option.getStartId());
19235                        if(option.getStartId() == 0) {
19236                                return type;
19237                        }
19238                        return type + "-" + (option.getStartId() + 1);
19239                } else {
19240                        long id = modelManager.DISPLAY_ID.get(type);
19241                        modelManager.DISPLAY_ID.put(type, id + 1);
19242                        return type + "-" + (id + 1);
19243                }
19244        }
19245
19246        private String getConstantName(Table tableModel) {
19247                if (modelManager.DISPLAY_NAME.containsKey(tableModel.getId())) {
19248                        return modelManager.DISPLAY_NAME.get(tableModel.getId());
19249                } else {
19250                        String name = getDisplayIdByType("SQL_CONSTANTS");
19251                        modelManager.DISPLAY_NAME.put(tableModel.getId(), name);
19252                        return name;
19253                }
19254        }
19255        
19256        private String getTempTableName(TTable table) {
19257                if (modelManager.DISPLAY_NAME.containsKey((long)System.identityHashCode(table))) {
19258                        return modelManager.DISPLAY_NAME.get((long)System.identityHashCode(table));
19259                } else {
19260                        String name = getDisplayIdByTypeFromZero(table.getName());
19261                        modelManager.DISPLAY_NAME.put((long)System.identityHashCode(table), name);
19262                        return name;
19263                }
19264        }
19265
19266        private String getResultSetName(ResultSet resultSetModel) {
19267
19268                if (modelManager.DISPLAY_NAME.containsKey(resultSetModel.getId())) {
19269                        return modelManager.DISPLAY_NAME.get(resultSetModel.getId());
19270                }
19271                
19272                if (resultSetModel.getGspObject() instanceof TFunctionCall && ((TFunctionCall)resultSetModel.getGspObject()).getFunctionType() == EFunctionType.array_t) {
19273                        String name = getResultSetDisplayId("ARRAY");
19274                        modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
19275                        if(option.containsResultSetType(ResultSetType.array)) {
19276                                resultSetModel.setTarget(true);
19277                        }
19278                        return name;
19279                }
19280                
19281                if (resultSetModel.getGspObject() instanceof TFunctionCall && ((TFunctionCall)resultSetModel.getGspObject()).getFunctionType() == EFunctionType.struct_t) {
19282                        String name = getResultSetDisplayId("STRUCT");
19283                        modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
19284                        if(option.containsResultSetType(ResultSetType.struct)) {
19285                                resultSetModel.setTarget(true);
19286                        }
19287                        return name;
19288                }
19289
19290                if (resultSetModel instanceof QueryTable) {
19291                        QueryTable table = (QueryTable) resultSetModel;
19292                        if (table.getAlias() != null && table.getAlias().trim().length() > 0) {
19293                                String name = getResultSetWithId("RESULT_OF_" + table.getAlias().trim());
19294                                if (table.getTableObject().getCTE() != null) {
19295                                        name = getResultSetWithId("RESULT_OF_" + table.getTableObject().getCTE().getTableName().toString()
19296                                                        + "_" + table.getAlias().trim());
19297                                }
19298                                modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
19299                                if(option.containsResultSetType(ResultSetType.result_of)) {
19300                                        resultSetModel.setTarget(true);
19301                                }
19302                                return name;
19303                        } else if (table.getTableObject().getCTE() != null) {
19304                                String name = getResultSetWithId("CTE-" + table.getTableObject().getCTE().getTableName().toString());
19305                                modelManager.DISPLAY_NAME.put(table.getId(), name);
19306                                if(option.containsResultSetType(ResultSetType.cte)) {
19307                                        resultSetModel.setTarget(true);
19308                                }
19309                                return name;
19310                        }
19311                }
19312
19313                if (resultSetModel instanceof SelectResultSet) {
19314                        if (((SelectResultSet) resultSetModel).getSelectStmt().getParentStmt() instanceof TInsertSqlStatement) {
19315                                String name = getResultSetDisplayId("INSERT-SELECT");
19316                                modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
19317                                if(option.containsResultSetType(ResultSetType.insert_select)) {
19318                                        resultSetModel.setTarget(true);
19319                                }
19320                                return name;
19321                        }
19322
19323                        if (((SelectResultSet) resultSetModel).getSelectStmt().getParentStmt() instanceof TUpdateSqlStatement) {
19324                                String name = getResultSetDisplayId("UPDATE-SELECT");
19325                                modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
19326                                if(option.containsResultSetType(ResultSetType.update_select)) {
19327                                        resultSetModel.setTarget(true);
19328                                }
19329                                return name;
19330                        }
19331                }
19332
19333                if (resultSetModel instanceof SelectSetResultSet) {
19334                        ESetOperatorType type = ((SelectSetResultSet) resultSetModel).getSetOperatorType();
19335                        String name = getResultSetDisplayId("RESULT_OF_" + type.name().toUpperCase());
19336                        modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
19337                        if(option.containsResultSetType(ResultSetType.result_of)) {
19338                                resultSetModel.setTarget(true);
19339                        }
19340                        return name;
19341                }
19342
19343                if (resultSetModel.getGspObject() instanceof TMergeUpdateClause) {
19344                        String name = getResultSetDisplayId("MERGE-UPDATE");
19345                        modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
19346                        if(option.containsResultSetType(ResultSetType.merge_update)) {
19347                                resultSetModel.setTarget(true);
19348                        }
19349                        return name;
19350                }
19351
19352                if (resultSetModel.getGspObject() instanceof TOutputClause) {
19353                        String name = getResultSetDisplayId("OUTPUT");
19354                        modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
19355                        if(option.containsResultSetType(ResultSetType.output)) {
19356                                resultSetModel.setTarget(true);
19357                        }
19358                        return name;
19359                }
19360
19361                if (resultSetModel.getGspObject() instanceof TMergeInsertClause) {
19362                        String name = getResultSetDisplayId("MERGE-INSERT");
19363                        modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
19364                        if(option.containsResultSetType(ResultSetType.merge_insert)) {
19365                                resultSetModel.setTarget(true);
19366                        }
19367                        return name;
19368                }
19369
19370                if (resultSetModel.getGspObject() instanceof TUpdateSqlStatement) {
19371                        String name = getResultSetDisplayId("UPDATE-SET");
19372                        modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
19373                        if(option.containsResultSetType(ResultSetType.update_set)) {
19374                                resultSetModel.setTarget(true);
19375                        }
19376                        return name;
19377                }
19378
19379                if (resultSetModel.getGspObject() instanceof TCaseExpression) {
19380                        String name = ((Function) resultSetModel).getFunctionName();
19381                        modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
19382                        if (option.containsResultSetType(ResultSetType.case_when) || option.containsResultSetType(ResultSetType.function)) {
19383                                resultSetModel.setTarget(true);
19384                        }
19385                        return name;
19386                }
19387                
19388                if (resultSetModel.getGspObject() instanceof TFunctionCall || resultSetModel instanceof Function) {
19389                        String name = ((Function) resultSetModel).getFunctionName();
19390                        modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
19391                        if(option.containsResultSetType(ResultSetType.function)) {
19392                                resultSetModel.setTarget(true);
19393                        }
19394                        return name;
19395                }
19396
19397                if (resultSetModel instanceof PivotedTable) {
19398                        String name = getResultSetDisplayId("PIVOT-TABLE");
19399                        if (((PivotedTable) resultSetModel).isUnpivoted()) {
19400                                name = getResultSetDisplayId("UNPIVOT-TABLE");
19401                                if(option.containsResultSetType(ResultSetType.unpivot_table)) {
19402                                        resultSetModel.setTarget(true);
19403                                }
19404                        }
19405                        else {
19406                                if(option.containsResultSetType(ResultSetType.pivot_table)) {
19407                                        resultSetModel.setTarget(true);
19408                                }
19409                        }
19410                        modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
19411                        return name;
19412                }
19413
19414                if (resultSetModel instanceof Alias) {
19415                        String name = getResultSetDisplayId("ALIAS");
19416                        modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
19417                        if(option.containsResultSetType(ResultSetType.alias)) {
19418                                resultSetModel.setTarget(true);
19419                        }
19420                        return name;
19421                }
19422
19423                String name = getResultSetDisplayId("RS");
19424                modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name);
19425                if(option.containsResultSetType(ResultSetType.select_list)) {
19426                        resultSetModel.setTarget(true);
19427                }
19428                return name;
19429        }
19430
19431        private String getResultSetWithId(String type) {
19432                type = DlineageUtil.getIdentifierNormalTableName(type);
19433                if (!modelManager.DISPLAY_ID.containsKey(type)) {
19434                        modelManager.DISPLAY_ID.put(type, option.getStartId() + 1);
19435                        return type + "-" + (option.getStartId() + 1);
19436                } else {
19437                        long id = modelManager.DISPLAY_ID.get(type);
19438                        modelManager.DISPLAY_ID.put(type, id + 1);
19439                        return type + "-" + (id + 1);
19440                }
19441        }
19442
19443        private String getResultSetDisplayId(String type) {
19444                if (!modelManager.DISPLAY_ID.containsKey(type)) {
19445                        modelManager.DISPLAY_ID.put(type, option.getStartId() + 1);
19446                        return type + "-" + (option.getStartId() + 1);
19447                } else {
19448                        long id = modelManager.DISPLAY_ID.get(type);
19449                        modelManager.DISPLAY_ID.put(type, id + 1);
19450                        return type + "-" + (id + 1);
19451                }
19452        }
19453
19454        private void appendViews(dataflow dataflow) {
19455                List<TCustomSqlStatement> views = modelManager.getViews();
19456                for (int i = 0; i < views.size(); i++) {
19457                        Table viewModel = (Table) modelManager.getViewModel(views.get(i));
19458                        if (!tableIds.contains(viewModel.getId())) {
19459                                appendViewModel(dataflow, viewModel);
19460                                tableIds.add(viewModel.getId());
19461                        }
19462                }
19463
19464                List<TTable> tables = modelManager.getBaseTables();
19465                for (int i = 0; i < tables.size(); i++) {
19466                        Object model = modelManager.getModel(tables.get(i));
19467                        if (model instanceof Table) {
19468                                Table tableModel = (Table) model;
19469                                if (tableModel.isView()) {
19470                                        if (!tableIds.contains(tableModel.getId())) {
19471                                                appendViewModel(dataflow, tableModel);
19472                                                tableIds.add(tableModel.getId());
19473                                        }
19474                                }
19475                        }
19476                }
19477
19478                List<Table> tableNames = modelManager.getTablesByName();
19479                for (int i = 0; i < tableNames.size(); i++) {
19480                        Table tableModel = tableNames.get(i);
19481                        if (tableModel.isView()) {
19482                                if (!tableIds.contains(tableModel.getId())) {
19483                                        appendViewModel(dataflow, tableModel);
19484                                        tableIds.add(tableModel.getId());
19485                                }
19486                        }
19487                }
19488        }
19489
19490        private void appendViewModel(dataflow dataflow, Table viewModel) {
19491                table viewElement = new table();
19492                viewElement.setId(String.valueOf(viewModel.getId()));
19493                if (!SQLUtil.isEmpty(viewModel.getDatabase())) {
19494                        viewElement.setDatabase(viewModel.getDatabase());
19495                }
19496                if (!SQLUtil.isEmpty(viewModel.getSchema())) {
19497                        viewElement.setSchema(viewModel.getSchema());
19498                }
19499                viewElement.setServer(viewModel.getServer());
19500                viewElement.setName(viewModel.getName());
19501                viewElement.setType("view");
19502                // Propagate the view sub type (e.g. temp_table for CREATE TEMPORARY VIEW)
19503                // so consumers can tell a temporary view apart from a regular one. Mantis 4538.
19504                if (viewModel.getSubType() != null) {
19505                        viewElement.setSubType(viewModel.getSubType().name());
19506                }
19507                viewElement.setStarStmt(viewModel.getStarStmt());
19508
19509                if(viewModel.isFromDDL()){
19510                        viewElement.setFromDDL(String.valueOf(viewModel.isFromDDL()));
19511                }
19512
19513        if(option.isTraceTablePosition()){
19514            for (Pair<Pair3<Long, Long, String>, Pair3<Long, Long, String>> position:viewModel.getPositions()){
19515                viewElement.setCoordinate(convertCoordinate(position.first)+","+convertCoordinate(position.second));
19516            }
19517        }
19518        else {
19519            viewElement.setCoordinate(convertCoordinate(viewModel.getStartPosition()) + ","
19520                    + convertCoordinate(viewModel.getEndPosition()));
19521        }
19522
19523                if (viewModel.getProcesses() != null) {
19524                        List<String> processIds = new ArrayList<String>();
19525                        for (Process process : viewModel.getProcesses()) {
19526                                processIds.add(String.valueOf(process.getId()));
19527                        }
19528                        viewElement.setProcessIds(processIds);
19529                }
19530                dataflow.getViews().add(viewElement);
19531
19532                List<TableColumn> columns = viewModel.getColumns();
19533
19534                if (containStarColumn(columns)) {
19535                        for (TableColumn column : columns) {
19536                                if (column.getName().endsWith("*")) {
19537                                        for (TableColumn starElement : columns) {
19538                                                if (starElement == column) {
19539                                                        continue;
19540                                                }
19541                                                TObjectName columnObject = starElement.getColumnObject();
19542                                                column.bindStarLinkColumn(columnObject);
19543                                        }
19544                                        if (viewModel.isCreateTable()) {
19545                                                column.setShowStar(false);
19546                                        }
19547                                }
19548                        }
19549                }
19550
19551                for (int j = 0; j < columns.size(); j++) {
19552                        TableColumn columnModel = (TableColumn) columns.get(j);
19553                        if (!columnModel.isPseduo() && columnModel.hasStarLinkColumn()) {
19554                                List<String> starLinkColumnList = columnModel.getStarLinkColumnNames();
19555                                for (int k = 0; k < starLinkColumnList.size(); k++) {
19556                                        column columnElement = new column();
19557                                        columnElement.setId(String.valueOf(columnModel.getId()) + "_" + k);
19558                                        String columnName = starLinkColumnList.get(k);
19559                                        if (containStarColumn(columns, columnName)) {
19560                                                continue;
19561                                        }
19562                                        columnElement.setName(columnName);
19563                                        if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
19564                                                columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
19565                                                                + convertCoordinate(columnModel.getEndPosition()));
19566                                        }
19567                                        viewElement.getColumns().add(columnElement);
19568                                }
19569
19570                                if (columnModel.isShowStar()) {
19571                                        column columnElement = new column();
19572                                        columnElement.setId(String.valueOf(columnModel.getId()));
19573                                        columnElement.setName(columnModel.getName());
19574                                        if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
19575                                                columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
19576                                                                + convertCoordinate(columnModel.getEndPosition()));
19577                                        }
19578                                        viewElement.getColumns().add(columnElement);
19579                                }
19580
19581                        } else {
19582                                column columnElement = new column();
19583                                columnElement.setId(String.valueOf(columnModel.getId()));
19584                                columnElement.setName(columnModel.getName());
19585                                if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
19586                                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
19587                                                        + convertCoordinate(columnModel.getEndPosition()));
19588                                }
19589                                if(columnModel.isPseduo()) {
19590                                        columnElement.setSource("system");
19591                                }
19592                                viewElement.getColumns().add(columnElement);
19593                        }
19594                }
19595
19596                TableRelationRows relationRows = viewModel.getRelationRows();
19597                if (relationRows.hasRelation()) {
19598                        column relationRowsElement = new column();
19599                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
19600                        relationRowsElement.setName(relationRows.getName());
19601                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
19602                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
19603                                                + convertCoordinate(relationRows.getEndPosition()));
19604                        }
19605                        relationRowsElement.setSource("system");
19606                        viewElement.getColumns().add(relationRowsElement);
19607                }
19608        }
19609
19610        private void appendStreamModel(dataflow dataflow, Table streamModel) {
19611                table streamElement = new table();
19612                streamElement.setId(String.valueOf(streamModel.getId()));
19613                if (!SQLUtil.isEmpty(streamModel.getDatabase())) {
19614                        streamElement.setDatabase(streamModel.getDatabase());
19615                }
19616                if (!SQLUtil.isEmpty(streamModel.getSchema())) {
19617                        streamElement.setSchema(streamModel.getSchema());
19618                }
19619                streamElement.setServer(streamModel.getServer());
19620                streamElement.setName(streamModel.getName());
19621                streamElement.setType("stream");
19622                if (streamModel.getFileType() != null) {
19623                        streamElement.setFileType(SQLUtil.trimColumnStringQuote(streamModel.getFileType()));
19624                }
19625
19626                if (streamModel.getStartPosition() != null && streamModel.getEndPosition() != null) {
19627                        streamElement.setCoordinate(convertCoordinate(streamModel.getStartPosition()) + ","
19628                                        + convertCoordinate(streamModel.getEndPosition()));
19629                }
19630
19631                if (streamModel.getProcesses() != null) {
19632                        List<String> processIds = new ArrayList<String>();
19633                        for (Process process : streamModel.getProcesses()) {
19634                                processIds.add(String.valueOf(process.getId()));
19635                        }
19636                        streamElement.setProcessIds(processIds);
19637                }
19638                dataflow.getStreams().add(streamElement);
19639
19640                List<TableColumn> columns = streamModel.getColumns();
19641
19642                for (int j = 0; j < columns.size(); j++) {
19643                        TableColumn columnModel = (TableColumn) columns.get(j);
19644                        column columnElement = new column();
19645                        columnElement.setId(String.valueOf(columnModel.getId()));
19646                        columnElement.setName(SQLUtil.trimColumnStringQuote(columnModel.getName()));
19647                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
19648                                        + convertCoordinate(columnModel.getEndPosition()));
19649                        streamElement.getColumns().add(columnElement);
19650                }
19651
19652                TableRelationRows relationRows = streamModel.getRelationRows();
19653                if (relationRows.hasRelation()) {
19654                        column relationRowsElement = new column();
19655                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
19656                        relationRowsElement.setName(relationRows.getName());
19657                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
19658                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
19659                                                + convertCoordinate(relationRows.getEndPosition()));
19660                        }
19661                        relationRowsElement.setSource("system");
19662                        streamElement.getColumns().add(relationRowsElement);
19663                }
19664        }
19665
19666        private void appendStageModel(dataflow dataflow, Table stageModel) {
19667                table stageElement = new table();
19668                stageElement.setId(String.valueOf(stageModel.getId()));
19669                if (!SQLUtil.isEmpty(stageModel.getDatabase())) {
19670                        stageElement.setDatabase(stageModel.getDatabase());
19671                }
19672                if (!SQLUtil.isEmpty(stageModel.getSchema())) {
19673                        stageElement.setSchema(stageModel.getSchema());
19674                }
19675                stageElement.setServer(stageModel.getServer());
19676                stageElement.setName(stageModel.getName());
19677                stageElement.setType("stage");
19678                stageElement.setLocation(stageModel.getLocation());
19679                if (stageModel.getFileType() != null) {
19680                        stageElement.setFileType(SQLUtil.trimColumnStringQuote(stageModel.getFileType()));
19681                }
19682
19683                if (stageModel.getStartPosition() != null && stageModel.getEndPosition() != null) {
19684                        stageElement.setCoordinate(convertCoordinate(stageModel.getStartPosition()) + ","
19685                                        + convertCoordinate(stageModel.getEndPosition()));
19686                }
19687
19688                if (stageModel.getProcesses() != null) {
19689                        List<String> processIds = new ArrayList<String>();
19690                        for (Process process : stageModel.getProcesses()) {
19691                                processIds.add(String.valueOf(process.getId()));
19692                        }
19693                        stageElement.setProcessIds(processIds);
19694                }
19695                dataflow.getStages().add(stageElement);
19696
19697                List<TableColumn> columns = stageModel.getColumns();
19698
19699                for (int j = 0; j < columns.size(); j++) {
19700                        TableColumn columnModel = (TableColumn) columns.get(j);
19701                        column columnElement = new column();
19702                        columnElement.setId(String.valueOf(columnModel.getId()));
19703                        columnElement.setName(SQLUtil.trimColumnStringQuote(columnModel.getName()));
19704                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
19705                                        + convertCoordinate(columnModel.getEndPosition()));
19706                        stageElement.getColumns().add(columnElement);
19707                }
19708
19709                TableRelationRows relationRows = stageModel.getRelationRows();
19710                if (relationRows.hasRelation()) {
19711                        column relationRowsElement = new column();
19712                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
19713                        relationRowsElement.setName(relationRows.getName());
19714                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
19715                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
19716                                                + convertCoordinate(relationRows.getEndPosition()));
19717                        }
19718                        relationRowsElement.setSource("system");
19719                        stageElement.getColumns().add(relationRowsElement);
19720                }
19721        }
19722
19723        private void appendSequenceModel(dataflow dataflow, Table sequenceModel) {
19724                table sequenceElement = new table();
19725                sequenceElement.setId(String.valueOf(sequenceModel.getId()));
19726                if (!SQLUtil.isEmpty(sequenceModel.getDatabase())) {
19727                        sequenceElement.setDatabase(sequenceModel.getDatabase());
19728                }
19729                if (!SQLUtil.isEmpty(sequenceModel.getSchema())) {
19730                        sequenceElement.setSchema(sequenceModel.getSchema());
19731                }
19732                sequenceElement.setServer(sequenceModel.getServer());
19733                sequenceElement.setName(sequenceModel.getName());
19734                sequenceElement.setType("sequence");
19735                sequenceElement.setLocation(sequenceModel.getLocation());
19736                if (sequenceModel.getFileType() != null) {
19737                        sequenceElement.setFileType(SQLUtil.trimColumnStringQuote(sequenceModel.getFileType()));
19738                }
19739
19740                if (sequenceModel.getStartPosition() != null && sequenceModel.getEndPosition() != null) {
19741                        sequenceElement.setCoordinate(convertCoordinate(sequenceModel.getStartPosition()) + ","
19742                                        + convertCoordinate(sequenceModel.getEndPosition()));
19743                }
19744
19745                if (sequenceModel.getProcesses() != null) {
19746                        List<String> processIds = new ArrayList<String>();
19747                        for (Process process : sequenceModel.getProcesses()) {
19748                                processIds.add(String.valueOf(process.getId()));
19749                        }
19750                        sequenceElement.setProcessIds(processIds);
19751                }
19752                dataflow.getSequences().add(sequenceElement);
19753
19754                List<TableColumn> columns = sequenceModel.getColumns();
19755
19756                for (int j = 0; j < columns.size(); j++) {
19757                        TableColumn columnModel = (TableColumn) columns.get(j);
19758                        column columnElement = new column();
19759                        columnElement.setId(String.valueOf(columnModel.getId()));
19760                        columnElement.setName(SQLUtil.trimColumnStringQuote(columnModel.getName()));
19761                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
19762                                        + convertCoordinate(columnModel.getEndPosition()));
19763                        sequenceElement.getColumns().add(columnElement);
19764                }
19765
19766                TableRelationRows relationRows = sequenceModel.getRelationRows();
19767                if (relationRows.hasRelation()) {
19768                        column relationRowsElement = new column();
19769                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
19770                        relationRowsElement.setName(relationRows.getName());
19771                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
19772                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
19773                                                + convertCoordinate(relationRows.getEndPosition()));
19774                        }
19775                        relationRowsElement.setSource("system");
19776                        sequenceElement.getColumns().add(relationRowsElement);
19777                }
19778        }
19779
19780        private void appendDataSourceModel(dataflow dataflow, Table datasourceModel) {
19781                table datasourceElement = new table();
19782                datasourceElement.setId(String.valueOf(datasourceModel.getId()));
19783                if (!SQLUtil.isEmpty(datasourceModel.getDatabase())) {
19784                        datasourceElement.setDatabase(datasourceModel.getDatabase());
19785                }
19786                if (!SQLUtil.isEmpty(datasourceModel.getSchema())) {
19787                        datasourceElement.setSchema(datasourceModel.getSchema());
19788                }
19789                datasourceElement.setServer(datasourceModel.getServer());
19790                datasourceElement.setName(datasourceModel.getName());
19791                datasourceElement.setType("datasource");
19792                datasourceElement.setLocation(datasourceModel.getLocation());
19793                if (datasourceModel.getFileType() != null) {
19794                        datasourceElement.setFileType(SQLUtil.trimColumnStringQuote(datasourceModel.getFileType()));
19795                }
19796
19797                if (datasourceModel.getStartPosition() != null && datasourceModel.getEndPosition() != null) {
19798                        datasourceElement.setCoordinate(convertCoordinate(datasourceModel.getStartPosition()) + ","
19799                                        + convertCoordinate(datasourceModel.getEndPosition()));
19800                }
19801
19802                if (datasourceModel.getProcesses() != null) {
19803                        List<String> processIds = new ArrayList<String>();
19804                        for (Process process : datasourceModel.getProcesses()) {
19805                                processIds.add(String.valueOf(process.getId()));
19806                        }
19807                        datasourceElement.setProcessIds(processIds);
19808                }
19809                dataflow.getDatasources().add(datasourceElement);
19810
19811                List<TableColumn> columns = datasourceModel.getColumns();
19812
19813                for (int j = 0; j < columns.size(); j++) {
19814                        TableColumn columnModel = (TableColumn) columns.get(j);
19815                        column columnElement = new column();
19816                        columnElement.setId(String.valueOf(columnModel.getId()));
19817                        columnElement.setName(SQLUtil.trimColumnStringQuote(columnModel.getName()));
19818                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
19819                                        + convertCoordinate(columnModel.getEndPosition()));
19820                        datasourceElement.getColumns().add(columnElement);
19821                }
19822
19823                TableRelationRows relationRows = datasourceModel.getRelationRows();
19824                if (relationRows.hasRelation()) {
19825                        column relationRowsElement = new column();
19826                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
19827                        relationRowsElement.setName(relationRows.getName());
19828                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
19829                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
19830                                                + convertCoordinate(relationRows.getEndPosition()));
19831                        }
19832                        relationRowsElement.setSource("system");
19833                        datasourceElement.getColumns().add(relationRowsElement);
19834                }
19835        }
19836
19837        private void appendDatabaseModel(dataflow dataflow, Table databaseModel) {
19838                table databaseElement = new table();
19839                databaseElement.setId(String.valueOf(databaseModel.getId()));
19840                if (!SQLUtil.isEmpty(databaseModel.getDatabase())) {
19841                        databaseElement.setDatabase(databaseModel.getDatabase());
19842                }
19843                if (!SQLUtil.isEmpty(databaseModel.getSchema())) {
19844                        databaseElement.setSchema(databaseModel.getSchema());
19845                }
19846                databaseElement.setServer(databaseModel.getServer());
19847                databaseElement.setName(databaseModel.getName());
19848                databaseElement.setType("database");
19849                if (databaseModel.getFileType() != null) {
19850                        databaseElement.setFileType(SQLUtil.trimColumnStringQuote(databaseModel.getFileType()));
19851                }
19852
19853                if (databaseModel.getStartPosition() != null && databaseModel.getEndPosition() != null) {
19854                        databaseElement.setCoordinate(convertCoordinate(databaseModel.getStartPosition()) + ","
19855                                        + convertCoordinate(databaseModel.getEndPosition()));
19856                }
19857
19858                if (databaseModel.getProcesses() != null) {
19859                        List<String> processIds = new ArrayList<String>();
19860                        for (Process process : databaseModel.getProcesses()) {
19861                                processIds.add(String.valueOf(process.getId()));
19862                        }
19863                        databaseElement.setProcessIds(processIds);
19864                }
19865                dataflow.getDatabases().add(databaseElement);
19866
19867                List<TableColumn> columns = databaseModel.getColumns();
19868
19869                for (int j = 0; j < columns.size(); j++) {
19870                        TableColumn columnModel = (TableColumn) columns.get(j);
19871                        column columnElement = new column();
19872                        columnElement.setId(String.valueOf(columnModel.getId()));
19873                        columnElement.setName(SQLUtil.trimColumnStringQuote(columnModel.getName()));
19874                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
19875                                        + convertCoordinate(columnModel.getEndPosition()));
19876                        databaseElement.getColumns().add(columnElement);
19877                }
19878
19879                TableRelationRows relationRows = databaseModel.getRelationRows();
19880                if (relationRows.hasRelation()) {
19881                        column relationRowsElement = new column();
19882                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
19883                        relationRowsElement.setName(relationRows.getName());
19884                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
19885                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
19886                                                + convertCoordinate(relationRows.getEndPosition()));
19887                        }
19888                        relationRowsElement.setSource("system");
19889                        databaseElement.getColumns().add(relationRowsElement);
19890                }
19891        }
19892
19893        private void appendSchemaModel(dataflow dataflow, Table schemaModel) {
19894                table schemaElement = new table();
19895                schemaElement.setId(String.valueOf(schemaModel.getId()));
19896                if (!SQLUtil.isEmpty(schemaModel.getDatabase())) {
19897                        schemaElement.setDatabase(schemaModel.getDatabase());
19898                }
19899                if (!SQLUtil.isEmpty(schemaModel.getSchema())) {
19900                        schemaElement.setSchema(schemaModel.getSchema());
19901                }
19902                schemaElement.setServer(schemaModel.getServer());
19903                schemaElement.setName(schemaModel.getName());
19904                schemaElement.setType("schema");
19905                if (schemaModel.getFileType() != null) {
19906                        schemaElement.setFileType(SQLUtil.trimColumnStringQuote(schemaModel.getFileType()));
19907                }
19908
19909                if (schemaModel.getStartPosition() != null && schemaModel.getEndPosition() != null) {
19910                        schemaElement.setCoordinate(convertCoordinate(schemaModel.getStartPosition()) + ","
19911                                        + convertCoordinate(schemaModel.getEndPosition()));
19912                }
19913
19914                if (schemaModel.getProcesses() != null) {
19915                        List<String> processIds = new ArrayList<String>();
19916                        for (Process process : schemaModel.getProcesses()) {
19917                                processIds.add(String.valueOf(process.getId()));
19918                        }
19919                        schemaElement.setProcessIds(processIds);
19920                }
19921                dataflow.getSchemas().add(schemaElement);
19922
19923                List<TableColumn> columns = schemaModel.getColumns();
19924
19925                for (int j = 0; j < columns.size(); j++) {
19926                        TableColumn columnModel = (TableColumn) columns.get(j);
19927                        column columnElement = new column();
19928                        columnElement.setId(String.valueOf(columnModel.getId()));
19929                        columnElement.setName(SQLUtil.trimColumnStringQuote(columnModel.getName()));
19930                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
19931                                        + convertCoordinate(columnModel.getEndPosition()));
19932                        schemaElement.getColumns().add(columnElement);
19933                }
19934
19935                TableRelationRows relationRows = schemaModel.getRelationRows();
19936                if (relationRows.hasRelation()) {
19937                        column relationRowsElement = new column();
19938                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
19939                        relationRowsElement.setName(relationRows.getName());
19940                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
19941                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
19942                                                + convertCoordinate(relationRows.getEndPosition()));
19943                        }
19944                        relationRowsElement.setSource("system");
19945                        schemaElement.getColumns().add(relationRowsElement);
19946                }
19947        }
19948
19949        private void appendPathModel(dataflow dataflow, Table pathModel) {
19950                table pathElement = new table();
19951                pathElement.setId(String.valueOf(pathModel.getId()));
19952                if (!SQLUtil.isEmpty(pathModel.getDatabase())) {
19953                        pathElement.setDatabase(pathModel.getDatabase());
19954                }
19955                if (!SQLUtil.isEmpty(pathModel.getSchema())) {
19956                        pathElement.setSchema(pathModel.getSchema());
19957                }
19958                pathElement.setServer(pathModel.getServer());
19959                pathElement.setName(pathModel.getName());
19960                pathElement.setType("path");
19961                if (pathModel.getFileFormat() != null) {
19962                        pathElement.setFileFormat(SQLUtil.trimColumnStringQuote(pathModel.getFileFormat()));
19963                }
19964
19965                if (pathModel.getStartPosition() != null && pathModel.getEndPosition() != null) {
19966                        pathElement.setCoordinate(convertCoordinate(pathModel.getStartPosition()) + ","
19967                                        + convertCoordinate(pathModel.getEndPosition()));
19968                }
19969
19970                if (pathModel.getProcesses() != null) {
19971                        List<String> processIds = new ArrayList<String>();
19972                        for (Process process : pathModel.getProcesses()) {
19973                                processIds.add(String.valueOf(process.getId()));
19974                        }
19975                        pathElement.setProcessIds(processIds);
19976                }
19977
19978                pathElement.setUri(pathModel.getName());
19979
19980                dataflow.getPaths().add(pathElement);
19981
19982                List<TableColumn> columns = pathModel.getColumns();
19983
19984                for (int j = 0; j < columns.size(); j++) {
19985                        TableColumn columnModel = (TableColumn) columns.get(j);
19986                        column columnElement = new column();
19987                        columnElement.setId(String.valueOf(columnModel.getId()));
19988                        columnElement.setName(SQLUtil.trimColumnStringQuote(columnModel.getName()));
19989                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
19990                                        + convertCoordinate(columnModel.getEndPosition()));
19991                        pathElement.getColumns().add(columnElement);
19992                }
19993
19994                TableRelationRows relationRows = pathModel.getRelationRows();
19995                if (relationRows.hasRelation()) {
19996                        column relationRowsElement = new column();
19997                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
19998                        relationRowsElement.setName(relationRows.getName());
19999                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
20000                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
20001                                                + convertCoordinate(relationRows.getEndPosition()));
20002                        }
20003                        relationRowsElement.setSource("system");
20004                        pathElement.getColumns().add(relationRowsElement);
20005                }
20006        }
20007
20008        private void appendVariableModel(dataflow dataflow, Table variableModel) {
20009                table variableElement = new table();
20010                variableElement.setId(String.valueOf(variableModel.getId()));
20011                if (!SQLUtil.isEmpty(variableModel.getDatabase())) {
20012                        variableElement.setDatabase(variableModel.getDatabase());
20013                }
20014                if (!SQLUtil.isEmpty(variableModel.getSchema())) {
20015                        variableElement.setSchema(variableModel.getSchema());
20016                }
20017                variableElement.setServer(variableModel.getServer());
20018                variableElement.setName(variableModel.getName());
20019                variableElement.setType("variable");
20020                variableElement.setParent(variableModel.getParent());
20021                if (variableModel.getProcedureId() != null) {
20022                        variableElement.setProcedureId(variableModel.getProcedureId());
20023                }
20024                variableElement.setIsTarget(String.valueOf(variableModel.isTarget()));
20025                if (variableModel.getSubType() != null) {
20026                        variableElement.setSubType(variableModel.getSubType().name());
20027                }
20028
20029                if (variableModel.getStartPosition() != null && variableModel.getEndPosition() != null) {
20030                        variableElement.setCoordinate(convertCoordinate(variableModel.getStartPosition()) + ","
20031                                        + convertCoordinate(variableModel.getEndPosition()));
20032                }
20033                dataflow.getVariables().add(variableElement);
20034
20035                List<TableColumn> columns = variableModel.getColumns();
20036
20037                if (containStarColumn(columns)) {
20038                        for (TableColumn column : columns) {
20039                                if (column.getName().endsWith("*")) {
20040                                        for (TableColumn starElement : columns) {
20041                                                if (starElement == column) {
20042                                                        continue;
20043                                                }
20044                                                TObjectName columnObject = starElement.getColumnObject();
20045                                                column.bindStarLinkColumn(columnObject);
20046                                        }
20047//                                      column.setShowStar(false);
20048                                }
20049                        }
20050                }
20051
20052                for (int j = 0; j < columns.size(); j++) {
20053                        TableColumn columnModel = columns.get(j);
20054                        if (columnModel.hasStarLinkColumn()) {
20055                                List<String> starLinkColumnList = columnModel.getStarLinkColumnNames();
20056                                for (int k = 0; k < starLinkColumnList.size(); k++) {
20057                                        column columnElement = new column();
20058                                        columnElement.setId(columnModel.getId() + "_" + k);
20059                                        String columnName = starLinkColumnList.get(k);
20060                                        if (containStarColumn(columns, columnName)) {
20061                                                continue;
20062                                        }
20063                                        columnElement.setName(columnName);
20064                                        if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
20065                                                columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
20066                                                                + convertCoordinate(columnModel.getEndPosition()));
20067                                        }
20068                                        variableElement.getColumns().add(columnElement);
20069                                }
20070
20071                                if (columnModel.isShowStar()) {
20072                                        column columnElement = new column();
20073                                        columnElement.setId(String.valueOf(columnModel.getId()));
20074                                        columnElement.setName(columnModel.getName());
20075                                        if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
20076                                                columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
20077                                                                + convertCoordinate(columnModel.getEndPosition()));
20078                                        }
20079                                        variableElement.getColumns().add(columnElement);
20080                                }
20081
20082                        } else {
20083                                column columnElement = new column();
20084                                columnElement.setId(String.valueOf(columnModel.getId()));
20085                                columnElement.setName(columnModel.getName());
20086                                if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
20087                                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
20088                                                        + convertCoordinate(columnModel.getEndPosition()));
20089                                }
20090                                variableElement.getColumns().add(columnElement);
20091                        }
20092                }
20093
20094                TableRelationRows relationRows = variableModel.getRelationRows();
20095                if (relationRows.hasRelation()) {
20096                        column relationRowsElement = new column();
20097                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
20098                        relationRowsElement.setName(relationRows.getName());
20099                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
20100                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
20101                                                + convertCoordinate(relationRows.getEndPosition()));
20102                        }
20103                        relationRowsElement.setSource("system");
20104                        variableElement.getColumns().add(relationRowsElement);
20105                }
20106        }
20107
20108        private void appendCursorModel(dataflow dataflow, Table cursorModel) {
20109                table cursorElement = new table();
20110                cursorElement.setId(String.valueOf(cursorModel.getId()));
20111                if (!SQLUtil.isEmpty(cursorModel.getDatabase())) {
20112                        cursorElement.setDatabase(cursorModel.getDatabase());
20113                }
20114                if (!SQLUtil.isEmpty(cursorModel.getSchema())) {
20115                        cursorElement.setSchema(cursorModel.getSchema());
20116                }
20117                cursorElement.setServer(cursorModel.getServer());
20118                cursorElement.setName(cursorModel.getName());
20119                cursorElement.setType("variable");
20120                if (cursorElement.getSubType() != null) {
20121                        cursorElement.setSubType(cursorModel.getSubType().name());
20122                }
20123
20124                if (cursorModel.getStartPosition() != null && cursorModel.getEndPosition() != null) {
20125                        cursorElement.setCoordinate(convertCoordinate(cursorModel.getStartPosition()) + ","
20126                                        + convertCoordinate(cursorModel.getEndPosition()));
20127                }
20128                dataflow.getVariables().add(cursorElement);
20129
20130                List<TableColumn> columns = cursorModel.getColumns();
20131
20132                if (containStarColumn(columns)) {
20133                        for (TableColumn column : columns) {
20134                                if (column.getName().endsWith("*")) {
20135                                        for (TableColumn starElement : columns) {
20136                                                if (starElement == column) {
20137                                                        continue;
20138                                                }
20139                                                TObjectName columnObject = starElement.getColumnObject();
20140                                                column.bindStarLinkColumn(columnObject);
20141                                        }
20142                                        column.setShowStar(false);
20143                                }
20144                        }
20145                }
20146
20147                for (int j = 0; j < columns.size(); j++) {
20148                        TableColumn columnModel = (TableColumn) columns.get(j);
20149                        if (columnModel.hasStarLinkColumn()) {
20150                                List<String> starLinkColumnList = columnModel.getStarLinkColumnNames();
20151                                for (int k = 0; k < starLinkColumnList.size(); k++) {
20152                                        column columnElement = new column();
20153                                        columnElement.setId(String.valueOf(columnModel.getId()) + "_" + k);
20154                                        String columnName = starLinkColumnList.get(k);
20155                                        if (containStarColumn(columns, columnName)) {
20156                                                continue;
20157                                        }
20158                                        columnElement.setName(columnName);
20159                                        if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
20160                                                columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
20161                                                                + convertCoordinate(columnModel.getEndPosition()));
20162                                        }
20163                                        cursorElement.getColumns().add(columnElement);
20164                                }
20165
20166                                if (columnModel.isShowStar()) {
20167                                        column columnElement = new column();
20168                                        columnElement.setId(String.valueOf(columnModel.getId()));
20169                                        columnElement.setName(columnModel.getName());
20170                                        if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
20171                                                columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
20172                                                                + convertCoordinate(columnModel.getEndPosition()));
20173                                        }
20174                                        cursorElement.getColumns().add(columnElement);
20175                                }
20176
20177                        } else {
20178                                column columnElement = new column();
20179                                columnElement.setId(String.valueOf(columnModel.getId()));
20180                                columnElement.setName(columnModel.getName());
20181                                if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
20182                                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
20183                                                        + convertCoordinate(columnModel.getEndPosition()));
20184                                }
20185                                cursorElement.getColumns().add(columnElement);
20186                        }
20187                }
20188
20189                TableRelationRows relationRows = cursorModel.getRelationRows();
20190                if (relationRows.hasRelation()) {
20191                        column relationRowsElement = new column();
20192                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
20193                        relationRowsElement.setName(relationRows.getName());
20194                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
20195                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
20196                                                + convertCoordinate(relationRows.getEndPosition()));
20197                        }
20198                        relationRowsElement.setSource("system");
20199                        cursorElement.getColumns().add(relationRowsElement);
20200                }
20201        }
20202
20203        private void appendErrors(dataflow dataflow) {
20204                List<ErrorInfo> errorInfos = this.getErrorMessages();
20205                if (metadataErrors != null) {
20206                        for (int i = 0; i < metadataErrors.size(); i++) {
20207                                errorInfos.add(i, metadataErrors.get(i));
20208                        }
20209                }
20210
20211                for (int i = 0; i < errorInfos.size(); ++i) {
20212                        ErrorInfo errorInfo = errorInfos.get(i);
20213                        error error = new error();
20214                        if (!SQLUtil.isEmpty(errorInfo.getErrorMessage())) {
20215                                error.setErrorMessage(errorInfo.getErrorMessage());
20216                        }
20217                        if (!SQLUtil.isEmpty(errorInfo.getErrorType())) {
20218                                error.setErrorType(errorInfo.getErrorType());
20219                        }
20220                        if (errorInfo.getStartPosition() != null && errorInfo.getEndPosition() != null) {
20221                                error.setCoordinate(convertCoordinate(errorInfo.getStartPosition()) + ","
20222                                                + convertCoordinate(errorInfo.getEndPosition()));
20223                        }
20224                        if (!SQLUtil.isEmpty(errorInfo.getFileName())) {
20225                                error.setFile(errorInfo.getFileName());
20226                        }
20227                        if (errorInfo.getOriginStartPosition() != null && errorInfo.getOriginEndPosition() != null) {
20228                                error.setOriginCoordinate(errorInfo.getOriginStartPosition() + "," + errorInfo.getOriginEndPosition());
20229                        }
20230                        dataflow.getErrors().add(error);
20231                }
20232        }
20233
20234        private void appendOraclePackages(dataflow dataflow) {
20235                List<OraclePackage> packages = this.modelManager.getOraclePackageModels();
20236
20237                for (int i = 0; i < packages.size(); ++i) {
20238                        OraclePackage model = packages.get(i);
20239                        oraclePackage oraclePackage = new oraclePackage();
20240                        oraclePackage.setId(String.valueOf(model.getId()));
20241                        if (!SQLUtil.isEmpty(model.getDatabase())) {
20242                                oraclePackage.setDatabase(model.getDatabase());
20243                        }
20244                        if (!SQLUtil.isEmpty(model.getSchema())) {
20245                                oraclePackage.setSchema(model.getSchema());
20246                        }
20247                        oraclePackage.setServer(model.getServer());
20248                        oraclePackage.setName(model.getName());
20249                        if (model.getType() != null) {
20250                                oraclePackage.setType(model.getType().name().replace("sst", ""));
20251                        }
20252                        if (model.getStartPosition() != null && model.getEndPosition() != null) {
20253                                oraclePackage.setCoordinate(
20254                                                convertCoordinate(model.getStartPosition()) + "," + convertCoordinate(model.getEndPosition()));
20255                        }
20256
20257                        dataflow.getPackages().add(oraclePackage);
20258
20259                        List<Argument> arguments = model.getArguments();
20260
20261                        for (int j = 0; j < arguments.size(); ++j) {
20262                                Argument argumentModel = (Argument) arguments.get(j);
20263                                argument argumentElement = new argument();
20264                                argumentElement.setId(String.valueOf(argumentModel.getId()));
20265                                argumentElement.setName(argumentModel.getName());
20266                                if (argumentModel.getStartPosition() != null && argumentModel.getEndPosition() != null) {
20267                                        argumentElement.setCoordinate(convertCoordinate(argumentModel.getStartPosition()) + ","
20268                                                        + convertCoordinate(argumentModel.getEndPosition()));
20269                                }
20270
20271                                argumentElement.setDatatype(argumentModel.getDataType().getDataTypeName());
20272                                argumentElement.setInout(argumentModel.getMode().name());
20273                                oraclePackage.getArguments().add(argumentElement);
20274                        }
20275
20276                        for (int j = 0; j < model.getProcedures().size(); j++) {
20277                                Procedure procedureModel = model.getProcedures().get(j);
20278                                procedure procedure = new procedure();
20279                                procedure.setId(String.valueOf(procedureModel.getId()));
20280                                if (!SQLUtil.isEmpty(procedureModel.getDatabase())) {
20281                                        procedure.setDatabase(procedureModel.getDatabase());
20282                                }
20283                                if (!SQLUtil.isEmpty(procedureModel.getSchema())) {
20284                                        procedure.setSchema(procedureModel.getSchema());
20285                                }
20286                                procedure.setServer(procedureModel.getServer());
20287                                procedure.setName(procedureModel.getName());
20288                                if (procedureModel.getType() != null) {
20289                                        procedure.setType(procedureModel.getType().name().replace("sst", ""));
20290                                }
20291                                if (procedureModel.getStartPosition() != null && procedureModel.getEndPosition() != null) {
20292                                        procedure.setCoordinate(convertCoordinate(procedureModel.getStartPosition()) + ","
20293                                                        + convertCoordinate(procedureModel.getEndPosition()));
20294                                }
20295
20296                                oraclePackage.getProcedures().add(procedure);
20297
20298                                List<Argument> procedureArguments = procedureModel.getArguments();
20299
20300                                for (int k = 0; k < procedureArguments.size(); ++k) {
20301                                        Argument argumentModel = (Argument) procedureArguments.get(k);
20302                                        argument argumentElement = new argument();
20303                                        argumentElement.setId(String.valueOf(argumentModel.getId()));
20304                                        argumentElement.setName(argumentModel.getName());
20305                                        if (argumentModel.getStartPosition() != null && argumentModel.getEndPosition() != null) {
20306                                                argumentElement.setCoordinate(convertCoordinate(argumentModel.getStartPosition()) + ","
20307                                                                + convertCoordinate(argumentModel.getEndPosition()));
20308                                        }
20309
20310                                        argumentElement.setDatatype(argumentModel.getDataType().getDataTypeName());
20311                                        argumentElement.setInout(argumentModel.getMode().name());
20312                                        procedure.getArguments().add(argumentElement);
20313                                }
20314                        }
20315                }
20316        }
20317
20318        private void appendProcedures(dataflow dataflow) {
20319                List<Procedure> procedures = this.modelManager.getProcedureModels();
20320
20321                for (int i = 0; i < procedures.size(); ++i) {
20322                        Procedure model = procedures.get(i);
20323                        if (model.getParentPackage() != null) {
20324                                continue;
20325                        }
20326                        procedure procedure = new procedure();
20327                        procedure.setId(String.valueOf(model.getId()));
20328                        if (!SQLUtil.isEmpty(model.getDatabase())) {
20329                                procedure.setDatabase(model.getDatabase());
20330                        }
20331                        if (!SQLUtil.isEmpty(model.getSchema())) {
20332                                procedure.setSchema(model.getSchema());
20333                        }
20334                        procedure.setServer(model.getServer());
20335                        procedure.setName(model.getName());
20336                        if (model.getType() != null) {
20337                                procedure.setType(model.getType().name().replace("sst", ""));
20338                        }
20339                        if (model.getStartPosition() != null && model.getEndPosition() != null) {
20340                                procedure.setCoordinate(
20341                                                convertCoordinate(model.getStartPosition()) + "," + convertCoordinate(model.getEndPosition()));
20342                        }
20343
20344                        dataflow.getProcedures().add(procedure);
20345
20346                        List<Argument> arguments = model.getArguments();
20347
20348                        for (int j = 0; j < arguments.size(); ++j) {
20349                                Argument argumentModel = (Argument) arguments.get(j);
20350                                argument argumentElement = new argument();
20351                                argumentElement.setId(String.valueOf(argumentModel.getId()));
20352                                argumentElement.setName(argumentModel.getName());
20353                                if (argumentModel.getStartPosition() != null && argumentModel.getEndPosition() != null) {
20354                                        argumentElement.setCoordinate(convertCoordinate(argumentModel.getStartPosition()) + ","
20355                                                        + convertCoordinate(argumentModel.getEndPosition()));
20356                                }
20357
20358                                argumentElement.setDatatype(argumentModel.getDataType().getDataTypeName());
20359                                argumentElement.setInout(argumentModel.getMode().name());
20360                                procedure.getArguments().add(argumentElement);
20361                        }
20362                }
20363        }
20364
20365        private void appendProcesses(dataflow dataflow) {
20366                List<Process> processes = this.modelManager.getProcessModels();
20367
20368                for (int i = 0; i < processes.size(); ++i) {
20369                        Process model = processes.get(i);
20370                        process process = new process();
20371                        process.setId(String.valueOf(model.getId()));
20372                        if (!SQLUtil.isEmpty(model.getDatabase())) {
20373                                process.setDatabase(model.getDatabase());
20374                        }
20375                        if (!SQLUtil.isEmpty(model.getSchema())) {
20376                                process.setSchema(model.getSchema());
20377                        }
20378                        process.setServer(model.getServer());
20379                        process.setName(getProcessName(model));
20380                        if (!SQLUtil.isEmpty(model.getProcedureName())) {
20381                                process.setProcedureName(model.getProcedureName());
20382                        }
20383                        if (model.getProcedureId() != null) {
20384                                process.setProcedureId(String.valueOf(model.getProcedureId()));
20385                        }
20386                        if (!SQLUtil.isEmpty(model.getQueryHashId())) {
20387                                process.setQueryHashId(model.getQueryHashId());
20388                        }
20389                        if (model.getGspObject() != null) {
20390                                process.setType(model.getGspObject().sqlstatementtype.name());
20391                        }
20392                        if (model.getStartPosition() != null && model.getEndPosition() != null) {
20393                                process.setCoordinate(
20394                                                convertCoordinate(model.getStartPosition()) + "," + convertCoordinate(model.getEndPosition()));
20395                        }
20396                        if (model.getTransforms() != null && !model.getTransforms().isEmpty()) {
20397                                for (Transform transformItem : model.getTransforms()) {
20398                                        process.addTransform(transformItem);
20399                                }
20400                        }
20401                        dataflow.getProcesses().add(process);
20402                }
20403        }
20404
20405        private void appendTables(dataflow dataflow) {
20406                List<TTable> tables = modelManager.getBaseTables();
20407                Map<String, table> tableMap = new HashMap<String, table>();
20408                Set<Long> tableModelIds = new HashSet<Long>();
20409                for (int i = 0; i < tables.size(); i++) {
20410                        Object model = modelManager.getModel(tables.get(i));
20411                        if (model instanceof Table) {
20412                                Table tableModel = (Table) model;
20413                                if(tableModelIds.contains(tableModel.getId())) {
20414                                        continue;
20415                                }
20416                                else {
20417                                        tableModelIds.add(tableModel.getId());
20418                                }
20419                                if (tableModel.isView()) {
20420                                        continue;
20421                                }
20422                                if (tableModel.isStage()) {
20423                                        appendStageModel(dataflow, tableModel);
20424                                        continue;
20425                                }
20426                                if (tableModel.isSequence()) {
20427                                        appendSequenceModel(dataflow, tableModel);
20428                                        continue;
20429                                }
20430                                if (tableModel.isDataSource()) {
20431                                        appendDataSourceModel(dataflow, tableModel);
20432                                        continue;
20433                                }
20434                                if (tableModel.isDatabase()) {
20435                                        appendDatabaseModel(dataflow, tableModel);
20436                                        continue;
20437                                }
20438                                if (tableModel.isSchema()) {
20439                                        appendSchemaModel(dataflow, tableModel);
20440                                        continue;
20441                                }
20442                                if (tableModel.isStream()) {
20443                                        appendStreamModel(dataflow, tableModel);
20444                                        continue;
20445                                }
20446                                if (tableModel.isPath()) {
20447                                        appendPathModel(dataflow, tableModel);
20448                                        continue;
20449                                }
20450                                if (tableModel.isVariable() && !tableModel.isCursor()) {
20451                                        appendVariableModel(dataflow, tableModel);
20452                                        continue;
20453                                }
20454                                if (tableModel.isCursor()) {
20455                                        appendCursorModel(dataflow, tableModel);
20456                                        continue;
20457                                }
20458                                if (tableModel.isConstant()) {
20459                                        appendConstantModel(dataflow, tableModel);
20460                                        continue;
20461                                }
20462                                if (!tableIds.contains(tableModel.getId())) {
20463                                        appendTableModel(dataflow, tableModel, tableMap);
20464                                        tableIds.add(tableModel.getId());
20465                                }
20466                        } else if (model instanceof QueryTable) {
20467                                QueryTable queryTable = (QueryTable) model;
20468                                if (!tableIds.contains(queryTable.getId())) {
20469                                        appendResultSet(dataflow, queryTable);
20470                                        tableIds.add(queryTable.getId());
20471                                }
20472                        }
20473                }
20474
20475                List<Table> tableNames = modelManager.getTablesByName();
20476                tableNames.addAll(modelManager.getDropTables());
20477                
20478                for (int i = 0; i < tableNames.size(); i++) {
20479                        Table tableModel = tableNames.get(i);
20480                        if(tableModelIds.contains(tableModel.getId())) {
20481                                continue;
20482                        }
20483                        else {
20484                                tableModelIds.add(tableModel.getId());
20485                        }
20486                        if (tableModel.isView()) {
20487                                continue;
20488                        }
20489                        if (tableModel.isDatabase()) {
20490                                appendDatabaseModel(dataflow, tableModel);
20491                                continue;
20492                        }
20493                        if (tableModel.isSchema()) {
20494                                appendSchemaModel(dataflow, tableModel);
20495                                continue;
20496                        }
20497                        if (tableModel.isStage()) {
20498                                appendStageModel(dataflow, tableModel);
20499                                continue;
20500                        }
20501                        if (tableModel.isSequence()) {
20502                                appendSequenceModel(dataflow, tableModel);
20503                                continue;
20504                        }
20505                        if (tableModel.isDataSource()) {
20506                                appendDataSourceModel(dataflow, tableModel);
20507                                continue;
20508                        }
20509                        if (tableModel.isStream()) {
20510                                appendStreamModel(dataflow, tableModel);
20511                                continue;
20512                        }
20513                        if (tableModel.isPath()) {
20514                                appendPathModel(dataflow, tableModel);
20515                                continue;
20516                        }
20517                        if (tableModel.isVariable()) {
20518                                appendVariableModel(dataflow, tableModel);
20519                                continue;
20520                        }
20521                        if (tableModel.isCursor()) {
20522                                appendCursorModel(dataflow, tableModel);
20523                                continue;
20524                        }
20525                        if (tableModel.isConstant()) {
20526                                appendConstantModel(dataflow, tableModel);
20527                                continue;
20528                        }
20529                        if (!tableIds.contains(tableModel.getId())) {
20530                                appendTableModel(dataflow, tableModel, tableMap);
20531                                tableIds.add(tableModel.getId());
20532                        }
20533                }
20534        }
20535
20536        private void appendConstantModel(dataflow dataflow, Table tableModel) {
20537                table constantElement = new table();
20538                constantElement.setId(String.valueOf(tableModel.getId()));
20539                if (!SQLUtil.isEmpty(tableModel.getDatabase())) {
20540                        constantElement.setDatabase(tableModel.getDatabase());
20541                }
20542                if (!SQLUtil.isEmpty(tableModel.getSchema())) {
20543                        constantElement.setSchema(tableModel.getSchema());
20544                }
20545                constantElement.setServer(tableModel.getServer());
20546                constantElement.setName(getConstantName(tableModel));
20547                constantElement.setType("constantTable");
20548
20549                if (tableModel.getStartPosition() != null && tableModel.getEndPosition() != null) {
20550                        constantElement.setCoordinate(convertCoordinate(tableModel.getStartPosition()) + ","
20551                                        + convertCoordinate(tableModel.getEndPosition()));
20552                }
20553                dataflow.getTables().add(constantElement);
20554
20555                List<TableColumn> columns = tableModel.getColumns();
20556                for (int j = 0; j < columns.size(); j++) {
20557                        TableColumn columnModel = (TableColumn) columns.get(j);
20558                        column columnElement = new column();
20559                        columnElement.setId(String.valueOf(columnModel.getId()));
20560                        columnElement.setName(columnModel.getName());
20561                        if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
20562                                columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
20563                                                + convertCoordinate(columnModel.getEndPosition()));
20564                        }
20565                        constantElement.getColumns().add(columnElement);
20566                }
20567        }
20568
20569        /**
20570         * Copies the authoritative endpoint classification from the internal model onto the
20571         * XML element, emitting each field ONLY when it carries information (kind/intro not
20572         * UNKNOWN, createdInSql only when true, normalizedName only when it differs from the
20573         * raw name). This keeps existing XML byte-identical for endpoints that aren't
20574         * classified, honouring the strictly-additive contract (§9 of
20575         * docs/tmp/dlineage-authoritative-endpoint-classification.md).
20576         */
20577        private void appendEndpointClassification(table tableElement, Table tableModel) {
20578                boolean created = tableModel.isCreatedInSql();
20579                EndpointKind kind = tableModel.getEndpointKind();
20580                // Emit endpointKind only when it carries signal: a non-catalog special kind
20581                // (temp / global-temp / table-variable / tempdb) always, or CATALOG_OBJECT only
20582                // when the object is created here. A plainly-referenced catalog table is the
20583                // default case and carries no attribute, so existing golden XML for ordinary
20584                // table references stays byte-identical.
20585                if (kind != null && kind != EndpointKind.UNKNOWN
20586                                && (created || kind != EndpointKind.CATALOG_OBJECT)) {
20587                        tableElement.setEndpointKind(kind.name());
20588                }
20589                EndpointIntroduction intro = tableModel.getEndpointIntroduction();
20590                if (intro != null && intro != EndpointIntroduction.UNKNOWN) {
20591                        tableElement.setEndpointIntroduction(intro.name());
20592                }
20593                if (created) {
20594                        tableElement.setCreatedInSql("true");
20595                }
20596                // Which name segments a dynamic fold assembled at runtime. Present only on endpoints
20597                // the analyzer marked inside a fold context, so ordinary output is untouched.
20598                if (tableModel.getTemplatedParts() != null) {
20599                        tableElement.setTemplatedParts(tableModel.getTemplatedParts());
20600                }
20601                // Emit normalizedName only when the raw name actually carries delimiters
20602                // (brackets / quotes / backticks) — that is when the normalized form is useful
20603                // and not trivially derivable. Avoids case-only churn on every plain name.
20604                // A templated endpoint always gets it: consumers match the delimiter-free skeleton
20605                // against later-resolved concrete names, and an unbracketed concatenation
20606                // ('RECON_' + @X + '_t') carries no delimiter to trigger the test above.
20607                String rawName = tableModel.getName();
20608                if (rawName != null && (hasIdentifierDelimiter(rawName) || tableModel.isDynamicTemplate())) {
20609                        String normalized = tableModel.getNormalizedName();
20610                        if (normalized != null && !normalized.equals(rawName)) {
20611                                tableElement.setNormalizedName(normalized);
20612                        }
20613                }
20614        }
20615
20616        private static boolean hasIdentifierDelimiter(String name) {
20617                return name.indexOf('[') != -1 || name.indexOf(']') != -1
20618                                || name.indexOf('`') != -1 || name.indexOf('"') != -1;
20619        }
20620
20621        private void appendTableModel(dataflow dataflow, Table tableModel, Map<String, table> tableMap) {
20622                if(tableModel.getSubType() == SubType.unnest) {}
20623                table tableElement = new table();
20624                tableElement.setId(String.valueOf(tableModel.getId()));
20625                // A one-part name that is ambiguous across two or more schemas (with no object
20626                // in the default schema) must NOT be bound to a fabricated default-schema
20627                // object: surface it unqualified and expose the candidate set instead.
20628                boolean ambiguousUnqualified = tableModel.isAmbiguousUnqualifiedTable()
20629                                && tableModel.getCandidateTables() != null
20630                                && !tableModel.getCandidateTables().isEmpty();
20631                if (!ambiguousUnqualified) {
20632                        if (!SQLUtil.isEmpty(tableModel.getDatabase())) {
20633                                tableElement.setDatabase(tableModel.getDatabase());
20634                        }
20635                        if (!SQLUtil.isEmpty(tableModel.getSchema())) {
20636                                tableElement.setSchema(tableModel.getSchema());
20637                        }
20638                }
20639                tableElement.setServer(tableModel.getServer());
20640                if (ambiguousUnqualified) {
20641                        tableElement.setName(DlineageUtil.getSimpleTableName(tableModel.getName()));
20642                        tableElement.setCandidateTables(new ArrayList<String>(tableModel.getCandidateTables()));
20643                } else {
20644                        tableElement.setName(tableModel.getName());
20645                }
20646                tableElement.setDisplayName(tableModel.getDisplayName());
20647                tableElement.setStarStmt(tableModel.getStarStmt());
20648                if(tableModel.isFromDDL()) {
20649                        tableElement.setFromDDL(String.valueOf(tableModel.isFromDDL()));
20650                }
20651                // Authoritative endpoint classification (additive). Emit only non-default
20652                // values so existing XML stays byte-identical for unclassified endpoints.
20653                // See docs/tmp/dlineage-authoritative-endpoint-classification.md.
20654                appendEndpointClassification(tableElement, tableModel);
20655
20656                if (tableModel.isPseudo()) {
20657                        tableElement.setType("pseudoTable");
20658                } else {
20659                        tableElement.setType("table");
20660                }
20661
20662                if (tableModel.getSubType() != null) {
20663                        if (tableModel.getSubType() == SubType.unnest) {
20664                                tableElement.setType(SubType.unnest.name());
20665                        }
20666                        else {
20667                                tableElement.setSubType(tableModel.getSubType().name());
20668                        }
20669                }
20670                if (tableModel.getParent() != null) {
20671                        tableElement.setParent(tableModel.getParent());
20672                }
20673                if (tableModel.getAlias() != null && tableModel.getAlias().trim().length() > 0) {
20674                        tableElement.setAlias(tableModel.getAlias());
20675                }
20676                if (tableModel.getStartPosition() != null && tableModel.getEndPosition() != null) {
20677            if(option.isTraceTablePosition()){
20678                for (Pair<Pair3<Long, Long, String>, Pair3<Long, Long, String>> position:tableModel.getPositions()){
20679                    tableElement.appendCoordinate(convertCoordinate(position.first)+","+convertCoordinate(position.second));
20680                }
20681            }
20682            else {
20683                tableElement.setCoordinate(convertCoordinate(tableModel.getStartPosition()) + ","
20684                    + convertCoordinate(tableModel.getEndPosition()));
20685            }
20686                }
20687                if (tableModel.getProcesses() != null) {
20688                        List<String> processIds = new ArrayList<String>();
20689                        for (Process process : tableModel.getProcesses()) {
20690                                processIds.add(String.valueOf(process.getId()));
20691                        }
20692                        tableElement.setProcessIds(processIds);
20693                }
20694
20695                table oldTableElement = null;
20696                String tableFullName = DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getQualifiedTableName(tableElement));
20697                
20698                if(tableMap.containsKey(tableFullName)) {
20699                        oldTableElement = tableMap.get(tableFullName);
20700                }
20701                else {
20702                        tableMap.put(tableFullName, tableElement);
20703                }
20704                
20705                if (tableModel.getSubType() == SubType.unnest) {
20706                        dataflow.getResultsets().add(tableElement);
20707                } else {
20708                        dataflow.getTables().add(tableElement);
20709                }
20710
20711                List<TableColumn> columns = tableModel.getColumns();
20712
20713                if (containStarColumn(columns)) {
20714                        for (TableColumn column : columns) {
20715                                if (column.getName().endsWith("*")) {
20716                                        for (TableColumn starElement : columns) {
20717                                                if (starElement == column) {
20718                                                        continue;
20719                                                }
20720                                                if (starElement.isNotBindStarLinkColumn()) {
20721                                                        continue;
20722                                                }
20723                                                TObjectName columnObject = starElement.getColumnObject();
20724                                                column.bindStarLinkColumn(columnObject);
20725                                        }
20726                                        if (tableModel.isCreateTable() && column.isExpandStar()) {
20727                                                column.setShowStar(false);
20728                                        }
20729                                }
20730                        }
20731                }
20732
20733                for (int j = 0; j < columns.size(); j++) {
20734                        TableColumn columnModel = columns.get(j);
20735
20736                        if(oldTableElement != null){
20737                                if(!CollectionUtil.isEmpty(oldTableElement.getColumns())){
20738                                        for(column oldColumnElement: oldTableElement.getColumns()){
20739                                                if(SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotColumn, oldColumnElement.getName(), columnModel.getName())){
20740                                                        if(columnModel.getDataType() == null && oldColumnElement.getDataType() != null){
20741                                                                columnModel.setDataType(oldColumnElement.getDataType());
20742                                                        }
20743                                                        if(columnModel.getPrimaryKey() == null && oldColumnElement.isPrimaryKey() != null){
20744                                                                columnModel.setPrimaryKey(oldColumnElement.isPrimaryKey());
20745                                                        }
20746                                                        if(columnModel.getIndexKey() == null && oldColumnElement.isIndexKey() != null){
20747                                                                columnModel.setIndexKey(oldColumnElement.isIndexKey());
20748                                                        }
20749                                                        if(columnModel.getUnqiueKey() == null && oldColumnElement.isUnqiueKey() != null){
20750                                                                columnModel.setUnqiueKey(oldColumnElement.isUnqiueKey());
20751                                                        }
20752                                                        if(columnModel.getForeignKey() == null && oldColumnElement.isForeignKey() != null){
20753                                                                columnModel.setForeignKey(oldColumnElement.isForeignKey());
20754                                                        }
20755
20756                                                        if(oldColumnElement.getDataType() == null && columnModel.getDataType() != null){
20757                                                                oldColumnElement.setDataType(columnModel.getDataType());
20758                                                        }
20759                                                        if(oldColumnElement.isPrimaryKey() == null && columnModel.getPrimaryKey() != null){
20760                                                                oldColumnElement.setPrimaryKey(columnModel.getPrimaryKey());
20761                                                        }
20762                                                        if(oldColumnElement.isIndexKey() == null && columnModel.getIndexKey() != null){
20763                                                                oldColumnElement.setIndexKey(columnModel.getIndexKey());
20764                                                        }
20765                                                        if(oldColumnElement.isUnqiueKey() == null && columnModel.getUnqiueKey() != null){
20766                                                                oldColumnElement.setUnqiueKey(columnModel.getUnqiueKey());
20767                                                        }
20768                                                        if(oldColumnElement.isForeignKey() == null && columnModel.getForeignKey() != null){
20769                                                                oldColumnElement.setForeignKey(columnModel.getForeignKey());
20770                                                        }
20771                                                        break;
20772                                                }
20773                                        }
20774                                }
20775
20776                        }
20777
20778                        if (!columnModel.isPseduo() && columnModel.hasStarLinkColumn() && !columnModel.isVariant()) {
20779                                List<String> starLinkColumnList = columnModel.getStarLinkColumnNames();
20780                                for (int k = 0; k < starLinkColumnList.size(); k++) {
20781                                        column columnElement = new column();
20782                                        columnElement.setId(String.valueOf(columnModel.getId()) + "_" + k);
20783                                        String columnName = starLinkColumnList.get(k);
20784                                        if (containStarColumn(columns, columnName)) {
20785                                                continue;
20786                                        }
20787                                        columnElement.setName(columnName);
20788                                        if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
20789                                                columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
20790                                                                + convertCoordinate(columnModel.getEndPosition()));
20791                                        }
20792                                        if (columnModel.getForeignKey()) {
20793                                                columnElement.setForeignKey(columnModel.getForeignKey());
20794                                        }
20795                                        if (columnModel.getIndexKey()) {
20796                                                columnElement.setIndexKey(columnModel.getIndexKey());
20797                                        }
20798                                        if (columnModel.getPrimaryKey()) {
20799                                                columnElement.setPrimaryKey(columnModel.getPrimaryKey());
20800                                        }
20801                                        if (columnModel.getUnqiueKey()) {
20802                                                columnElement.setUnqiueKey(columnModel.getUnqiueKey());
20803                                        }
20804                                        if (columnModel.getDataType() != null) {
20805                                                columnElement.setDataType(columnModel.getDataType());
20806                                        }
20807                                        tableElement.getColumns().add(columnElement);
20808                                }
20809                                if (columnModel.isShowStar()) {
20810                                        column columnElement = new column();
20811                                        columnElement.setId(String.valueOf(columnModel.getId()));
20812                                        columnElement.setName(columnModel.getName());
20813                                        if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
20814                                                columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
20815                                                                + convertCoordinate(columnModel.getEndPosition()));
20816                                        }
20817                                        if (columnModel.getForeignKey()) {
20818                                                columnElement.setForeignKey(columnModel.getForeignKey());
20819                                        }
20820                                        if (columnModel.getIndexKey()) {
20821                                                columnElement.setIndexKey(columnModel.getIndexKey());
20822                                        }
20823                                        if (columnModel.getPrimaryKey()) {
20824                                                columnElement.setPrimaryKey(columnModel.getPrimaryKey());
20825                                        }
20826                                        if (columnModel.getUnqiueKey()) {
20827                                                columnElement.setUnqiueKey(columnModel.getUnqiueKey());
20828                                        }
20829                                        if (columnModel.getDataType() != null) {
20830                                                columnElement.setDataType(columnModel.getDataType());
20831                                        }
20832                                        tableElement.getColumns().add(columnElement);
20833                                }
20834                        } else {
20835                                column columnElement = new column();
20836                                columnElement.setId(String.valueOf(columnModel.getId()));
20837                                columnElement.setName(columnModel.getName());
20838                                columnElement.setDisplayName(columnModel.getDisplayName());
20839                                if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) {
20840                                        columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + ","
20841                                                        + convertCoordinate(columnModel.getEndPosition()));
20842                                }
20843                                if (columnModel.isPseduo()) {
20844                                        columnElement.setSource("system");
20845                                }
20846                                if (columnModel.getForeignKey()) {
20847                                        columnElement.setForeignKey(columnModel.getForeignKey());
20848                                }
20849                                if (columnModel.getIndexKey()) {
20850                                        columnElement.setIndexKey(columnModel.getIndexKey());
20851                                }
20852                                if (columnModel.getPrimaryKey()) {
20853                                        columnElement.setPrimaryKey(columnModel.getPrimaryKey());
20854                                }
20855                                if (columnModel.getUnqiueKey()) {
20856                                        columnElement.setUnqiueKey(columnModel.getUnqiueKey());
20857                                }
20858                                if (columnModel.getDataType() != null) {
20859                                        columnElement.setDataType(columnModel.getDataType());
20860                                }
20861                                tableElement.getColumns().add(columnElement);
20862                        }
20863                }
20864
20865                TableRelationRows relationRows = tableModel.getRelationRows();
20866                if (relationRows.hasRelation()) {
20867                        column relationRowsElement = new column();
20868                        relationRowsElement.setId(String.valueOf(relationRows.getId()));
20869                        relationRowsElement.setName(relationRows.getName());
20870                        if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) {
20871                                relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + ","
20872                                                + convertCoordinate(relationRows.getEndPosition()));
20873                        }
20874                        relationRowsElement.setSource("system");
20875                        tableElement.getColumns().add(relationRowsElement);
20876                }
20877        }
20878
20879        private boolean containStarColumn(List<TableColumn> columns, String qualifiedColumnName) {
20880                for (TableColumn tableColumn : columns) {
20881                        if (DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()).equals(qualifiedColumnName)) {
20882                                return true;
20883                        }
20884                }
20885                return false;
20886        }
20887
20888        private TableColumn searchTableColumn(List<TableColumn> columns, String qualifiedColumnName) {
20889                for (TableColumn tableColumn : columns) {
20890                        if (DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()).equals(qualifiedColumnName)) {
20891                                return tableColumn;
20892                        }
20893                }
20894                return null;
20895        }
20896
20897        private boolean containStarColumn(Collection<RelationshipElement<?>> elements, String qualifiedColumnName) {
20898                for (RelationshipElement element : elements) {
20899                        if (element.getElement() instanceof TableColumn) {
20900                                if (DlineageUtil.getIdentifierNormalColumnName(((TableColumn) element.getElement()).getName())
20901                                                .equals(qualifiedColumnName)) {
20902                                        return true;
20903                                }
20904                        } else if (element.getElement() instanceof ResultColumn) {
20905                                if (DlineageUtil.getIdentifierNormalColumnName(((ResultColumn) element.getElement()).getName())
20906                                                .equals(qualifiedColumnName)) {
20907                                        return true;
20908                                }
20909                        }
20910                }
20911                return false;
20912        }
20913
20914        /**
20915         * Returns true if there is a non-star source element matching the column name
20916         * whose parent (table/resultset) does NOT also contribute a star source in
20917         * the same relationship. This identifies "definitive" explicit sources like
20918         * subquery columns (e.g., "coalesce(...) as col_a") vs incidental references
20919         * (e.g., "aTab.id" where aTab.* is also a source).
20920         */
20921        private boolean hasDefinitiveNonStarSource(Collection<RelationshipElement<?>> elements, String qualifiedColumnName) {
20922                // First, collect parent IDs of all star (*) sources
20923                Set<Long> starSourceParentIds = new HashSet<Long>();
20924                for (RelationshipElement<?> element : elements) {
20925                        if (element.getElement() instanceof TableColumn) {
20926                                TableColumn tc = (TableColumn) element.getElement();
20927                                if ("*".equals(tc.getName()) && tc.getTable() != null) {
20928                                        starSourceParentIds.add(tc.getTable().getId());
20929                                }
20930                        } else if (element.getElement() instanceof ResultColumn) {
20931                                ResultColumn rc = (ResultColumn) element.getElement();
20932                                if ("*".equals(rc.getName()) && rc.getResultSet() != null) {
20933                                        starSourceParentIds.add(rc.getResultSet().getId());
20934                                }
20935                        }
20936                }
20937
20938                // Then check if any non-star source matching the column name has a parent
20939                // that does NOT also have a star source
20940                for (RelationshipElement<?> element : elements) {
20941                        if (element.getElement() instanceof TableColumn) {
20942                                TableColumn tc = (TableColumn) element.getElement();
20943                                if (!"*".equals(tc.getName())
20944                                                && DlineageUtil.getIdentifierNormalColumnName(tc.getName()).equals(qualifiedColumnName)) {
20945                                        if (tc.getTable() != null && !starSourceParentIds.contains(tc.getTable().getId())) {
20946                                                return true;
20947                                        }
20948                                }
20949                        } else if (element.getElement() instanceof ResultColumn) {
20950                                ResultColumn rc = (ResultColumn) element.getElement();
20951                                if (!"*".equals(rc.getName())
20952                                                && DlineageUtil.getIdentifierNormalColumnName(rc.getName()).equals(qualifiedColumnName)) {
20953                                        if (rc.getResultSet() != null && !starSourceParentIds.contains(rc.getResultSet().getId())) {
20954                                                return true;
20955                                        }
20956                                }
20957                        }
20958                }
20959                return false;
20960        }
20961
20962        private void analyzeSelectStmt(TSelectSqlStatement stmt) {
20963                if (!accessedSubqueries.contains(stmt)) {
20964                        accessedSubqueries.add(stmt);
20965                } else {
20966                        if (modelManager.getModel(stmt) != null) {
20967                                return;
20968                        }
20969                }
20970
20971                // ClickHouse external table functions must be modelled BEFORE the
20972                // select-list relations are analyzed: the column path resolves its
20973                // source through modelManager.getModel(table), and without an early
20974                // binding it materializes a generic function node and the external
20975                // upstream silently vanishes.
20976                if (option.getVendor() == EDbVendor.dbvclickhouse && stmt.getRelations() != null) {
20977                        for (TTable lcTable : stmt.getRelations()) {
20978                                TFunctionCall lcFuncCall = lcTable.getFuncCall();
20979                                if (lcFuncCall == null && lcTable.getTableExpr() != null) {
20980                                        lcFuncCall = lcTable.getTableExpr().getFunctionCall();
20981                                }
20982                                if (lcFuncCall != null) {
20983                                        modelClickhouseExternalTableFunction(lcTable, lcFuncCall);
20984                                }
20985                        }
20986                }
20987
20988                if (stmt.getParentStmt() == null && stmt.getIntoClause() == null && stmt.getIntoTableClause() == null) {
20989                        if(option.isIgnoreTopSelect() && (option.isIgnoreRecordSet() || option.isSimpleOutput())){
20990                                if(option.getAnalyzeMode() == null || option.getAnalyzeMode() == AnalyzeMode.dataflow){
20991                                        return;
20992                                }
20993                        }
20994                }
20995
20996                if (stmt.getSetOperatorType() != ESetOperatorType.none) {
20997
20998                        // Iteratively analyze all descendant UNION branches before processing this node.
20999                        // Uses iterative post-order traversal to preserve the original left-right-self
21000                        // processing order, avoiding StackOverflow with deeply nested UNION trees.
21001                        {
21002                                Deque<TSelectSqlStatement> stack1 = new ArrayDeque<>();
21003                                List<TSelectSqlStatement> postOrder = new ArrayList<>();
21004                                stack1.push(stmt);
21005                                while (!stack1.isEmpty()) {
21006                                        TSelectSqlStatement node = stack1.pop();
21007                                        postOrder.add(node);
21008                                        if (node.getSetOperatorType() != ESetOperatorType.none) {
21009                                                // Push left first, then right, so that after reversal left comes first
21010                                                if (node.getLeftStmt() != null) stack1.push(node.getLeftStmt());
21011                                                if (node.getRightStmt() != null) stack1.push(node.getRightStmt());
21012                                        }
21013                                }
21014                                Collections.reverse(postOrder);
21015                                // Process all descendants in post-order, skip stmt itself (processed below)
21016                                for (int pi = 0; pi < postOrder.size() - 1; pi++) {
21017                                        TSelectSqlStatement node = postOrder.get(pi);
21018                                        if (!accessedStatements.contains(node)) {
21019                                                accessedStatements.add(node);
21020                                                analyzeSelectStmt(node);
21021                                        }
21022                                }
21023                        }
21024
21025                        stmtStack.push(stmt);
21026                        SelectSetResultSet resultSet = modelFactory.createSelectSetResultSet(stmt);
21027
21028                        ResultSet leftResultSetModel = (ResultSet) modelManager.getModel(stmt.getLeftStmt());
21029                        if (leftResultSetModel != null && leftResultSetModel != resultSet
21030                                        && !leftResultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
21031                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
21032                                impactRelation.setEffectType(EffectType.select);
21033                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
21034                                                leftResultSetModel.getRelationRows()));
21035                                impactRelation.setTarget(
21036                                                new RelationRowsRelationshipElement<ResultSetRelationRows>(resultSet.getRelationRows()));
21037                        }
21038                        
21039                        ResultSet rightResultSetModel = (ResultSet) modelManager.getModel(stmt.getRightStmt());
21040                        if (rightResultSetModel != null && rightResultSetModel != resultSet
21041                                        && !rightResultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
21042                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
21043                                impactRelation.setEffectType(EffectType.select);
21044                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
21045                                                rightResultSetModel.getRelationRows()));
21046                                impactRelation.setTarget(
21047                                                new RelationRowsRelationshipElement<ResultSetRelationRows>(resultSet.getRelationRows()));
21048                        }
21049                        
21050                        if ((leftResultSetModel != null && leftResultSetModel.isDetermined())
21051                                        || (rightResultSetModel != null && rightResultSetModel.isDetermined())) {
21052                                resultSet.setDetermined(true);
21053                        }
21054                        
21055                        if (resultSet.getColumns() == null || resultSet.getColumns().isEmpty()) {
21056                                if (getResultColumnList(stmt.getLeftStmt()) != null) {
21057                                        createSelectSetResultColumns(resultSet, stmt.getLeftStmt());
21058                                } else if (getResultColumnList(stmt.getRightStmt()) != null) {
21059                                        createSelectSetResultColumns(resultSet, stmt.getRightStmt());
21060                                }
21061                        }
21062
21063                        List<ResultColumn> columns = resultSet.getColumns();
21064                        for (int i = 0; i < columns.size(); i++) {
21065                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
21066                                relation.setEffectType(EffectType.select);
21067                                relation.setTarget(new ResultColumnRelationshipElement(columns.get(i)));
21068
21069                                if (!stmt.getLeftStmt().isCombinedQuery()) {
21070                                        ResultSet sourceResultSet = (ResultSet) modelManager
21071                                                        .getModel(stmt.getLeftStmt().getResultColumnList());
21072                                        if (sourceResultSet!=null && sourceResultSet.getColumns().size() > i) {
21073                                                if (columns.get(i).getName().endsWith("*")) {
21074                                                        for (ResultColumn column : sourceResultSet.getColumns()) {
21075                                                                relation.addSource(new ResultColumnRelationshipElement(column));
21076                                                        }
21077                                                } else {
21078                                                        relation.addSource(
21079                                                                        new ResultColumnRelationshipElement(sourceResultSet.getColumns().get(i)));
21080                                                }
21081                                        }
21082                                } else {
21083                                        ResultSet sourceResultSet = (ResultSet) modelManager.getModel(stmt.getLeftStmt());
21084                                        if (sourceResultSet != null && sourceResultSet.getColumns().size() > i) {
21085                                                if (columns.get(i).getName().endsWith("*")) {
21086                                                        for (ResultColumn column : sourceResultSet.getColumns()) {
21087                                                                relation.addSource(new ResultColumnRelationshipElement(column));
21088                                                        }
21089                                                } else {
21090                                                        relation.addSource(
21091                                                                        new ResultColumnRelationshipElement(sourceResultSet.getColumns().get(i)));
21092                                                }
21093                                        }
21094                                }
21095
21096                                if (!stmt.getRightStmt().isCombinedQuery()) {
21097                                        ResultSet sourceResultSet = (ResultSet) modelManager
21098                                                        .getModel(stmt.getRightStmt().getResultColumnList());
21099                                        if (sourceResultSet != null && sourceResultSet.getColumns().size() > i) {
21100                                                if (columns.get(i).getName().endsWith("*")) {
21101                                                        for (ResultColumn column : sourceResultSet.getColumns()) {
21102                                                                relation.addSource(new ResultColumnRelationshipElement(column));
21103                                                        }
21104                                                } else {
21105                                                        relation.addSource(
21106                                                                        new ResultColumnRelationshipElement(sourceResultSet.getColumns().get(i)));
21107                                                }
21108                                        } else if (sourceResultSet != null) {
21109                                                for (ResultColumn column : sourceResultSet.getColumns()) {
21110                                                        if (column.hasStarLinkColumn()) {
21111                                                                relation.addSource(new ResultColumnRelationshipElement(column));
21112                                                        }
21113                                                }
21114                                        }
21115                                } else {
21116                                        ResultSet sourceResultSet = (ResultSet) modelManager.getModel(stmt.getRightStmt());
21117                                        if (sourceResultSet != null && sourceResultSet.getColumns().size() > i) {
21118                                                relation.addSource(new ResultColumnRelationshipElement(sourceResultSet.getColumns().get(i)));
21119                                        } else if (sourceResultSet != null) {
21120                                                for (ResultColumn column : sourceResultSet.getColumns()) {
21121                                                        if (column.hasStarLinkColumn()) {
21122                                                                relation.addSource(new ResultColumnRelationshipElement(column));
21123                                                        }
21124                                                }
21125                                        }
21126                                }
21127                        }
21128                        
21129                        analyzeSelectIntoClause(stmt);
21130                        
21131                        stmtStack.pop();
21132                } else {
21133
21134                        // handle hive stmt, issue_id I3SGZB
21135                        if (stmt.getHiveBodyList() != null && stmt.getHiveBodyList().size() > 0) {
21136                                stmtStack.push(stmt);
21137                                hiveFromTables = stmt.tables;
21138                                if (hiveFromTables != null) {
21139                                        for (int i = 0; i < hiveFromTables.size(); i++) {
21140                                                modelFactory.createTable(hiveFromTables.getTable(i));
21141                                        }
21142                                }
21143                                for (int i = 0; i < stmt.getHiveBodyList().size(); i++) {
21144                                        analyzeCustomSqlStmt(stmt.getHiveBodyList().get(i));
21145                                }
21146                                stmtStack.pop();
21147                                return;
21148                        }
21149                        
21150                        if (stmt.getTransformClause() != null) {
21151                                analyzeHiveTransformClause(stmt, stmt.getTransformClause());
21152                                return;
21153                        }
21154
21155                        if (stmt.getResultColumnList() == null) {
21156                                return;
21157                        }
21158
21159                        stmtStack.push(stmt);
21160
21161                        TTableList fromTables = stmt.tables;
21162                        if ((fromTables == null || fromTables.size() == 0)
21163                                        && (hiveFromTables != null && hiveFromTables.size() > 0)) {
21164                                fromTables = hiveFromTables;
21165                        }
21166
21167                        for (int i = 0; i < fromTables.size(); i++) {
21168                                TTable table = fromTables.getTable(i);
21169                                if (table.getLateralViewList() != null && !table.getLateralViewList().isEmpty()) {
21170                                        analyzeTableSubquery(table);
21171                                        analyzeLateralView(stmt, table, table.getLateralViewList());
21172                                        stmtStack.pop();
21173                                        return;
21174                                }
21175                                if (table.getUnnestClause() != null && option.getVendor() == EDbVendor.dbvpresto) {
21176                                        analyzeTableSubquery(table);
21177                                        analyzePrestoUnnest(stmt, table);
21178                                        stmtStack.pop();
21179                                        return;
21180                                }
21181                                if (table.getUnnestClause() != null && option.getVendor() == EDbVendor.dbvbigquery) {
21182                                        analyzeBigQueryUnnest(stmt, table);
21183                                }
21184                        }
21185                        
21186                        //用来获取 pivotedTable 对应的columns
21187                        TPivotedTable pivotedTable = null; 
21188                        if (stmt.getJoins() != null && stmt.getJoins().size() > 0) {
21189                                for (TJoin join : stmt.getJoins()) {
21190                                        if (join.getTable() != null && join.getTable().getPivotedTable() != null) {
21191                                                if (isUnPivotedTable(join.getTable().getPivotedTable())) {
21192                                                        analyzeUnPivotedTable(stmt, join.getTable().getPivotedTable());
21193                                                        pivotedTable = join.getTable().getPivotedTable();
21194                                                } else {
21195                                                        analyzePivotedTable(stmt, join.getTable().getPivotedTable());
21196                                                        pivotedTable = join.getTable().getPivotedTable();
21197                                                }
21198                                        }
21199                                }
21200                        }
21201
21202                        for (int i = 0; i < fromTables.size(); i++) {
21203                                TTable table = fromTables.getTable(i);
21204                                // Handle TABLE(func()) which has tableType=tableExpr and funcCall=null
21205                                if (table.getTableType() == ETableSource.tableExpr && table.getFuncCall() == null && pipelinedAnalyzer != null) {
21206                                        try {
21207                                                pipelinedAnalyzer.tryStitchTableExpr(table);
21208                                        } catch (Exception e) {
21209                                                // Don't let pipelined stitching failure break main flow
21210                                        }
21211                                }
21212
21213                                if (table.getFuncCall() != null || (table.getTableExpr()!=null && table.getTableExpr().getFunctionCall()!=null)) {
21214                                        TFunctionCall functionCall = table.getFuncCall();
21215                                        if (functionCall == null) {
21216                                                functionCall = table.getTableExpr().getFunctionCall();
21217                                        }
21218                                        Procedure callee = modelManager.getProcedureByName(
21219                                                        DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
21220                                        if (callee == null && procedureDDLMap.containsKey(DlineageUtil.getFunctionNameWithArgNum(functionCall))) {
21221                                                analyzeCustomSqlStmt(procedureDDLMap.get(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
21222                                                callee = modelManager.getProcedureByName(
21223                                                                DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall)));
21224                                        }
21225
21226                                        if(callee!=null) {
21227                                                if (callee.getArguments() != null) {
21228                                                        for (int j = 0; j < callee.getArguments().size(); j++) {
21229                                                                Argument argument = callee.getArguments().get(j);
21230                                                                Variable variable = resolveFormalVariable(callee, argument);
21231                                                                if(variable!=null) {
21232                                                                        if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
21233                                                                                Transform transform = new Transform();
21234                                                                                transform.setType(Transform.FUNCTION);
21235                                                                                transform.setCode(functionCall);
21236                                                                                compositeBindingColumn(variable).setTransform(transform);
21237                                                                        }
21238                                                                        Process process = modelFactory.createProcess(functionCall);
21239                                                                        variable.addProcess(process);
21240                                                                        analyzeFunctionArgumentsDataFlowRelation(compositeBindingColumn(variable), functionCall, j, process);
21241                                                                }
21242                                                        }
21243                                                }
21244                                                Set<Object> functionTableModelObjs = modelManager.getFunctionTable(DlineageUtil
21245                                                                .getIdentifierNormalTableName(functionCall.getFunctionName().toString()));
21246                                                if (functionTableModelObjs != null) {
21247                                                        modelManager.bindModel(table, functionTableModelObjs.iterator().next());
21248                                                }
21249                                                // Try pipelined function stitching
21250                                                if (pipelinedAnalyzer != null) {
21251                                                        try {
21252                                                                pipelinedAnalyzer.tryStitchCallSite(table, functionCall);
21253                                                        } catch (Exception e) {
21254                                                                // Don't let pipelined stitching failure break main flow
21255                                                        }
21256                                                }
21257                                                continue;
21258                                        } else {
21259                                                Set<Object> functionTableModelObjs = modelManager.getFunctionTable(DlineageUtil
21260                                                                .getIdentifierNormalTableName(functionCall.getFunctionName().toString()));
21261                                                if (functionTableModelObjs == null) {
21262                                                        // ClickHouse external table functions expose real
21263                                                        // external data; a generic function node silently
21264                                                        // dropped their upstream (missing lineage is
21265                                                        // invisible to the user — the correctness ruling).
21266                                                        if (modelClickhouseExternalTableFunction(table, functionCall)) {
21267                                                                continue;
21268                                                        }
21269//                                                      Procedure procedure = modelManager.getProcedureByName(DlineageUtil
21270//                                                                      .getIdentifierNormalTableName(functionCall.getFunctionName().toString()));
21271//                                                      if (procedure != null) {
21272                                                                createFunction(functionCall);
21273                                                                continue;
21274//                                                      }
21275                                                }
21276                                                if (functionTableModelObjs!=null && functionTableModelObjs.iterator().next() instanceof Table) {
21277                                                        Table functionTableModel = (Table) functionTableModelObjs.iterator().next();
21278                                                        if (functionTableModel.getColumns() != null) {
21279                                                                Table functionTable = modelFactory.createTableFromCreateDDL(table, true);
21280                                                                if (functionTable == functionTableModel)
21281                                                                        continue;
21282                                                                for (int j = 0; j < functionTableModel.getColumns().size(); j++) {
21283                                                                        TableColumn column = modelFactory.createTableColumn(functionTable,
21284                                                                                        functionTableModel.getColumns().get(j).getColumnObject(), true);
21285                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
21286                                                                        relation.setEffectType(EffectType.select);
21287                                                                        relation.setTarget(new TableColumnRelationshipElement(column));
21288                                                                        relation.addSource(
21289                                                                                        new TableColumnRelationshipElement(functionTableModel.getColumns().get(j)));
21290                                                                }
21291                                                        }
21292                                                } else if (functionTableModelObjs!=null && functionTableModelObjs.iterator().next() instanceof ResultSet) {
21293                                                        ResultSet functionTableModel = (ResultSet) functionTableModelObjs.iterator().next();
21294                                                        if (functionTableModel.getColumns() != null) {
21295                                                                Table functionTable = modelFactory.createTableFromCreateDDL(table, true);
21296                                                                for (int j = 0; j < functionTableModel.getColumns().size(); j++) {
21297                                                                        TParseTreeNode columnObj = functionTableModel.getColumns().get(j).getColumnObject();
21298                                                                        if (columnObj instanceof TObjectName) {
21299                                                                                TableColumn column = modelFactory.createTableColumn(functionTable,
21300                                                                                                (TObjectName) columnObj, true);
21301                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
21302                                                                                relation.setEffectType(EffectType.select);
21303                                                                                relation.setTarget(new TableColumnRelationshipElement(column));
21304                                                                                relation.addSource(new ResultColumnRelationshipElement(
21305                                                                                                functionTableModel.getColumns().get(j)));
21306                                                                        } else if (columnObj instanceof TResultColumn) {
21307                                                                                TableColumn column = modelFactory.createTableColumn(functionTable,
21308                                                                                                (TResultColumn) columnObj);
21309                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
21310                                                                                relation.setEffectType(EffectType.select);
21311                                                                                relation.setTarget(new TableColumnRelationshipElement(column));
21312                                                                                relation.addSource(new ResultColumnRelationshipElement(
21313                                                                                                functionTableModel.getColumns().get(j)));
21314                                                                        }
21315                                                                }
21316                                                        }
21317                                                }
21318                                                // Try pipelined function stitching for unresolved function calls
21319                                                if (pipelinedAnalyzer != null) {
21320                                                        try {
21321                                                                pipelinedAnalyzer.tryStitchCallSite(table, functionCall);
21322                                                        } catch (Exception e) {
21323                                                                // Don't let pipelined stitching failure break main flow
21324                                                        }
21325                                                }
21326                                        }
21327                                }
21328
21329                                if (table.getPartitionExtensionClause() != null
21330                                                && table.getPartitionExtensionClause().getKeyValues() != null) {
21331                                        TExpressionList values = table.getPartitionExtensionClause().getKeyValues();
21332                                        Table tableModel = modelFactory.createTable(table);
21333                                        for (TExpression value : values) {
21334                                                if (value.getExpressionType() == EExpressionType.simple_constant_t) {
21335                                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
21336                                                        impactRelation.setEffectType(EffectType.select);
21337                                                        Table constantTable = modelFactory.createConstantsTable(stmtStack.peek());
21338                                                        TableColumn constantColumn = modelFactory.createTableColumn(constantTable, value.getConstantOperand());
21339                                                        impactRelation.addSource(new TableColumnRelationshipElement(constantColumn));
21340                                                        impactRelation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(
21341                                                                        tableModel.getRelationRows()));
21342                                                }
21343                                        }
21344                                }
21345
21346                                if (table.getSubquery() != null) {
21347                                        QueryTable queryTable = modelFactory.createQueryTable(table);
21348                                        TSelectSqlStatement subquery = table.getSubquery();
21349                                        analyzeSelectStmt(subquery);
21350
21351                                        ResultSet resultSetModel = (ResultSet) modelManager.getModel(subquery);
21352                                        if (resultSetModel != null && resultSetModel.isDetermined()) {
21353                                                queryTable.setDetermined(true);
21354                                        }
21355
21356                                        if (resultSetModel != null && resultSetModel != queryTable
21357                                                        && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
21358                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
21359                                                impactRelation.setEffectType(EffectType.select);
21360                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
21361                                                                resultSetModel.getRelationRows()));
21362                                                impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
21363                                                                queryTable.getRelationRows()));
21364                                        }
21365
21366                                        if (resultSetModel != null && resultSetModel != queryTable
21367                                                        && queryTable.getTableObject().getAliasClause() != null
21368                                                        && queryTable.getTableObject().getAliasClause().getColumns() != null) {
21369                                                for (int j = 0; j < queryTable.getColumns().size()
21370                                                                && j < resultSetModel.getColumns().size(); j++) {
21371                                                        ResultColumn sourceColumn = resultSetModel.getColumns().get(j);
21372                                                        ResultColumn targetColumn = queryTable.getColumns().get(j);
21373
21374                                                        DataFlowRelationship queryRalation = modelFactory.createDataFlowRelation();
21375                                                        queryRalation.setEffectType(EffectType.select);
21376                                                        queryRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
21377                                                        queryRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
21378                                                }
21379                                        } else if (subquery.getSetOperatorType() != ESetOperatorType.none) {
21380                                                SelectSetResultSet selectSetResultSetModel = (SelectSetResultSet) modelManager
21381                                                                .getModel(subquery);
21382                                                for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) {
21383                                                        ResultColumn sourceColumn = selectSetResultSetModel.getColumns().get(j);
21384                                                        ResultColumn targetColumn = modelFactory.createSelectSetResultColumn(queryTable,
21385                                                                        sourceColumn);
21386                                                        for (TObjectName starLinkColumn : sourceColumn.getStarLinkColumnList()) {
21387                                                                targetColumn.bindStarLinkColumn(starLinkColumn);
21388                                                        }
21389                                                        DataFlowRelationship selectSetRalation = modelFactory.createDataFlowRelation();
21390                                                        selectSetRalation.setEffectType(EffectType.select);
21391                                                        selectSetRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
21392                                                        selectSetRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
21393                                                }
21394                                        }
21395                                } else if (table.getOutputMerge() != null) {
21396                                        QueryTable queryTable = modelFactory.createQueryTable(table);
21397                                        TMergeSqlStatement subquery = table.getOutputMerge();
21398                                        analyzeMergeStmt(subquery);
21399
21400                                        for (TResultColumn column : subquery.getOutputClause().getSelectItemList()) {
21401                                                modelFactory.createResultColumn(queryTable, column);
21402                                                analyzeResultColumn(column, EffectType.select);
21403                                        }
21404
21405                                        ResultSet resultSetModel = (ResultSet) modelManager
21406                                                        .getModel(subquery.getOutputClause().getSelectItemList());
21407
21408                                        if (resultSetModel != null && resultSetModel != queryTable
21409                                                        && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
21410                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
21411                                                impactRelation.setEffectType(EffectType.select);
21412                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
21413                                                                resultSetModel.getRelationRows()));
21414                                                impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
21415                                                                queryTable.getRelationRows()));
21416                                        }
21417
21418                                        if (resultSetModel != null && resultSetModel != queryTable
21419                                                        && queryTable.getTableObject().getAliasClause() != null
21420                                                        && queryTable.getTableObject().getAliasClause().getColumns() != null) {
21421                                                for (int j = 0; j < queryTable.getColumns().size()
21422                                                                && j < resultSetModel.getColumns().size(); j++) {
21423                                                        ResultColumn sourceColumn = resultSetModel.getColumns().get(j);
21424                                                        ResultColumn targetColumn = queryTable.getColumns().get(j);
21425
21426                                                        DataFlowRelationship queryRalation = modelFactory.createDataFlowRelation();
21427                                                        queryRalation.setEffectType(EffectType.select);
21428                                                        queryRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
21429                                                        queryRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
21430                                                }
21431                                        }
21432                                } else if (table.getTableExpr() != null && table.getTableExpr().getSubQuery() != null) {
21433                                        QueryTable queryTable = modelFactory.createQueryTable(table);
21434                                        TSelectSqlStatement subquery = table.getTableExpr().getSubQuery();
21435                                        analyzeSelectStmt(subquery);
21436
21437                                        ResultSet resultSetModel = (ResultSet) modelManager.getModel(subquery);
21438
21439                                        if (resultSetModel != null && resultSetModel != queryTable
21440                                                        && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
21441                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
21442                                                impactRelation.setEffectType(EffectType.select);
21443                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
21444                                                                resultSetModel.getRelationRows()));
21445                                                impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
21446                                                                queryTable.getRelationRows()));
21447                                        }
21448
21449                                        if (resultSetModel != null && resultSetModel != queryTable
21450                                                        && queryTable.getTableObject().getAliasClause() != null) {
21451                                                for (int j = 0; j < resultSetModel.getColumns().size(); j++) {
21452                                                        ResultColumn sourceColumn = resultSetModel.getColumns().get(j);
21453                                                        ResultColumn targetColumn = modelFactory.createSelectSetResultColumn(queryTable,
21454                                                                        sourceColumn);
21455
21456                                                        DataFlowRelationship queryRalation = modelFactory.createDataFlowRelation();
21457                                                        queryRalation.setEffectType(EffectType.select);
21458                                                        queryRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
21459                                                        queryRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
21460                                                }
21461                                        } else if (subquery.getSetOperatorType() != ESetOperatorType.none) {
21462                                                SelectSetResultSet selectSetResultSetModel = (SelectSetResultSet) modelManager
21463                                                                .getModel(subquery);
21464                                                for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) {
21465                                                        ResultColumn sourceColumn = selectSetResultSetModel.getColumns().get(j);
21466                                                        ResultColumn targetColumn = modelFactory.createSelectSetResultColumn(queryTable,
21467                                                                        sourceColumn);
21468                                                        for (TObjectName starLinkColumn : sourceColumn.getStarLinkColumnList()) {
21469                                                                targetColumn.bindStarLinkColumn(starLinkColumn);
21470                                                        }
21471                                                        DataFlowRelationship selectSetRalation = modelFactory.createDataFlowRelation();
21472                                                        selectSetRalation.setEffectType(EffectType.select);
21473                                                        selectSetRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
21474                                                        selectSetRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
21475                                                }
21476                                        }
21477                                } else if (table.getCTE() != null) {
21478                                        QueryTable queryTable = modelFactory.createQueryTable(table);
21479
21480                                        TObjectNameList cteColumns = table.getCTE().getColumnList();
21481                                        if (cteColumns != null) {
21482                                                for (int j = 0; j < cteColumns.size(); j++) {
21483                                                        modelFactory.createResultColumn(queryTable, cteColumns.getObjectName(j));
21484                                                }
21485                                                queryTable.setDetermined(true);
21486                                        }
21487                                        TSelectSqlStatement subquery = table.getCTE().getSubquery();
21488                                        if (subquery != null && !stmtStack.contains(subquery) && subquery.getResultColumnList() != null) {
21489                                                analyzeSelectStmt(subquery);
21490
21491                                                ResultSet resultSetModel = (ResultSet) modelManager.getModel(subquery);
21492                                                if (resultSetModel != null && resultSetModel != queryTable
21493                                                                && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
21494                                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
21495                                                        impactRelation.setEffectType(EffectType.select);
21496                                                        impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
21497                                                                        resultSetModel.getRelationRows()));
21498                                                        impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
21499                                                                        queryTable.getRelationRows()));
21500                                                }
21501
21502                                                if (subquery.getSetOperatorType() != ESetOperatorType.none) {
21503                                                        SelectSetResultSet selectSetResultSetModel = (SelectSetResultSet) modelManager
21504                                                                        .getModel(subquery);
21505                                                        int x = 0;
21506                                                        for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) {
21507                                                                ResultColumn sourceColumn = selectSetResultSetModel.getColumns().get(j);
21508                                                                ResultColumn targetColumn = null;
21509                                                                if (cteColumns != null) {
21510                                                                        if (queryTable.getColumns().size() <= j) {
21511                                                                                for (int k = 0; k < queryTable.getColumns().size(); k++) {
21512                                                                                        if (queryTable.getColumns().get(k).getName().equals(sourceColumn.getName())
21513                                                                                                        || queryTable.getColumns().get(k).getName()
21514                                                                                                                        .equals(sourceColumn.getAlias())) {
21515                                                                                                targetColumn = queryTable.getColumns().get(k);
21516                                                                                        }
21517                                                                                }
21518                                                                        } else {
21519                                                                                if (x < j) {
21520                                                                                        x = j;
21521                                                                                }
21522                                                                                targetColumn = queryTable.getColumns().get(x);
21523                                                                                x++;
21524                                                                                if (resultSetModel.getColumns().get(j).getName().contains("*")
21525                                                                                                && x < cteColumns.size()) {
21526                                                                                        j--;
21527                                                                                }
21528                                                                        }
21529                                                                } else {
21530                                                                        targetColumn = modelFactory.createSelectSetResultColumn(queryTable, sourceColumn);
21531                                                                }
21532                                                                for (TObjectName starLinkColumn : sourceColumn.getStarLinkColumnList()) {
21533                                                                        targetColumn.bindStarLinkColumn(starLinkColumn);
21534                                                                }
21535                                                                createCteSelectRelation(null, targetColumn, sourceColumn);
21536                                                        }
21537                                                        if (!queryTable.isDetermined()) {
21538                                                                queryTable.setDetermined(selectSetResultSetModel.isDetermined());
21539                                                        }
21540                                                } else {
21541                                                        int x = 0;
21542                                                        for (int j = 0; j < resultSetModel.getColumns().size(); j++) {
21543                                                                ResultColumn sourceColumn = resultSetModel.getColumns().get(j);
21544                                                                ResultColumn targetColumn = null;
21545                                                                if (cteColumns != null) {
21546                                                                        if (queryTable.getColumns().size() <= j) {
21547                                                                                for (int k = 0; k < queryTable.getColumns().size(); k++) {
21548                                                                                        if (queryTable.getColumns().get(k).getName().equals(sourceColumn.getName())
21549                                                                                                        || queryTable.getColumns().get(k).getName()
21550                                                                                                                        .equals(sourceColumn.getAlias())) {
21551                                                                                                targetColumn = queryTable.getColumns().get(k);
21552                                                                                        }
21553                                                                                }
21554                                                                        } else {
21555                                                                                if (x < j) {
21556                                                                                        x = j;
21557                                                                                }
21558                                                                                targetColumn = queryTable.getColumns().get(x);
21559                                                                                x++;
21560                                                                                if (resultSetModel.getColumns().get(j).getName().contains("*")
21561                                                                                                && x < cteColumns.size()) {
21562                                                                                        j--;
21563                                                                                }
21564                                                                        }
21565                                                                } else {
21566                                                                        targetColumn = modelFactory.createSelectSetResultColumn(queryTable, sourceColumn);
21567                                                                }
21568                                                                for (TObjectName starLinkColumn : sourceColumn.getStarLinkColumnList()) {
21569                                                                        targetColumn.bindStarLinkColumn(starLinkColumn);
21570                                                                }
21571                                                                createCteSelectRelation(null, targetColumn, sourceColumn);
21572                                                        }
21573                                                        if (!queryTable.isDetermined()) {
21574                                                                queryTable.setDetermined(resultSetModel.isDetermined());
21575                                                        }
21576                                                }
21577                                        } else if (table.getCTE().getUpdateStmt() != null) {
21578                                                analyzeCustomSqlStmt(table.getCTE().getUpdateStmt());
21579                                        } else if (table.getCTE().getInsertStmt() != null) {
21580                                                analyzeCustomSqlStmt(table.getCTE().getInsertStmt());
21581                                        } else if (table.getCTE().getDeleteStmt() != null) {
21582                                                analyzeCustomSqlStmt(table.getCTE().getDeleteStmt());
21583                                        }
21584                                } else if (table.getTableType().name().startsWith("open")) {
21585                                        continue;
21586                                } else if (table.getTableType() == ETableSource.jsonTable) {
21587                                        Table functionTable = modelFactory.createJsonTable(table);
21588                                        TJsonTable jsonTable = table.getJsonTable();
21589                                        TColumnDefinitionList definitions = jsonTable.getColumnDefinitions();
21590                                        if (definitions != null) {
21591                                                for (int j = 0; j < definitions.size(); j++) {
21592                            TColumnDefinitionList nestDefinitions = definitions.getColumn(j).getNestedTableColumns();
21593                                                        if(nestDefinitions!=null) {
21594                                                                for(int k=0;k<nestDefinitions.size();k++){
21595                                                                        if (nestDefinitions.getColumn(k).getColumnName() == null) {
21596                                                                                continue;
21597                                                                        }
21598                                                                        TableColumn column = modelFactory.createTableColumn(functionTable,
21599                                            nestDefinitions.getColumn(k).getColumnName(), true);
21600                                    if (nestDefinitions.getColumn(k).getColumnPath() != null) {
21601                                        column.setDisplayName(
21602                                                column.getName() + ":" + nestDefinitions.getColumn(k).getColumnPath());
21603                                    }
21604                                                                }
21605                                                        }
21606                                                        else {
21607                                TableColumn column = modelFactory.createTableColumn(functionTable,
21608                                        definitions.getColumn(j).getColumnName(), true);
21609                                if (definitions.getColumn(j).getColumnPath() != null) {
21610                                    column.setDisplayName(
21611                                            column.getName() + ":" + definitions.getColumn(j).getColumnPath());
21612                                }
21613                            }
21614                                                }
21615                                        } else {
21616                                                TObjectName keyColumn = new TObjectName();
21617                                                keyColumn.setString("key");
21618                                                modelFactory.createJsonTableColumn(functionTable, keyColumn);
21619                                                TObjectName valueColumn = new TObjectName();
21620                                                valueColumn.setString("value");
21621                                                modelFactory.createJsonTableColumn(functionTable, valueColumn);
21622                                                TObjectName typeColumn = new TObjectName();
21623                                                typeColumn.setString("type");
21624                                                modelFactory.createJsonTableColumn(functionTable, typeColumn);
21625                                        }
21626
21627                                        functionTable.setCreateTable(true);
21628                                        functionTable.setSubType(SubType.function);
21629                                        modelManager.bindCreateModel(table, functionTable);
21630
21631                                        if (jsonTable.getJsonExpression() == null) {
21632                                                ErrorInfo errorInfo = new ErrorInfo();
21633                                                errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
21634                                                errorInfo.setErrorMessage("Can't handle the json table: " + jsonTable.toString());
21635                                                errorInfo.setStartPosition(new Pair3<Long, Long, String>(jsonTable.getStartToken().lineNo,
21636                                                                jsonTable.getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
21637                                                errorInfo.setEndPosition(new Pair3<Long, Long, String>(jsonTable.getEndToken().lineNo,
21638                                                                jsonTable.getEndToken().columnNo + jsonTable.getEndToken().getAstext().length(),
21639                                                                ModelBindingManager.getGlobalHash()));
21640                                                errorInfo.fillInfo(this);
21641                                                errorInfos.add(errorInfo);
21642                                        } else {
21643                                                String jsonName = jsonTable.getJsonExpression().toString();
21644                                                if (!jsonName.startsWith("@")) {
21645                                                        columnsInExpr visitor = new columnsInExpr();
21646                                                        jsonTable.getJsonExpression().inOrderTraverse(visitor);
21647                                                        List<TObjectName> objectNames = visitor.getObjectNames();
21648                                                        List<TParseTreeNode> functions = visitor.getFunctions();
21649                                                        List<TParseTreeNode> constants = visitor.getConstants();
21650                                                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
21651
21652                                                        for (int j = 0; j < functionTable.getColumns().size(); j++) {
21653                                                                TableColumn tableColumn = functionTable.getColumns().get(j);
21654                                                                if (functions != null && !functions.isEmpty()) {
21655                                                                        analyzeFunctionDataFlowRelation(tableColumn, functions, EffectType.function);
21656                                                                }
21657                                                                if (subquerys != null && !subquerys.isEmpty()) {
21658                                                                        analyzeSubqueryDataFlowRelation(tableColumn, subquerys, EffectType.select);
21659                                                                }
21660                                                                if (objectNames != null && !objectNames.isEmpty()) {
21661                                                                        analyzeDataFlowRelation(tableColumn, objectNames, EffectType.select, functions);
21662                                                                }
21663                                                                if (constants != null && !constants.isEmpty()) {
21664                                                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select,
21665                                                                                        functions);
21666                                                                }
21667                                                        }
21668                                                } else {
21669                                                        TStatementList stmts = stmt.getGsqlparser().getSqlstatements();
21670                                                        for (int j = 0; j < stmts.size(); j++) {
21671                                                                TCustomSqlStatement item = stmts.get(j);
21672                                                                if (item instanceof TMssqlDeclare) {
21673                                                                        if (analyzeMssqlJsonDeclare((TMssqlDeclare) item, jsonName, functionTable)) {
21674                                                                                break;
21675                                                                        }
21676                                                                }
21677                                                        }
21678                                                }
21679                                        }
21680                                } else if (table.getTableType() == ETableSource.xmltable) {
21681                                        Table functionTable = modelFactory.createXmlTable(table);
21682                                        TXmlTable xmlTable = table.getXmlTable();
21683                                        TXmlTableParameter param = xmlTable != null ? xmlTable.getArg() : null;
21684
21685                                        // Output columns from COLUMNS clause
21686                                        if (param != null && param.getXmlTableColumns() != null) {
21687                                                TColumnDefinitionList defs = param.getXmlTableColumns();
21688                                                for (int j = 0; j < defs.size(); j++) {
21689                                                        TColumnDefinition def = defs.getColumn(j);
21690                                                        TableColumn column = modelFactory.createTableColumn(
21691                                                                        functionTable, def.getColumnName(), true);
21692                                                        if (def.getXmlTableColumnPath() != null) {
21693                                                                column.setDisplayName(
21694                                                                                column.getName() + ":" + def.getXmlTableColumnPath());
21695                                                        }
21696                                                }
21697                                        }
21698                                        functionTable.setCreateTable(true);
21699                                        functionTable.setSubType(SubType.function);
21700                                        modelManager.bindCreateModel(table, functionTable);
21701
21702                                        // Source columns from PASSING (v1: many-to-many)
21703                                        TResultColumnList passing = null;
21704                                        if (param != null && param.getXmlPassingClause() != null) {
21705                                                passing = param.getXmlPassingClause().getPassingList();
21706                                        }
21707                                        if (passing != null && passing.size() > 0) {
21708                                                columnsInExpr visitor = new columnsInExpr();
21709                                                for (int p = 0; p < passing.size(); p++) {
21710                                                        TExpression e = passing.getResultColumn(p).getExpr();
21711                                                        if (e != null) e.inOrderTraverse(visitor);
21712                                                }
21713                                                List<TObjectName> objectNames = visitor.getObjectNames();
21714                                                List<TParseTreeNode> functions = visitor.getFunctions();
21715                                                List<TParseTreeNode> constants = visitor.getConstants();
21716                                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
21717
21718                                                for (int j = 0; j < functionTable.getColumns().size(); j++) {
21719                                                        TableColumn tc = functionTable.getColumns().get(j);
21720                                                        if (functions != null && !functions.isEmpty())
21721                                                                analyzeFunctionDataFlowRelation(tc, functions, EffectType.function);
21722                                                        if (subquerys != null && !subquerys.isEmpty())
21723                                                                analyzeSubqueryDataFlowRelation(tc, subquerys, EffectType.select);
21724                                                        if (objectNames != null && !objectNames.isEmpty())
21725                                                                analyzeDataFlowRelation(tc, objectNames, EffectType.select, functions);
21726                                                        if (constants != null && !constants.isEmpty())
21727                                                                analyzeConstantDataFlowRelation(tc, constants, EffectType.select, functions);
21728                                                }
21729                                        } else {
21730                                                ErrorInfo errorInfo = new ErrorInfo();
21731                                                errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR);
21732                                                errorInfo.setErrorMessage("Can't analyze XMLTABLE: missing PASSING clause.");
21733                                                errorInfos.add(errorInfo);
21734                                        }
21735                                } else if (getTableLinkedColumns(table) != null && getTableLinkedColumns(table).size() > 0) {
21736                                        if (table.getTableType() == ETableSource.rowList && table.getRowList() != null
21737                                                        && table.getRowList().size() > 0) {
21738                                                Table tableModel = modelFactory.createTable(table);
21739                                                for (int j = 0; j < table.getRowList().size(); j++) {
21740                                                        TMultiTarget rowList = table.getRowList().getMultiTarget(j);
21741                                                        for (int k = 0; k < rowList.getColumnList().size(); k++) {
21742                                                                TResultColumn column = rowList.getColumnList().getResultColumn(k);
21743                                                                if (column.getFieldAttr() == null)
21744                                                                        continue;
21745                                                                TableColumn tableColumn = modelFactory.createTableColumn(tableModel,
21746                                                                                column.getFieldAttr(), true);
21747
21748                                                                columnsInExpr visitor = new columnsInExpr();
21749                                                                column.getExpr().inOrderTraverse(visitor);
21750                                                                List<TObjectName> objectNames = visitor.getObjectNames();
21751                                                                List<TParseTreeNode> functions = visitor.getFunctions();
21752                                                                List<TParseTreeNode> constants = visitor.getConstants();
21753                                                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
21754
21755                                                                if (functions != null && !functions.isEmpty()) {
21756                                                                        analyzeFunctionDataFlowRelation(tableColumn, functions, EffectType.function);
21757                                                                }
21758                                                                if (subquerys != null && !subquerys.isEmpty()) {
21759                                                                        analyzeSubqueryDataFlowRelation(tableColumn, subquerys, EffectType.select);
21760                                                                }
21761                                                                if (objectNames != null && !objectNames.isEmpty()) {
21762                                                                        analyzeDataFlowRelation(tableColumn, objectNames, EffectType.select, functions);
21763                                                                }
21764                                                                if (constants != null && !constants.isEmpty()) {
21765                                                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select,
21766                                                                                        functions);
21767                                                                }
21768                                                        }
21769                                                }
21770                                        } else {
21771                                                if(table.getTableType() == ETableSource.pivoted_table) {
21772                                                        continue;
21773                                                }
21774                                                if (table.getTableType() == ETableSource.rowList) {
21775                                                        List<TResultColumnList> rowList = table.getValueClause().getRows();
21776                                                        
21777                                                        QueryTable tableModel = modelFactory.createQueryTable(table);
21778                                                        TAliasClause aliasClause = table.getAliasClause();
21779                                                        if (aliasClause != null && aliasClause.getColumns() != null) {
21780                                                                for (TObjectName column : aliasClause.getColumns()) {
21781                                                                        modelFactory.createResultColumn(tableModel, column, true);
21782                                                                }       
21783                                                        } else {        
21784                                                                int columnCount = rowList.get(0).size();
21785                                                                for (int j = 1; j <= columnCount; j++) {
21786                                                                        TObjectName columnName = new TObjectName();
21787                                                                        columnName.setString("column" + j);
21788                                                                        modelFactory.createResultColumn(tableModel, columnName, true);
21789                                                                }
21790                                                        }
21791                                                        tableModel.setDetermined(true);
21792                                                        
21793                                                        for (TResultColumnList resultColumnList : rowList) {
21794                                                                for (int j = 0; j < resultColumnList.size(); j++) {
21795                                                                        TResultColumn resultColumn = resultColumnList.getResultColumn(j);
21796                                                                        analyzeValueColumn(tableModel.getColumns().get(j), resultColumn, EffectType.select);
21797                                                                }
21798                                                        }
21799                                                        
21800                                                } else {
21801                                                        Table tableModel = modelFactory.createTable(table);
21802                                                        for (int j = 0; j < getTableLinkedColumns(table).size(); j++) {
21803                                                                TObjectName object = getTableLinkedColumns(table).getObjectName(j);
21804
21805                                                                if (object.getDbObjectType() == EDbObjectType.variable) {
21806                                                                        continue;
21807                                                                }
21808
21809                                                                if (object.getColumnNameOnly().startsWith("@")
21810                                                                                && (option.getVendor() == EDbVendor.dbvmssql
21811                                                                                                || option.getVendor() == EDbVendor.dbvazuresql)) {
21812                                                                        continue;
21813                                                                }
21814
21815                                                                if (object.getColumnNameOnly().startsWith(":")
21816                                                                                && (option.getVendor() == EDbVendor.dbvhana
21817                                                                                                || option.getVendor() == EDbVendor.dbvteradata)) {
21818                                                                        continue;
21819                                                                }
21820
21821                                                                if (isBuiltInFunctionName(object) && isFromFunction(object)) {
21822                                                                        continue;
21823                                                                }
21824
21825                                                                if (!"*".equals(getColumnName(object))) {
21826                                                                        if (isStructColumn(object)) {
21827
21828                                                                        } else {
21829                                                                                if (pivotedTable != null) {
21830                                                                                        ResultColumn resultColumn = getPivotedTableColumn(pivotedTable, object);
21831                                                                                        if (resultColumn != null) {
21832                                                                                                continue;
21833                                                                                        }
21834                                                                                }
21835                                                                                TableColumn tableColumn = modelFactory.createTableColumn(tableModel, object,
21836                                                                                                false);
21837                                                                                if(tableColumn == null) {
21838                                                                                        continue;
21839                                                                                }
21840                                                                                if (table.getUnnestClause() != null
21841                                                                                                && table.getUnnestClause().getArrayExpr() != null) {
21842                                                                                        columnsInExpr visitor = new columnsInExpr();
21843                                                                                        table.getUnnestClause().getArrayExpr().inOrderTraverse(visitor);
21844
21845                                                                                        List<TObjectName> objectNames = visitor.getObjectNames();
21846                                                                                        List<TParseTreeNode> functions = visitor.getFunctions();
21847
21848                                                                                        if (functions != null && !functions.isEmpty()) {
21849                                                                                                analyzeFunctionDataFlowRelation(tableColumn, functions,
21850                                                                                                                EffectType.select);
21851
21852                                                                                        }
21853
21854                                                                                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
21855                                                                                        if (subquerys != null && !subquerys.isEmpty()) {
21856                                                                                                analyzeSubqueryDataFlowRelation(tableColumn, subquerys,
21857                                                                                                                EffectType.select);
21858                                                                                        }
21859
21860                                                                                        analyzeDataFlowRelation(tableColumn, objectNames, EffectType.select,
21861                                                                                                        functions);
21862
21863                                                                                        List<TParseTreeNode> constants = visitor.getConstants();
21864                                                                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select,
21865                                                                                                        functions);
21866                                                                                }
21867                                                                        }
21868                                                                } else {
21869                                                                        boolean flag = false;
21870                                                                        for (TObjectName column : getTableLinkedColumns(table)) {
21871                                                                                if ("*".equals(getColumnName(column))) {
21872                                                                                        continue;
21873                                                                                }
21874                                                                                if (column.getLocation() != ESqlClause.where
21875                                                                                                && column.getLocation() != ESqlClause.joinCondition) {
21876                                                                                        flag = true;
21877                                                                                }
21878                                                                        }
21879                                                                        if (!flag) {
21880                                                                                TableColumn tableColumn = modelFactory.createTableColumn(tableModel, object,
21881                                                                                                false);
21882                                                                                if (tableColumn != null && !tableModel.hasSQLEnv()) {
21883                                                                                        tableColumn.setShowStar(true);
21884                                                                                }
21885                                                                        }
21886                                                                }
21887
21888                                                        }
21889                                                }
21890                                        }
21891                                }
21892                                else {
21893                                        modelFactory.createTable(table);
21894                                }
21895                        }
21896
21897                        if (pivotedTable!=null) {
21898                                for (TJoin join : stmt.getJoins()) {
21899                                        if (join.getTable() != null && join.getTable().getPivotedTable() != null) {
21900                                                if (isUnPivotedTable(join.getTable().getPivotedTable())) {
21901                                                        analyzeUnPivotedTable(stmt, join.getTable().getPivotedTable());
21902                                                } else {
21903                                                        analyzePivotedTable(stmt, join.getTable().getPivotedTable());
21904                                                }
21905                                                stmtStack.pop();
21906                                                return;
21907                                        }
21908                                }
21909                        }
21910
21911                        if (!stmt.isCombinedQuery()) {
21912                                Object queryModel = modelManager.getModel(stmt.getResultColumnList());
21913
21914                                if (queryModel == null) {
21915                                        TSelectSqlStatement parentStmt = getParentSetSelectStmt(stmt);
21916                                        if (isTopResultSet(stmt) || parentStmt == null) {
21917                                                ResultSet resultSetModel = modelFactory.createResultSet(stmt,
21918                                                                isTopResultSet(stmt) && isShowTopSelectResultSet() && stmt.getIntoClause() == null);
21919
21920                                                createPseudoImpactRelation(stmt, resultSetModel, EffectType.select);
21921
21922                                                boolean isDetermined = true;
21923                                                Map<String, AtomicInteger> keyMap = new HashMap<String, AtomicInteger>();
21924                                                Set<String> columnNames = new HashSet<String>();
21925                                                for (int i = 0; i < stmt.getResultColumnList().size(); i++) {
21926                                                        TResultColumn column = stmt.getResultColumnList().getResultColumn(i);
21927
21928                                                        if (column.getExpr().getComparisonType() == EComparisonType.equals
21929                                                                        && column.getExpr().getLeftOperand().getObjectOperand() != null) {
21930                                                                TObjectName columnObject = column.getExpr().getLeftOperand().getObjectOperand();
21931
21932                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel,
21933                                                                                columnObject);
21934                                                                if (columnObject.getDbObjectType() == EDbObjectType.variable) {
21935                                                                        Table variable = modelManager
21936                                                                                        .getTableByName(DlineageUtil.getTableFullName(columnObject.toString()));
21937                                                                        if (variable != null) {
21938                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
21939                                                                                relation.setEffectType(EffectType.select);
21940                                                                                TableColumn columnModel = variable instanceof Variable
21941                                                                                                ? compositeBindingColumn((Variable) variable)
21942                                                                                                : variable.getColumns().get(0);
21943                                                                                relation.setTarget(new TableColumnRelationshipElement(columnModel));
21944                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
21945                                                                        } else {
21946                                                                                variable = modelFactory.createVariable(columnObject);
21947                                                                                variable.setCreateTable(true);
21948                                                                                variable.setSubType(SubType.record);
21949                                                                                TObjectName variableProperties = new TObjectName();
21950                                                                                variableProperties.setString("*");
21951                                                                                TableColumn variableProperty = modelFactory.createTableColumn(variable,
21952                                                                                                variableProperties, true);
21953                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
21954                                                                                relation.setEffectType(EffectType.select);
21955                                                                                relation.setTarget(new TableColumnRelationshipElement(variableProperty));
21956                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
21957                                                                        }
21958                                                                }
21959
21960                                                                columnsInExpr visitor = new columnsInExpr();
21961                                                                column.getExpr().getRightOperand().inOrderTraverse(visitor);
21962
21963                                                                List<TObjectName> objectNames = visitor.getObjectNames();
21964                                                                List<TParseTreeNode> functions = visitor.getFunctions();
21965
21966                                                                if (functions != null && !functions.isEmpty()) {
21967                                                                        analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.select);
21968
21969                                                                }
21970
21971                                                                List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
21972                                                                if (subquerys != null && !subquerys.isEmpty()) {
21973                                                                        analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.select);
21974                                                                }
21975
21976                                                                analyzeDataFlowRelation(resultColumn, objectNames, EffectType.select, functions);
21977
21978                                                                List<TParseTreeNode> constants = visitor.getConstants();
21979                                                                analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.select, functions);
21980                                                        } else {
21981                                                                if (column.getFieldAttr() != null && isStructColumn(column.getFieldAttr())) {
21982                                                                        Table table = modelFactory.createTable(column.getFieldAttr().getSourceTable());
21983                                                                        for (int j = 0; j < table.getColumns().size(); j++) {
21984                                                                                TObjectName columnName = new TObjectName();
21985                                                                                if (table.getColumns().get(j).getName().equals(table.getAlias())) {
21986                                                                                        if (!SQLUtil.isEmpty(column.getColumnAlias())) {
21987                                                                                                columnName.setString(column.getColumnAlias());
21988                                                                                        } else {
21989                                                                                                columnName.setString(table.getColumns().get(j).getName());
21990                                                                                        }
21991                                                                                } else {
21992                                                                                        columnName.setString(
21993                                                                                                        table.getAlias() + "." + table.getColumns().get(j).getName());
21994                                                                                }
21995                                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel,
21996                                                                                                columnName);
21997                                                                                DataFlowRelationship relationship = modelFactory.createDataFlowRelation();
21998                                                                                relationship.setTarget(new ResultColumnRelationshipElement(resultColumn));
21999                                                                                relationship.addSource(
22000                                                                                                new TableColumnRelationshipElement(table.getColumns().get(j)));
22001                                                                        }
22002                                                                } else if (column.getExpr().getFunctionCall() != null && column.getExpr().getFunctionCall().getFunctionType() == EFunctionType.struct_t) {
22003                                                                        Function function = (Function) createFunction(column.getExpr().getFunctionCall());
22004                                                                        String functionName = getResultSetName(function);
22005                                                                        for (int j = 0; j < function.getColumns().size(); j++) {
22006                                                                                TObjectName columnName = new TObjectName();
22007                                                                                if (column.getAliasClause() != null) {
22008                                                                                        columnName.setString(column.getAliasClause() + "."
22009                                                                                                        + function.getColumns().get(j).getName());
22010                                                                                }
22011                                                                                else {
22012                                                                                        columnName.setString(functionName + "."
22013                                                                                                        + function.getColumns().get(j).getName());
22014                                                                                }
22015                                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel,
22016                                                                                                columnName);
22017                                                                                resultColumn.setStruct(true);
22018                                                                                DataFlowRelationship relationship = modelFactory.createDataFlowRelation();
22019                                                                                relationship.setTarget(new ResultColumnRelationshipElement(resultColumn));
22020                                                                                relationship.addSource(
22021                                                                                                new ResultColumnRelationshipElement(function.getColumns().get(j)));
22022                                                                        }
22023                                                                } else if (column.getExpr().getFunctionCall() != null && column.getExpr().getFunctionCall().getFunctionType() == EFunctionType.array_t) {
22024                                                                        Function function = (Function) createFunction(column.getExpr().getFunctionCall());
22025                                                                        String functionName = getResultSetName(function);
22026                                                                        if (function.getColumns().size() == 1) {
22027                                                                                // Scalar result inside ARRAY() - use just alias as column name
22028                                                                                TObjectName columnName = new TObjectName();
22029                                                                                if (column.getAliasClause() != null) {
22030                                                                                        columnName.setString(column.getAliasClause().toString());
22031                                                                                } else {
22032                                                                                        columnName.setString(functionName);
22033                                                                                }
22034                                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel,
22035                                                                                                columnName);
22036                                                                                DataFlowRelationship relationship = modelFactory.createDataFlowRelation();
22037                                                                                relationship.setTarget(new ResultColumnRelationshipElement(resultColumn));
22038                                                                                relationship.addSource(
22039                                                                                                new ResultColumnRelationshipElement(function.getColumns().get(0)));
22040                                                                        } else {
22041                                                                                for (int j = 0; j < function.getColumns().size(); j++) {
22042                                                                                        TObjectName columnName = new TObjectName();
22043                                                                                        if (column.getAliasClause() != null) {
22044                                                                                                columnName.setString(column.getAliasClause() + "."
22045                                                                                                                + getColumnNameOnly(function.getColumns().get(j).getName()));
22046                                                                                        }
22047                                                                                        else {
22048                                                                                                columnName.setString(functionName + "."
22049                                                                                                                + getColumnNameOnly(function.getColumns().get(j).getName()));
22050                                                                                        }
22051                                                                                        ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel,
22052                                                                                                        columnName);
22053                                                                                        resultColumn.setStruct(true);
22054                                                                                        DataFlowRelationship relationship = modelFactory.createDataFlowRelation();
22055                                                                                        relationship.setTarget(new ResultColumnRelationshipElement(resultColumn));
22056                                                                                        relationship.addSource(
22057                                                                                                        new ResultColumnRelationshipElement(function.getColumns().get(j)));
22058                                                                                }
22059                                                                        }
22060                                                                } else if (column.getExpr().getExprList() != null && column.getExpr().getExpressionType() == EExpressionType.array_t) {
22061                                                                        if(column.getExpr().getObjectOperand()!=null) {
22062                                                                                TObjectName exprColumnName = column.getExpr().getObjectOperand();
22063                                                                                Table table = modelFactory.createTable(exprColumnName.getSourceTable());
22064                                                                                for (int z = 0; z < table.getColumns().size(); z++) {
22065                                                                                        TableColumn tableColumn = table.getColumns().get(z);
22066                                                                                        if(getColumnName(exprColumnName.toString()).equals(getColumnName(tableColumn.getName()))) {
22067                                                                                                TObjectName columnName = new TObjectName();
22068                                                                                                if (column.getAliasClause() != null) {
22069                                                                                                        columnName.setString(column.getAliasClause().toString());
22070                                                                                                } else {
22071                                                                                                        columnName
22072                                                                                                                        .setString(getColumnNameOnly(columnName.toString()));
22073                                                                                                }
22074                                                                                                ResultColumn resultColumn = modelFactory
22075                                                                                                                .createResultColumn(resultSetModel, columnName);
22076                                                                                                resultColumn.setStruct(true);
22077                                                                                                DataFlowRelationship relationship = modelFactory
22078                                                                                                                .createDataFlowRelation();
22079                                                                                                relationship.setTarget(
22080                                                                                                                new ResultColumnRelationshipElement(resultColumn));
22081                                                                                                relationship.addSource(new TableColumnRelationshipElement(
22082                                                                                                                tableColumn));
22083                                                                                        }
22084                                                                                }
22085                                                                        }
22086                                                                        else {
22087                                                                                for (int j = 0; j < column.getExpr().getExprList().size(); j++) {
22088                                                                                        TExpression expression = column.getExpr().getExprList().getExpression(j);
22089                                                                                        if(expression.getExpressionType() == EExpressionType.function_t) {
22090                                                                                                Function function = (Function)createFunction(expression.getFunctionCall());
22091                                                                                                String functionName = getResultSetName(function);
22092                                                                                                if (function != null && function.getColumns() != null) {
22093                                                                                                        if (function.getColumns().size() == 1) {
22094                                                                                                                // Scalar function inside ARRAY[] (e.g., ARRAY[TO_JSON_STRING(col)])
22095                                                                                                                // Use just the alias as column name, not alias.functionName
22096                                                                                                                TObjectName columnName = new TObjectName();
22097                                                                                                                if (column.getAliasClause() != null) {
22098                                                                                                                        columnName.setString(column.getAliasClause().toString());
22099                                                                                                                } else {
22100                                                                                                                        columnName.setString(functionName);
22101                                                                                                                }
22102                                                                                                                ResultColumn resultColumn = modelFactory
22103                                                                                                                                .createResultColumn(resultSetModel, columnName);
22104                                                                                                                DataFlowRelationship relationship = modelFactory
22105                                                                                                                                .createDataFlowRelation();
22106                                                                                                                relationship.setTarget(
22107                                                                                                                                new ResultColumnRelationshipElement(resultColumn));
22108                                                                                                                relationship.addSource(new ResultColumnRelationshipElement(
22109                                                                                                                                function.getColumns().get(0)));
22110                                                                                                        } else {
22111                                                                                                                // Multi-column (struct-like) function - use dotted naming
22112                                                                                                                for (int x = 0; x < function.getColumns().size(); x++) {
22113                                                                                                                        TObjectName columnName = new TObjectName();
22114                                                                                                                        if (column.getAliasClause() != null) {
22115                                                                                                                                columnName.setString(column.getAliasClause() + "."
22116                                                                                                                                                + getColumnNameOnly(
22117                                                                                                                                                function.getColumns().get(x).getName()));
22118                                                                                                                        } else {
22119                                                                                                                                columnName.setString(
22120                                                                                                                                                functionName + "." + getColumnNameOnly(
22121                                                                                                                                                                function.getColumns().get(x).getName()));
22122                                                                                                                        }
22123                                                                                                                        ResultColumn resultColumn = modelFactory
22124                                                                                                                                        .createResultColumn(resultSetModel, columnName);
22125                                                                                                                        resultColumn.setStruct(true);
22126                                                                                                                        DataFlowRelationship relationship = modelFactory
22127                                                                                                                                        .createDataFlowRelation();
22128                                                                                                                        relationship.setTarget(
22129                                                                                                                                        new ResultColumnRelationshipElement(resultColumn));
22130                                                                                                                        relationship.addSource(new ResultColumnRelationshipElement(
22131                                                                                                                                        function.getColumns().get(x)));
22132                                                                                                                }
22133                                                                                                        }
22134                                                                                                }
22135                                                                                        }
22136                                                                                        else if(expression.getExpressionType() == EExpressionType.simple_object_name_t) {
22137                                                                                                TObjectName exprColumnName = expression.getObjectOperand();
22138                                                                                                Table table = modelFactory.createTable(exprColumnName.getSourceTable());
22139                                                                                                for (int z = 0; z < table.getColumns().size(); z++) {
22140                                                                                                        TableColumn tableColumn = table.getColumns().get(z);
22141                                                                                                        if(getColumnName(exprColumnName.toString()).equals(getColumnName(tableColumn.getName()))) {
22142                                                                                                                TObjectName columnName = new TObjectName();
22143                                                                                                                if (column.getAliasClause() != null) {
22144                                                                                                                        columnName.setString(column.getAliasClause().toString());
22145                                                                                                                } else {
22146                                                                                                                        columnName
22147                                                                                                                                        .setString(getColumnNameOnly(columnName.toString()));
22148                                                                                                                }
22149                                                                                                                ResultColumn resultColumn = modelFactory
22150                                                                                                                                .createResultColumn(resultSetModel, columnName);
22151                                                                                                                resultColumn.setStruct(true);
22152                                                                                                                DataFlowRelationship relationship = modelFactory
22153                                                                                                                                .createDataFlowRelation();
22154                                                                                                                relationship.setTarget(
22155                                                                                                                                new ResultColumnRelationshipElement(resultColumn));
22156                                                                                                                relationship.addSource(new TableColumnRelationshipElement(
22157                                                                                                                                tableColumn));
22158                                                                                                        }
22159                                                                                                }
22160                                                                                        }
22161                                                                                }
22162                                                                        }
22163                                                                } else {
22164                                                                        if ("*".equals(column.getColumnNameOnly())) {
22165                                                                                Map<String, Pair<String, TExpression>> replaceAsIdentifierMap = new HashMap<String, Pair<String, TExpression>>();
22166                                                                                Map<String, TObjectName> replaceColumnMap = new HashMap<String, TObjectName>();
22167                                                                                if(column.getReplaceExprAsIdentifiers()!=null && column.getReplaceExprAsIdentifiers().size()>0) {
22168                                                                                        for(TReplaceExprAsIdentifier replace: column.getReplaceExprAsIdentifiers()) {
22169                                                                                                replaceAsIdentifierMap.put(replace.getIdentifier().toString(), new Pair<String, TExpression>(column.getExpr().getExceptReplaceClause().toString(), replace.getExpr()));
22170                                                                                                replaceColumnMap.put(replace.getIdentifier().toString(), replace.getIdentifier());
22171                                                                                        }
22172                                                                                }
22173                                                                                
22174                                                                                TObjectName columnObject = column.getFieldAttr();
22175                                                                                List<TTable> sourceTables = columnObject.getSourceTableList();
22176                                                                                if (sourceTables != null && !sourceTables.isEmpty()) {
22177                                                                                        boolean[] determine = new boolean[sourceTables.size()];
22178                                                                                        for (int k = 0; k < sourceTables.size(); k++) {
22179                                                                                                TTable sourceTable = sourceTables.get(k);
22180                                                                                                Object tableModel = modelManager.getModel(sourceTable);
22181                                                                                                if (tableModel instanceof Table && ((Table) tableModel).isCreateTable()) {
22182                                                                                                        Table table = (Table) tableModel;
22183                                                                                                        for (int j = 0; j < table.getColumns().size(); j++) {
22184                                                                                                                TableColumn tableColumn = table.getColumns().get(j);
22185                                                                                                                if (column.getExceptColumnList() != null) {
22186                                                                                                                        boolean except = false;
22187                                                                                                                        for (TObjectName objectName : column.getExceptColumnList()) {
22188                                                                                                                                if(getColumnName(objectName.toString()).equals(getColumnName(tableColumn.getName()))) {
22189                                                                                                                                        except = true;
22190                                                                                                                                        break;
22191                                                                                                                                }
22192                                                                                                                        }
22193                                                                                                                        if (!except && tableColumn.isStruct()) {
22194                                                                                                                                List<String> names = SQLUtil
22195                                                                                                                                                .parseNames(tableColumn.getName());
22196                                                                                                                                for (String name : names) {
22197                                                                                                                                        for (TObjectName objectName : column
22198                                                                                                                                                        .getExceptColumnList()) {
22199                                                                                                                                                if (getColumnName(objectName.toString())
22200                                                                                                                                                                .equals(getColumnName(name))) {
22201                                                                                                                                                        except = true;
22202                                                                                                                                                        break;
22203                                                                                                                                                }
22204                                                                                                                                        }
22205                                                                                                                                        if (except) {
22206                                                                                                                                                break;
22207                                                                                                                                        }
22208                                                                                                                                }
22209                                                                                                                        }
22210                                                                                                                        if(except){
22211                                                                                                                                continue;
22212                                                                                                                        }
22213                                                                                                                }
22214                                                                                                                
22215                                                                                                                if (replaceAsIdentifierMap.containsKey(tableColumn.getName())) {
22216                                                                                                                        Pair<String, TExpression> expr = replaceAsIdentifierMap.get(tableColumn.getName());
22217                                                                                                                        ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel, replaceColumnMap.get(tableColumn.getName()));
22218                                                                                                                        Transform transform = new Transform();
22219                                                                                                                        transform.setType(Transform.EXPRESSION);
22220                                                                                                                        TObjectName expression = new TObjectName();
22221                                                                                                                        expression.setString(expr.first);
22222                                                                                                                transform.setCode(expression);
22223                                                                                                                        resultColumn.setTransform(transform);
22224                                                                                                                        analyzeResultColumnExpressionRelation(resultColumn, expr.second);
22225                                                                                                                }
22226                                                                                                                else {
22227                                                                                                                        String columnName = DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName());
22228                                                                                                                        //bigquery
22229                                                                                                                        boolean exist = false;
22230                                                                                                                        if (!columnName.matches("(?i)f\\d+_")) {
22231                                                                                                                                if (!keyMap.containsKey(columnName)) {
22232                                                                                                                                        keyMap.put(columnName, new AtomicInteger(0));
22233                                                                                                                                } else {
22234                                                                                                                                        while (columnNames.contains(columnName)) {
22235                                                                                                                                                if(columnName.indexOf("*") == -1 && stmt.toString().matches("(?is).*using\\s*\\(\\s*"+columnName+"\\s*\\).*")) {
22236                                                                                                                                                        exist = true;
22237                                                                                                                                                        break;
22238                                                                                                                                                } else if (keyMap.containsKey(columnName)) {
22239                                                                                                                                                        int index = keyMap.get(columnName)
22240                                                                                                                                                                        .incrementAndGet();
22241                                                                                                                                                        columnName = columnName + index;
22242                                                                                                                                                }
22243                                                                                                                                        }
22244                                                                                                                                }
22245                                                                                                                                columnNames.add(columnName);
22246                                                                                                                        }
22247                                                                                                                        if (exist) {
22248                                                                                                                                String targetColumn = columnName;
22249                                                                                                                                DataFlowRelationship relation = modelFactory
22250                                                                                                                                                .createDataFlowRelation();
22251                                                                                                                                relation.setEffectType(EffectType.select);
22252                                                                                                                                relation.setTarget(new ResultColumnRelationshipElement(
22253                                                                                                                                                resultSetModel.getColumns().stream()
22254                                                                                                                                                                .filter(t -> SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotColumn,
22255                                                                                                                                                                                t.getName(), targetColumn))
22256                                                                                                                                                                .findFirst().get()));
22257                                                                                                                                relation.addSource(new TableColumnRelationshipElement(
22258                                                                                                                                                tableColumn));
22259                                                                                                                                continue;
22260                                                                                                                        }
22261                                                                                                                        // Normal-form bookkeeping over synthetic collision-renamed keys
22262                                                                                                                        // (columnName may carry an appended index) — NOT identifier
22263                                                                                                                        // equality; stays on normalized-key compare by design (P0d.3c2).
22264                                                                                                                        if (DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName())
22265                                                                                                                                        .equalsIgnoreCase(DlineageUtil.getIdentifierNormalColumnName(columnName))) {
22266                                                                                                                                columnName = tableColumn.getName();
22267                                                                                                                        }
22268                                                                                                                        ResultColumn resultColumn = modelFactory.createStarResultColumn(resultSetModel, column, columnName);
22269                                                                                                                        resultColumn.setStruct(tableColumn.isStruct());
22270                                                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
22271                                                                                                                        relation.setEffectType(EffectType.select);
22272                                                                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
22273                                                                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
22274                                                                                                                }
22275                                                                                                        }
22276                                                                                                        determine[k] = true;
22277                                                                                                        continue;
22278                                                                                                }
22279                                                                                                else if (tableModel instanceof ResultSet && ((ResultSet) tableModel).isDetermined()) {
22280                                                                                                        ResultSet table = (ResultSet) tableModel;
22281                                                                                                        for (int j = 0; j < table.getColumns().size(); j++) {
22282                                                                                                                ResultColumn tableColumn = table.getColumns().get(j);
22283                                                                                                                if (column.getExceptColumnList() != null) {
22284                                                                                                                        boolean except = false;
22285                                                                                                                        for (TObjectName objectName : column.getExceptColumnList()) {
22286                                                                                                                                if(getColumnName(objectName.toString()).equals(getColumnName(tableColumn.getName()))) {
22287                                                                                                                                        except = true;
22288                                                                                                                                        break;
22289                                                                                                                                }
22290                                                                                                                        }
22291                                                                                                                        if (!except && tableColumn.isStruct()) {
22292                                                                                                                                List<String> names = SQLUtil
22293                                                                                                                                                .parseNames(tableColumn.getName());
22294                                                                                                                                for (String name : names) {
22295                                                                                                                                        for (TObjectName objectName : column
22296                                                                                                                                                        .getExceptColumnList()) {
22297                                                                                                                                                if (getColumnName(objectName.toString())
22298                                                                                                                                                                .equals(getColumnName(name))) {
22299                                                                                                                                                        except = true;
22300                                                                                                                                                        break;
22301                                                                                                                                                }
22302                                                                                                                                        }
22303                                                                                                                                        if (except) {
22304                                                                                                                                                break;
22305                                                                                                                                        }
22306                                                                                                                                }
22307                                                                                                                        }
22308                                                                                                                        if(except){
22309                                                                                                                                continue;
22310                                                                                                                        }
22311                                                                                                                }
22312                                                                                                                if (replaceAsIdentifierMap.containsKey(tableColumn.getName())) {
22313                                                                                                                        Pair<String, TExpression> expr = replaceAsIdentifierMap.get(tableColumn.getName());
22314                                                                                                                        ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel, replaceColumnMap.get(tableColumn.getName()));
22315                                                                                                                        Transform transform = new Transform();
22316                                                                                                                        transform.setType(Transform.EXPRESSION);
22317                                                                                                                        TObjectName expression = new TObjectName();
22318                                                                                                                        expression.setString(expr.first);
22319                                                                                                                transform.setCode(expression);
22320                                                                                                                        resultColumn.setTransform(transform);
22321                                                                                                                        analyzeResultColumnExpressionRelation(resultColumn, expr.second);
22322                                                                                                                }
22323                                                                                                                else if(tableColumn.getRefColumnName()!=null) {
22324                                                                                                                        ResultColumn resultColumn = modelFactory.createStarResultColumn(resultSetModel, column, tableColumn.getRefColumnName());
22325                                                                                                                        if(tableColumn.isStruct()) {
22326                                                                                                                                resultColumn.setStruct(true);
22327                                                                                                                        }
22328                                                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
22329                                                                                                                        relation.setEffectType(EffectType.select);
22330                                                                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
22331                                                                                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn));
22332                                                                                                                }
22333                                                                                                                else {
22334                                                                                                                        String columnName = DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName());
22335                                                                                                                        //bigquery
22336                                                                                                                        if (!columnName.matches("(?i)f\\d+_")) {
22337                                                                                                                                if (!keyMap.containsKey(columnName)) {
22338                                                                                                                                        keyMap.put(columnName, new AtomicInteger(0));
22339                                                                                                                                } else {
22340                                                                                                                                        while (columnNames.contains(columnName)) {
22341                                                                                                                                                int index = keyMap.get(columnName)
22342                                                                                                                                                                .incrementAndGet();
22343                                                                                                                                                columnName = columnName + index;
22344                                                                                                                                        }
22345                                                                                                                                }
22346                                                                                                                                columnNames.add(columnName);
22347                                                                                                                        }
22348                                                                                                                        // Normal-form bookkeeping over synthetic collision-renamed keys
22349                                                                                                                        // (columnName may carry an appended index) — NOT identifier
22350                                                                                                                        // equality; stays on normalized-key compare by design (P0d.3c2).
22351                                                                                                                        if (DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName())
22352                                                                                                                                        .equalsIgnoreCase(DlineageUtil.getIdentifierNormalColumnName(columnName))) {
22353                                                                                                                                columnName = tableColumn.getName();
22354                                                                                                                        }
22355                                                                                                                        if (modelManager.getModel(column) instanceof ResultColumn) {
22356                                                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
22357                                                                                                                                relation.setEffectType(EffectType.select);
22358                                                                                                                                relation.setTarget(new ResultColumnRelationshipElement((ResultColumn)modelManager.getModel(column)));
22359                                                                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
22360                                                                                                                        }
22361                                                                                                                        else {
22362                                                                                                                                ResultColumn resultColumn = modelFactory.createStarResultColumn(resultSetModel, column, columnName);
22363                                                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
22364                                                                                                                                relation.setEffectType(EffectType.select);
22365                                                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
22366                                                                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
22367                                                                                                                        }
22368                                                                                                                }
22369                                                                                                        }
22370                                                                                                        determine[k] = true;
22371                                                                                                        continue;
22372                                                                                                }
22373                                                                                                else {
22374                                                                                                        ResultColumn resultColumn = modelFactory
22375                                                                                                                        .createResultColumn(resultSetModel, column);
22376                                                                                                        if (tableModel instanceof Function) {
22377                                                                                                
22378                                                                                                        } else {
22379                                                                                                                TObjectName[] columns = modelManager
22380                                                                                                                                .getTableColumns(sourceTable);
22381                                                                                                                for (int j = 0; j < columns.length; j++) {
22382                                                                                                                        TObjectName columnName = columns[j];
22383                                                                                                                        if (columnName == null
22384                                                                                                                                        || "*".equals(getColumnName(columnName))) {
22385                                                                                                                                continue;
22386                                                                                                                        }
22387                                                                                                                        if (isStructColumn(columnName)) {
22388                                                                                                                                continue;
22389                                                                                                                        }
22390
22391                                                                                                                        resultColumn.bindStarLinkColumn(columnName);
22392                                                                                                                        if (column.getExceptColumnList() != null) {
22393                                                                                                                                for (TObjectName objectName : column
22394                                                                                                                                                .getExceptColumnList()) {
22395                                                                                                                                        resultColumn.unbindStarLinkColumn(objectName);
22396                                                                                                                                }
22397                                                                                                                        }
22398                                                                                                                }
22399                                                                                                                if (tableModel instanceof ResultSet) {
22400                                                                                                                        ResultSet queryTable = (ResultSet) tableModel;
22401                                                                                                                        if (!containStarColumn(queryTable)) {
22402                                                                                                                                resultColumn.setShowStar(false);
22403                                                                                                                        }
22404                                                                                                                }
22405                                                                                                                if (tableModel instanceof Table) {
22406                                                                                                                        Table table = (Table) tableModel;
22407                                                                                                                        if (table.isCreateTable()) {
22408                                                                                                                                resultColumn.setShowStar(false);
22409                                                                                                                        }
22410                                                                                                                }
22411                                                                                                        }
22412                                                                                                }
22413                                                                                        }
22414                                                                                        if(!Arrays.toString(determine).contains("false")) {
22415                                                                                                continue;
22416                                                                                        }
22417                                                                                } else {
22418                                                                                        ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel, column);
22419                                                                                        TTableList tables = stmt.getTables();
22420                                                                                        for (int k = 0; k < tables.size(); k++) {
22421                                                                                                TTable table = tables.getTable(k);
22422                                                                                                TObjectName[] columns = modelManager.getTableColumns(table);
22423                                                                                                for (int j = 0; j < columns.length; j++) {
22424                                                                                                        TObjectName columnName = columns[j];
22425                                                                                                        if (columnName == null) {
22426                                                                                                                continue;
22427                                                                                                        }
22428                                                                                                        if ("*".equals(getColumnName(columnName))) {
22429                                                                                                                if (modelManager.getModel(table) instanceof Table) {
22430                                                                                                                        Table tableModel = (Table) modelManager.getModel(table);
22431                                                                                                                        if (tableModel != null
22432                                                                                                                                        && !tableModel.getColumns().isEmpty()) {
22433                                                                                                                                for (TableColumn item : tableModel.getColumns()) {
22434                                                                                                                                        resultColumn
22435                                                                                                                                                        .bindStarLinkColumn(item.getColumnObject());
22436                                                                                                                                        if (table.getSubquery() == null
22437                                                                                                                                                        && table.getCTE() == null
22438                                                                                                                                                        && !tableModel.isCreateTable()) {
22439                                                                                                                                                resultColumn.setShowStar(true);
22440                                                                                                                                        }
22441                                                                                                                                }
22442                                                                                                                        }
22443                                                                                                                } else if (modelManager.getModel(table) instanceof QueryTable) {
22444                                                                                                                        QueryTable tableModel = (QueryTable) modelManager
22445                                                                                                                                        .getModel(table);
22446                                                                                                                        if (tableModel != null
22447                                                                                                                                        && !tableModel.getColumns().isEmpty()) {
22448                                                                                                                                for (ResultColumn item : tableModel.getColumns()) {
22449                                                                                                                                        if (item.hasStarLinkColumn()) {
22450                                                                                                                                                for (TObjectName starLinkColumn : item
22451                                                                                                                                                                .getStarLinkColumnList()) {
22452                                                                                                                                                        resultColumn
22453                                                                                                                                                                        .bindStarLinkColumn(starLinkColumn);
22454                                                                                                                                                }
22455                                                                                                                                        } else if (item
22456                                                                                                                                                        .getColumnObject() instanceof TObjectName) {
22457                                                                                                                                                resultColumn.bindStarLinkColumn(
22458                                                                                                                                                                (TObjectName) item.getColumnObject());
22459                                                                                                                                        } else if (item
22460                                                                                                                                                        .getColumnObject() instanceof TResultColumn) {
22461                                                                                                                                                TResultColumn queryTableColumn = (TResultColumn) item
22462                                                                                                                                                                .getColumnObject();
22463                                                                                                                                                TObjectName tableColumnObject = queryTableColumn
22464                                                                                                                                                                .getFieldAttr();
22465                                                                                                                                                if (tableColumnObject != null) {
22466                                                                                                                                                        resultColumn.bindStarLinkColumn(
22467                                                                                                                                                                        tableColumnObject);
22468                                                                                                                                                } else if (queryTableColumn
22469                                                                                                                                                                .getAliasClause() != null && !item.isStruct()) {
22470                                                                                                                                                        resultColumn.bindStarLinkColumn(
22471                                                                                                                                                                        queryTableColumn.getAliasClause()
22472                                                                                                                                                                                        .getAliasName());
22473                                                                                                                                                }
22474                                                                                                                                        }
22475                                                                                                                                }
22476                                                                                                                        }
22477                                                                                                                }
22478                                                                                                                continue;
22479                                                                                                        }
22480                                                                                                        resultColumn.bindStarLinkColumn(columnName);
22481                                                                                                }
22482                                                                                        }
22483                                                                                }
22484                                                                                isDetermined = false;
22485                                                                        }
22486                                                                        else {
22487                                                                                if(column.getAliasClause()!=null && column.getAliasClause().getColumns()!=null) {
22488                                                                                        for(TObjectName aliasColumn: column.getAliasClause().getColumns()) {
22489                                                                                                modelFactory.createResultColumn(resultSetModel, aliasColumn);
22490                                                                                        }
22491                                                                                }
22492                                                                                else {
22493                                                                                        modelFactory.createResultColumn(resultSetModel, column);
22494                                                                                }
22495                                                                        }
22496                                                                        analyzeResultColumn(column, EffectType.select);
22497                                                                }
22498                                                        }
22499                                                }
22500                                                if (isDetermined) {
22501                                                        resultSetModel.setDetermined(isDetermined);
22502                                                }
22503                                        }
22504
22505                                        TSelectSqlStatement parent = getParentSetSelectStmt(stmt);
22506                                        if (parent != null && parent.getSetOperatorType() != ESetOperatorType.none) {
22507                                                ResultSet resultSetModel = modelFactory.createResultSet(stmt, false);
22508                                                if(queryModel == null) {
22509                                                        queryModel = resultSetModel;
22510                                                } 
22511
22512                                                createPseudoImpactRelation(stmt, resultSetModel, EffectType.select);
22513
22514                                                boolean isDetermined = true;
22515                                                for (int i = 0; i < stmt.getResultColumnList().size(); i++) {
22516                                                        TResultColumn column = stmt.getResultColumnList().getResultColumn(i);
22517                                                        if ("*".equals(column.getColumnNameOnly())) {
22518                                                                
22519                                                                Map<String, Pair<String, TExpression>> replaceAsIdentifierMap = new HashMap<String, Pair<String, TExpression>>();
22520                                                                Map<String, TObjectName> replaceColumnMap = new HashMap<String, TObjectName>();
22521                                                                if(column.getReplaceExprAsIdentifiers()!=null && column.getReplaceExprAsIdentifiers().size()>0) {
22522                                                                        for(TReplaceExprAsIdentifier replace: column.getReplaceExprAsIdentifiers()) {
22523                                                                                replaceAsIdentifierMap.put(replace.getIdentifier().toString(), new Pair<String, TExpression>(column.getExpr().getExceptReplaceClause().toString(), replace.getExpr()));
22524                                                                                replaceColumnMap.put(replace.getIdentifier().toString(), replace.getIdentifier());
22525                                                                        }
22526                                                                }
22527                                                                
22528                                                                TObjectName columnObject = column.getFieldAttr();
22529                                                                TTable sourceTable = columnObject.getSourceTable();
22530                                                                if (sourceTable != null) {
22531                                                                        Object tableModel = modelManager.getModel(sourceTable);
22532                                                                        if (tableModel instanceof Table && ((Table) tableModel).isCreateTable()) {
22533                                                                                Table table = (Table) tableModel;
22534                                                                                for (int j = 0; j < table.getColumns().size(); j++) {
22535                                                                                        TableColumn tableColumn = table.getColumns().get(j);
22536                                                                                        if (column.getExceptColumnList() != null) {
22537                                                                                                boolean except = false;
22538                                                                                                for (TObjectName objectName : column.getExceptColumnList()) {
22539                                                                                                        if (getColumnName(objectName.toString())
22540                                                                                                                        .equals(getColumnName(tableColumn.getName()))) {
22541                                                                                                                except = true;
22542                                                                                                                break;
22543                                                                                                        }
22544                                                                                                }
22545                                                                                                if (!except && tableColumn.isStruct()) {
22546                                                                                                        List<String> names = SQLUtil
22547                                                                                                                        .parseNames(tableColumn.getName());
22548                                                                                                        for (String name : names) {
22549                                                                                                                for (TObjectName objectName : column
22550                                                                                                                                .getExceptColumnList()) {
22551                                                                                                                        if (getColumnName(objectName.toString())
22552                                                                                                                                        .equals(getColumnName(name))) {
22553                                                                                                                                except = true;
22554                                                                                                                                break;
22555                                                                                                                        }
22556                                                                                                                }
22557                                                                                                                if (except) {
22558                                                                                                                        break;
22559                                                                                                                }
22560                                                                                                        }
22561                                                                                                }
22562                                                                                                if (except) {
22563                                                                                                        continue;
22564                                                                                                }
22565                                                                                        }
22566                                                                                        
22567                                                                                        if (replaceAsIdentifierMap.containsKey(tableColumn.getName())) {
22568                                                                                                Pair<String, TExpression> expr = replaceAsIdentifierMap.get(tableColumn.getName());
22569                                                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel, replaceColumnMap.get(tableColumn.getName()));
22570                                                                                                Transform transform = new Transform();
22571                                                                                                transform.setType(Transform.EXPRESSION);
22572                                                                                                TObjectName expression = new TObjectName();
22573                                                                                                expression.setString(expr.first);
22574                                                                                        transform.setCode(expression);
22575                                                                                                resultColumn.setTransform(transform);
22576                                                                                                analyzeResultColumnExpressionRelation(resultColumn, expr.second);
22577                                                                                        }
22578                                                                                        else {
22579                                                                                                ResultColumn resultColumn = modelFactory.createStarResultColumn(
22580                                                                                                                resultSetModel, column, tableColumn.getName());
22581                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
22582                                                                                                relation.setEffectType(EffectType.select);
22583                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
22584                                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
22585                                                                                        }
22586                                                                                }
22587                                                                                continue;
22588                                                                        } else if (tableModel instanceof ResultSet
22589                                                                                        && ((ResultSet) tableModel).isDetermined()) {
22590                                                                                ResultSet table = (ResultSet) tableModel;
22591                                                                                for (int j = 0; j < table.getColumns().size(); j++) {
22592                                                                                        ResultColumn tableColumn = table.getColumns().get(j);
22593                                                                                        if (column.getExceptColumnList() != null) {
22594                                                                                                boolean except = false;
22595                                                                                                for (TObjectName objectName : column.getExceptColumnList()) {
22596                                                                                                        if (getColumnName(objectName.toString())
22597                                                                                                                        .equals(getColumnName(tableColumn.getName()))) {
22598                                                                                                                except = true;
22599                                                                                                                break;
22600                                                                                                        }
22601                                                                                                }
22602                                                                                                if (!except && tableColumn.isStruct()) {
22603                                                                                                        List<String> names = SQLUtil
22604                                                                                                                        .parseNames(tableColumn.getName());
22605                                                                                                        for (String name : names) {
22606                                                                                                                for (TObjectName objectName : column
22607                                                                                                                                .getExceptColumnList()) {
22608                                                                                                                        if (getColumnName(objectName.toString())
22609                                                                                                                                        .equals(getColumnName(name))) {
22610                                                                                                                                except = true;
22611                                                                                                                                break;
22612                                                                                                                        }
22613                                                                                                                }
22614                                                                                                                if (except) {
22615                                                                                                                        break;
22616                                                                                                                }
22617                                                                                                        }
22618                                                                                                }
22619                                                                                                if (except) {
22620                                                                                                        continue;
22621                                                                                                }
22622                                                                                        }
22623                                                                                        
22624                                                                                        if (replaceAsIdentifierMap.containsKey(tableColumn.getName())) {
22625                                                                                                Pair<String, TExpression> expr = replaceAsIdentifierMap.get(tableColumn.getName());
22626                                                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel, replaceColumnMap.get(tableColumn.getName()));
22627                                                                                                Transform transform = new Transform();
22628                                                                                                transform.setType(Transform.EXPRESSION);
22629                                                                                                TObjectName expression = new TObjectName();
22630                                                                                                expression.setString(expr.first);
22631                                                                                        transform.setCode(expression);
22632                                                                                                resultColumn.setTransform(transform);
22633                                                                                                analyzeResultColumnExpressionRelation(resultColumn, expr.second);
22634                                                                                        }
22635                                                                                        else if (tableColumn.getRefColumnName() != null) {
22636                                                                                                ResultColumn resultColumn = modelFactory.createStarResultColumn(
22637                                                                                                                (ResultSet)queryModel, column, tableColumn.getRefColumnName());
22638                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
22639                                                                                                relation.setEffectType(EffectType.select);
22640                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
22641                                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
22642                                                                                        } else {
22643                                                                                                ResultColumn resultColumn = modelFactory.createStarResultColumn(
22644                                                                                                                resultSetModel, column, tableColumn.getName());
22645                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
22646                                                                                                relation.setEffectType(EffectType.select);
22647                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
22648                                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
22649                                                                                        }
22650                                                                                }
22651                                                                                continue;
22652                                                                        }
22653                                                                        else {
22654                                                                                isDetermined = false;
22655                                                                        }
22656                                                                }
22657                                                                
22658                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel, column);
22659                                                                
22660                                                                if (columnObject.getTableToken() != null && sourceTable != null) {
22661                                                                        TObjectName[] columns = modelManager.getTableColumns(sourceTable);
22662                                                                        for (int j = 0; j < columns.length; j++) {
22663                                                                                TObjectName columnName = columns[j];
22664                                                                                if (columnName == null || "*".equals(getColumnName(columnName))) {
22665                                                                                        continue;
22666                                                                                }
22667                                                                                resultColumn.bindStarLinkColumn(columnName);
22668                                                                        }
22669
22670                                                                        if (modelManager.getModel(sourceTable) instanceof Table) {
22671                                                                                Table tableModel = (Table) modelManager.getModel(sourceTable);
22672                                                                                if (tableModel != null && !tableModel.getColumns().isEmpty()) {
22673                                                                                        for (TableColumn item : tableModel.getColumns()) {
22674                                                                                                if ("*".equals(getColumnName(item.getColumnObject()))) {
22675                                                                                                        continue;
22676                                                                                                }
22677                                                                                                resultColumn.bindStarLinkColumn(item.getColumnObject());
22678                                                                                        }
22679                                                                                }
22680                                                                        } else if (modelManager.getModel(sourceTable) instanceof QueryTable) {
22681                                                                                QueryTable tableModel = (QueryTable) modelManager.getModel(sourceTable);
22682                                                                                if (tableModel != null && !tableModel.getColumns().isEmpty()) {
22683                                                                                        for (ResultColumn item : tableModel.getColumns()) {
22684                                                                                                if (item.hasStarLinkColumn()) {
22685                                                                                                        for (TObjectName starLinkColumn : item.getStarLinkColumnList()) {
22686                                                                                                                if ("*".equals(getColumnName(starLinkColumn))) {
22687                                                                                                                        continue;
22688                                                                                                                }
22689                                                                                                                resultColumn.bindStarLinkColumn(starLinkColumn);
22690                                                                                                        }
22691                                                                                                } else if (item.getColumnObject() instanceof TObjectName) {
22692                                                                                                        TObjectName starLinkColumn = (TObjectName) item.getColumnObject();
22693                                                                                                        if ("*".equals(getColumnName(starLinkColumn))) {
22694                                                                                                                continue;
22695                                                                                                        }
22696                                                                                                        resultColumn.bindStarLinkColumn(starLinkColumn);
22697                                                                                                }
22698                                                                                        }
22699                                                                                }
22700                                                                        }
22701
22702                                                                } else {
22703                                                                        TTableList tables = stmt.getTables();
22704                                                                        for (int k = 0; k < tables.size(); k++) {
22705                                                                                TTable table = tables.getTable(k);
22706                                                                                TObjectName[] columns = modelManager.getTableColumns(table);
22707                                                                                for (int j = 0; j < columns.length; j++) {
22708                                                                                        TObjectName columnName = columns[j];
22709                                                                                        if (columnName == null) {
22710                                                                                                continue;
22711                                                                                        }
22712                                                                                        if ("*".equals(getColumnName(columnName))) {
22713                                                                                                if (modelManager.getModel(table) instanceof Table) {
22714                                                                                                        Table tableModel = (Table) modelManager.getModel(table);
22715                                                                                                        if (tableModel != null && !tableModel.getColumns().isEmpty()) {
22716                                                                                                                for (TableColumn item : tableModel.getColumns()) {
22717                                                                                                                        resultColumn.bindStarLinkColumn(item.getColumnObject());
22718                                                                                                                }
22719                                                                                                        }
22720                                                                                                } else if (modelManager.getModel(table) instanceof QueryTable) {
22721                                                                                                        QueryTable tableModel = (QueryTable) modelManager.getModel(table);
22722                                                                                                        if (tableModel != null && !tableModel.getColumns().isEmpty()) {
22723                                                                                                                for (ResultColumn item : tableModel.getColumns()) {
22724                                                                                                                        if (item.hasStarLinkColumn()) {
22725                                                                                                                                for (TObjectName starLinkColumn : item
22726                                                                                                                                                .getStarLinkColumnList()) {
22727                                                                                                                                        resultColumn.bindStarLinkColumn(starLinkColumn);
22728                                                                                                                                }
22729                                                                                                                        } else if (item.getColumnObject() instanceof TObjectName) {
22730                                                                                                                                resultColumn.bindStarLinkColumn(
22731                                                                                                                                                (TObjectName) item.getColumnObject());
22732                                                                                                                        } else if (item
22733                                                                                                                                        .getColumnObject() instanceof TResultColumn) {
22734                                                                                                                                TResultColumn queryTableColumn = (TResultColumn) item
22735                                                                                                                                                .getColumnObject();
22736                                                                                                                                TObjectName tableColumnObject = queryTableColumn
22737                                                                                                                                                .getFieldAttr();
22738                                                                                                                                if (tableColumnObject != null) {
22739                                                                                                                                        resultColumn.bindStarLinkColumn(tableColumnObject);
22740                                                                                                                                } else if (queryTableColumn.getAliasClause() != null) {
22741                                                                                                                                        resultColumn.bindStarLinkColumn(queryTableColumn
22742                                                                                                                                                        .getAliasClause().getAliasName());
22743                                                                                                                                }
22744                                                                                                                        }
22745                                                                                                                }
22746                                                                                                        }
22747                                                                                                }
22748
22749                                                                                                continue;
22750                                                                                        }
22751                                                                                        resultColumn.bindStarLinkColumn(columnName);
22752                                                                                }
22753                                                                        }
22754                                                                }
22755                                                        }
22756                                                        else {
22757                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel, column);
22758                                                        }
22759                                                        analyzeResultColumn(column, EffectType.select);
22760
22761                                                }
22762                                                
22763                                                resultSetModel.setDetermined(isDetermined);
22764                                        }
22765                                } else {
22766                                        for (int i = 0; i < stmt.getResultColumnList().size(); i++) {
22767                                                TResultColumn column = stmt.getResultColumnList().getResultColumn(i);
22768
22769                                                if (!(queryModel instanceof ResultSet)) {
22770                                                        continue;
22771                                                }
22772                                                
22773                                                ResultSet resultSetModel = (ResultSet)queryModel;
22774                                                        
22775                                                if ("*".equals(column.getColumnNameOnly())) {
22776                                                        TObjectName columnObject = column.getFieldAttr();
22777                                                        TTable sourceTable = columnObject.getSourceTable();
22778                                                        if (column.toString().indexOf(".") == -1 && stmt.getTables().size() > 1) {
22779                                                                sourceTable = null;
22780                                                        }
22781                                                        if (sourceTable != null) {
22782                                                                {
22783                                                                        Object tableModel = modelManager.getModel(sourceTable);
22784                                                                        if (tableModel instanceof Table && ((Table) tableModel).isCreateTable()) {
22785                                                                                Table table = (Table) tableModel;
22786                                                                                for (int j = 0; j < table.getColumns().size(); j++) {
22787                                                                                        TableColumn tableColumn = table.getColumns().get(j);
22788                                                                                        if (column.getExceptColumnList() != null) {
22789                                                                                                boolean except = false;
22790                                                                                                for (TObjectName objectName : column.getExceptColumnList()) {
22791                                                                                                        if (getColumnName(objectName.toString())
22792                                                                                                                        .equals(getColumnName(tableColumn.getName()))) {
22793                                                                                                                except = true;
22794                                                                                                                break;
22795                                                                                                        }
22796                                                                                                }
22797                                                                                                if (!except && tableColumn.isStruct()) {
22798                                                                                                        List<String> names = SQLUtil
22799                                                                                                                        .parseNames(tableColumn.getName());
22800                                                                                                        for (String name : names) {
22801                                                                                                                for (TObjectName objectName : column
22802                                                                                                                                .getExceptColumnList()) {
22803                                                                                                                        if (getColumnName(objectName.toString())
22804                                                                                                                                        .equals(getColumnName(name))) {
22805                                                                                                                                except = true;
22806                                                                                                                                break;
22807                                                                                                                        }
22808                                                                                                                }
22809                                                                                                                if (except) {
22810                                                                                                                        break;
22811                                                                                                                }
22812                                                                                                        }
22813                                                                                                }
22814                                                                                                if (except) {
22815                                                                                                        continue;
22816                                                                                                }
22817                                                                                        }
22818                                                                                        ResultColumn resultColumn = modelFactory.createStarResultColumn(
22819                                                                                                        resultSetModel, column, tableColumn.getName());
22820                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
22821                                                                                        relation.setEffectType(EffectType.select);
22822                                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
22823                                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
22824                                                                                }
22825                                                                                if(stmt.getResultColumnList().size() == 1) {
22826                                                                                        resultSetModel.setDetermined(true);
22827                                                                                }
22828                                                                                else {
22829                                                                                        int starCount = 0;
22830                                                                                        for (int j = 0; j < stmt.getResultColumnList().size(); j++) {
22831                                                                                                if (stmt.getResultColumnList().getResultColumn(j).getColumnNameOnly()
22832                                                                                                                .endsWith("*")) {
22833                                                                                                        starCount += 1;
22834                                                                                                }
22835                                                                                        }
22836                                                                                        if (starCount <= 1) {
22837                                                                                                resultSetModel.setDetermined(true);
22838                                                                                        }
22839                                                                                }
22840                                                                                continue;
22841                                                                        } else if (tableModel instanceof ResultSet
22842                                                                                        && ((ResultSet) tableModel).isDetermined()) {
22843                                                                                ResultSet table = (ResultSet) tableModel;
22844                                                                                for (int j = 0; j < table.getColumns().size(); j++) {
22845                                                                                        ResultColumn tableColumn = table.getColumns().get(j);
22846                                                                                        if (column.getExceptColumnList() != null) {
22847                                                                                                boolean except = false;
22848                                                                                                for (TObjectName objectName : column.getExceptColumnList()) {
22849                                                                                                        if (getColumnName(objectName.toString())
22850                                                                                                                        .equals(getColumnName(tableColumn.getName()))) {
22851                                                                                                                except = true;
22852                                                                                                                break;
22853                                                                                                        }
22854                                                                                                }
22855                                                                                                if (!except && tableColumn.isStruct()) {
22856                                                                                                        List<String> names = SQLUtil
22857                                                                                                                        .parseNames(tableColumn.getName());
22858                                                                                                        for (String name : names) {
22859                                                                                                                for (TObjectName objectName : column
22860                                                                                                                                .getExceptColumnList()) {
22861                                                                                                                        if (getColumnName(objectName.toString())
22862                                                                                                                                        .equals(getColumnName(name))) {
22863                                                                                                                                except = true;
22864                                                                                                                                break;
22865                                                                                                                        }
22866                                                                                                                }
22867                                                                                                                if (except) {
22868                                                                                                                        break;
22869                                                                                                                }
22870                                                                                                        }
22871                                                                                                }
22872                                                                                                if (except) {
22873                                                                                                        continue;
22874                                                                                                }
22875                                                                                        }
22876                                                                                        if (tableColumn.getRefColumnName() != null) {
22877                                                                                                ResultColumn resultColumn = modelFactory.createStarResultColumn(
22878                                                                                                                (ResultSet)queryModel, column, tableColumn.getRefColumnName());
22879                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
22880                                                                                                relation.setEffectType(EffectType.select);
22881                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
22882                                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
22883                                                                                        } else {
22884                                                                                                ResultColumn resultColumn = modelFactory.createStarResultColumn(
22885                                                                                                                resultSetModel, column, tableColumn.getName());
22886                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
22887                                                                                                relation.setEffectType(EffectType.select);
22888                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
22889                                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
22890                                                                                        }
22891                                                                                }
22892                                                                                if(stmt.getResultColumnList().size() == 1) {
22893                                                                                        resultSetModel.setDetermined(true);
22894                                                                                }
22895                                                                                else {
22896                                                                                        int starCount = 0;
22897                                                                                        for (int j = 0; j < stmt.getResultColumnList().size(); j++) {
22898                                                                                                if (stmt.getResultColumnList().getResultColumn(j).getColumnNameOnly()
22899                                                                                                                .endsWith("*")) {
22900                                                                                                        starCount += 1;
22901                                                                                                }
22902                                                                                        }
22903                                                                                        if (starCount <= 1) {
22904                                                                                                resultSetModel.setDetermined(true);
22905                                                                                        }
22906                                                                                }
22907                                                                                continue;
22908                                                                        }
22909                                                                }
22910                                                                
22911                                                                ResultColumn resultColumn  = modelFactory.createResultColumn(resultSetModel, column);   
22912                                                                if (modelManager.getModel(sourceTable) instanceof Table) {
22913                                                                        Table tableModel = (Table) modelManager.getModel(sourceTable);
22914                                                                        if (tableModel != null) {
22915                                                                                modelFactory.createTableColumn(tableModel, columnObject, false);
22916                                                                        }
22917                                                                        TObjectName[] columns = modelManager.getTableColumns(sourceTable);
22918                                                                        for (int j = 0; j < columns.length; j++) {
22919                                                                                TObjectName columnName = columns[j];
22920                                                                                if (columnName == null || "*".equals(getColumnName(columnName))) {
22921                                                                                        continue;
22922                                                                                }
22923                                                                                resultColumn.bindStarLinkColumn(columnName);
22924                                                                        }
22925
22926                                                                        if (tableModel.getColumns() != null) {
22927                                                                                for (int j = 0; j < tableModel.getColumns().size(); j++) {
22928                                                                                        TableColumn tableColumn = tableModel.getColumns().get(j);
22929                                                                                        TObjectName columnName = tableColumn.getColumnObject();
22930                                                                                        if (columnName == null || "*".equals(getColumnName(columnName))) {
22931                                                                                                continue;
22932                                                                                        }
22933                                                                                        resultColumn.bindStarLinkColumn(columnName);
22934                                                                                }
22935                                                                        }
22936                                                                } else if (modelManager.getModel(sourceTable) instanceof QueryTable) {
22937                                                                        QueryTable tableModel = (QueryTable) modelManager.getModel(sourceTable);
22938                                                                        if (tableModel != null && !tableModel.getColumns().isEmpty()) {
22939                                                                                for (ResultColumn item : tableModel.getColumns()) {
22940                                                                                        if (item.hasStarLinkColumn()) {
22941                                                                                                for (TObjectName starLinkColumn : item.getStarLinkColumnList()) {
22942                                                                                                        resultColumn.bindStarLinkColumn(starLinkColumn);
22943                                                                                                }
22944                                                                                        } else if (item.getColumnObject() instanceof TObjectName) {
22945                                                                                                resultColumn.bindStarLinkColumn((TObjectName) item.getColumnObject());
22946                                                                                        } else if (item.getColumnObject() instanceof TResultColumn) {
22947                                                                                                TResultColumn queryTableColumn = (TResultColumn) item.getColumnObject();
22948                                                                                                TObjectName tableColumnObject = queryTableColumn.getFieldAttr();
22949                                                                                                if (tableColumnObject != null) {
22950                                                                                                        resultColumn.bindStarLinkColumn(tableColumnObject);
22951                                                                                                } else if (queryTableColumn.getAliasClause() != null) {
22952                                                                                                        resultColumn.bindStarLinkColumn(
22953                                                                                                                        queryTableColumn.getAliasClause().getAliasName());
22954                                                                                                }
22955                                                                                        }
22956                                                                                }
22957                                                                        }
22958                                                                }
22959                                                        } else {
22960                                                                ResultColumn resultColumn  = modelFactory.createResultColumn(resultSetModel, column);   
22961                                                                TTableList tables = stmt.getTables();
22962                                                                for (int k = 0; k < tables.size(); k++) {
22963                                                                        TTable table = tables.getTable(k);
22964                                                                        TObjectName[] columns = modelManager.getTableColumns(table);
22965                                                                        for (int j = 0; j < columns.length; j++) {
22966                                                                                TObjectName columnName = columns[j];
22967                                                                                if (columnName == null) {
22968                                                                                        continue;
22969                                                                                }
22970                                                                                if ("*".equals(getColumnName(columnName))) {
22971                                                                                        if (modelManager.getModel(table) instanceof Table) {
22972                                                                                                Table tableModel = (Table) modelManager.getModel(table);
22973                                                                                                if (tableModel != null) {
22974                                                                                                        modelFactory.createTableColumn(tableModel, columnName, false);
22975                                                                                                }
22976                                                                                                if (tableModel != null && !tableModel.getColumns().isEmpty()) {
22977                                                                                                        for (int z = 0; z < tableModel.getColumns().size(); z++) {
22978                                                                                                                resultColumn.bindStarLinkColumn(
22979                                                                                                                                tableModel.getColumns().get(z).getColumnObject());
22980                                                                                                        }
22981                                                                                                }
22982                                                                                        } else if (modelManager.getModel(table) instanceof QueryTable) {
22983                                                                                                QueryTable tableModel = (QueryTable) modelManager.getModel(table);
22984                                                                                                if (tableModel != null && !tableModel.getColumns().isEmpty()) {
22985                                                                                                        for (ResultColumn item : tableModel.getColumns()) {
22986                                                                                                                if (item.hasStarLinkColumn()) {
22987                                                                                                                        for (TObjectName starLinkColumn : item
22988                                                                                                                                        .getStarLinkColumnList()) {
22989                                                                                                                                resultColumn.bindStarLinkColumn(starLinkColumn);
22990                                                                                                                        }
22991                                                                                                                } else if (item.getColumnObject() instanceof TObjectName) {
22992                                                                                                                        resultColumn.bindStarLinkColumn(
22993                                                                                                                                        (TObjectName) item.getColumnObject());
22994                                                                                                                } else if (item.getColumnObject() instanceof TResultColumn) {
22995                                                                                                                        TResultColumn queryTableColumn = (TResultColumn) item
22996                                                                                                                                        .getColumnObject();
22997                                                                                                                        TObjectName tableColumnObject = queryTableColumn
22998                                                                                                                                        .getFieldAttr();
22999                                                                                                                        if (tableColumnObject != null) {
23000                                                                                                                                resultColumn.bindStarLinkColumn(tableColumnObject);
23001                                                                                                                        } else if (queryTableColumn.getAliasClause() != null) {
23002                                                                                                                                resultColumn.bindStarLinkColumn(queryTableColumn
23003                                                                                                                                                .getAliasClause().getAliasName());
23004                                                                                                                        }
23005                                                                                                                }
23006                                                                                                        }
23007                                                                                                }
23008                                                                                        }
23009                                                                                        continue;
23010                                                                                }
23011                                                                                resultColumn.bindStarLinkColumn(columnName);
23012                                                                        }
23013                                                                }
23014                                                        }
23015                                                }
23016                                                else {
23017                                                        ResultColumn resultColumn  = modelFactory.createResultColumn(resultSetModel, column);   
23018                                                }
23019                                                
23020                                                analyzeResultColumn(column, EffectType.select);
23021
23022                                        }
23023                                        
23024                                        if (queryModel instanceof ResultSet) {
23025                                                boolean isDetermined = true;
23026                                                ResultSet resultSet = (ResultSet) queryModel;
23027                                                for (ResultColumn column : resultSet.getColumns()) {
23028                                                        if (column.getName().endsWith("*")) {
23029                                                                isDetermined = false;
23030                                                                break;
23031                                                        }
23032                                                }
23033                                                if (isDetermined) {
23034                                                        resultSet.setDetermined(isDetermined);
23035                                                }
23036                                        }
23037                                }
23038                        }
23039
23040                
23041                        analyzeSelectIntoClause(stmt);
23042                
23043
23044                        if (stmt.getJoins() != null && stmt.getJoins().size() > 0) {
23045                                for (int i = 0; i < stmt.getJoins().size(); i++) {
23046                                        TJoin join = stmt.getJoins().getJoin(i);
23047                                        ResultSet topResultSet = (ResultSet) modelManager.getModel(stmt);
23048                                        if (join.getJoinItems() != null && join.getJoinItems().size() > 0) {
23049                                                for (int k = 0; k < join.getJoinItems().size(); k++) {
23050                                                        TTable table = join.getJoinItems().getJoinItem(k).getTable();
23051                                                        if (table != null && table.getSubquery() != null) {
23052                                                                
23053                                                                ResultSet joinResultSet = (ResultSet) modelManager.getModel(table.getSubquery());
23054                                                                for (int x = 0; x < joinResultSet.getColumns().size(); x++) {
23055                                                                        ResultColumn sourceColumn = joinResultSet.getColumns().get(x);
23056                                                                        ResultColumn resultColumn = matchResultColumn(topResultSet.getColumns(),
23057                                                                                        sourceColumn);
23058                                                                        if (resultColumn != null
23059                                                                                        && resultColumn.getColumnObject() instanceof TResultColumn) {
23060                                                                                TResultColumn column = (TResultColumn) resultColumn.getColumnObject();
23061                                                                                if (column.getAliasClause() == null && column.getFieldAttr() != null) {
23062                                                                                        TObjectName resultObject = column.getFieldAttr();
23063                                                                                        if (resultObject.getSourceTable() == null
23064                                                                                                        || resultObject.getSourceTable().equals(table)) {
23065                                                                                                DataFlowRelationship combinedQueryRelation = modelFactory
23066                                                                                                                .createDataFlowRelation();
23067                                                                                                combinedQueryRelation.setEffectType(EffectType.select);
23068                                                                                                combinedQueryRelation
23069                                                                                                                .setTarget(new ResultColumnRelationshipElement(resultColumn));
23070                                                                                                combinedQueryRelation
23071                                                                                                                .addSource(new ResultColumnRelationshipElement(sourceColumn));
23072                                                                                        }
23073                                                                                }
23074                                                                        }
23075                                                                }
23076                                                        }
23077                                                        
23078                                                        if(join.getJoinItems().getJoinItem(k).getJoin()!=null) {
23079                                                                analyzeJoin(join.getJoinItems().getJoinItem(k).getJoin(), EffectType.select);
23080                                                        }
23081                                                }
23082                                        }
23083                                        analyzeJoin(join, EffectType.select);
23084                                }
23085                        }
23086
23087                        if (stmt.getWhereClause() != null) {
23088                                TExpression expr = stmt.getWhereClause().getCondition();
23089                                if (expr != null) {
23090                                        analyzeFilterCondition(null, expr, null, JoinClauseType.where, EffectType.select);
23091                                }
23092                        }
23093
23094                        stmtStack.pop();
23095                }
23096        }
23097
23098        protected TObjectNameList getTableLinkedColumns(TTable table) {
23099                if(structObjectMap.containsKey(table)) {
23100                        return structObjectMap.get(table);
23101                }
23102                return table.getLinkedColumns();
23103        }
23104
23105        protected boolean isTopResultSet(TSelectSqlStatement stmt) {
23106                TCustomSqlStatement parent = stmt.getParentStmt();
23107                if (parent == null)
23108                        return true;
23109                if (parent instanceof TMssqlReturn) {
23110                        return true;
23111                }
23112                if (parent instanceof TReturnStmt) {
23113                        return true;
23114                }
23115                if (parent instanceof TCommonBlock) {
23116                        TCommonBlock block = (TCommonBlock) parent;
23117                        if (block.getStatements() != null) {
23118                                for (int i = 0; i < block.getStatements().size(); i++) {
23119                                        TCustomSqlStatement child = block.getStatements().get(i);
23120                                        if(stmt == child) {
23121                                                return true;
23122                                        }
23123                                }
23124                        }
23125                }
23126                if (parent instanceof TMssqlBlock) {
23127                        TMssqlBlock block = (TMssqlBlock) parent;
23128                        if (block.getStatements() != null) {
23129                                for (int i = 0; i < block.getStatements().size(); i++) {
23130                                        TCustomSqlStatement child = block.getStatements().get(i);
23131                                        if(stmt == child) {
23132                                                return true;
23133                                        }
23134                                }
23135                        }
23136                }
23137                if (parent instanceof TStoredProcedureSqlStatement) {
23138                        TStoredProcedureSqlStatement block = (TStoredProcedureSqlStatement) parent;
23139                        if (block.getStatements() != null) {
23140                                for (int i = 0; i < block.getStatements().size(); i++) {
23141                                        TCustomSqlStatement child = block.getStatements().get(i);
23142                                        if(child == stmt) {
23143                                                return true;
23144                                        }
23145                                        if (child instanceof TReturnStmt) {
23146                                                TReturnStmt returnStmt = (TReturnStmt) child;
23147                                                if (returnStmt.getStatements() != null) {
23148                                                        for (int j = 0; j < returnStmt.getStatements().size(); j++) {
23149                                                                TCustomSqlStatement child1 = returnStmt.getStatements().get(j);
23150                                                                if(child1 == stmt) {
23151                                                                        return true;
23152                                                                }
23153                                                        }
23154                                                }
23155                                        }
23156                                        if (child instanceof TMssqlReturn) {
23157                                                TMssqlReturn returnStmt = (TMssqlReturn) child;
23158                                                if (returnStmt.getStatements() != null) {
23159                                                        for (int j = 0; j < returnStmt.getStatements().size(); j++) {
23160                                                                TCustomSqlStatement child1 = returnStmt.getStatements().get(j);
23161                                                                if(child1 == stmt) {
23162                                                                        return true;
23163                                                                }
23164                                                        }
23165                                                }
23166                                        }
23167                                }
23168                        }
23169                }
23170                return false;
23171        }
23172
23173        protected void analyzeTableSubquery(TTable table) {
23174                if(table.getSubquery()!=null) {
23175                        QueryTable queryTable = modelFactory.createQueryTable(table);
23176                        TSelectSqlStatement subquery = table.getSubquery();
23177                        analyzeSelectStmt(subquery);
23178
23179                        ResultSet resultSetModel = (ResultSet) modelManager.getModel(subquery);
23180
23181                        if (resultSetModel != null && resultSetModel != queryTable
23182                                        && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
23183                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
23184                                impactRelation.setEffectType(EffectType.select);
23185                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
23186                                                resultSetModel.getRelationRows()));
23187                                impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
23188                                                queryTable.getRelationRows()));
23189                        }
23190
23191                        if (resultSetModel != null && resultSetModel != queryTable
23192                                        && queryTable.getTableObject().getAliasClause() != null
23193                                        && queryTable.getTableObject().getAliasClause().getColumns() != null) {
23194                                for (int j = 0; j < queryTable.getColumns().size()
23195                                                && j < resultSetModel.getColumns().size(); j++) {
23196                                        ResultColumn sourceColumn = resultSetModel.getColumns().get(j);
23197                                        ResultColumn targetColumn = queryTable.getColumns().get(j);
23198
23199                                        DataFlowRelationship queryRalation = modelFactory.createDataFlowRelation();
23200                                        queryRalation.setEffectType(EffectType.select);
23201                                        queryRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
23202                                        queryRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
23203                                }
23204                        } else if (subquery.getSetOperatorType() != ESetOperatorType.none) {
23205                                SelectSetResultSet selectSetResultSetModel = (SelectSetResultSet) modelManager
23206                                                .getModel(subquery);
23207                                for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) {
23208                                        ResultColumn sourceColumn = selectSetResultSetModel.getColumns().get(j);
23209                                        ResultColumn targetColumn = modelFactory.createSelectSetResultColumn(queryTable,
23210                                                        sourceColumn);
23211                                        for (TObjectName starLinkColumn : sourceColumn.getStarLinkColumnList()) {
23212                                                targetColumn.bindStarLinkColumn(starLinkColumn);
23213                                        }
23214                                        DataFlowRelationship selectSetRalation = modelFactory.createDataFlowRelation();
23215                                        selectSetRalation.setEffectType(EffectType.select);
23216                                        selectSetRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
23217                                        selectSetRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
23218                                }
23219                        }
23220                }
23221        }
23222
23223        private ResultColumn getPivotedTableColumn(TPivotedTable pivotedTable, TObjectName columnName) {
23224                List<TPivotClause> pivotClauses = new ArrayList<TPivotClause>();
23225                if (pivotedTable.getPivotClause() != null) {
23226                        pivotClauses.add(pivotedTable.getPivotClause());
23227                }
23228                if (pivotedTable.getPivotClauseList() != null) {
23229                        for (int i = 0; i < pivotedTable.getPivotClauseList().size(); i++) {
23230                                pivotClauses.add(pivotedTable.getPivotClauseList().getElement(i));
23231                        }
23232                }
23233                for (TPivotClause clause : pivotClauses) {
23234                        Object model = modelManager.getModel(clause);
23235                        if (model instanceof PivotedTable) {
23236                                PivotedTable pivotedTableModel = (PivotedTable) model;
23237                                if (pivotedTableModel.getColumns() != null) {
23238                                        for (ResultColumn column : pivotedTableModel.getColumns()) {
23239                                                if (DlineageUtil.compareColumnIdentifier(getColumnName(columnName),
23240                                                                getColumnName(SQLUtil.trimColumnStringQuote(column.getName())))) {
23241                                                        return column;
23242                                                }
23243                                        }
23244                                }
23245                        }
23246                }
23247                return null;
23248        }
23249
23250        private void analyzeHiveTransformClause(TSelectSqlStatement stmt, THiveTransformClause transformClause) {
23251                Table mapSourceTable = null;
23252                QueryTable mapQueryTable = null;
23253                if(stmt.getTables()!=null) {
23254                        for(int i=0;i<stmt.getTables().size();i++) {
23255                                TTable table = stmt.getTables().getTable(i);
23256                                if (table.getSubquery() != null) {
23257                                        if (transformClause.getTransformType() == ETransformType.ettReduce) {
23258                                                mapQueryTable = modelFactory.createQueryTable(table);
23259                                        }
23260                                        analyzeSelectStmt(table.getSubquery());
23261                                }
23262                                else {
23263                                        mapSourceTable = modelFactory.createTable(table);
23264                                }
23265                        }
23266                }
23267                
23268                if (transformClause.getTransformType() == ETransformType.ettReduce) {
23269                        modelFactory.createResultSet(stmt, false);
23270                }
23271                
23272                List<TableColumn> mapTableColumns = new ArrayList<TableColumn>();
23273                List<ResultColumn> mapResultSetColumns = new ArrayList<ResultColumn>();
23274                List<ResultColumn> redueResultSetColumns = new ArrayList<ResultColumn>();
23275                
23276                if(transformClause.getExpressionList()!=null) {
23277                        for(TExpression expression: transformClause.getExpressionList()) {
23278                                if(expression.getObjectOperand()!=null) {
23279                                        if (transformClause.getTransformType() == ETransformType.ettMap || transformClause.getTransformType() == ETransformType.ettSelect) {
23280                                                if (mapSourceTable != null) {
23281                                                        TableColumn tableColumn = modelFactory.createTableColumn(mapSourceTable,
23282                                                                        expression.getObjectOperand(), false);
23283                                                        if (tableColumn != null) {
23284                                                                mapTableColumns.add(tableColumn);
23285                                                        }
23286                                                }
23287                                        }
23288                                        else if (transformClause.getTransformType() == ETransformType.ettReduce) {
23289                                                if (mapQueryTable != null) {
23290                                                        ResultColumn resultColumn = modelFactory.createResultColumn(mapQueryTable,
23291                                                                        expression.getObjectOperand(), false);
23292                                                        if (resultColumn != null) {
23293                                                                mapResultSetColumns.add(resultColumn);
23294                                                        }
23295                                                }
23296                                        }
23297                                }
23298                        }
23299                }
23300                
23301                if (transformClause.getAliasClause() != null) {
23302                        Object model = modelManager.getModel(stmt);
23303                        if (model instanceof ResultSet) {
23304                                ResultSet result = (ResultSet) model;
23305                                if (result!=null && transformClause.getAliasClause().getColumns() != null) {
23306                                        for (TObjectName column : transformClause.getAliasClause().getColumns()) {
23307                                                ResultColumn resultColumn = modelFactory.createResultColumn(result, column);
23308                                                if (resultColumn != null) {
23309                                                        if (transformClause.getTransformType() == ETransformType.ettMap 
23310                                                                        || transformClause.getTransformType() == ETransformType.ettSelect) {
23311                                                                mapResultSetColumns.add(resultColumn);
23312                                                        }
23313                                                        else if (transformClause.getTransformType() == ETransformType.ettReduce) {
23314                                                                redueResultSetColumns.add(resultColumn);
23315                                                        }
23316                                                }
23317                                        }
23318                                }
23319                        }
23320                }
23321                
23322                if (transformClause.getTransformType() == ETransformType.ettMap 
23323                                || transformClause.getTransformType() == ETransformType.ettSelect) {
23324                        if (!mapTableColumns.isEmpty() && !mapResultSetColumns.isEmpty()) {
23325                                for (ResultColumn resultColumn : mapResultSetColumns) {
23326                                        for (TableColumn tableColumn : mapTableColumns) {
23327                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
23328                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
23329                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
23330                                                relation.setEffectType(EffectType.select);
23331                                        }
23332                                }
23333                        }
23334                }
23335                else if (transformClause.getTransformType() == ETransformType.ettReduce) {
23336                        if (!redueResultSetColumns.isEmpty() && !mapResultSetColumns.isEmpty()) {
23337                                for (ResultColumn reduceResultColumn : redueResultSetColumns) {
23338                                        for (ResultColumn mapResultColumn : mapResultSetColumns) {
23339                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
23340                                                relation.setTarget(new ResultColumnRelationshipElement(reduceResultColumn));
23341                                                relation.addSource(new ResultColumnRelationshipElement(mapResultColumn));
23342                                                relation.setEffectType(EffectType.select);
23343                                        }
23344                                }
23345                        }
23346                }
23347        }
23348
23349        protected boolean isStructColumn(TObjectName columnName) {
23350                return columnName.getSourceTable() != null && columnName.getSourceTable().getAliasClause() != null
23351                                && columnName.getSourceTable().getUnnestClause() != null
23352                                && DlineageUtil.compareColumnIdentifier(getColumnName(columnName),
23353                                                getColumnName(columnName.getSourceTable().getAliasClause().getAliasName()));
23354        }
23355
23356        private TObjectName getObjectName(ResultColumn column) {
23357                if (column.getColumnObject() instanceof TResultColumn) {
23358                        TResultColumn resultColumn = (TResultColumn) column.getColumnObject();
23359                        if (resultColumn.getAliasClause() != null && resultColumn.getAliasClause().getAliasName() != null) {
23360                                return resultColumn.getAliasClause().getAliasName();
23361                        }
23362                        if (resultColumn.getFieldAttr() != null) {
23363                                return resultColumn.getFieldAttr();
23364                        }
23365                        if (resultColumn.getExpr() != null
23366                                        && resultColumn.getExpr().getExpressionType() == EExpressionType.simple_object_name_t) {
23367                                return resultColumn.getExpr().getObjectOperand();
23368                        }
23369                } else if (column.getColumnObject() instanceof TObjectName) {
23370                        return (TObjectName) column.getColumnObject();
23371                }
23372                return null;
23373        }
23374
23375        private boolean isShowTopSelectResultSet() {
23376                if (option.isSimpleOutput() && !option.isSimpleShowTopSelectResultSet())
23377                        return false;
23378                return true;
23379        }
23380
23381        /**
23382         * The record field the i-th SELECT ... INTO item binds to (#695).
23383         * SELECT INTO a record is POSITIONAL: item k fills field k. Binding
23384         * get(0) put every select item on the first field once records expanded
23385         * to per-field columns — five sources asserted on one field, none on the
23386         * others. Items beyond the known fields land on the record's star
23387         * (an unresolved component), never on a wrong field.
23388         */
23389        private TableColumn selectIntoRecordColumn(Table tableModel, TableColumn variableColumn, int ordinal) {
23390                if (variableColumn != null) {
23391                        return variableColumn;
23392                }
23393                List<TableColumn> columns = tableModel.getColumns();
23394                if (columns == null || columns.isEmpty()) {
23395                        return null;
23396                }
23397                if (columns.size() == 1) {
23398                        return columns.get(0);
23399                }
23400                if (!(tableModel instanceof Variable) || !((Variable) tableModel).isTypeOrderedFields()) {
23401                        // Fields discovered from consumption carry no positional meaning,
23402                        // and expanding a star here fans every select item onto every
23403                        // consumed pseudo-field. Keep the historical whole-record binding.
23404                        return columns.get(0);
23405                }
23406                int k = 0;
23407                for (TableColumn column : columns) {
23408                        if ("*".equals(column.getName())) {
23409                                continue;
23410                        }
23411                        if (k == ordinal) {
23412                                return column;
23413                        }
23414                        k++;
23415                }
23416                return tableStarColumn(tableModel);
23417        }
23418
23419        private static boolean hasRealTokenPosition(Pair3<Long, Long, String> start) {
23420                return start != null && start.first != null && start.second != null && start.first > 0
23421                                && !(start.first == 1L && start.second == 1L);
23422        }
23423
23424        /**
23425         * A SELECT INTO target column minted from a name-only object (star-expanded
23426         * columns of SELECT * INTO #t, set-operation and expanded-star branches) has
23427         * no source token, so its position renders as the synthetic
23428         * "[1,1,0],[1,n,0]" start-of-module coordinate. Give it the producing
23429         * result column's span instead (for a star column, the '*' token), so
23430         * definition-site consumers see an honest coordinate.
23431         */
23432        private static void fillSyntheticColumnPosition(TableColumn tableColumn, ResultColumn producedBy) {
23433                if (tableColumn == null || producedBy == null) {
23434                        return;
23435                }
23436                if (hasRealTokenPosition(tableColumn.getStartPosition())) {
23437                        return;
23438                }
23439                if (!hasRealTokenPosition(producedBy.getStartPosition())) {
23440                        return;
23441                }
23442                tableColumn.setStartPosition(producedBy.getStartPosition());
23443                tableColumn.setEndPosition(producedBy.getEndPosition());
23444        }
23445
23446        private void analyzeSelectIntoClause(TSelectSqlStatement stmt) {
23447                if (stmt.getParentStmt() instanceof TSelectSqlStatement) {
23448                        return;
23449                }
23450                
23451                // Keep Oracle trigger pseudo-record fields aligned with their INTO
23452                // expressions. A single shared slot made every source position target the
23453                // last trigger correlation field in a multi-column list (MantisBT 4707).
23454                List<TObjectName> oracleIntoColumnNames = new ArrayList<TObjectName>();
23455                
23456                TIntoClause intoClause = stmt.getIntoClause();
23457                
23458                TSelectSqlStatement leftStmt = DlineageUtil.getLeftStmt(stmt);
23459                
23460                if (intoClause == null && leftStmt != null) {
23461                        intoClause = leftStmt.getIntoClause();
23462                }
23463                
23464                if (intoClause != null) {
23465                        List<TObjectName> tableNames = new ArrayList<TObjectName>();
23466                        if (intoClause.getExprList() != null) {
23467                                for (int j = 0; j < intoClause.getExprList().size(); j++) {
23468                                        TObjectName tableName = intoClause.getExprList().getExpression(j).getObjectOperand();
23469                                        if (tableName != null) {
23470                                                if (tableName.toString().startsWith(":") && option.getVendor() == EDbVendor.dbvoracle
23471                                                                && tableName.getDbObjectType() == EDbObjectType.column) {
23472                                                        TObjectName tableAlias = new TObjectName();
23473                                                        if (!SQLUtil.isEmpty(tableName.getTableString())) {
23474                                                                tableAlias.setString(tableName.getTableString());
23475                                                        }
23476                                                        else {
23477                                                                tableAlias.setString(tableName.toString());
23478                                                        }
23479                                                        tableNames.add(tableAlias);
23480                                                        TTable sourceTable = tableName.getSourceTable();
23481                                                        Table boundTable = modelManager.getTableByName(
23482                                                                        DlineageUtil.getTableFullName(tableAlias.toString()));
23483                                                        boolean sourceMatchesBinding = sourceTable != null && boundTable != null
23484                                                                        && SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotTable,
23485                                                                                        boundTable.getName(), sourceTable.getTableName().toString());
23486                                                        if (sourceTable != null && (boundTable == null || sourceMatchesBinding)) {
23487                                                                // Prefer a trigger-correlation binding registered by
23488                                                                // bindTriggerCorrelationNames(). Resolver fallback can
23489                                                                // otherwise misclassify a custom REFERENCING alias as the
23490                                                                // SELECT's only FROM table.
23491                                                                modelFactory.createTable(sourceTable, tableAlias);
23492                                                        }
23493                                                        // A bare host bind such as :first_name is a scalar
23494                                                        // variable, not a pseudo-record field. Preserve its
23495                                                        // historical result-column-derived name.
23496                                                        oracleIntoColumnNames.add(SQLUtil.isEmpty(tableName.getTableString())
23497                                                                        ? null : tableName);
23498                                                } else {
23499                                                        if (tableName != null) {
23500                                                                tableNames.add(tableName);
23501                                                                oracleIntoColumnNames.add(null);
23502                                                        }
23503                                                }
23504                                        }
23505                                        else if(intoClause.getExprList().getExpression(j).getFunctionCall()!=null) {
23506                                                TObjectName variableName = intoClause.getExprList().getExpression(j).getFunctionCall().getFunctionName();
23507                                                tableNames.add(variableName);
23508                                                oracleIntoColumnNames.add(null);
23509                                                Variable variable = modelFactory.createVariable(variableName);
23510                                                variable.setSubType(SubType.record);
23511                                                TObjectName variableProperties = new TObjectName();
23512                                                variableProperties.setString("*");
23513                                                modelFactory.createTableColumn(variable, variableProperties, true);
23514                                        }
23515                                }
23516                        } else if (intoClause.getVariableList() != null) {
23517                                for (int j = 0; j < intoClause.getVariableList().size(); j++) {
23518                                        TObjectName tableName = intoClause.getVariableList().getObjectName(j);
23519                                        if (tableName != null) {
23520                                                tableNames.add(tableName);
23521                                                oracleIntoColumnNames.add(null);
23522                                        }
23523                                }
23524                        } else if (intoClause.getIntoName() != null) {
23525                                tableNames.add(intoClause.getIntoName());
23526                                oracleIntoColumnNames.add(null);
23527                        }
23528
23529                        ResultSet queryModel = (ResultSet) modelManager.getModel(stmt.getResultColumnList());
23530                        if (stmt.getSetOperatorType() != ESetOperatorType.none) {
23531                                queryModel = (ResultSet) modelManager.getModel(stmt);
23532                        }
23533                        for (int j = 0; j < tableNames.size(); j++) {
23534                                TObjectName tableName = tableNames.get(j);
23535                                TObjectName oracleIntoColumnName = oracleIntoColumnNames.get(j);
23536                                if (tableName.getColumnNameOnly().startsWith("@")
23537                                                && (option.getVendor() == EDbVendor.dbvmssql || option.getVendor() == EDbVendor.dbvazuresql)) {
23538                                        continue;
23539                                }
23540
23541                                if (tableName.getColumnNameOnly().startsWith(":")
23542                                                && (option.getVendor() == EDbVendor.dbvhana || option.getVendor() == EDbVendor.dbvteradata)) {
23543                                        continue;
23544                                }
23545
23546                                Table tableModel;
23547                                TableColumn variableColumn = null;
23548
23549                                if (tableName.getDbObjectType() == EDbObjectType.variable) {
23550                                        if (tableName.toString().indexOf(".") != -1) {
23551                                                List<String> splits = SQLUtil.parseNames(tableName.toString());
23552                                                tableModel = modelFactory.createVariable(splits.get(splits.size() - 2));
23553                                                // INTO rec.field names its target field explicitly (#695);
23554                                                // bind that field, never an ordinal of the record.
23555                                                variableColumn = modelFactory.createTableColumn(tableModel, tableName, true);
23556                                        } else {
23557                                                tableModel = modelFactory.createVariable(tableName);
23558                                        }
23559                                        if (tableModel.getSubType() == null) {
23560                                                tableModel.setSubType(SubType.record);
23561                                        }
23562                                        if(variableColumn == null
23563                                                        && (tableModel.getColumns() == null || tableModel.getColumns().isEmpty())) {
23564                                                TObjectName variableProperties = new TObjectName();
23565                                                variableProperties.setString("*");
23566                                                variableColumn = modelFactory.createTableColumn(tableModel, variableProperties, true);
23567                                        }
23568                                } else {
23569                                        tableModel = modelFactory.createTableByName(tableName);
23570                                        // SELECT ... INTO <table> creates the target. Parse fact, set
23571                                        // regardless of isDetermined() (unlike the gated setCreateTable
23572                                        // below). See dlineage-authoritative-endpoint-classification.md.
23573                                        tableModel.setCreatedInSql(true);
23574                                        tableModel.setEndpointIntroduction(EndpointIntroduction.SELECT_INTO);
23575                                }
23576                                TableColumn oracleIntoTableColumn = oracleIntoColumnName == null ? null
23577                                                : modelFactory.createTableColumn(tableModel, oracleIntoColumnName, true);
23578
23579                                if (queryModel instanceof ResultSet && (stmt.getWhereClause() != null || hasJoin(stmt))) {
23580                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
23581                                        impactRelation.setEffectType(EffectType.insert);
23582                                        impactRelation.addSource(
23583                                                        new RelationRowsRelationshipElement<ResultSetRelationRows>(((ResultSet)queryModel).getRelationRows()));
23584                                        impactRelation.setTarget(
23585                                                        new RelationRowsRelationshipElement<TableRelationRows>(tableModel.getRelationRows()));
23586                                }
23587                                
23588                                Process process = modelFactory.createProcess(stmt);
23589                                tableModel.addProcess(process);
23590                                
23591                                if (stmt.getSetOperatorType() != ESetOperatorType.none) {
23592                                        for (int setopOrdinal = 0; setopOrdinal < queryModel.getColumns().size(); setopOrdinal++) {
23593                                                ResultColumn resultColumn = queryModel.getColumns().get(setopOrdinal);
23594                                                if (tableNames.size() > 1
23595                                                                && (tableName.getDbObjectType() == EDbObjectType.variable
23596                                                                                || oracleIntoTableColumn != null)
23597                                                                && setopOrdinal != j) {
23598                                                        continue;
23599                                                }
23600                                                TableColumn tableColumn;
23601                                                if (oracleIntoTableColumn != null) {
23602                                                        tableColumn = oracleIntoTableColumn;
23603                                                } else if (tableModel.isVariable()) {
23604                                                        // SELECT ... INTO variables and typed records binds
23605                                                        // positionally even through a set operation (codex-696 F4).
23606                                                        tableColumn = selectIntoRecordColumn(tableModel, variableColumn, setopOrdinal);
23607                                                } else {
23608                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
23609                                                                        resultColumn.getName());
23610                                                        fillSyntheticColumnPosition(tableColumn, resultColumn);
23611                                                }
23612
23613                                                if (DlineageUtil.isTempTable(tableModel, option.getVendor()) && sqlenv != null
23614                                                                && tableModel.getDatabase() != null && tableModel.getSchema() != null) {
23615                                                        TSQLSchema schema = sqlenv
23616                                                                        .getSQLSchema(tableModel.getDatabase() + "." + tableModel.getSchema(), true);
23617                                                        if (schema != null) {
23618                                                                TSQLTable tempTable = schema
23619                                                                                .createTable(DlineageUtil.getSimpleTableName(tableModel.getName()));
23620                                                                tempTable.addColumn(tableColumn.getName());
23621                                                        }
23622                                                }
23623
23624                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
23625                                                relation.setEffectType(EffectType.insert);
23626                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
23627                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
23628                                                relation.setProcess(process);
23629                                        }
23630                                        
23631                                        tableModel.setDetermined(queryModel.isDetermined());
23632                                        if(queryModel.isDetermined() && DlineageUtil.isTempTable(tableModel, option.getVendor())) {
23633                                                tableModel.setCreateTable(true, false);
23634                                        }
23635                                        if (j + 1 == tableNames.size()) {
23636                                                return;
23637                                        }
23638                                        continue;
23639                                }
23640
23641                                boolean isDetermined = true;
23642                                for (int i = 0; i < stmt.getResultColumnList().size(); i++) {
23643                                        if (tableNames.size() > 1
23644                                                        && (tableName.getDbObjectType() == EDbObjectType.variable
23645                                                                        || oracleIntoTableColumn != null)) {
23646                                                if (i != j) {
23647                                                        continue;
23648                                                }
23649                                        }
23650                                        TResultColumn column = stmt.getResultColumnList().getResultColumn(i);
23651
23652                                        if ("*".equals(column.getColumnNameOnly()) && column.getFieldAttr() != null
23653                                                        && column.getFieldAttr().getSourceTable() != null) {
23654                                                Object model = modelManager.getModel(column);
23655                                                if(model instanceof LinkedHashMap) {
23656                                                        LinkedHashMap<String, ResultColumn> columns = (LinkedHashMap<String, ResultColumn>)model;
23657                                                        int mapOrdinal = 0;
23658                                                        for(String key: columns.keySet()) {
23659                                                                ResultColumn sourceColumn = columns.get(key);
23660                                                                TableColumn tableColumn;
23661                                                                if (tableModel instanceof Variable
23662                                                                                && ((Variable) tableModel).isTypeOrderedFields()) {
23663                                                                        tableColumn = selectIntoRecordColumn(tableModel, null, mapOrdinal);
23664                                                                } else {
23665                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
23666                                                                                        sourceColumn.getName());
23667                                                                        fillSyntheticColumnPosition(tableColumn, sourceColumn);
23668                                                                }
23669                                                                mapOrdinal++;
23670                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
23671                                                                relation.setEffectType(EffectType.insert);
23672                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
23673                                                                relation.addSource(new ResultColumnRelationshipElement(sourceColumn));
23674                                                                relation.setProcess(process);
23675                                                        }
23676                                                }
23677                                                else if(model instanceof ResultColumn) {
23678                                                        isDetermined = false;
23679                                                        ResultColumn resultColumn = (ResultColumn) model;
23680                                                        List<TObjectName> columns = resultColumn.getStarLinkColumnList();
23681                                                        if (columns.size() > 0) {
23682                                                                for (int k = 0; k < columns.size(); k++) {
23683        
23684                                                                        TableColumn tableColumn;
23685                                                                        if (tableModel instanceof Variable
23686                                                                                        && ((Variable) tableModel).isTypeOrderedFields()) {
23687                                                                                // SELECT * INTO a typed record fills its
23688                                                                                // DECLARED fields by position — creating
23689                                                                                // fields from source names left the real
23690                                                                                // fields with no upstream (codex-696 F4).
23691                                                                                tableColumn = selectIntoRecordColumn(tableModel, null, k);
23692                                                                        } else {
23693                                                                                tableColumn = modelFactory.createInsertTableColumn(tableModel,
23694                                                                                                columns.get(k));
23695                                                                                // Star-link names are synthesized without
23696                                                                                // tokens; the '*' span is the honest
23697                                                                                // definition site for the expanded column.
23698                                                                                fillSyntheticColumnPosition(tableColumn, resultColumn);
23699                                                                        }
23700
23701                                                                        if (DlineageUtil.isTempTable(tableModel, option.getVendor()) && sqlenv != null
23702                                                                                        && tableModel.getDatabase() != null && tableModel.getSchema() != null) {
23703                                                                                TSQLSchema schema = sqlenv.getSQLSchema(
23704                                                                                                tableModel.getDatabase() + "." + tableModel.getSchema(), true);
23705                                                                                if (schema != null) {
23706                                                                                        TSQLTable tempTable = schema.createTable(DlineageUtil.getSimpleTableName(tableModel.getName()));
23707                                                                                        tempTable.addColumn(tableColumn.getName());
23708                                                                                }
23709                                                                        }
23710        
23711                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
23712                                                                        relation.setEffectType(EffectType.insert);
23713                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
23714                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
23715                                                                        relation.setProcess(process);
23716                                                                }
23717                                                                if (resultColumn.isShowStar()) {
23718                                                                        TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
23719                                                                                        column.getFieldAttr());
23720                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
23721                                                                        relation.setEffectType(EffectType.insert);
23722                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
23723                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
23724                                                                        relation.setProcess(process);
23725                                                                }
23726                                                        } else {
23727                                                                TObjectName columnObject = column.getFieldAttr();
23728                                                                if (column.getAliasClause() != null) {
23729                                                                        columnObject = column.getAliasClause().getAliasName();
23730                                                                }
23731                                                                if (columnObject != null) {
23732                                                                        TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
23733                                                                                        columnObject);
23734                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
23735                                                                        relation.setEffectType(EffectType.insert);
23736                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
23737                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
23738                                                                        relation.setProcess(process);
23739                                                                } else if (!SQLUtil.isEmpty(column.getColumnAlias())) {
23740                                                                        TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel,
23741                                                                                        column.getAliasClause().getAliasName());
23742                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
23743                                                                        relation.setEffectType(EffectType.insert);
23744                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
23745                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
23746                                                                        relation.setProcess(process);
23747                                                                }
23748                                                        }
23749                                                }
23750                                        } else {
23751                                                ResultColumn resultColumn = null;
23752
23753                                                if (queryModel instanceof QueryTable) {
23754                                                        resultColumn = (ResultColumn) modelManager.getModel(column);
23755                                                } else if (queryModel instanceof ResultSet) {
23756                                                        resultColumn = (ResultColumn) modelManager.getModel(column);
23757                                                } else {
23758                                                        continue;
23759                                                }
23760
23761                                                if (resultColumn == null && column.getAliasClause() != null) {
23762                                                        resultColumn = (ResultColumn) modelManager.getModel(column.getAliasClause().getAliasName());
23763                                                }
23764
23765                                                if (resultColumn != null) {
23766                                                        TObjectName columnObject = column.getFieldAttr();
23767                                                        if (column.getAliasClause() != null) {
23768                                                                columnObject = column.getAliasClause().getAliasName();
23769                                                        }
23770                                                        TableColumn tableColumn = null;
23771                                                        if (columnObject != null) {
23772                                                                if (tableModel.isVariable()) {
23773                                                                        tableColumn = selectIntoRecordColumn(tableModel, variableColumn, i);
23774                                                                } 
23775                                                                else if (oracleIntoTableColumn != null) {
23776                                                                        tableColumn = oracleIntoTableColumn;
23777                                                                }
23778                                                                else {
23779                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel, columnObject);
23780                                                                        if (containStarColumn(queryModel)) {
23781                                                                                tableColumn.notBindStarLinkColumn(true);
23782                                                                        }
23783                                                                }
23784                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
23785                                                                relation.setEffectType(EffectType.insert);
23786                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
23787                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
23788                                                                relation.setProcess(process);
23789                                                        } else if (!SQLUtil.isEmpty(column.getColumnAlias())) {
23790                                                                if (tableModel.isVariable()) {
23791                                                                        tableColumn = selectIntoRecordColumn(tableModel, variableColumn, i);
23792                                                                } else {
23793                                                                        tableColumn = modelFactory.createInsertTableColumn(tableModel,
23794                                                                                        column.getAliasClause().getAliasName());
23795                                                                }
23796                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
23797                                                                relation.setEffectType(EffectType.insert);
23798                                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
23799                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
23800                                                                relation.setProcess(process);
23801                                                        } else {
23802                                                                if (tableModel.isVariable()) {
23803                                                                        tableColumn = selectIntoRecordColumn(tableModel, variableColumn, i);
23804                                                                }
23805                                                                if (tableColumn != null) {
23806                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
23807                                                                        relation.setEffectType(EffectType.insert);
23808                                                                        relation.setTarget(new TableColumnRelationshipElement(tableColumn));
23809                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
23810                                                                        relation.setProcess(process);
23811                                                                }
23812                                                        }
23813                                                }
23814                                        }
23815                                }
23816                                tableModel.setDetermined(isDetermined);
23817                                if(isDetermined && DlineageUtil.isTempTable(tableModel, option.getVendor())) {
23818                                        tableModel.setCreateTable(true, false);
23819                                }
23820                        }
23821                }
23822        }
23823
23824        private boolean isUnPivotedTable(TPivotedTable pivotedTable) {
23825                if (pivotedTable.getPivotClauseList() != null && pivotedTable.getPivotClauseList().size() > 0) {
23826                        return pivotedTable.getPivotClauseList().getElement(0).getType() == TPivotClause.unpivot;
23827                } else {
23828                        TPivotClause pivotClause = pivotedTable.getPivotClause();
23829                        return pivotClause.getType() == TPivotClause.unpivot;
23830                }
23831        }
23832
23833        private void analyzeUnPivotedTable(TSelectSqlStatement stmt, TPivotedTable pivotedTable) {
23834                List<Object> tables = new ArrayList<Object>();
23835                Set<Object> pivotedColumns = new HashSet<Object>();
23836                TTable fromTable = pivotedTable.getTableSource();
23837                Object table = modelManager.getModel(fromTable);
23838                List<TPivotClause> pivotClauses = new ArrayList<TPivotClause>();
23839                if (pivotedTable.getPivotClauseList() != null && pivotedTable.getPivotClauseList().size() > 0) {
23840                        for (int i = 0; i < pivotedTable.getPivotClauseList().size(); i++) {
23841                                pivotClauses.add(pivotedTable.getPivotClauseList().getElement(i));
23842                        }
23843                } else {
23844                        TPivotClause pivotClause = pivotedTable.getPivotClause();
23845                        pivotClauses.add(pivotClause);
23846                }
23847
23848                for (int y = 0; y < pivotClauses.size(); y++) {
23849                        TPivotClause pivotClause = pivotClauses.get(y);
23850                        PivotedTable pivotTable = modelFactory.createPivotdTable(pivotClause);
23851                        pivotTable.setUnpivoted(true);
23852
23853                        if (pivotClause.getValueColumnList() != null) {
23854                                for (int j = 0; j < pivotClause.getValueColumnList().size(); j++) {
23855                                        modelFactory.createResultColumn(pivotTable, pivotClause.getValueColumnList().getObjectName(j));
23856                                }
23857                        }
23858                        if (pivotClause.getPivotColumnList() != null) {
23859                                for (int j = 0; j < pivotClause.getPivotColumnList().size(); j++) {
23860                                        modelFactory.createResultColumn(pivotTable, pivotClause.getPivotColumnList().getObjectName(j));
23861                                }
23862                        }
23863                        if (pivotClause.getUnpivotInClause()!=null && pivotClause.getUnpivotInClause().getItems() != null) {
23864                                for (int j = 0; j < pivotClause.getUnpivotInClause().getItems().size(); j++) {
23865                                        TObjectName columnName = pivotClause.getUnpivotInClause().getItems().getElement(j).getColumn();
23866                                        if (columnName != null) {
23867                                                if (table instanceof QueryTable) {
23868                                                        for (ResultColumn tableColumn : ((QueryTable) table).getColumns()) {
23869                                                                if (getColumnName(columnName)
23870                                                                                .equals(DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
23871                                                                        for (ResultColumn resultColumn : pivotTable.getColumns()) {
23872                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
23873                                                                                relation.setEffectType(EffectType.select);
23874                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
23875                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
23876                                                                                pivotedColumns.add(tableColumn);
23877                                                                        }
23878                                                                        break;
23879                                                                }
23880                                                        }
23881                                                } else if (table instanceof Table) {
23882                                                        for (TableColumn tableColumn : ((Table) table).getColumns()) {
23883                                                                if (getColumnName(columnName)
23884                                                                                .equals(DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
23885                                                                        for (ResultColumn resultColumn : pivotTable.getColumns()) {
23886                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
23887                                                                                relation.setEffectType(EffectType.select);
23888                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
23889                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
23890                                                                                pivotedColumns.add(tableColumn);
23891                                                                        }
23892                                                                        break;
23893                                                                }
23894                                                        }
23895                                                }
23896                                        } else {
23897                                                TObjectNameList columnNames = pivotClause.getUnpivotInClause().getItems().getElement(j)
23898                                                                .getColumnList();
23899                                                for (TObjectName columnName1 : columnNames) {
23900                                                        if (table instanceof QueryTable) {
23901                                                                for (ResultColumn tableColumn : ((QueryTable) table).getColumns()) {
23902                                                                        if (getColumnName(columnName1).equals(
23903                                                                                        DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
23904                                                                                for (ResultColumn resultColumn : pivotTable.getColumns()) {
23905                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
23906                                                                                        relation.setEffectType(EffectType.select);
23907                                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
23908                                                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn));
23909                                                                                        pivotedColumns.add(tableColumn);
23910                                                                                }
23911                                                                                break;
23912                                                                        }
23913                                                                }
23914                                                        } else if (table instanceof Table) {
23915                                                                for (TableColumn tableColumn : ((Table) table).getColumns()) {
23916                                                                        if (getColumnName(columnName1).equals(
23917                                                                                        DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
23918                                                                                for (ResultColumn resultColumn : pivotTable.getColumns()) {
23919                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
23920                                                                                        relation.setEffectType(EffectType.select);
23921                                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
23922                                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
23923                                                                                        pivotedColumns.add(tableColumn);
23924                                                                                }
23925                                                                                break;
23926                                                                        }
23927                                                                }
23928                                                        }
23929                                                }
23930                                        }
23931                                }
23932                        }
23933                        tables.add(pivotTable);
23934                        tables.add(table);
23935                }
23936
23937                ResultSet resultSet = modelFactory.createResultSet(stmt,
23938                                isTopResultSet(stmt) && isShowTopSelectResultSet());
23939                TResultColumnList columnList = stmt.getResultColumnList();
23940                for (int i = 0; i < columnList.size(); i++) {
23941                        TResultColumn column = columnList.getResultColumn(i);
23942                        ResultColumn resultColumn = modelFactory.createAndBindingSelectSetResultColumn(resultSet, column, i);
23943                        if (resultColumn.getColumnObject() instanceof TResultColumn) {
23944                                TResultColumn columnObject = (TResultColumn) resultColumn.getColumnObject();
23945                                if (columnObject.getFieldAttr() != null) {
23946                                        if ("*".equals(getColumnName(columnObject.getFieldAttr()))) {
23947                                                resultColumn.setShowStar(false);
23948                                                int index = 0;
23949                                                for (int k = 0; k < tables.size(); k++) {
23950                                                        Object tableItem = tables.get(k);
23951                                                        if (tableItem instanceof ResultSet) {
23952                                                                for (int x = 0; x < ((ResultSet) tableItem).getColumns().size(); x++) {
23953                                                                        ResultColumn tableColumn = ((ResultSet) tableItem).getColumns().get(x);
23954                                                                        if (pivotedColumns.contains(tableColumn)) {
23955                                                                                continue;
23956                                                                        }
23957                                                                        if (tableColumn.getColumnObject() instanceof TObjectName) {
23958                                                                                resultColumn.bindStarLinkColumn((TObjectName) tableColumn.getColumnObject());
23959                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
23960                                                                                relation.setEffectType(EffectType.select);
23961                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn, (TObjectName) tableColumn.getColumnObject()));
23962                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
23963                                                                        } else if (tableColumn.getColumnObject() instanceof TResultColumn) {
23964                                                                                if (((TResultColumn) tableColumn.getColumnObject()).getFieldAttr() != null) {
23965                                                                                        if (tableColumn.hasStarLinkColumn()) {
23966                                                                                                for (int z = 0; z < tableColumn.getStarLinkColumnList().size(); z++) {
23967                                                                                                        ResultColumn resultColumn1 = modelFactory.createResultColumn(
23968                                                                                                                        (ResultSet) tableItem,
23969                                                                                                                        tableColumn.getStarLinkColumnList().get(z));
23970                                                                                                        DataFlowRelationship relation = modelFactory
23971                                                                                                                        .createDataFlowRelation();
23972                                                                                                        relation.setEffectType(EffectType.select);
23973                                                                                                        relation.setTarget(
23974                                                                                                                        new ResultColumnRelationshipElement(resultColumn));
23975                                                                                                        relation.addSource(
23976                                                                                                                        new ResultColumnRelationshipElement(resultColumn1));
23977                                                                                                        tableColumn.getStarLinkColumns().remove(
23978                                                                                                                        getColumnName(tableColumn.getStarLinkColumnList().get(z)));
23979                                                                                                        z--;
23980                                                                                                }
23981                                                                                        } else {
23982                                                                                                resultColumn.bindStarLinkColumn(
23983                                                                                                                ((TResultColumn) tableColumn.getColumnObject()).getFieldAttr());
23984                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
23985                                                                                                relation.setEffectType(EffectType.select);
23986                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn, ((TResultColumn) tableColumn.getColumnObject()).getFieldAttr()));
23987                                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
23988                                                                                        }
23989                                                                                } else if (((TResultColumn) tableColumn.getColumnObject()).getExpr() != null) {
23990                                                                                        TExpression expr = ((TResultColumn) tableColumn.getColumnObject())
23991                                                                                                        .getExpr();
23992                                                                                        if (expr.getExpressionType() == EExpressionType.simple_constant_t) {
23993                                                                                                TObjectName columnName = new TObjectName();
23994                                                                                                columnName.setString(expr.toString());
23995                                                                                                resultColumn.bindStarLinkColumn(columnName);
23996                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
23997                                                                                                relation.setEffectType(EffectType.select);
23998                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn, columnName));
23999                                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
24000                                                                                        }
24001                                                                                }
24002                                                                        }
24003                                                                }
24004                                                        } else if (tableItem instanceof Table) {
24005                                                                for (TableColumn tableColumn : ((Table) tableItem).getColumns()) {
24006                                                                        if (pivotedColumns.contains(tableColumn)) {
24007                                                                                continue;
24008                                                                        }
24009                                                                        resultColumn.bindStarLinkColumn((TObjectName) tableColumn.getColumnObject(), index);
24010                                                                        index++;
24011                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
24012                                                                        relation.setEffectType(EffectType.select);
24013                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn, (TObjectName) tableColumn.getColumnObject()));
24014                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
24015                                                                }
24016                                                        }
24017                                                }
24018                                        } else {
24019                                                for (int k = 0; k < tables.size(); k++) {
24020                                                        Object tableItem = tables.get(k);
24021                                                        if (tableItem instanceof ResultSet) {
24022                                                                for (ResultColumn tableColumn : ((ResultSet) tableItem).getColumns()) {
24023                                                                        if (getColumnName(columnObject.getFieldAttr()).equals(
24024                                                                                        DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
24025                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
24026                                                                                relation.setEffectType(EffectType.select);
24027                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
24028                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
24029                                                                                break;
24030                                                                        }
24031                                                                }
24032                                                        } else if (tableItem instanceof Table) {
24033                                                                for (TableColumn tableColumn : ((Table) tableItem).getColumns()) {
24034                                                                        if (getColumnName(columnObject.getFieldAttr()).equals(
24035                                                                                        DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
24036                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
24037                                                                                relation.setEffectType(EffectType.select);
24038                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
24039                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
24040                                                                                break;
24041                                                                        }
24042                                                                }
24043                                                        }
24044                                                }
24045                                        }
24046                                } else if (columnObject.getExpr() != null) {
24047                                        analyzeResultColumn(column, EffectType.select);
24048                                }
24049                        }
24050                }
24051        }
24052
24053        private void analyzePivotedTable(TSelectSqlStatement stmt, TPivotedTable pivotedTable) {
24054                List<Object> tables = new ArrayList<Object>();
24055                Set<Object> pivotedColumns = new HashSet<Object>();
24056                TTable fromTable = pivotedTable.getTableSource();
24057                Object table = modelManager.getModel(fromTable);
24058                if(table == null && fromTable.getSubquery()!=null) {
24059                        table = modelFactory.createQueryTable(fromTable);
24060                        TSelectSqlStatement subquery = fromTable.getSubquery();
24061                        analyzeSelectStmt(subquery);
24062                }
24063                List<TPivotClause> pivotClauses = new ArrayList<TPivotClause>();
24064                if (pivotedTable.getPivotClauseList() != null && pivotedTable.getPivotClauseList().size() > 0) {
24065                        for (int i = 0; i < pivotedTable.getPivotClauseList().size(); i++) {
24066                                pivotClauses.add(pivotedTable.getPivotClauseList().getElement(i));
24067                        }
24068                } else {
24069                        TPivotClause pivotClause = pivotedTable.getPivotClause();
24070                        pivotClauses.add(pivotClause);
24071                }
24072
24073                for (int y = 0; y < pivotClauses.size(); y++) {
24074                        TPivotClause pivotClause = pivotClauses.get(y);
24075                        List<TFunctionCall> functionCalls = new ArrayList<TFunctionCall>();
24076                        if (pivotClause.getAggregation_function() != null || pivotClause.getAggregation_function_list() != null) {
24077                                if (pivotClause.getAggregation_function() != null) {
24078                                        functionCalls.add((TFunctionCall) pivotClause.getAggregation_function());
24079                                } else if (pivotClause.getAggregation_function_list() != null) {
24080                                        for (int i = 0; i < pivotClause.getAggregation_function_list().size(); i++) {
24081                                                functionCalls.add((TFunctionCall) pivotClause.getAggregation_function_list().getResultColumn(i)
24082                                                                .getExpr().getFunctionCall());
24083                                        }
24084                                }
24085
24086                                if (functionCalls.isEmpty()) {
24087                                        return;
24088                                }
24089
24090                                if (pivotClause.getPivotColumnList() == null) {
24091                                        return;
24092                                }
24093
24094                                if (pivotClause.getPivotInClause() == null) {
24095                                        return;
24096                                }
24097
24098                                for (int x = 0; x < functionCalls.size(); x++) {
24099                                        TFunctionCall functionCall = functionCalls.get(x);
24100                                        Function function = modelFactory.createFunction(functionCall);
24101                                        ResultColumn column = modelFactory.createFunctionResultColumn(function,
24102                                                        ((TFunctionCall) functionCall).getFunctionName());
24103
24104                                        List<TExpression> expressions = new ArrayList<TExpression>();
24105                                        getFunctionExpressions(expressions, new ArrayList<TExpression>(), functionCall);
24106
24107                                        for (int j = 0; j < expressions.size(); j++) {
24108                                                columnsInExpr visitor = new columnsInExpr();
24109                                                expressions.get(j).inOrderTraverse(visitor);
24110                                                List<TObjectName> objectNames = visitor.getObjectNames();
24111                                                if (objectNames == null) {
24112                                                        continue;
24113                                                }
24114                                                for (TObjectName columnName : objectNames) {
24115                                                        if (table instanceof QueryTable) {
24116                                                                for (int i = 0; i < ((QueryTable) table).getColumns().size(); i++) {
24117                                                                        boolean find = false;
24118                                                                        ResultColumn tableColumn = ((QueryTable) table).getColumns().get(i);
24119                                                                        if (tableColumn.hasStarLinkColumn()) {
24120                                                                                for (int k = 0; k < tableColumn.getStarLinkColumnList().size(); k++) {
24121                                                                                        TObjectName objectName = tableColumn.getStarLinkColumnList().get(k);
24122                                                                                        if (getColumnName(columnName).equals(getColumnName(objectName))) {
24123                                                                                                ResultColumn resultColumn = modelFactory
24124                                                                                                                .createResultColumn((QueryTable) table, objectName);
24125                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
24126                                                                                                relation.setEffectType(EffectType.select);
24127                                                                                                relation.setTarget(new ResultColumnRelationshipElement(column));
24128                                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
24129                                                                                                pivotedColumns.add(resultColumn);
24130                                                                                                tableColumn.getStarLinkColumns()
24131                                                                                                                .remove(DlineageUtil.getColumnName(objectName));
24132                                                                                                find = true;
24133                                                                                                break;
24134                                                                                        }
24135                                                                                }
24136                                                                        } else if (getColumnName(columnName).equals(
24137                                                                                        DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
24138                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
24139                                                                                relation.setEffectType(EffectType.select);
24140                                                                                relation.setTarget(new ResultColumnRelationshipElement(column));
24141                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
24142                                                                                pivotedColumns.add(tableColumn);
24143                                                                                find = true;
24144                                                                                break;
24145                                                                        } 
24146                                                                        
24147                                                                        if (!find && tableColumn.getName().endsWith("*")) {
24148                                                                                QueryTable queryTable = (QueryTable)table;
24149                                                                                ResultColumn resultColumn = modelFactory.createResultColumn(queryTable, columnName);
24150                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
24151                                                                                relation.setEffectType(EffectType.select);
24152                                                                                relation.setTarget(new ResultColumnRelationshipElement(column));
24153                                                                                relation.addSource(new ResultColumnRelationshipElement(resultColumn));
24154                                                                                tableColumn.bindStarLinkColumn(columnName);
24155                                                                        }
24156                                                                }
24157                                                        } else if (table instanceof Table) {
24158                                                                for (TableColumn tableColumn : ((Table) table).getColumns()) {
24159                                                                        if (getColumnName(columnName).equals(
24160                                                                                        DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
24161                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
24162                                                                                relation.setEffectType(EffectType.select);
24163                                                                                relation.setTarget(new ResultColumnRelationshipElement(column));
24164                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
24165                                                                                pivotedColumns.add(tableColumn);
24166                                                                                break;
24167                                                                        }
24168                                                                }
24169                                                        }
24170                                                }
24171                                        }
24172
24173                                        PivotedTable pivotTable = modelFactory.createPivotdTable(pivotClause);
24174                                        pivotTable.setUnpivoted(false);
24175
24176                                        if (pivotClause.getPivotInClause().getItems() != null) {
24177                                                for (int j = 0; j < pivotClause.getPivotInClause().getItems().size(); j++) {
24178                                                        ResultColumn resultColumn = null;
24179                                                        if (pivotClause.getAggregation_function_list() != null
24180                                                                        && pivotClause.getAggregation_function_list().size() > 1) {
24181                                                                TResultColumn functionColumn = pivotClause.getAggregation_function_list()
24182                                                                                .getResultColumn(x);
24183                                                                TObjectName tableColumn = new TObjectName();
24184                                                                if (option.getVendor() == EDbVendor.dbvbigquery) {
24185                                                                        tableColumn.setString(getResultColumnString(functionColumn) + "_"
24186                                                                                        + SQLUtil.trimColumnStringQuote(getResultColumnString(
24187                                                                                                        pivotClause.getPivotInClause().getItems().getResultColumn(j))));
24188                                                                }
24189                                                                else {
24190                                                                        tableColumn.setString(SQLUtil
24191                                                                                        .trimColumnStringQuote(getResultColumnString(
24192                                                                                                        pivotClause.getPivotInClause().getItems().getResultColumn(j)))
24193                                                                                        + "_" + getResultColumnString(functionColumn));
24194                                                                }
24195                                                                resultColumn = modelFactory.createResultColumn(pivotTable, tableColumn);
24196                                                        } else {
24197                                                                resultColumn = modelFactory.createSelectSetResultColumn(pivotTable,
24198                                                                                pivotClause.getPivotInClause().getItems().getResultColumn(j), j);
24199                                                        }
24200                                                        {
24201                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
24202                                                                relation.setEffectType(EffectType.select);
24203                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
24204                                                                relation.addSource(new ResultColumnRelationshipElement(column));
24205                                                        }
24206                                                        {
24207                                                                for (TObjectName columnName : pivotClause.getPivotColumnList()) {
24208                                                                        if (table instanceof QueryTable) {
24209                                                                                for (int i = 0; i < ((QueryTable) table).getColumns().size(); i++) {
24210                                                                                        ResultColumn tableColumn = ((QueryTable) table).getColumns().get(i);
24211                                                                                        if (tableColumn.hasStarLinkColumn()) {
24212                                                                                                for (int k = 0; k < tableColumn.getStarLinkColumnList().size(); k++) {
24213                                                                                                        TObjectName objectName = tableColumn.getStarLinkColumnList().get(k);
24214                                                                                                        if (getColumnName(columnName).equals(getColumnName(objectName))) {
24215                                                                                                                ResultColumn resultColumn1 = modelFactory
24216                                                                                                                                .createResultColumn((QueryTable) table, objectName);
24217                                                                                                                DataFlowRelationship relation = modelFactory
24218                                                                                                                                .createDataFlowRelation();
24219                                                                                                                relation.setEffectType(EffectType.select);
24220                                                                                                                relation.setTarget(new ResultColumnRelationshipElement(column));
24221                                                                                                                relation.addSource(
24222                                                                                                                                new ResultColumnRelationshipElement(resultColumn1));
24223                                                                                                                pivotedColumns.add(resultColumn1);
24224                                                                                                                tableColumn.getStarLinkColumns()
24225                                                                                                                                .remove(DlineageUtil.getColumnName(objectName));
24226                                                                                                                break;
24227                                                                                                        }
24228                                                                                                }
24229                                                                                        } else if (getColumnName(columnName).equals(DlineageUtil
24230                                                                                                        .getIdentifierNormalColumnName(tableColumn.getName()))) {
24231                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
24232                                                                                                relation.setEffectType(EffectType.select);
24233                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
24234                                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
24235                                                                                                pivotedColumns.add(tableColumn);
24236                                                                                                break;
24237                                                                                        }
24238                                                                                }
24239                                                                        } else if (table instanceof Table) {
24240                                                                                for (TableColumn tableColumn : ((Table) table).getColumns()) {
24241                                                                                        if (getColumnName(columnName).equals(DlineageUtil
24242                                                                                                        .getIdentifierNormalColumnName(tableColumn.getName()))) {
24243                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
24244                                                                                                relation.setEffectType(EffectType.select);
24245                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
24246                                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
24247                                                                                                pivotedColumns.add(tableColumn);
24248                                                                                                break;
24249                                                                                        }
24250                                                                                }
24251                                                                        }
24252                                                                }
24253                                                        }
24254                                                }
24255                                        } else if (pivotClause.getPivotInClause().getSubQuery() != null) {
24256                                                TSelectSqlStatement subquery = pivotClause.getPivotInClause().getSubQuery();
24257                                                analyzeSelectStmt(subquery);
24258                                                ResultSet selectSetResultSetModel = (ResultSet) modelManager.getModel(subquery);
24259                                                for (int j = 0; j < subquery.getResultColumnList().size(); j++) {
24260                                                        ResultColumn resultColumn = modelFactory.createSelectSetResultColumn(pivotTable,
24261                                                                        subquery.getResultColumnList().getResultColumn(j), j);
24262                                                        {
24263                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
24264                                                                relation.setEffectType(EffectType.select);
24265                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
24266                                                                relation.addSource(new ResultColumnRelationshipElement(column));
24267                                                                relation.addSource(new ResultColumnRelationshipElement(
24268                                                                                selectSetResultSetModel.getColumns().get(j)));
24269                                                        }
24270                                                        {
24271                                                                for (TObjectName columnName : pivotClause.getPivotColumnList()) {
24272                                                                        if (table instanceof QueryTable) {
24273                                                                                for (ResultColumn tableColumn : ((QueryTable) table).getColumns()) {
24274                                                                                        if (getColumnName(columnName).equals(DlineageUtil
24275                                                                                                        .getIdentifierNormalColumnName(tableColumn.getName()))) {
24276                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
24277                                                                                                relation.setEffectType(EffectType.select);
24278                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
24279                                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
24280                                                                                                pivotedColumns.add(tableColumn);
24281                                                                                                break;
24282                                                                                        }
24283                                                                                }
24284                                                                        } else if (table instanceof Table) {
24285                                                                                for (TableColumn tableColumn : ((Table) table).getColumns()) {
24286                                                                                        if (getColumnName(columnName).equals(DlineageUtil
24287                                                                                                        .getIdentifierNormalColumnName(tableColumn.getName()))) {
24288                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
24289                                                                                                relation.setEffectType(EffectType.select);
24290                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
24291                                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
24292                                                                                                pivotedColumns.add(tableColumn);
24293                                                                                                break;
24294                                                                                        }
24295                                                                                }
24296                                                                        }
24297                                                                }
24298                                                        }
24299                                                }
24300                                        }
24301                                        tables.add(pivotTable);
24302                                        tables.add(table);
24303                                }
24304                        }
24305                }
24306
24307                TPivotClause pivotClause = pivotClauses.get(0);
24308                boolean hasAlias = pivotClause.getAliasClause() != null && pivotClause.getAliasClause().getColumns() != null
24309                                && pivotClause.getAliasClause().getColumns().size() > 0;
24310                if (hasAlias) {
24311                        Alias alias = modelFactory.createAlias(pivotClause.getAliasClause());
24312                        List<TObjectName> aliasColumns = new ArrayList<TObjectName>();
24313                        int index = 0;
24314                        for (int k = 0; k < tables.size(); k++) {
24315                                Object tableItem = tables.get(k);
24316                                if (tableItem instanceof ResultSet) {
24317                                        for (ResultColumn tableColumn : ((ResultSet) tableItem).getColumns()) {
24318                                                if (pivotedColumns.contains(tableColumn)) {
24319                                                        continue;
24320                                                }
24321                                                if (tableColumn.getColumnObject() instanceof TObjectName) {
24322                                                        aliasColumns.add((TObjectName) tableColumn.getColumnObject());
24323                                                } else if (tableColumn.getColumnObject() instanceof TResultColumn) {
24324                                                        if (((TResultColumn) tableColumn.getColumnObject()).getFieldAttr() != null) {
24325                                                                aliasColumns.add(((TResultColumn) tableColumn.getColumnObject()).getFieldAttr());
24326                                                        } else {
24327                                                                TExpression expr = ((TResultColumn) tableColumn.getColumnObject()).getExpr();
24328                                                                if (expr.getExpressionType() == EExpressionType.simple_constant_t) {
24329                                                                        TObjectName columnName = new TObjectName();
24330                                                                        columnName.setString(expr.toString());
24331                                                                        aliasColumns.add(columnName);
24332                                                                }
24333                                                        }
24334                                                }
24335                                        }
24336                                } else if (tableItem instanceof Table) {
24337                                        for (TableColumn tableColumn : ((Table) tableItem).getColumns()) {
24338                                                if (pivotedColumns.contains(tableColumn)) {
24339                                                        continue;
24340                                                }
24341                                                aliasColumns.add(index, (TObjectName) tableColumn.getColumnObject());
24342                                                index++;
24343                                        }
24344                                }
24345                        }
24346
24347                        IndexedLinkedHashMap<String, ResultColumn> aliasColumnMap = new IndexedLinkedHashMap<String, ResultColumn>();
24348                        int diffCount = pivotClause.getAliasClause().getColumns().size() -  aliasColumns.size();
24349                        for (int k = 0; k < pivotClause.getAliasClause().getColumns().size(); k++) {
24350                                if (pivotClause.getAliasClause().getColumns().size() > aliasColumns.size()) {
24351                                        if (k < diffCount) {
24352                                                continue;
24353                                        }
24354                                        ResultColumn resultColumn = modelFactory.createResultColumn(alias,
24355                                                        pivotClause.getAliasClause().getColumns().getObjectName(k));
24356                                        if ((k - diffCount) < aliasColumns.size()) {
24357                                                aliasColumnMap.put(aliasColumns.get(k - diffCount).toString(), resultColumn);
24358                                        }
24359                                } else {
24360                                        ResultColumn resultColumn = modelFactory.createResultColumn(alias,
24361                                                        pivotClause.getAliasClause().getColumns().getObjectName(k));
24362                                        if (k < aliasColumns.size()) {
24363                                                aliasColumnMap.put(aliasColumns.get(k).toString(), resultColumn);
24364                                        }
24365                                }
24366                        }
24367
24368                        for (int k = 0; k < tables.size(); k++) {
24369                                Object tableItem = tables.get(k);
24370                                if (tableItem instanceof ResultSet) {
24371                                        int resultColumnSize = ((ResultSet) tableItem).getColumns().size();
24372                                        for (int x = 0; x < resultColumnSize; x++) {
24373                                                ResultColumn tableColumn = ((ResultSet) tableItem).getColumns().get(x);
24374                                                if (pivotedColumns.contains(tableColumn)) {
24375                                                        continue;
24376                                                }
24377                                                if (tableColumn.getColumnObject() instanceof TObjectName) {
24378                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
24379                                                        relation.setEffectType(EffectType.select);
24380                                                        relation.setTarget(new ResultColumnRelationshipElement(
24381                                                                        aliasColumnMap.get(tableColumn.getColumnObject().toString())));
24382                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn));
24383                                                } else if (tableColumn.getColumnObject() instanceof TResultColumn) {
24384                                                        if (((TResultColumn) tableColumn.getColumnObject()).getFieldAttr() != null) {
24385                                                                ResultColumn targetColumn = aliasColumnMap.get(
24386                                                                                ((TResultColumn) tableColumn.getColumnObject()).getFieldAttr().toString());
24387                                                                if(targetColumn == null) {
24388                                                                        continue;
24389                                                                }
24390                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
24391                                                                relation.setEffectType(EffectType.select);
24392                                                                relation.setTarget(new ResultColumnRelationshipElement(targetColumn));
24393                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
24394                                                        } else if (((TResultColumn) tableColumn.getColumnObject()).getExpr() != null) {
24395                                                                TExpression expr = ((TResultColumn) tableColumn.getColumnObject()).getExpr();
24396                                                                if (expr.getExpressionType() == EExpressionType.simple_constant_t) {
24397                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
24398                                                                        relation.setEffectType(EffectType.select);
24399                                                                        ResultColumn targetColumn = (ResultColumn) aliasColumnMap
24400                                                                                        .getValueAtIndex(aliasColumnMap.size() - resultColumnSize + x);
24401                                                                        relation.setTarget(new ResultColumnRelationshipElement(targetColumn));
24402                                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn));
24403                                                                }
24404                                                        }
24405                                                }
24406                                        }
24407                                } else if (tableItem instanceof Table) {
24408                                        for (TableColumn tableColumn : ((Table) tableItem).getColumns()) {
24409                                                if (pivotedColumns.contains(tableColumn)) {
24410                                                        continue;
24411                                                }
24412                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
24413                                                relation.setEffectType(EffectType.select);
24414                                                relation.setTarget(new ResultColumnRelationshipElement(
24415                                                                aliasColumnMap.get(tableColumn.getColumnObject().toString())));
24416                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
24417                                        }
24418                                }
24419                        }
24420
24421                        ResultSet resultSet = modelFactory.createResultSet(stmt,
24422                                        isTopResultSet(stmt) && isShowTopSelectResultSet());
24423                        TResultColumnList columnList = stmt.getResultColumnList();
24424                        for (int i = 0; i < columnList.size(); i++) {
24425                                TResultColumn column = columnList.getResultColumn(i);
24426                                ResultColumn resultColumn = modelFactory.createAndBindingSelectSetResultColumn(resultSet, column, i);
24427                                if (resultColumn.getColumnObject() instanceof TResultColumn) {
24428                                        TResultColumn columnObject = (TResultColumn) resultColumn.getColumnObject();
24429                                        if (columnObject.getFieldAttr() != null) {
24430                                                if ("*".equals(getColumnName(columnObject.getFieldAttr()))) {
24431                                                        resultColumn.setShowStar(false);
24432                                                        for (ResultColumn tableColumn : ((ResultSet) alias).getColumns()) {
24433                                                                resultColumn.bindStarLinkColumn((TObjectName) tableColumn.getColumnObject());
24434                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
24435                                                                relation.setEffectType(EffectType.select);
24436                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn, (TObjectName) tableColumn.getColumnObject()));
24437                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
24438                                                        }
24439                                                } else {
24440                                                        for (ResultColumn tableColumn : ((ResultSet) alias).getColumns()) {
24441                                                                if (getColumnName(columnObject.getFieldAttr())
24442                                                                                .equals(DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
24443                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
24444                                                                        relation.setEffectType(EffectType.select);
24445                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
24446                                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn));
24447                                                                        break;
24448                                                                }
24449                                                        }
24450                                                }
24451                                        } else if (columnObject.getExpr() != null && columnObject.getExpr()
24452                                                        .getExpressionType() == EExpressionType.sqlserver_proprietary_column_alias_t) {
24453                                                for (ResultColumn tableColumn : ((ResultSet) alias).getColumns()) {
24454                                                        if (getColumnName(columnObject.getExpr().getRightOperand().getObjectOperand())
24455                                                                        .equals(DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
24456                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
24457                                                                relation.setEffectType(EffectType.select);
24458                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
24459                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
24460                                                                break;
24461                                                        }
24462                                                }
24463                                        } else if (columnObject.getExpr() != null && columnObject.getExpr()
24464                                                        .getExpressionType() == EExpressionType.function_t) {
24465                                                Function function = (Function) createFunction(columnObject.getExpr().getFunctionCall());
24466                                                for (ResultColumn arg : function.getColumns()) {
24467                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
24468                                                        relation.setEffectType(EffectType.select);
24469                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
24470                                                        relation.addSource(new ResultColumnRelationshipElement(arg));
24471                                                }
24472                                        }
24473                                }
24474                        }
24475                } else {
24476                        ResultSet resultSet = modelFactory.createResultSet(stmt, isTopResultSet(stmt));
24477                        TResultColumnList columnList = stmt.getResultColumnList();
24478                        for (int i = 0; i < columnList.size(); i++) {
24479                                TResultColumn column = columnList.getResultColumn(i);
24480                                ResultColumn resultColumn = modelFactory.createAndBindingSelectSetResultColumn(resultSet, column, i);
24481                                if (resultColumn.getColumnObject() instanceof TResultColumn) {
24482                                        boolean fromFunction = false;
24483                                        TResultColumn columnObject = (TResultColumn) resultColumn.getColumnObject();
24484                                        TObjectName resultColumnFieldAttr = columnObject.getFieldAttr();
24485                                        List<TObjectName> resultColumnNames = new ArrayList<TObjectName>();
24486                                        if (resultColumnFieldAttr != null) {
24487                                                resultColumnNames.add(resultColumnFieldAttr);
24488                                        } else if (columnObject.getExpr() != null
24489                                                        && column.getExpr().getExpressionType() == EExpressionType.function_t) {
24490                                                extractFunctionObjectNames(column.getExpr().getFunctionCall(), resultColumnNames);
24491                                                fromFunction = true;
24492                                        }
24493                                        
24494                                        if (!resultColumnNames.isEmpty()) {
24495                                                for (TObjectName resultColumnName : resultColumnNames) {
24496                                                        if ("*".equals(getColumnName(resultColumnName))) {
24497                                                                resultColumn.setShowStar(false);
24498                                                                int index = 0;
24499                                                                for (int k = 0; k < tables.size(); k++) {
24500                                                                        Object tableItem = tables.get(k);
24501                                                                        if (tableItem instanceof ResultSet && !(tableItem instanceof QueryTable)) {
24502                                                                                for (int x = 0; x < ((ResultSet) tableItem).getColumns().size(); x++) {
24503                                                                                        ResultColumn tableColumn = ((ResultSet) tableItem).getColumns().get(x);
24504                                                                                        if (pivotedColumns.contains(tableColumn)) {
24505                                                                                                continue;
24506                                                                                        }
24507                                                                                        if (tableColumn.getColumnObject() instanceof TObjectName) {
24508                                                                                                resultColumn.bindStarLinkColumn(
24509                                                                                                                (TObjectName) tableColumn.getColumnObject());
24510                                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
24511                                                                                                relation.setEffectType(EffectType.select);
24512                                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn, (TObjectName) tableColumn.getColumnObject()));
24513                                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
24514                                                                                        } else if (tableColumn.getColumnObject() instanceof TResultColumn) {
24515                                                                                                if (((TResultColumn) tableColumn.getColumnObject())
24516                                                                                                                .getFieldAttr() != null) {
24517                                                                                                        if (tableColumn.hasStarLinkColumn()) {
24518                                                                                                                for (int z = 0; z < tableColumn.getStarLinkColumnList()
24519                                                                                                                                .size(); z++) {
24520                                                                                                                        ResultColumn resultColumn1 = modelFactory
24521                                                                                                                                        .createResultColumn((ResultSet) tableItem,
24522                                                                                                                                                        tableColumn.getStarLinkColumnList().get(z));
24523                                                                                                                        DataFlowRelationship relation = modelFactory
24524                                                                                                                                        .createDataFlowRelation();
24525                                                                                                                        relation.setEffectType(EffectType.select);
24526                                                                                                                        relation.setTarget(
24527                                                                                                                                        new ResultColumnRelationshipElement(resultColumn));
24528                                                                                                                        relation.addSource(
24529                                                                                                                                        new ResultColumnRelationshipElement(resultColumn1));
24530                                                                                                                        tableColumn.getStarLinkColumns().remove(getColumnName(
24531                                                                                                                                        tableColumn.getStarLinkColumnList().get(z)));
24532                                                                                                                        z--;
24533                                                                                                                }
24534                                                                                                        } else {
24535                                                                                                                resultColumn.bindStarLinkColumn(
24536                                                                                                                                ((TResultColumn) tableColumn.getColumnObject())
24537                                                                                                                                                .getFieldAttr());
24538                                                                                                                DataFlowRelationship relation = modelFactory
24539                                                                                                                                .createDataFlowRelation();
24540                                                                                                                relation.setEffectType(EffectType.select);
24541                                                                                                                relation.setTarget(
24542                                                                                                                                new ResultColumnRelationshipElement(resultColumn, ((TResultColumn) tableColumn.getColumnObject())
24543                                                                                                                                                .getFieldAttr()));
24544                                                                                                                relation.addSource(
24545                                                                                                                                new ResultColumnRelationshipElement(tableColumn));
24546                                                                                                        }
24547                                                                                                } else if (((TResultColumn) tableColumn.getColumnObject())
24548                                                                                                                .getExpr() != null) {
24549                                                                                                        TExpression expr = ((TResultColumn) tableColumn.getColumnObject())
24550                                                                                                                        .getExpr();
24551                                                                                                        if (expr.getExpressionType() == EExpressionType.simple_constant_t) {
24552                                                                                                                TObjectName columnName = new TObjectName();
24553                                                                                                                columnName.setString(expr.toString());
24554                                                                                                                resultColumn.bindStarLinkColumn(columnName);
24555                                                                                                                DataFlowRelationship relation = modelFactory
24556                                                                                                                                .createDataFlowRelation();
24557                                                                                                                relation.setEffectType(EffectType.select);
24558                                                                                                                relation.setTarget(
24559                                                                                                                                new ResultColumnRelationshipElement(resultColumn, columnName));
24560                                                                                                                relation.addSource(
24561                                                                                                                                new ResultColumnRelationshipElement(tableColumn));
24562                                                                                                        }
24563                                                                                                }
24564                                                                                        }
24565                                                                                }
24566                                                                        } else if (tableItem instanceof Table) {
24567                                                                                for (TableColumn tableColumn : ((Table) tableItem).getColumns()) {
24568                                                                                        if (pivotedColumns.contains(tableColumn)) {
24569                                                                                                continue;
24570                                                                                        }
24571                                                                                        resultColumn.bindStarLinkColumn((TObjectName) tableColumn.getColumnObject(),
24572                                                                                                        index);
24573                                                                                        index++;
24574                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
24575                                                                                        relation.setEffectType(EffectType.select);
24576                                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn, (TObjectName) tableColumn.getColumnObject()));
24577                                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
24578                                                                                }
24579                                                                        } else if (tableItem instanceof QueryTable) {
24580                                                                                for (ResultColumn tableColumn : ((QueryTable) tableItem).getColumns()) {
24581                                                                                        if (pivotedColumns.contains(tableColumn)) {
24582                                                                                                continue;
24583                                                                                        }
24584                                                                                        TObjectName column1 = new TObjectName();
24585                                                                                        column1.setString(tableColumn.getName());
24586                                                                                        resultColumn.bindStarLinkColumn(column1, index);
24587                                                                                        index++;
24588                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
24589                                                                                        relation.setEffectType(EffectType.select);
24590                                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn, column1));
24591                                                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn), false);
24592                                                                                }
24593                                                                        }
24594                                                                }
24595                                                        } else {
24596                                                                ResultColumn pivotedTableColumn = getPivotedTableColumn(pivotedTable, resultColumnName);
24597                                                                        if (pivotedTableColumn != null) {
24598                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
24599                                                                                relation.setEffectType(EffectType.select);
24600                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
24601                                                                                relation.addSource(new ResultColumnRelationshipElement(pivotedTableColumn));
24602                                                                        } else {
24603                                                                                for (int k = 0; k < tables.size(); k++) {
24604                                                                                        Object tableItem = tables.get(k);
24605                                                                                        if (tableItem instanceof ResultSet) {
24606                                                                                                for (ResultColumn tableColumn : ((ResultSet) tableItem).getColumns()) {
24607                                                                                                        if (DlineageUtil
24608                                                                                                                        .getIdentifierNormalColumnName(tableColumn.getName()).equals(getColumnName(resultColumnName))) {
24609                                                                                                                if (fromFunction) {
24610                                                                                                                        Function function = (Function)createPivotedFunction(column.getExpr().getFunctionCall(), tableColumn);
24611                                                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
24612                                                                                                                        relation.setEffectType(EffectType.select);
24613                                                                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
24614                                                                                                                        if (function.getColumns() != null && !function.getColumns().isEmpty()) {
24615                                                                                                                                for (ResultColumn functionColumn : function.getColumns()) {
24616                                                                                                                                        relation.addSource(new ResultColumnRelationshipElement(functionColumn));
24617                                                                                                                                }
24618                                                                                                                        }
24619                                                                                                                } else {
24620                                                                                                                        DataFlowRelationship relation = modelFactory
24621                                                                                                                                        .createDataFlowRelation();
24622                                                                                                                        relation.setEffectType(EffectType.select);
24623                                                                                                                        relation.setTarget(
24624                                                                                                                                        new ResultColumnRelationshipElement(resultColumn));
24625                                                                                                                        relation.addSource(
24626                                                                                                                                        new ResultColumnRelationshipElement(tableColumn));
24627                                                                                                                }
24628                                                                                                                break;
24629                                                                                                        }
24630                                                                                                }
24631                                                                                        } else if (tableItem instanceof Table) {
24632                                                                                                for (TableColumn tableColumn : ((Table) tableItem).getColumns()) {
24633                                                                                                        if (getColumnName(resultColumnName).equals(DlineageUtil
24634                                                                                                                        .getIdentifierNormalColumnName(tableColumn.getName()))) {
24635                                                                                                                DataFlowRelationship relation = modelFactory
24636                                                                                                                                .createDataFlowRelation();
24637                                                                                                                relation.setEffectType(EffectType.select);
24638                                                                                                                relation.setTarget(
24639                                                                                                                                new ResultColumnRelationshipElement(resultColumn));
24640                                                                                                                relation.addSource(
24641                                                                                                                                new TableColumnRelationshipElement(tableColumn));
24642                                                                                                                break;
24643                                                                                                        }
24644                                                                                                }
24645                                                                                        }
24646                                                                                }
24647                                                                        }
24648                                                                }
24649                                                }
24650                                        } else if (columnObject.getExpr() != null && columnObject.getExpr()
24651                                                        .getExpressionType() == EExpressionType.sqlserver_proprietary_column_alias_t) {
24652                                                for (int k = 0; k < tables.size(); k++) {
24653                                                        Object tableItem = tables.get(k);
24654                                                        if (tableItem instanceof ResultSet) {
24655                                                                for (ResultColumn tableColumn : ((ResultSet) tableItem).getColumns()) {
24656                                                                        if (columnObject.getExpr().getRightOperand().getObjectOperand() != null
24657                                                                                        && getColumnName(
24658                                                                                                        columnObject.getExpr().getRightOperand().getObjectOperand())
24659                                                                                                                        .equals(DlineageUtil.getIdentifierNormalColumnName(
24660                                                                                                                                        tableColumn.getName()))) {
24661                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
24662                                                                                relation.setEffectType(EffectType.select);
24663                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
24664                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
24665                                                                                break;
24666                                                                        } else if (columnObject.getExpr().getRightOperand()
24667                                                                                        .getExpressionType() == EExpressionType.function_t) {
24668                                                                                List<TExpression> expressions = new ArrayList<TExpression>();
24669                                                                                getFunctionExpressions(expressions, new ArrayList<TExpression>(),
24670                                                                                                columnObject.getExpr().getRightOperand().getFunctionCall());
24671                                                                                for (int j = 0; j < expressions.size(); j++) {
24672                                                                                        columnsInExpr visitor = new columnsInExpr();
24673                                                                                        expressions.get(j).inOrderTraverse(visitor);
24674                                                                                        List<TObjectName> objectNames = visitor.getObjectNames();
24675                                                                                        if (objectNames == null) {
24676                                                                                                continue;
24677                                                                                        }
24678                                                                                        for (TObjectName columnName : objectNames) {
24679                                                                                                if (getColumnName(columnName).equals(DlineageUtil
24680                                                                                                                .getIdentifierNormalColumnName(tableColumn.getName()))) {
24681                                                                                                        DataFlowRelationship relation = modelFactory
24682                                                                                                                        .createDataFlowRelation();
24683                                                                                                        relation.setEffectType(EffectType.select);
24684                                                                                                        relation.setTarget(
24685                                                                                                                        new ResultColumnRelationshipElement(resultColumn));
24686                                                                                                        relation.addSource(
24687                                                                                                                        new ResultColumnRelationshipElement(tableColumn));
24688                                                                                                        break;
24689                                                                                                }
24690                                                                                        }
24691                                                                                }
24692                                                                        }
24693                                                                }
24694                                                        } else if (tableItem instanceof Table) {
24695                                                                for (TableColumn tableColumn : ((Table) tableItem).getColumns()) {
24696                                                                        if (getColumnName(columnObject.getExpr().getRightOperand().getObjectOperand())
24697                                                                                        .equals(DlineageUtil
24698                                                                                                        .getIdentifierNormalColumnName(tableColumn.getName()))) {
24699                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
24700                                                                                relation.setEffectType(EffectType.select);
24701                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
24702                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
24703                                                                                break;
24704                                                                        }
24705                                                                }
24706                                                        }
24707                                                }
24708                                        }
24709                                }
24710                        }
24711                }
24712
24713                analyzeSelectIntoClause(stmt);
24714        }
24715
24716        private Function createPivotedFunction(TFunctionCall functionCall, ResultColumn sourceColumn) {
24717                Function function = modelFactory.createFunction((TFunctionCall) functionCall);
24718                ResultColumn column = modelFactory.createFunctionResultColumn(function,
24719                                ((TFunctionCall) functionCall).getFunctionName());
24720                if ("COUNT".equalsIgnoreCase(((TFunctionCall) functionCall).getFunctionName().toString())) {
24721                        // @see https://e.gitee.com/gudusoft/issues/list?issue=I40NUP
24722                        // COUNT特殊处理,不和参数关联
24723                        if (option.isShowCountTableColumn()) {
24724                                analyzePivotedFunctionArgumentsDataFlowRelation(column, functionCall, sourceColumn);
24725                        }
24726                } else {
24727                        analyzePivotedFunctionArgumentsDataFlowRelation(column, functionCall, sourceColumn);
24728                        Set<Object> functionTableModelObjs = modelManager.getFunctionTable(getIdentifiedFunctionName(function));
24729                        if (functionTableModelObjs!=null && functionTableModelObjs.iterator().next() instanceof ResultSet) {
24730                                ResultSet functionTableModel = (ResultSet) functionTableModelObjs.iterator().next();
24731                                if (functionTableModel.getColumns() != null) {
24732                                        for (int j = 0; j < functionTableModel.getColumns().size(); j++) {
24733                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
24734                                                        relation.setEffectType(EffectType.select);
24735                                                        relation.setTarget(new ResultColumnRelationshipElement(column));
24736                                                        relation.addSource(new ResultColumnRelationshipElement(
24737                                                                        functionTableModel.getColumns().get(j)));
24738                                        }
24739                                }
24740                        }
24741                }
24742                return function;
24743        }
24744
24745        private void analyzePivotedFunctionArgumentsDataFlowRelation(ResultColumn column, TFunctionCall functionCall,
24746                        ResultColumn sourceColumn) {
24747                List<TExpression> directExpressions = new ArrayList<TExpression>();
24748                List<TExpression> indirectExpressions = new ArrayList<TExpression>();
24749
24750                getFunctionExpressions(directExpressions, indirectExpressions, functionCall);
24751
24752                for (int j = 0; j < directExpressions.size(); j++) {
24753                        columnsInExpr visitor = new columnsInExpr();
24754                        directExpressions.get(j).inOrderTraverse(visitor);
24755
24756                        List<TObjectName> objectNames = visitor.getObjectNames();
24757                        List<TParseTreeNode> constants = visitor.getConstants();
24758
24759                        if (objectNames != null) {
24760                                for (TObjectName name : objectNames) {
24761                                        if (DlineageUtil.compareColumnIdentifier(name.toString(), sourceColumn.getName())) {
24762                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
24763                                                relation.setEffectType(EffectType.select);
24764                                                relation.setTarget(new ResultColumnRelationshipElement(column));
24765                                                relation.addSource(new ResultColumnRelationshipElement(sourceColumn));
24766                                        }
24767                                }
24768                        }
24769                        if (constants != null) {
24770                                for (TParseTreeNode name : constants) {
24771                                        if (name.toString().equals(sourceColumn.getName())) {
24772                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
24773                                                relation.setEffectType(EffectType.select);
24774                                                relation.setTarget(new ResultColumnRelationshipElement(column));
24775                                                relation.addSource(new ResultColumnRelationshipElement(sourceColumn));
24776                                        }
24777                                }
24778                        }
24779                }
24780        }
24781        
24782        private void extractFunctionObjectNames(TFunctionCall functionCall, List<TObjectName> resultColumnNames) {
24783                List<TExpression> directExpressions = new ArrayList<TExpression>();
24784                List<TExpression> indirectExpressions = new ArrayList<TExpression>();
24785
24786                getFunctionExpressions(directExpressions, indirectExpressions, functionCall);
24787
24788                for (int j = 0; j < directExpressions.size(); j++) {
24789                        columnsInExpr visitor = new columnsInExpr();
24790                        directExpressions.get(j).inOrderTraverse(visitor);
24791
24792                        List<TObjectName> objectNames = visitor.getObjectNames();
24793                        List<TParseTreeNode> functions = visitor.getFunctions();
24794                        List<TParseTreeNode> constants = visitor.getConstants(); 
24795
24796                        if (objectNames != null) {
24797                                resultColumnNames.addAll(objectNames);
24798                        }
24799                        
24800                        if (constants != null) {
24801                                for(TParseTreeNode item: constants) {
24802                                        if(item instanceof TConstant && ((TConstant) item).getLiteralType().getText().equals(ELiteralType.string_et.getText())) {
24803                                                TObjectName object = new TObjectName();
24804                                                object.setString(item.toString());
24805                                                resultColumnNames.add(object);
24806                                        }
24807                                }
24808                        }
24809
24810                        if (functions != null && !functions.isEmpty()) {
24811                                for (TParseTreeNode function : functions) {
24812                                        if (function instanceof TFunctionCall) {
24813                                                extractFunctionObjectNames((TFunctionCall) function, resultColumnNames);
24814                                        }
24815                                }
24816                        }
24817                }
24818        }
24819
24820        private String getResultColumnString(TResultColumn resultColumn) {
24821                if (resultColumn.getAliasClause() != null) {
24822                        return resultColumn.getAliasClause().toString();
24823                }
24824                return resultColumn.toString();
24825        }
24826
24827        private void analyzeBigQueryUnnest(TSelectSqlStatement stmt, TTable table) {
24828                Table unnestTable = modelFactory.createTableFromCreateDDL(table, false, getTempTableName(table));
24829                unnestTable.setSubType(SubType.unnest);
24830                TUnnestClause clause = table.getUnnestClause();
24831                TExpression arrayExpr = clause.getArrayExpr();
24832                if (arrayExpr == null){
24833                        if (clause.getColumns() != null) {
24834                                for (TObjectName column : clause.getColumns()) {
24835                                        if (clause.getDerivedColumnList() != null) {
24836                                                unnestTable.setCreateTable(true);
24837                                                for (int i = 0; i < clause.getDerivedColumnList().size(); i++) {
24838                                                        TObjectName columnName = new TObjectName();
24839                                                        columnName.setString(column.getColumnNameOnly() + "."
24840                                                                        + clause.getDerivedColumnList().getObjectName(i).getColumnNameOnly());
24841                                                        TableColumn tableColumn = modelFactory.createTableColumn(unnestTable, columnName, true);
24842                                                        List<TObjectName> columns = new ArrayList<TObjectName>();
24843                                                        columns.add(column);
24844                                                        analyzeDataFlowRelation(tableColumn, columns, EffectType.select, null);
24845                                                }
24846                                        }
24847                                        else {
24848                                                unnestTable.setCreateTable(true);
24849                                                boolean find = false;
24850                                                if (column.getSourceTable() != null && modelManager.getModel(column.getSourceTable()) instanceof Table) {
24851                                                        Table sourceTable = (Table)modelManager.getModel(column.getSourceTable());
24852                                                        if(sourceTable!=null) {
24853                                                                for(TableColumn tableColumn: sourceTable.getColumns()) {
24854                                                                        if(tableColumn.isStruct()) {
24855                                                                                List<String> names = SQLUtil.parseNames(tableColumn.getName());
24856                                                                                if (SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotColumn, names.get(0), column.getColumnNameOnly())) {
24857                                                                                        TObjectName columnName = new TObjectName();
24858                                                                                        columnName.setString(tableColumn.getName());
24859                                                                                        TableColumn unnestTableColumn = modelFactory.createTableColumn(unnestTable,
24860                                                                                                        columnName, true);
24861                                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
24862                                                                                        relation.setEffectType(EffectType.select);
24863                                                                                        relation.setTarget(new TableColumnRelationshipElement(unnestTableColumn));
24864                                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
24865                                                                                        find = true;
24866                                                                                }
24867                                                                        }
24868                                                                }
24869                                                        }
24870                                                }
24871                                                if (!find) {
24872                                                        TObjectName colName = column;
24873                                                        if (table.getAliasClause() != null && table.getAliasClause().getAliasName() != null) {
24874                                                                colName = table.getAliasClause().getAliasName();
24875                                                        }
24876                                                        TableColumn tableColumn = modelFactory.createTableColumn(unnestTable, colName, true);
24877                                                        List<TObjectName> columns = new ArrayList<TObjectName>();
24878                                                        columns.add(column);
24879                                                        analyzeDataFlowRelation(tableColumn, columns, EffectType.select, null);
24880                                                }
24881                                        }
24882                                }
24883                        }
24884                        return;
24885                }
24886                List<TExpression> expressions = new ArrayList<TExpression>();
24887                TExpressionList values = arrayExpr.getExprList();
24888                if (values == null) {
24889                        expressions.add(arrayExpr);
24890                }
24891                else {
24892                        for(TExpression value: values) {
24893                                expressions.add(value);
24894                        }
24895                }
24896                for (TExpression value : expressions) {
24897                        unnestTable.setCreateTable(true);
24898                        if (value.getExpressionType() == EExpressionType.simple_object_name_t) {
24899                                if (table.getAliasClause() != null) {
24900                                        TableColumn tableColumn = modelFactory.createTableColumn(unnestTable,
24901                                                        table.getAliasClause().getAliasName(), true);
24902                                        columnsInExpr visitor = new columnsInExpr();
24903                                        value.inOrderTraverse(visitor);
24904                                        List<TObjectName> columns = visitor.getObjectNames();
24905                                        analyzeDataFlowRelation(tableColumn, columns, EffectType.select, null);
24906                                } else {
24907                                        TResultColumnList resultColumnList = stmt.getResultColumnList();
24908                                        for (int i = 0; i < resultColumnList.size(); i++) {
24909                                                TObjectName firstColumn = new TObjectName();
24910                                                firstColumn.setString(resultColumnList.getResultColumn(0).getColumnNameOnly());
24911                                                TableColumn tableColumn = modelFactory.createTableColumn(unnestTable, firstColumn, true);
24912                                                columnsInExpr visitor = new columnsInExpr();
24913                                                value.inOrderTraverse(visitor);
24914                                                List<TObjectName> columns = visitor.getObjectNames();
24915                                                analyzeDataFlowRelation(tableColumn, columns, EffectType.select, null);
24916                                        }
24917                                }
24918                        } else if (value.getExpressionType() == EExpressionType.function_t) {
24919                                if (table.getAliasClause() != null) {
24920                                        TableColumn tableColumn = modelFactory.createTableColumn(unnestTable,
24921                                                        table.getAliasClause().getAliasName(), true);
24922                                        columnsInExpr visitor = new columnsInExpr();
24923                                        value.inOrderTraverse(visitor);
24924                                        List<TParseTreeNode> functions = visitor.getFunctions();
24925                                        analyzeFunctionDataFlowRelation(tableColumn, functions, EffectType.select, null);
24926                                } else {
24927                                        TResultColumnList resultColumnList = stmt.getResultColumnList();
24928                                        for (int i = 0; i < resultColumnList.size(); i++) {
24929                                                TObjectName firstColumn = new TObjectName();
24930                                                firstColumn.setString(resultColumnList.getResultColumn(0).getColumnNameOnly());
24931                                                TableColumn tableColumn = modelFactory.createTableColumn(unnestTable, firstColumn, true);
24932                                                columnsInExpr visitor = new columnsInExpr();
24933                                                value.inOrderTraverse(visitor);
24934                                                List<TObjectName> columns = visitor.getObjectNames();
24935                                                analyzeDataFlowRelation(tableColumn, columns, EffectType.select, null);
24936                                        }
24937                                }
24938                        } else if (value.getExpressionType() == EExpressionType.simple_constant_t) {
24939                                if (table.getAliasClause() != null) {
24940                                        TableColumn tableColumn = modelFactory.createTableColumn(unnestTable,
24941                                                        table.getAliasClause().getAliasName(), true);
24942                                        columnsInExpr visitor = new columnsInExpr();
24943                                        value.inOrderTraverse(visitor);
24944                                        List<TParseTreeNode> constants = visitor.getConstants();
24945                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select, null);
24946                                } else {
24947                                        TObjectName firstColumn = new TObjectName();
24948                                        firstColumn.setString("f0_");
24949                                        TableColumn tableColumn = modelFactory.createTableColumn(unnestTable, firstColumn, true);
24950                                        columnsInExpr visitor = new columnsInExpr();
24951                                        value.inOrderTraverse(visitor);
24952                                        List<TParseTreeNode> constants = visitor.getConstants();
24953                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select, null);
24954                                }
24955                        } else if (value.getExpressionType() == EExpressionType.list_t) {
24956                                if (arrayExpr.getTypeName() != null && arrayExpr.getTypeName().getColumnDefList() != null) {
24957                                        for (int i = 0; i < arrayExpr.getTypeName().getColumnDefList().size(); i++) {
24958                                                TColumnDefinition column = arrayExpr.getTypeName().getColumnDefList().getColumn(i);
24959                                                if (column != null && column.getColumnName() != null) {
24960                                                        TableColumn tableColumn = modelFactory.createTableColumn(unnestTable,
24961                                                                        column.getColumnName(), true);
24962                                                        columnsInExpr visitor = new columnsInExpr();
24963                                                        value.getExprList().getExpression(i).inOrderTraverse(visitor);
24964                                                        List<TParseTreeNode> constants = visitor.getConstants();
24965                                                        analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select, null);
24966                                                }
24967                                        }
24968                                } else {
24969                                        for (int i = 0; i < value.getExprList().size(); i++) {
24970                                                TObjectName firstColumn = new TObjectName();
24971                                                firstColumn.setString("f" + i + "_");
24972                                                TableColumn tableColumn = modelFactory.createTableColumn(unnestTable, firstColumn, true);
24973                                                columnsInExpr visitor = new columnsInExpr();
24974                                                value.getExprList().getExpression(i).inOrderTraverse(visitor);
24975                                                List<TParseTreeNode> constants = visitor.getConstants();
24976                                                analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select, null);
24977                                        }
24978                                }
24979                        } else if (value.getExpressionType() == EExpressionType.subquery_t) {
24980                                analyzeSelectStmt(value.getSubQuery());
24981                                ResultSet resultSet = (ResultSet) modelManager.getModel(value.getSubQuery());
24982                                if (resultSet != null) {
24983                                        for (int i = 0; i < resultSet.getColumns().size(); i++) {
24984                                                TObjectName columnName =  new TObjectName();
24985                                                if(resultSet.getColumns().get(i).getAlias()!=null) {
24986                                                        columnName.setString(resultSet.getColumns().get(i).getAlias());
24987                                                }
24988                                                else {
24989                                                        columnName.setString(getColumnName(resultSet.getColumns().get(i).getName()));
24990                                                }
24991                                                TableColumn tableColumn = modelFactory.createTableColumn(unnestTable, columnName, true);
24992                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
24993                                                relation.setEffectType(EffectType.select);
24994                                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
24995                                                relation.addSource(
24996                                                                new ResultColumnRelationshipElement(resultSet.getColumns().get(i)));
24997                                        }
24998                                }
24999                        }
25000                }
25001        }
25002
25003        private void analyzePrestoUnnest(TSelectSqlStatement stmt, TTable table) {
25004                List<Table> tables = new ArrayList<Table>();
25005                TTable targetTable = stmt.getTables().getTable(0);
25006                Table stmtTable = modelFactory.createTable(targetTable);
25007                Object sourceTable = null;
25008                if (targetTable.getSubquery() != null) {
25009                        analyzeSelectStmt(targetTable.getSubquery());
25010                        if (targetTable.getSubquery().isCombinedQuery()) {
25011                                ResultSet sourceResultSet = (ResultSet) modelManager.getModel(targetTable.getSubquery());
25012                                sourceTable = sourceResultSet;
25013                        } else if (targetTable.getSubquery().getResultColumnList() != null) {
25014                                ResultSet sourceResultSet = (ResultSet) modelManager
25015                                                .getModel(targetTable.getSubquery().getResultColumnList());
25016                                sourceTable = sourceResultSet;
25017                        } else if (targetTable.getSubquery().getValueClause() != null) {
25018                                List<TResultColumnList> rowList = targetTable.getSubquery().getValueClause().getRows();
25019                                if (rowList != null && rowList.size() > 0) {
25020                                        Table valuesTable = modelFactory.createTableByName("Values-Table", true);
25021                                        int columnCount = rowList.get(0).size();
25022                                        for (int j = 1; j <= columnCount; j++) {
25023                                                TObjectName columnName = new TObjectName();
25024                                                TResultColumn columnObject = rowList.get(0).getResultColumn(j - 1);
25025                                                if (columnObject.getExpr().getExpressionType() == EExpressionType.typecast_t) {
25026                                                        columnName.setString(columnObject.getExpr().getLeftOperand().toString());
25027                                                } else {
25028                                                        columnName.setString(columnObject.getExpr().toString());
25029                                                }
25030                                                modelFactory.createTableColumn(valuesTable, columnName, true);
25031                                        }
25032                                        valuesTable.setCreateTable(true);
25033                                        valuesTable.setSubType(SubType.values_table);
25034                                        sourceTable = valuesTable;
25035                                }
25036                        }
25037                }
25038                tables.add(stmtTable);
25039                Table unnestTable = modelFactory.createTable(table);
25040                unnestTable.setSubType(SubType.unnest);
25041                tables.add(unnestTable);
25042                if (table.getAliasClause() != null && table.getAliasClause().getColumns() != null
25043                                && table.getUnnestClause().getColumns() != null) {
25044                        int unnestTableSize = table.getUnnestClause().getColumns().size();
25045                        for (int i = 0; i < table.getAliasClause().getColumns().size(); i++) {
25046                                TableColumn sourceColumn = null;
25047                                if (unnestTableSize > i) {
25048                                        sourceColumn = modelFactory.createTableColumn(stmtTable,
25049                                                        table.getUnnestClause().getColumns().getObjectName(i), true);
25050                                        if (sourceTable instanceof Table) {
25051                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
25052                                                relation.setEffectType(EffectType.select);
25053                                                relation.setTarget(new TableColumnRelationshipElement(sourceColumn));
25054                                                relation.addSource(
25055                                                                new TableColumnRelationshipElement(((Table) sourceTable).getColumns().get(i)));
25056                                        } else if (sourceTable instanceof ResultSet) {
25057                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
25058                                                relation.setEffectType(EffectType.select);
25059                                                relation.setTarget(new TableColumnRelationshipElement(sourceColumn));
25060                                                relation.addSource(
25061                                                                new ResultColumnRelationshipElement(((ResultSet) sourceTable).getColumns().get(i)));
25062                                        }
25063                                } else {
25064                                        sourceColumn = modelFactory.createTableColumn(stmtTable,
25065                                                        table.getUnnestClause().getColumns().getObjectName(unnestTableSize - 1), true);
25066                                }
25067                                TableColumn targetColumn = modelFactory.createTableColumn(unnestTable,
25068                                                table.getAliasClause().getColumns().getObjectName(i), true);
25069                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
25070                                relation.setEffectType(EffectType.select);
25071                                relation.setTarget(new TableColumnRelationshipElement(targetColumn));
25072                                relation.addSource(new TableColumnRelationshipElement(sourceColumn));
25073                        }
25074                }
25075
25076                ResultSet resultSet = modelFactory.createResultSet(stmt,
25077                                isTopResultSet(stmt) && isShowTopSelectResultSet());
25078                TResultColumnList columnList = stmt.getResultColumnList();
25079                for (int i = 0; i < columnList.size(); i++) {
25080                        TResultColumn column = columnList.getResultColumn(i);
25081                        ResultColumn resultColumn = modelFactory.createSelectSetResultColumn(resultSet, column, i);
25082                        if (resultColumn.getColumnObject() instanceof TResultColumn) {
25083                                TResultColumn columnObject = (TResultColumn) resultColumn.getColumnObject();
25084                                if (columnObject.getFieldAttr() != null) {
25085                                        if ("*".equals(getColumnName(columnObject.getFieldAttr()))) {
25086                                                for (int k = 0; k < tables.size(); k++) {
25087                                                        Table tableItem = tables.get(k);
25088                                                        for (TableColumn tableColumn : tableItem.getColumns()) {
25089                                                                resultColumn.bindStarLinkColumn(tableColumn.getColumnObject());
25090                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
25091                                                                relation.setEffectType(EffectType.select);
25092                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn, tableColumn.getColumnObject()));
25093                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
25094                                                        }
25095                                                }
25096                                        } else {
25097                                                boolean match = false;
25098                                                for (int k = 0; k < tables.size(); k++) {
25099                                                        Table tableItem = tables.get(k);
25100                                                        for (TableColumn tableColumn : tableItem.getColumns()) {
25101                                                                if (getColumnName(columnObject.getFieldAttr())
25102                                                                                .equals(DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
25103                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
25104                                                                        relation.setEffectType(EffectType.select);
25105                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
25106                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
25107                                                                        match = true;
25108                                                                }
25109                                                        }
25110                                                }
25111                                                if (!match) {
25112                                                        TableColumn tableColumn = modelFactory.createTableColumn(stmtTable,
25113                                                                        columnObject.getFieldAttr(), false);
25114                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
25115                                                        relation.setEffectType(EffectType.select);
25116                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
25117                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
25118                                                }
25119                                        }
25120                                }
25121                        }
25122                }
25123        }
25124
25125        private void analyzeLateralView(TSelectSqlStatement stmt, TTable table, ArrayList<TLateralView> lateralViews) {
25126                List<Object> tables = new ArrayList<Object>();
25127                Object stmtTable = null;
25128                if(table.getSubquery()!=null) {
25129                        stmtTable = modelFactory.createQueryTable(table);
25130                        tables.add(stmtTable);
25131                }
25132                else {
25133                        stmtTable = modelFactory.createTable(table);
25134                        tables.add(stmtTable);
25135                }
25136                for (int i = 0; i < lateralViews.size(); i++) {
25137                        TLateralView lateralView = lateralViews.get(i);
25138                        TFunctionCall functionCall = lateralView.getUdtf();
25139                        List<TExpression> expressions = new ArrayList<TExpression>();
25140                        if (functionCall == null) {
25141                                continue;
25142                        }
25143                        Function function = modelFactory.createFunction(functionCall);
25144                        ResultColumn column = modelFactory.createFunctionResultColumn(function,
25145                                        ((TFunctionCall) functionCall).getFunctionName());
25146
25147                        Table lateralTable = null;
25148                        if (lateralView.getTableAlias() != null) {
25149                                lateralTable = modelFactory.createTableByName(lateralView.getTableAlias().getAliasName(), true);
25150                        } else {
25151                                lateralTable = modelFactory.createTableByName(functionCall.toString(), true);
25152                        }
25153
25154                        for (int j = 0; j < lateralView.getColumnAliasList().size(); j++) {
25155                                TObjectName viewColumn = lateralView.getColumnAliasList().getObjectName(j);
25156                                TableColumn tableColumn = modelFactory.createTableColumn(lateralTable, viewColumn, true);
25157
25158                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
25159                                relation.setEffectType(EffectType.select);
25160                                relation.setTarget(new TableColumnRelationshipElement(tableColumn));
25161                                relation.addSource(new ResultColumnRelationshipElement(column));
25162                        }
25163
25164                        getFunctionExpressions(expressions, new ArrayList<TExpression>(), functionCall);
25165                        for (int j = 0; j < expressions.size(); j++) {
25166                                columnsInExpr visitor = new columnsInExpr();
25167                                expressions.get(j).inOrderTraverse(visitor);
25168                                List<TObjectName> objectNames = visitor.getObjectNames();
25169                                if (objectNames == null) {
25170                                        continue;
25171                                }
25172                                for (TObjectName columnName : objectNames) {
25173                                        boolean match = false;
25174                                        for (int k = 0; k < tables.size(); k++) {
25175                                                Object item = tables.get(k);
25176                                                if(item instanceof Table) {
25177                                                        Table tableItem = (Table)item;
25178                                                        for (TableColumn tableColumn : tableItem.getColumns()) {
25179                                                                if (getColumnName(columnName)
25180                                                                                .equals(DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
25181                                                                        match = true;
25182                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
25183                                                                        relation.setEffectType(EffectType.select);
25184                                                                        relation.setTarget(new ResultColumnRelationshipElement(column));
25185                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
25186                                                                }
25187                                                        }
25188                                                }
25189                                                else {
25190                                                        ResultSet tableItem = (ResultSet)item;
25191                                                        for (ResultColumn tableColumn : tableItem.getColumns()) {
25192                                                                if (getColumnName(columnName)
25193                                                                                .equals(DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
25194                                                                        match = true;
25195                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
25196                                                                        relation.setEffectType(EffectType.select);
25197                                                                        relation.setTarget(new ResultColumnRelationshipElement(column));
25198                                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn));
25199                                                                }
25200                                                        }
25201                                                }
25202                                        }
25203                                        if (!match) {
25204                                                if(stmtTable instanceof Table) {
25205                                                        TableColumn tableColumn = modelFactory.createTableColumn((Table)stmtTable, columnName, false);
25206                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
25207                                                        relation.setEffectType(EffectType.select);
25208                                                        relation.setTarget(new ResultColumnRelationshipElement(column));
25209                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
25210                                                }
25211                                                else {
25212                                                        ResultColumn tableColumn = modelFactory.createResultColumn((ResultSet)stmtTable, columnName, false);
25213                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
25214                                                        relation.setEffectType(EffectType.select);
25215                                                        relation.setTarget(new ResultColumnRelationshipElement(column));
25216                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn));
25217                                                }
25218                                        }
25219                                }
25220                                List<TParseTreeNode> constants = visitor.getConstants();
25221                                if (!constants.isEmpty()) {
25222                                        if (option.isShowConstantTable()) {
25223                                                Table constantTable = modelFactory.createConstantsTable(stmtStack.peek());
25224                                                for (TParseTreeNode constant : constants) {
25225                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
25226                                                        relation.setEffectType(EffectType.select);
25227                                                        relation.setTarget(new ResultColumnRelationshipElement(column));
25228                                                        if (constant instanceof TConstant) {
25229                                                                TableColumn constantColumn = modelFactory.createTableColumn(constantTable,
25230                                                                                (TConstant) constant);
25231                                                                relation.addSource(new ConstantRelationshipElement(constantColumn));
25232                                                        } else if (constant instanceof TObjectName) {
25233                                                                TableColumn constantColumn = modelFactory.createTableColumn(constantTable,
25234                                                                                (TObjectName) constant, false);
25235                                                                relation.addSource(new ConstantRelationshipElement(constantColumn));
25236                                                        }
25237                                                }
25238                                        }
25239                                }
25240                                
25241                                List<TParseTreeNode> functions = visitor.getFunctions();
25242                                if (functions != null && !functions.isEmpty()) {
25243                                        analyzeFunctionDataFlowRelation(column, functions, EffectType.function);
25244                                }
25245                        }
25246                        tables.add(lateralTable);
25247                }
25248
25249                ResultSet resultSet = modelFactory.createResultSet(stmt,
25250                                isTopResultSet(stmt) && isShowTopSelectResultSet());
25251                TResultColumnList columnList = stmt.getResultColumnList();
25252                for (int i = 0; i < columnList.size(); i++) {
25253                        TResultColumn column = columnList.getResultColumn(i);
25254                        ResultColumn resultColumn = modelFactory.createSelectSetResultColumn(resultSet, column, i);
25255                        if (resultColumn.getColumnObject() instanceof TResultColumn) {
25256                                TResultColumn columnObject = (TResultColumn) resultColumn.getColumnObject();
25257                                if (columnObject.getFieldAttr() != null) {
25258                                        if ("*".equals(getColumnName(columnObject.getFieldAttr()))) {
25259                                                for (int k = 0; k < tables.size(); k++) {
25260                                                        Object item = tables.get(k);
25261                                                        if(item instanceof Table) {
25262                                                                Table tableItem = (Table)item;
25263                                                                for (TableColumn tableColumn : tableItem.getColumns()) {
25264                                                                        resultColumn.bindStarLinkColumn(tableColumn.getColumnObject());
25265                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
25266                                                                        relation.setEffectType(EffectType.select);
25267                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn, tableColumn.getColumnObject()));
25268                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
25269                                                                }
25270                                                        }
25271                                                        else {
25272                                                                ResultSet tableItem = (ResultSet)item;
25273                                                                for (ResultColumn tableColumn : tableItem.getColumns()) {
25274                                                                        TObjectName linkColumn = new TObjectName();
25275                                                                        linkColumn.setString(tableColumn.getName());
25276                                                                        resultColumn.bindStarLinkColumn(linkColumn);
25277                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
25278                                                                        relation.setEffectType(EffectType.select);
25279                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn, linkColumn));
25280                                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn));
25281                                                                }
25282                                                        }
25283                                                }
25284                                        } else {
25285                                                boolean match = false;
25286                                                for (int k = 0; k < tables.size(); k++) {
25287                                                        Object item = tables.get(k);
25288                                                        if(item instanceof Table) {
25289                                                                Table tableItem = (Table)item;
25290                                                                for (TableColumn tableColumn : tableItem.getColumns()) {
25291                                                                        if (getColumnName(columnObject.getFieldAttr())
25292                                                                                        .equals(DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
25293                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
25294                                                                                relation.setEffectType(EffectType.select);
25295                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
25296                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
25297                                                                                match = true;
25298                                                                        }
25299                                                                }
25300                                                        }
25301                                                        else {
25302                                                                ResultSet tableItem = (ResultSet)item;
25303                                                                for (ResultColumn tableColumn : tableItem.getColumns()) {
25304                                                                        if (getColumnName(columnObject.getFieldAttr())
25305                                                                                        .equals(DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
25306                                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
25307                                                                                relation.setEffectType(EffectType.select);
25308                                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
25309                                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
25310                                                                                match = true;
25311                                                                        }
25312                                                                }
25313                                                        }
25314                                                }
25315                                                if (!match) {
25316                                                        if(stmtTable instanceof Table) {
25317                                                                TableColumn tableColumn = modelFactory.createTableColumn((Table)stmtTable,
25318                                                                                columnObject.getFieldAttr(), false);
25319                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
25320                                                                relation.setEffectType(EffectType.select);
25321                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
25322                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
25323                                                        }
25324                                                        else {
25325                                                                ResultColumn tableColumn = modelFactory.createResultColumn((ResultSet)stmtTable,
25326                                                                                columnObject.getFieldAttr(), false);
25327                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
25328                                                                relation.setEffectType(EffectType.select);
25329                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
25330                                                                relation.addSource(new ResultColumnRelationshipElement(tableColumn));
25331                                                        }
25332                                                }
25333                                        }
25334                                }
25335                                else if (columnObject.getExpr() != null
25336                                                && columnObject.getExpr().getExpressionType() == EExpressionType.function_t) {
25337                                        analyzeResultColumn(column, EffectType.select);
25338                                        for (int k = 0; k < tables.size(); k++) {
25339                                                Object item = tables.get(k);
25340                                                if (item instanceof Table) {
25341                                                        Table tableItem = (Table) item;
25342                                                        for (TableColumn tableColumn : tableItem.getColumns()) {
25343                                                                if (getColumnName(resultColumn.getName()).equals(
25344                                                                                DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
25345                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
25346                                                                        relation.setEffectType(EffectType.select);
25347                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
25348                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
25349                                                                }
25350                                                        }
25351                                                } else {
25352                                                        ResultSet tableItem = (ResultSet) item;
25353                                                        for (ResultColumn tableColumn : tableItem.getColumns()) {
25354                                                                if (getColumnName(resultColumn.getName()).equals(
25355                                                                                DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) {
25356                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
25357                                                                        relation.setEffectType(EffectType.select);
25358                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
25359                                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn));
25360                                                                }
25361                                                        }
25362                                                }
25363                                        }
25364                                }
25365                        }
25366                }
25367        }
25368
25369        private boolean isFromFunction(TObjectName object) {
25370
25371                Stack<TParseTreeNode> nodes = object.getStartToken().getNodesStartFromThisToken();
25372                if (nodes != null) {
25373                        for (int i = 0; i < nodes.size(); i++) {
25374                                if (nodes.get(i) instanceof TFunctionCall) {
25375                                        return true;
25376                                }
25377                        }
25378                }
25379                return false;
25380        }
25381
25382        private TResultColumnList getResultColumnList(TSelectSqlStatement stmt) {
25383                // Iterative DFS (left-first) to find the first non-combined query's result column list.
25384                // Avoids StackOverflow with deeply nested UNION trees.
25385                Deque<TSelectSqlStatement> stack = new ArrayDeque<>();
25386                stack.push(stmt);
25387                while (!stack.isEmpty()) {
25388                        TSelectSqlStatement current = stack.pop();
25389                        if (current.isCombinedQuery()) {
25390                                // Push right first so left is processed first (stack is LIFO)
25391                                if (current.getRightStmt() != null) stack.push(current.getRightStmt());
25392                                if (current.getLeftStmt() != null) stack.push(current.getLeftStmt());
25393                        } else {
25394                                if (current.getResultColumnList() != null) {
25395                                        return current.getResultColumnList();
25396                                }
25397                        }
25398                }
25399                return null;
25400        }
25401
25402        private void createPseudoImpactRelation(TCustomSqlStatement stmt, ResultSet resultSetModel, EffectType effectType) {
25403                if (stmt.getTables() != null) {
25404                        for (int i = 0; i < stmt.getTables().size(); i++) {
25405                                TTable table = stmt.getTables().getTable(i);
25406                                if (modelManager.getModel(table) instanceof ResultSet) {
25407                                        ResultSet tableModel = (ResultSet) modelManager.getModel(table);
25408                                        if (tableModel != resultSetModel && !tableModel.getRelationRows().getHoldRelations().isEmpty()) {
25409                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
25410                                                impactRelation.setEffectType(effectType);
25411                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
25412                                                                tableModel.getRelationRows()));
25413                                                impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
25414                                                                resultSetModel.getRelationRows()));
25415                                        }
25416                                } else if (modelManager.getModel(table) instanceof Table) {
25417                                        Table tableModel = (Table) modelManager.getModel(table);
25418                                        if (!tableModel.getRelationRows().getHoldRelations().isEmpty()) {
25419                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
25420                                                impactRelation.setEffectType(effectType);
25421                                                impactRelation.addSource(
25422                                                                new RelationRowsRelationshipElement<TableRelationRows>(tableModel.getRelationRows()));
25423                                                impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
25424                                                                resultSetModel.getRelationRows()));
25425                                        }
25426                                }
25427                        }
25428                }
25429        }
25430
25431        private void analyzeFunctionDataFlowRelation(Object gspObject, List<TParseTreeNode> functions,
25432                        EffectType effectType) {
25433                for (int i = 0; i < functions.size(); i++) {
25434                        TParseTreeNode functionCall = functions.get(i);
25435                        if (functionCall instanceof TFunctionCall) {
25436                                String functionName = DlineageUtil.getIdentifierNormalTableName(
25437                                                DlineageUtil.getFunctionNameWithArgNum((TFunctionCall) functionCall));
25438                                Procedure procedure = modelManager.getProcedureByName(functionName);
25439                                if (procedure != null) {
25440                                        String procedureParent = getProcedureParentName(stmtStack.peek());
25441                                        if (procedureParent != null) {
25442                                                Procedure caller = modelManager
25443                                                                .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
25444                                                if (caller != null) {
25445                                                        CallRelationship callRelation = modelFactory.createCallRelation();
25446                                                        callRelation.setCallObject(functionCall);
25447                                                        callRelation.setTarget(new ProcedureRelationshipElement(caller));
25448                                                        callRelation.addSource(new ProcedureRelationshipElement(procedure));
25449                                                        if (isBuiltInFunctionName(((TFunctionCall)functionCall).getFunctionName())||isKeyword(((TFunctionCall)functionCall).getFunctionName())) {
25450                                                                callRelation.setBuiltIn(true);
25451                                                        }
25452                                                }
25453                                        }
25454                                        if (procedure.getProcedureObject() instanceof TCreateFunctionStmt) {
25455                                                TCreateFunctionStmt createFunction = (TCreateFunctionStmt)procedure.getProcedureObject();
25456                                                TTypeName dataType = createFunction.getReturnDataType();
25457                                                if (dataType!=null && dataType.getTypeOfList() != null && dataType.getTypeOfList().getColumnDefList() != null) {
25458                                                        Object modelObject = modelManager.getModel(gspObject);
25459                                                        if(modelObject instanceof ResultColumn) {
25460                                                                ResultColumn resultColumn = (ResultColumn)modelObject;
25461                                                                ResultSet resultSet = resultColumn.getResultSet();
25462                                                                for (int j = 0; j < dataType.getTypeOfList().getColumnDefList().size(); j++) {
25463                                                                        TObjectName columnName = new TObjectName();
25464                                                                        if( dataType.getDataType() == EDataType.array_t) {
25465//                                                                              columnName.setString(resultColumn.getName() + ".array."
25466//                                                                                              + dataType.getTypeOfList().getColumnDefList().getColumn(j)
25467//                                                                                                              .getColumnName().getColumnNameOnly());
25468                                                                                columnName.setString(resultColumn.getName() + "."
25469                                                                                                + dataType.getTypeOfList().getColumnDefList().getColumn(j)
25470                                                                                                .getColumnName().getColumnNameOnly());
25471                                                                        }
25472                                                                        else {
25473                                                                                columnName.setString(resultColumn.getName() + "."
25474                                                                                                + dataType.getTypeOfList().getColumnDefList().getColumn(j)
25475                                                                                                                .getColumnName().getColumnNameOnly());
25476                                                                        }
25477                                                                        ResultColumn sturctColumn = modelFactory.createResultColumn(resultSet, columnName,
25478                                                                                        true);
25479                                                                        sturctColumn.setStruct(true);
25480                                                                        Function sourceFunction = (Function)createFunction(functionCall);
25481                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
25482                                                                        relation.setEffectType(effectType);
25483                                                                        relation.setTarget(new ResultColumnRelationshipElement(sturctColumn));
25484                                                                        if (sourceFunction.getColumns() != null && !sourceFunction.getColumns().isEmpty()) {
25485                                                                                for (ResultColumn column : sourceFunction.getColumns()) {
25486                                                                                        relation.addSource(new ResultColumnRelationshipElement(column));
25487                                                                                }
25488                                                                        }
25489                                                                }
25490                                                                resultSet.getColumns().remove(resultColumn);
25491                                                        }
25492                                                        return;
25493                                                }
25494                                        }
25495                                }
25496                        }
25497                        
25498                        if(gspObject instanceof TResultColumn) {
25499                                TResultColumn resultColumn = (TResultColumn)gspObject;
25500                                if(resultColumn.getAliasClause()!=null && resultColumn.getAliasClause().getColumns()!=null) {
25501                                        for(TObjectName columnName: resultColumn.getAliasClause().getColumns()) {
25502                                                analyzeFunctionDataFlowRelation(columnName, Arrays.asList(functionCall), effectType, null);
25503                                        }
25504                                        return;
25505                                }
25506                        }
25507                        analyzeFunctionDataFlowRelation(gspObject, Arrays.asList(functionCall), effectType, null);
25508                }
25509        }
25510
25511        private void analyzeFunctionDataFlowRelation(Object gspObject, List<TParseTreeNode> functions,
25512                        EffectType effectType, Process process) {
25513
25514                Object modelObject = modelManager.getModel(gspObject);
25515                if (modelObject == null) {
25516                        if (gspObject instanceof ResultColumn || gspObject instanceof TableColumn) {
25517                                modelObject = gspObject;
25518                        }
25519                }
25520
25521                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
25522                relation.setEffectType(effectType);
25523                relation.setProcess(process);
25524
25525                if (modelObject instanceof ResultColumn) {
25526                        relation.setTarget(new ResultColumnRelationshipElement((ResultColumn) modelObject));
25527
25528                } else if (modelObject instanceof TableColumn) {
25529                        relation.setTarget(new TableColumnRelationshipElement((TableColumn) modelObject));
25530
25531                } else {
25532                        throw new UnsupportedOperationException();
25533                }
25534
25535                for (int i = 0; i < functions.size(); i++) {
25536                        TParseTreeNode functionCall = functions.get(i);
25537
25538                        if (functionCall instanceof TFunctionCall) {
25539                                TFunctionCall call = (TFunctionCall) functionCall;
25540                                Procedure callee = modelManager.getProcedureByName(
25541                                                DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(call)));
25542                                if (callee == null && procedureDDLMap.containsKey(DlineageUtil.getFunctionNameWithArgNum(call))) {
25543                                        analyzeCustomSqlStmt(procedureDDLMap.get(DlineageUtil.getFunctionNameWithArgNum(call)));
25544                                        callee = modelManager.getProcedureByName(
25545                                                        DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(call)));
25546                                }
25547
25548                                if (callee != null) {
25549                                        String procedureParent = getProcedureParentName(stmtStack.peek());
25550                                        if (procedureParent != null) {
25551                                                Procedure caller = modelManager
25552                                                                .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
25553                                                if (caller != null) {
25554                                                        CallRelationship callRelation = modelFactory.createCallRelation();
25555                                                        callRelation.setCallObject(functionCall);
25556                                                        callRelation.setTarget(new ProcedureRelationshipElement(caller));
25557                                                        callRelation.addSource(new ProcedureRelationshipElement(callee));
25558                                                        if (isBuiltInFunctionName(call.getFunctionName()) || isKeyword(call.getFunctionName())) {
25559                                                                callRelation.setBuiltIn(true);
25560                                                        }
25561                                                }
25562                                        }
25563                                        if (callee.getArguments() != null) {
25564                                                for (int j = 0; j < callee.getArguments().size(); j++) {
25565                                                        Argument argument = callee.getArguments().get(j);
25566                                                        Variable variable = resolveFormalVariable(callee, argument);
25567                                                        if(variable!=null) {
25568                                                                if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
25569                                                                        Transform transform = new Transform();
25570                                                                        transform.setType(Transform.FUNCTION);
25571                                                                        transform.setCode(call);
25572                                                                        compositeBindingColumn(variable).setTransform(transform);
25573                                                                }
25574                                                                Process callProcess = modelFactory.createProcess(call);
25575                                                                variable.addProcess(callProcess);
25576                                                                analyzeFunctionArgumentsDataFlowRelation(compositeBindingColumn(variable), call, j, callProcess);
25577                                                        }
25578                                                }
25579                                        }
25580                                        Set<Object> functionTableModelObjs = modelManager.getFunctionTable(DlineageUtil
25581                                                        .getIdentifierNormalTableName(call.getFunctionName().toString()));
25582                                        if (functionTableModelObjs != null) {
25583                                                modelManager.bindModel(call, functionTableModelObjs.iterator().next());
25584                                                for (Object functionTableModelObj : functionTableModelObjs) {
25585                                                        if (functionTableModelObj instanceof ResultSet) {
25586                                                                ResultSet resultSet = (ResultSet) functionTableModelObj;
25587                                                                for (ResultColumn column : resultSet.getColumns()) {
25588                                                                        relation.addSource(new ResultColumnRelationshipElement(column));
25589                                                                }
25590                                                        }
25591                                                }
25592                                        }
25593                                        continue;
25594                                }
25595                        }
25596
25597
25598                        Object functionModel = createFunction(functionCall);
25599                        if (functionModel instanceof Function) {
25600                                Function sourceFunction = (Function)functionModel;
25601                                if (sourceFunction.getColumns() != null && !sourceFunction.getColumns().isEmpty()) {
25602                                        for (ResultColumn column : sourceFunction.getColumns()) {
25603                                                relation.addSource(new ResultColumnRelationshipElement(column));
25604                                        }
25605                                }
25606                                else if (functionCall instanceof TFunctionCall) {
25607                                        relation.addSource(new ResultColumnRelationshipElement((FunctionResultColumn) modelManager
25608                                                        .getModel(((TFunctionCall) functionCall).getFunctionName())));
25609                                } else if (functionCall instanceof TCaseExpression) {
25610                                        relation.addSource(new ResultColumnRelationshipElement((FunctionResultColumn) modelManager
25611                                                        .getModel(((TCaseExpression) functionCall).getWhenClauseItemList())));
25612                                }
25613                                
25614                                if (sourceFunction != null && !sourceFunction.getRelationRows().getHoldRelations().isEmpty()) {
25615                                        boolean find = false;
25616                                        if (modelObject instanceof ResultColumn) {
25617                                                ResultSetRelationRows targetRelationRows = ((ResultColumn) modelObject).getResultSet().getRelationRows();
25618                                                if(targetRelationRows.hasRelation()) {
25619                                                        for(Relationship relationship: targetRelationRows.getHoldRelations()) {
25620                                                                if(relationship.getSources().contains(sourceFunction.getRelationRows())) {
25621                                                                        find = true;
25622                                                                        break;
25623                                                                }
25624                                                        }
25625                                                }
25626                                        }
25627                                        else if (modelObject instanceof TableColumn) {
25628                                                TableRelationRows targetRelationRows = ((TableColumn) modelObject).getTable().getRelationRows();
25629                                                if(targetRelationRows.hasRelation()) {
25630                                                        for(Relationship relationship: targetRelationRows.getHoldRelations()) {
25631                                                                if(relationship.getSources().contains(sourceFunction.getRelationRows())) {
25632                                                                        find = true;
25633                                                                        break;
25634                                                                }
25635                                                        }
25636                                                }
25637                                        }
25638                                        
25639                                        if (!find) {
25640                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
25641                                                impactRelation.setEffectType(EffectType.select);
25642                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
25643                                                                sourceFunction.getRelationRows()));
25644                                                if (modelObject instanceof ResultColumn) {
25645                                                        impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
25646                                                                        ((ResultColumn) modelObject).getResultSet().getRelationRows()));
25647                                                } else if (modelObject instanceof TableColumn) {
25648                                                        impactRelation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(
25649                                                                        ((TableColumn) modelObject).getTable().getRelationRows()));
25650                                                }
25651                                        }
25652                                }
25653                        } else if (functionModel instanceof Table) {
25654                                TFunctionCall call = (TFunctionCall) functionCall;
25655                                String functionName = call.getFunctionName().toString();
25656                                boolean flag = false;
25657                                if (functionName.indexOf(".") != -1) {
25658                                        String columnName = functionName.substring(functionName.indexOf(".") + 1);
25659                                        for (TableColumn tableColumn : ((Table) functionModel).getColumns()) {
25660                                                if (DlineageUtil.sameName(ESQLDataObjectType.dotColumn, DlineageUtil.columnSegmentOf(tableColumn.getName()), columnName)) {
25661                                                        TableColumnRelationshipElement element = new TableColumnRelationshipElement(tableColumn);
25662                                                        relation.addSource(element);
25663                                                        flag = true;
25664                                                        break;
25665                                                }
25666                                        }
25667                                }
25668
25669                                if (!flag) {
25670                                        TableColumn tableColumn = modelFactory.createTableColumn((Table) functionModel,
25671                                                        ((TFunctionCall) functionCall));
25672                                        TableColumnRelationshipElement element = new TableColumnRelationshipElement(tableColumn);
25673                                        relation.addSource(element);
25674                                }
25675                        }
25676                }
25677
25678        }
25679
25680        private void analyzeSubqueryDataFlowRelation(Object gspObject, List<TSelectSqlStatement> subquerys,
25681                        EffectType effectType) {
25682                analyzeSubqueryDataFlowRelation(gspObject, subquerys, effectType, null);
25683        }
25684
25685        private void analyzeSubqueryDataFlowRelation(Object gspObject, List<TSelectSqlStatement> subquerys,
25686                        EffectType effectType, Process process) {
25687
25688                Object modelObject = modelManager.getModel(gspObject);
25689                if (modelObject == null) {
25690                        if (gspObject instanceof ResultColumn || gspObject instanceof TableColumn) {
25691                                modelObject = gspObject;
25692                        }
25693                }
25694
25695                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
25696                relation.setEffectType(effectType);
25697                relation.setProcess(process);
25698
25699                if (modelObject instanceof ResultColumn) {
25700                        relation.setTarget(new ResultColumnRelationshipElement((ResultColumn) modelObject));
25701
25702                } else if (modelObject instanceof TableColumn) {
25703                        relation.setTarget(new TableColumnRelationshipElement((TableColumn) modelObject));
25704
25705                } else {
25706                        throw new UnsupportedOperationException();
25707                }
25708
25709                for (int i = 0; i < subquerys.size(); i++) {
25710                        TSelectSqlStatement subquery = subquerys.get(i);
25711                        ResultSet resultSetModel = (ResultSet) modelManager.getModel(subquery);
25712                        if (resultSetModel != null && resultSetModel.getColumns() != null) {
25713                                for (ResultColumn column : resultSetModel.getColumns()) {
25714                                        relation.addSource(new ResultColumnRelationshipElement(column));
25715                                }
25716                        }
25717                        
25718                        if (resultSetModel != null && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
25719                                boolean find = false;
25720                                if (modelObject instanceof ResultColumn) {
25721                                        ResultSetRelationRows targetRelationRows = ((ResultColumn) modelObject).getResultSet().getRelationRows();
25722                                        if(targetRelationRows.hasRelation()) {
25723                                                for(Relationship relationship: targetRelationRows.getHoldRelations()) {
25724                                                        if(relationship.getSources().contains(resultSetModel.getRelationRows())) {
25725                                                                find = true;
25726                                                                break;
25727                                                        }
25728                                                }
25729                                        }
25730                                }
25731                                else if (modelObject instanceof TableColumn) {
25732                                        TableRelationRows targetRelationRows = ((TableColumn) modelObject).getTable().getRelationRows();
25733                                        if(targetRelationRows.hasRelation()) {
25734                                                for(Relationship relationship: targetRelationRows.getHoldRelations()) {
25735                                                        if(relationship.getSources().contains(resultSetModel.getRelationRows())) {
25736                                                                find = true;
25737                                                                break;
25738                                                        }
25739                                                }
25740                                        }
25741                                }
25742                                
25743                                if (!find) {
25744                                        ImpactRelationship impactRelation = modelFactory.createImpactRelation();
25745                                        impactRelation.setEffectType(EffectType.select);
25746                                        impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
25747                                                        resultSetModel.getRelationRows()));
25748                                        if (modelObject instanceof ResultColumn) {
25749                                                impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
25750                                                                ((ResultColumn) modelObject).getResultSet().getRelationRows()));
25751                                        } else if (modelObject instanceof TableColumn) {
25752                                                impactRelation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(
25753                                                                ((TableColumn) modelObject).getTable().getRelationRows()));
25754                                        }
25755                                }
25756                        }
25757                }
25758
25759        }
25760
25761    private Object createFunction(TParseTreeNode functionCall) {
25762        if (!processingFunctions.add(functionCall)) {
25763            return modelManager.getModel(functionCall);
25764        }
25765        try {
25766
25767            if (functionCall instanceof TFunctionCall) {
25768                TFunctionCall functionObj = (TFunctionCall) functionCall;
25769                // Generic structured-dataflow dispatch. When a vendor adapter (e.g.
25770                // Spark from_json+explode) returns a descriptor, model the function
25771                // as a structured generator: per-field result columns linked to
25772                // exact structured source paths such as nodes[*].key. When no
25773                // adapter matches, fall through to the existing function logic
25774                // unchanged.
25775                StructuredAdapterContext sctx = new StructuredAdapterContext(option.getVendor());
25776                StructuredDataflowDescriptor sdescriptor =
25777                        StructuredDataflowRegistry.defaultRegistry().describe(functionObj, sctx);
25778                if (sdescriptor != null) {
25779                    Function structuredFn = createStructuredDataflowFunction(sdescriptor);
25780                    if (structuredFn != null) {
25781                        return structuredFn;
25782                    }
25783                }
25784
25785                if (!isBuiltInFunctionName(functionObj.getFunctionName())) {
25786                    TCustomSqlStatement stmt = stmtStack.peek();
25787                    String procedureParent = getProcedureParentName(stmt);
25788                    if (procedureParent != null) {
25789                        Procedure procedureCallee = modelManager.getProcedureByName(DlineageUtil
25790                                .getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionObj)));
25791                        if (procedureCallee != null) {
25792                            if (procedureParent != null) {
25793                                Procedure caller = modelManager
25794                                        .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
25795                                if (caller != null) {
25796                                    CallRelationship callRelation = modelFactory.createCallRelation();
25797                                    callRelation.setCallObject(functionCall);
25798                                    callRelation.setTarget(new ProcedureRelationshipElement(caller));
25799                                    callRelation.addSource(new ProcedureRelationshipElement(procedureCallee));
25800                                    if (isBuiltInFunctionName(functionObj.getFunctionName()) || isKeyword(functionObj.getFunctionName())) {
25801                                        callRelation.setBuiltIn(true);
25802                                    }
25803                                }
25804                            }
25805                            if (procedureCallee.getArguments() != null) {
25806                                for (int j = 0; j < procedureCallee.getArguments().size(); j++) {
25807                                    Argument argument = procedureCallee.getArguments().get(j);
25808                                    Variable variable = modelFactory.createVariable(procedureCallee, argument.getName(), false);
25809                                    if (variable != null) {
25810                                        if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) {
25811                                            Transform transform = new Transform();
25812                                            transform.setType(Transform.FUNCTION);
25813                                            transform.setCode(functionObj);
25814                                            compositeBindingColumn(variable).setTransform(transform);
25815                                        }
25816                                        Process process = modelFactory.createProcess(functionObj);
25817                                        variable.addProcess(process);
25818                                        analyzeFunctionArgumentsDataFlowRelation(compositeBindingColumn(variable), functionObj, j, process);
25819                                    }
25820                                }
25821                            }
25822                        } else {
25823                            TFunctionCall call = (TFunctionCall) functionCall;
25824                            String functionName = call.getFunctionName().toString();
25825                            if (functionName.indexOf(".") != -1) {
25826                                Table functionTable = modelManager
25827                                        .getTableByName(functionName.substring(0, functionName.indexOf(".")));
25828                                if (functionTable != null) {
25829                                    String columnName = functionName.substring(functionName.indexOf(".") + 1);
25830                                    for (TableColumn tableColumn : functionTable.getColumns()) {
25831                                        if (DlineageUtil.sameName(ESQLDataObjectType.dotColumn, DlineageUtil.columnSegmentOf(tableColumn.getName()), columnName)) {
25832                                            return functionTable;
25833                                        }
25834                                    }
25835                                }
25836                            }
25837                            Function function = modelFactory.createFunction(call);
25838                            if (procedureParent != null) {
25839                                Procedure caller = modelManager
25840                                        .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent));
25841                                if (caller != null) {
25842                                    CallRelationship callRelation = modelFactory.createCallRelation();
25843                                    callRelation.setCallObject(functionCall);
25844                                    callRelation.setTarget(new ProcedureRelationshipElement(caller));
25845                                    callRelation.addSource(new FunctionRelationshipElement(function));
25846                                    if (isBuiltInFunctionName(functionObj.getFunctionName()) || isKeyword(functionObj.getFunctionName())) {
25847                                        callRelation.setBuiltIn(true);
25848                                    }
25849                                }
25850                            }
25851                        }
25852                    }
25853                } else if (isConstantFunction(functionObj.getFunctionName())
25854                        && (functionObj.getArgs() == null || functionObj.getArgs().size() == 0)) {
25855                    if (option.isShowConstantTable()) {
25856                        Table constantTable = modelFactory.createConstantsTable(stmtStack.peek());
25857                        modelFactory.createTableColumn(constantTable, functionObj);
25858                        return constantTable;
25859                    } else {
25860                        return null;
25861                    }
25862                }
25863
25864                if (functionObj.getFunctionType() == EFunctionType.struct_t) {
25865                    Function function = modelFactory.createFunction((TFunctionCall) functionCall);
25866                    if (functionObj instanceof TTableFunction) {
25867                        TTableFunction tableFunction = (TTableFunction) functionObj;
25868                        if (tableFunction.getFieldValues() != null) {
25869                            for (int i = 0; i < tableFunction.getFieldValues().size(); i++) {
25870                                TResultColumn resultColumn = tableFunction.getFieldValues().getResultColumn(i);
25871                                if (resultColumn.getAliasClause() != null) {
25872                                    ResultColumn column = modelFactory.createFunctionResultColumn(function,
25873                                            resultColumn.getAliasClause().getAliasName());
25874                                    columnsInExpr visitor = new columnsInExpr();
25875                                    resultColumn.getExpr().inOrderTraverse(visitor);
25876                                    List<TObjectName> objectNames = visitor.getObjectNames();
25877                                    List<TParseTreeNode> functions = visitor.getFunctions();
25878                                    if (functions != null && !functions.isEmpty()) {
25879                                        analyzeFunctionDataFlowRelation(column, functions, EffectType.function);
25880                                    }
25881                                    List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
25882                                    if (subquerys != null && !subquerys.isEmpty()) {
25883                                        analyzeSubqueryDataFlowRelation(column, subquerys, EffectType.function);
25884                                    }
25885                                    analyzeDataFlowRelation(column, objectNames, EffectType.function, functions);
25886                                    List<TParseTreeNode> constants = visitor.getConstants();
25887                                    analyzeConstantDataFlowRelation(column, constants, EffectType.function, functions);
25888                                } else if (resultColumn.getFieldAttr() != null) {
25889                                    ResultColumn column = modelFactory.createFunctionResultColumn(function,
25890                                            resultColumn.getFieldAttr());
25891                                    analyzeDataFlowRelation(column, Arrays.asList(resultColumn.getFieldAttr()), EffectType.function, null);
25892                                } else if (resultColumn.getExpr() != null) {
25893                                    if (resultColumn.getExpr().getFunctionCall() != null) {
25894                                        Function resultColumnFunction = (Function) createFunction(
25895                                                resultColumn.getExpr().getFunctionCall());
25896                                        String functionName = getResultSetName(function);
25897                                        for (int j = 0; j < resultColumnFunction.getColumns().size(); j++) {
25898                                            TObjectName columnName = new TObjectName();
25899                                            if (resultColumn.getAliasClause() != null) {
25900                                                columnName.setString(resultColumn.getAliasClause() + "." + getColumnNameOnly(
25901                                                        resultColumnFunction.getColumns().get(j).getName()));
25902                                            } else {
25903                                                columnName.setString(functionName + "." + getColumnNameOnly(
25904                                                        resultColumnFunction.getColumns().get(j).getName()));
25905                                            }
25906                                            ResultColumn functionResultColumn = modelFactory.createResultColumn(function,
25907                                                    columnName);
25908                                            DataFlowRelationship relationship = modelFactory.createDataFlowRelation();
25909                                            relationship.setTarget(new ResultColumnRelationshipElement(functionResultColumn));
25910                                            relationship.addSource(new ResultColumnRelationshipElement(
25911                                                    resultColumnFunction.getColumns().get(j)));
25912                                        }
25913                                    } else if (resultColumn.getExpr().getCaseExpression() != null) {
25914                                        function = modelFactory.createFunction(resultColumn.getExpr().getCaseExpression());
25915                                        ResultColumn column = modelFactory.createFunctionResultColumn(function,
25916                                                ((TCaseExpression) resultColumn.getExpr().getCaseExpression()).getWhenClauseItemList());
25917                                        analyzeFunctionArgumentsDataFlowRelation(column, functionCall);
25918                                    }
25919                                }
25920                            }
25921                            return function;
25922                        }
25923                    } else if (functionObj instanceof TFunctionCall) {
25924
25925                    }
25926                }
25927
25928                // BigQuery array_agg(table_alias) row-reference expansion
25929                if (functionObj.getFunctionType() == EFunctionType.array_agg_t
25930                        && option.getVendor() == EDbVendor.dbvbigquery
25931                        && functionObj.getArgs() != null
25932                        && functionObj.getArgs().size() == 1) {
25933
25934                    TExpression arg0 = functionObj.getArgs().getExpression(0);
25935                    if (arg0 != null
25936                            && arg0.getExpressionType() == EExpressionType.simple_object_name_t) {
25937
25938                        TObjectName on = arg0.getObjectOperand();
25939                        if (isArrayAggRowReference(on)) {
25940                            Function function = (Function) modelManager.getModel(functionCall);
25941                            if (function == null) {
25942                                function = modelFactory.createFunction((TFunctionCall) functionCall);
25943                            }
25944
25945                            // Only bind once (avoid duplicate processing when
25946                            // createFunction is called from analyzeFunctionDataFlowRelation)
25947                            if (function.getColumns() == null || function.getColumns().isEmpty()) {
25948                                TTable srcTable = on.getSourceTable();
25949                                String tableAlias = srcTable.getAliasName() != null
25950                                        ? srcTable.getAliasName().toString() : null;
25951
25952                                Table sourceTable = (Table) modelManager.getModel(srcTable);
25953                                sourceTable.removeColumn(tableAlias);
25954
25955                                // Collect inferred column names from ORDER BY (in function)
25956                                // and GROUP BY (in enclosing SELECT). Excludes the table alias.
25957                                List<TObjectName> inferredColumns = collectRowReferenceInferredColumns(
25958                                        functionObj, tableAlias);
25959
25960                                // Create a single * function result column carrying star-link
25961                                // metadata for the inferred columns and direct source edges
25962                                // for * and each inferred column.
25963                                createRowReferenceStarColumn(function, srcTable, inferredColumns);
25964                            }
25965
25966                            return function;
25967                        }
25968                    }
25969                }
25970
25971                if (functionObj.getFunctionType() == EFunctionType.array_t || functionObj.getFunctionType() == EFunctionType.array_agg_t) {
25972                    Function function = modelFactory.createFunction((TFunctionCall) functionCall);
25973                    if (functionObj.getArgs() != null) {
25974                        if (functionObj.getArgs().getExpression(0).getSubQuery() != null) {
25975                            TSelectSqlStatement stmt = functionObj.getArgs().getExpression(0).getSubQuery();
25976                            analyzeSelectStmt(stmt);
25977                            ResultSet resultset = (ResultSet) modelManager.getModel(stmt);
25978                            for (int i = 0; i < resultset.getColumns().size(); i++) {
25979                                ResultColumn sourceColumn = resultset.getColumns().get(i);
25980                                TObjectName columnName = new TObjectName();
25981                                columnName.setString(sourceColumn.getName());
25982                                ResultColumn resultColumn = modelFactory.createFunctionResultColumn(function,
25983                                        columnName);
25984                                DataFlowRelationship relationship = modelFactory.createDataFlowRelation();
25985                                relationship.setTarget(new ResultColumnRelationshipElement(resultColumn));
25986                                relationship.addSource(
25987                                        new ResultColumnRelationshipElement(sourceColumn));
25988                            }
25989                            return function;
25990                        } else if (functionObj.getArgs().getExpression(0).getExpressionType() == EExpressionType.function_t) {
25991                            Object functionTableModelObj = createFunction(functionObj.getArgs().getExpression(0).getFunctionCall());
25992                            if (functionTableModelObj instanceof ResultSet) {
25993                                ResultSet resultset = (ResultSet) functionTableModelObj;
25994                                for (int i = 0; i < resultset.getColumns().size(); i++) {
25995                                    ResultColumn sourceColumn = resultset.getColumns().get(i);
25996                                    TObjectName columnName = new TObjectName();
25997                                    columnName.setString(sourceColumn.getName());
25998                                    ResultColumn resultColumn = modelFactory.createFunctionResultColumn(function, columnName);
25999                                    DataFlowRelationship relationship = modelFactory.createDataFlowRelation();
26000                                    relationship.setTarget(new ResultColumnRelationshipElement(resultColumn));
26001                                    relationship.addSource(new ResultColumnRelationshipElement(sourceColumn));
26002                                }
26003                                return function;
26004                            }
26005                        }
26006                    }
26007                }
26008
26009                Function function = modelFactory.createFunction((TFunctionCall) functionCall);
26010                ResultColumn column = modelFactory.createFunctionResultColumn(function,
26011                        ((TFunctionCall) functionCall).getFunctionName());
26012                if ("COUNT".equalsIgnoreCase(((TFunctionCall) functionCall).getFunctionName().toString())) {
26013                    // @see https://e.gitee.com/gudusoft/issues/list?issue=I40NUP
26014                    // COUNT特殊处理,不和参数关联
26015                    if (option.isShowCountTableColumn()) {
26016                        analyzeFunctionArgumentsDataFlowRelation(column, functionCall);
26017                    }
26018                } else {
26019                    boolean isCustomFunction = analyzeCustomFunctionCall((TFunctionCall) functionCall);
26020//                              if(!isCustomFunction) 
26021                    {
26022                        analyzeFunctionArgumentsDataFlowRelation(column, functionCall);
26023                    }
26024                    Set<Object> functionTableModelObjs = modelManager.getFunctionTable(getIdentifiedFunctionName(function));
26025                    if (functionTableModelObjs != null) {
26026                        for (Object functionTableModelObj : functionTableModelObjs) {
26027                            if (functionTableModelObj instanceof ResultSet) {
26028                                ResultSet functionTableModel = (ResultSet) functionTableModelObj;
26029                                if (functionTableModel.getColumns() != null) {
26030                                    for (int j = 0; j < functionTableModel.getColumns().size(); j++) {
26031                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
26032                                        relation.setEffectType(EffectType.select);
26033                                        relation.setTarget(new ResultColumnRelationshipElement(column));
26034                                        relation.addSource(new ResultColumnRelationshipElement(
26035                                                functionTableModel.getColumns().get(j)));
26036                                    }
26037                                }
26038                            }
26039                        }
26040                    }
26041                }
26042                return function;
26043            } else if (functionCall instanceof TCaseExpression) {
26044                Function function = modelFactory.createFunction((TCaseExpression) functionCall);
26045                ResultColumn column = modelFactory.createFunctionResultColumn(function,
26046                        ((TCaseExpression) functionCall).getWhenClauseItemList());
26047                analyzeFunctionArgumentsDataFlowRelation(column, functionCall);
26048                return function;
26049            } else if (functionCall instanceof TObjectName) {
26050                Function function = modelFactory.createFunction((TObjectName) functionCall);
26051                TObjectName columnName = new TObjectName();
26052                columnName.setString(function.getFunctionName());
26053                ResultColumn column = modelFactory.createResultColumn(function,
26054                        columnName);
26055                analyzeFunctionArgumentsDataFlowRelation(column, functionCall);
26056                return function;
26057            }
26058            return null;
26059        } finally {
26060            processingFunctions.remove(functionCall);
26061        }
26062    }
26063
26064        protected String getIdentifiedFunctionName(Function function) {
26065                return DlineageUtil.getIdentifierNormalFunctionName(function.getFunctionName());
26066        }
26067        
26068        private boolean isConstantFunction(TObjectName functionName) {
26069                boolean result = CONSTANT_BUILTIN_FUNCTIONS.contains(functionName.toString().toUpperCase());
26070                if (result) {
26071                        return true;
26072                }
26073                return false;
26074        }
26075
26076        private void analyzeFunctionArgumentsDataFlowRelation(Object resultColumn, TParseTreeNode gspObject) {
26077                List<TExpression> directExpressions = new ArrayList<TExpression>();
26078                List<TExpression> indirectExpressions = new ArrayList<TExpression>();
26079                List<TExpression> conditionExpressions = new ArrayList<TExpression>();
26080                if (gspObject instanceof TFunctionCall) {
26081                        TFunctionCall functionCall = (TFunctionCall) gspObject;
26082                        getFunctionExpressions(directExpressions, indirectExpressions, functionCall);
26083                } else if (gspObject instanceof TCaseExpression) {
26084                        TCaseExpression expr = (TCaseExpression) gspObject;
26085                        TExpression inputExpr = expr.getInput_expr();
26086                        if (inputExpr != null) {
26087                                if(option.isShowCaseWhenAsDirect()){
26088                                        directExpressions.add(inputExpr);
26089                                }
26090                                else {
26091                                        conditionExpressions.add(inputExpr);
26092                                }
26093                        }
26094                        TExpression defaultExpr = expr.getElse_expr();
26095                        if (defaultExpr != null) {
26096                                directExpressions.add(defaultExpr);
26097                        }
26098                        TWhenClauseItemList list = expr.getWhenClauseItemList();
26099                        for (int i = 0; i < list.size(); i++) {
26100                                TWhenClauseItem element = list.getWhenClauseItem(i);
26101                                if(option.isShowCaseWhenAsDirect()){
26102                                        directExpressions.add(element.getComparison_expr());
26103                                }
26104                                else {
26105                                        conditionExpressions.add(element.getComparison_expr());
26106                                }
26107                                directExpressions.add(element.getReturn_expr());
26108                        }
26109                }
26110
26111                for (int j = 0; j < directExpressions.size(); j++) {
26112                        columnsInExpr visitor = new columnsInExpr();
26113                        directExpressions.get(j).inOrderTraverse(visitor);
26114
26115                        List<TObjectName> objectNames = visitor.getObjectNames();
26116                        List<TParseTreeNode> functions = visitor.getFunctions();
26117
26118                        if (functions != null && !functions.isEmpty()) {
26119                                analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function);
26120                        }
26121
26122                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
26123                        if (subquerys != null && !subquerys.isEmpty()) {
26124                                analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.function);
26125                        }
26126
26127                        analyzeDataFlowRelation(resultColumn, objectNames, EffectType.function, functions);
26128
26129                        List<TParseTreeNode> constants = visitor.getConstants();
26130                        analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.function, functions);
26131                }
26132
26133                conditionExpressions.addAll(indirectExpressions);
26134                for (int j = 0; j < conditionExpressions.size(); j++) {
26135                        analyzeFilterCondition(resultColumn, conditionExpressions.get(j), null, null, EffectType.function);
26136                }
26137        }
26138        
26139        private void analyzeFunctionArgumentsDataFlowRelation(Object resultColumn, TCallStatement callStatment, int argumentIndex, Process process) {
26140                List<TExpression> directExpressions = new ArrayList<TExpression>();
26141                List<TExpression> indirectExpressions = new ArrayList<TExpression>();
26142                List<TExpression> conditionExpressions = new ArrayList<TExpression>();
26143
26144                getFunctionExpressions(directExpressions, indirectExpressions, callStatment, argumentIndex);
26145
26146                for (int j = 0; j < directExpressions.size(); j++) {
26147                        columnsInExpr visitor = new columnsInExpr();
26148                        directExpressions.get(j).inOrderTraverse(visitor);
26149
26150                        List<TObjectName> objectNames = visitor.getObjectNames();
26151                        List<TParseTreeNode> functions = visitor.getFunctions();
26152
26153                        if (functions != null && !functions.isEmpty()) {
26154                                analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function);
26155                        }
26156
26157                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
26158                        if (subquerys != null && !subquerys.isEmpty()) {
26159                                analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.function);
26160                        }
26161
26162                        DataFlowRelationship relation = analyzeDataFlowRelation(resultColumn, objectNames, EffectType.function, functions);
26163                        if (relation != null) {
26164                                relation.setProcess(process);
26165                        }
26166
26167                        List<TParseTreeNode> constants = visitor.getConstants();
26168                        analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.function, functions);
26169                }
26170
26171                conditionExpressions.addAll(indirectExpressions);
26172                for (int j = 0; j < conditionExpressions.size(); j++) {
26173                        analyzeFilterCondition(resultColumn, conditionExpressions.get(j), null, null, EffectType.function);
26174                }
26175        }
26176
26177        private void analyzeFunctionArgumentsDataFlowRelation(Object resultColumn, TDb2CallStmt callStatment, int argumentIndex, Process process) {
26178                List<TExpression> directExpressions = new ArrayList<TExpression>();
26179                List<TExpression> indirectExpressions = new ArrayList<TExpression>();
26180                List<TExpression> conditionExpressions = new ArrayList<TExpression>();
26181
26182                getFunctionExpressions(directExpressions, indirectExpressions, callStatment, argumentIndex);
26183
26184                for (int j = 0; j < directExpressions.size(); j++) {
26185                        columnsInExpr visitor = new columnsInExpr();
26186                        directExpressions.get(j).inOrderTraverse(visitor);
26187
26188                        List<TObjectName> objectNames = visitor.getObjectNames();
26189                        List<TParseTreeNode> functions = visitor.getFunctions();
26190
26191                        if (functions != null && !functions.isEmpty()) {
26192                                analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function);
26193                        }
26194
26195                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
26196                        if (subquerys != null && !subquerys.isEmpty()) {
26197                                analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.function);
26198                        }
26199
26200                        DataFlowRelationship relation = analyzeDataFlowRelation(resultColumn, objectNames, EffectType.function, functions);
26201                        if (relation != null) {
26202                                relation.setProcess(process);
26203                        }
26204
26205                        List<TParseTreeNode> constants = visitor.getConstants();
26206                        analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.function, functions);
26207                }
26208
26209                conditionExpressions.addAll(indirectExpressions);
26210                for (int j = 0; j < conditionExpressions.size(); j++) {
26211                        analyzeFilterCondition(resultColumn, conditionExpressions.get(j), null, null, EffectType.function);
26212                }
26213        }
26214
26215        private void analyzeFunctionArgumentsDataFlowRelation(Object resultColumn, TFunctionCall functionCall, int argumentIndex, Process process) {
26216                List<TExpression> directExpressions = new ArrayList<TExpression>();
26217                List<TExpression> indirectExpressions = new ArrayList<TExpression>();
26218                List<TExpression> conditionExpressions = new ArrayList<TExpression>();
26219
26220                getFunctionExpressions(directExpressions, indirectExpressions, functionCall, argumentIndex);
26221
26222                for (int j = 0; j < directExpressions.size(); j++) {
26223                        columnsInExpr visitor = new columnsInExpr();
26224                        directExpressions.get(j).inOrderTraverse(visitor);
26225
26226                        List<TObjectName> objectNames = visitor.getObjectNames();
26227                        List<TParseTreeNode> functions = visitor.getFunctions();
26228
26229                        if (functions != null && !functions.isEmpty()) {
26230                                analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function);
26231                        }
26232
26233                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
26234                        if (subquerys != null && !subquerys.isEmpty()) {
26235                                analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.function);
26236                        }
26237
26238                        DataFlowRelationship relation = analyzeDataFlowRelation(resultColumn, objectNames, EffectType.function, functions);
26239                        if (relation != null) {
26240                                relation.setProcess(process);
26241                        }
26242
26243                        List<TParseTreeNode> constants = visitor.getConstants();
26244                        analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.function, functions);
26245                }
26246
26247                conditionExpressions.addAll(indirectExpressions);
26248                for (int j = 0; j < conditionExpressions.size(); j++) {
26249                        analyzeFilterCondition(resultColumn, conditionExpressions.get(j), null, null, EffectType.function);
26250                }
26251        }
26252
26253        private void analyzeFunctionArgumentsDataFlowRelation(Object resultColumn, TMssqlExecute functionCall, String argumentName, int argumentIndex, Process process) {
26254                List<TExpression> directExpressions = new ArrayList<TExpression>();
26255                List<TExpression> indirectExpressions = new ArrayList<TExpression>();
26256                List<TExpression> conditionExpressions = new ArrayList<TExpression>();
26257
26258                getFunctionExpressions(directExpressions, indirectExpressions, functionCall, argumentName, argumentIndex);
26259
26260                for (int j = 0; j < directExpressions.size(); j++) {
26261                        columnsInExpr visitor = new columnsInExpr();
26262                        directExpressions.get(j).inOrderTraverse(visitor);
26263
26264                        List<TObjectName> objectNames = visitor.getObjectNames();
26265                        List<TParseTreeNode> functions = visitor.getFunctions();
26266
26267                        if (functions != null && !functions.isEmpty()) {
26268                                analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function);
26269                        }
26270
26271                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
26272                        if (subquerys != null && !subquerys.isEmpty()) {
26273                                analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.function);
26274                        }
26275
26276                        DataFlowRelationship relation = analyzeDataFlowRelation(resultColumn, objectNames, EffectType.function, functions);
26277                        if (relation != null) {
26278                                relation.setProcess(process);
26279                        }
26280
26281                        List<TParseTreeNode> constants = visitor.getConstants();
26282                        analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.function, functions);
26283                }
26284
26285                conditionExpressions.addAll(indirectExpressions);
26286                for (int j = 0; j < conditionExpressions.size(); j++) {
26287                        analyzeFilterCondition(resultColumn, conditionExpressions.get(j), null, null, EffectType.function);
26288                }
26289        }
26290
26291
26292        private static void addResultColumnListExpressions(
26293                        TResultColumnList list, List<TExpression> target) {
26294                if (list == null) return;
26295                for (int k = 0; k < list.size(); k++) {
26296                        TExpression e = list.getResultColumn(k).getExpr();
26297                        if (e != null) target.add(e);
26298                }
26299        }
26300
26301        private static void addPassingClauseExpressions(
26302                        TFunctionCall fc, List<TExpression> target) {
26303                TResultColumnList list = null;
26304                if (fc.getXmlPassingClause() != null
26305                                && fc.getXmlPassingClause().getPassingList() != null) {
26306                        list = fc.getXmlPassingClause().getPassingList();
26307                } else if (fc.getPassingClause() != null
26308                                && fc.getPassingClause().getPassingList() != null) {
26309                        list = fc.getPassingClause().getPassingList();
26310                }
26311                addResultColumnListExpressions(list, target);
26312        }
26313
26314        private void collectOracleXmlFunctionExpressions(
26315                        TFunctionCall fc,
26316                        List<TExpression> direct,
26317                        List<TExpression> indirect) {
26318
26319                if (option.getVendor() != EDbVendor.dbvoracle) return;
26320
26321                // XMLELEMENT: value exprs + XMLATTRIBUTES exprs (direct)
26322                addResultColumnListExpressions(fc.getXMLElementValueExprList(), direct);
26323                if (fc.getXMLAttributesClause() != null) {
26324                        addResultColumnListExpressions(
26325                                        fc.getXMLAttributesClause().getValueExprList(), direct);
26326                }
26327
26328                // XMLFOREST (direct)
26329                addResultColumnListExpressions(fc.getXMLForestValueList(), direct);
26330
26331                // EXTRACT(XML) — AST-shape disambiguation (V3)
26332                // XML form populates getXMLType_Instance(); scalar EXTRACT(YEAR FROM d) does not.
26333                if (fc.getXMLType_Instance() != null) {
26334                        direct.add(fc.getXMLType_Instance());
26335                }
26336
26337                String name = fc.getFunctionName() == null
26338                                ? "" : fc.getFunctionName().toString().toUpperCase();
26339
26340                // XMLEXISTS indirect (filter) from PASSING expressions
26341                if ("XMLEXISTS".equals(name)) {
26342                        addPassingClauseExpressions(fc, indirect);
26343                }
26344
26345                // XMLAGG / SYS_XMLAGG ORDER BY → indirect
26346                if (("XMLAGG".equals(name) || "SYS_XMLAGG".equals(name))
26347                                && fc.getSortClause() != null) {
26348                        TOrderByItemList orderByList = fc.getSortClause().getItems();
26349                        if (orderByList != null) {
26350                                for (int k = 0; k < orderByList.size(); k++) {
26351                                        TExpression e = orderByList.getOrderByItem(k).getSortKey();
26352                                        if (e != null) indirect.add(e);
26353                                }
26354                        }
26355                }
26356
26357                // DELIBERATELY NOT HARVESTED (metadata, not data):
26358                //   fc.getTypeExpression()       — target type for XMLCAST
26359                //   fc.getXMLElementNameExpr()   — element tag identifier
26360                //   XPath / XQuery literal strings that appear as function args
26361                //   datatype / format / style tokens
26362        }
26363
26364        private void collectArrayAggOrderByExpressions(
26365                        TFunctionCall fc,
26366                        List<TExpression> direct,
26367                        List<TExpression> indirect) {
26368                if (fc.getFunctionType() != EFunctionType.array_agg_t) return;
26369
26370                TOrderByItemList items = null;
26371                if (fc.getSortClause() != null) items = fc.getSortClause().getItems();
26372                if (items == null || items.size() == 0) {
26373                        items = fc.getOrderByList();
26374                }
26375                if (items == null) return;
26376
26377                // Resolve source tables for ORDER BY columns that the parser
26378                // doesn't link (they're inside a function call, outside the
26379                // resolver's normal scope)
26380                TTable fallbackTable = null;
26381                if (!stmtStack.isEmpty() && stmtStack.peek() instanceof TSelectSqlStatement) {
26382                        TTableList tables = ((TSelectSqlStatement) stmtStack.peek()).tables;
26383                        if (tables != null && tables.size() > 0) {
26384                                fallbackTable = tables.getTable(0);
26385                        }
26386                }
26387
26388                for (int k = 0; k < items.size(); k++) {
26389                        TExpression e = items.getOrderByItem(k).getSortKey();
26390                        if (e != null) {
26391                                if (fallbackTable != null
26392                                                && e.getExpressionType() == EExpressionType.simple_object_name_t
26393                                                && e.getObjectOperand() != null
26394                                                && e.getObjectOperand().getSourceTable() == null) {
26395                                        e.getObjectOperand().setSourceTable(fallbackTable);
26396                                }
26397                                direct.add(e);
26398                        }
26399                }
26400        }
26401
26402        private boolean isArrayAggRowReference(TObjectName on) {
26403                if (on == null) return false;
26404                TTable src = on.getSourceTable();
26405                if (src == null) return false;
26406
26407                // Raw equivalent of the old getColumnName extraction (toString fallback preserved).
26408                String raw = on.getColumnNameOnly();
26409                if (raw == null || raw.trim().isEmpty()) {
26410                        raw = on.toString() == null ? "" : on.toString().trim();
26411                } else {
26412                        raw = raw.trim();
26413                }
26414                if (raw.isEmpty()) return false;
26415
26416                // Check if identifier text matches the table alias or table name (canonical dotTable).
26417                boolean matchesAlias =
26418                        (src.getAliasName() != null
26419                                && SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotTable, raw, src.getAliasName().toString()))
26420                        || (src.getTableName() != null
26421                                && SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotTable, raw, src.getTableName().toString()));
26422                return matchesAlias;
26423        }
26424
26425        /**
26426         * Collects column names inferred from a row-reference array_agg:
26427         *  - ORDER BY keys inside the function call
26428         *  - GROUP BY keys in the enclosing SELECT
26429         * Excludes the table alias itself (so `original` in `array_agg(original)`
26430         * does not leak into the column list).
26431         */
26432        private List<TObjectName> collectRowReferenceInferredColumns(
26433                        TFunctionCall functionCall, String tableAlias) {
26434                List<TObjectName> columns = new ArrayList<TObjectName>();
26435
26436                TOrderByItemList orderItems = null;
26437                if (functionCall.getSortClause() != null) {
26438                        orderItems = functionCall.getSortClause().getItems();
26439                }
26440                if (orderItems == null || orderItems.size() == 0) {
26441                        orderItems = functionCall.getOrderByList();
26442                }
26443                if (orderItems != null) {
26444                        for (int k = 0; k < orderItems.size(); k++) {
26445                                TExpression sortKey = orderItems.getOrderByItem(k).getSortKey();
26446                                collectInferredColumn(sortKey, tableAlias, columns);
26447                        }
26448                }
26449
26450                if (!stmtStack.isEmpty() && stmtStack.peek() instanceof TSelectSqlStatement) {
26451                        TSelectSqlStatement select = (TSelectSqlStatement) stmtStack.peek();
26452                        if (select.getGroupByClause() != null) {
26453                                TGroupByItemList groupByList = select.getGroupByClause().getItems();
26454                                for (int k = 0; k < groupByList.size(); k++) {
26455                                        TExpression expr = groupByList.getGroupByItem(k).getExpr();
26456                                        collectInferredColumn(expr, tableAlias, columns);
26457                                }
26458                        }
26459                }
26460                return columns;
26461        }
26462
26463        private void collectInferredColumn(TExpression expr, String tableAlias,
26464                        List<TObjectName> columns) {
26465                if (expr == null) return;
26466                if (expr.getExpressionType() != EExpressionType.simple_object_name_t) return;
26467                if (expr.getObjectOperand() == null) return;
26468                String colName = DlineageUtil.getColumnName(expr.getObjectOperand());
26469                if (colName == null || colName.isEmpty()) return;
26470                if (tableAlias != null && SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotTable, tableAlias, colName)) return;
26471                if(!columns.contains(expr.getObjectOperand())) {
26472                        columns.add(expr.getObjectOperand());
26473                }
26474        }
26475
26476        /**
26477         * Creates a `*` FunctionResultColumn plus one FunctionResultColumn per
26478         * inferred column. Each column gets its own source-edge back to
26479         * the source table (source.* -> function.*, source.id -> function.id, etc).
26480         *
26481         * Downstream, the `*` column collects all three edges into the inner
26482         * `unique` ResultColumn, and post-processing (rewriteArrayAggRowReferenceTargets)
26483         * splits the collapsed CTAS relation into one per column.
26484         */
26485        private void createRowReferenceStarColumn(Function function, TTable src,
26486                        List<TObjectName> inferredColumns) {
26487                Object tableModel = modelManager.getModel(src);
26488
26489                // Create function.* and bind source.* -> function.*
26490                TObjectName starName = new TObjectName();
26491                starName.setString("*");
26492                ResultColumn starColumn = modelFactory.createFunctionResultColumn(function, starName);
26493                starColumn.setStruct(true);
26494                starColumn.setShowStar(true);
26495
26496                if (tableModel instanceof Table) {
26497                        Table sourceTable = (Table) tableModel;
26498                        TableColumn sourceCol = findOrCreateTableColumn(sourceTable, "*");
26499                        
26500                        for (TObjectName colName : inferredColumns) {
26501                                TableColumn tableColumn = modelFactory.createTableColumn(sourceTable, colName, false);
26502                                if (tableColumn != null) {
26503                                        //order by and group by fdr relation
26504                                        AbstractRelationship relation = modelFactory.createImpactRelation();
26505                                        relation.setEffectType(EffectType.function);
26506                                        relation.setTarget(new ResultColumnRelationshipElement(starColumn));
26507                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
26508                                        
26509                                        // star column fdd relation
26510                                        relation = modelFactory.createDataFlowRelation();
26511                                        relation.setEffectType(EffectType.function);
26512                                        relation.setTarget(new ResultColumnRelationshipElement(starColumn));
26513                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
26514                                }
26515                        }
26516
26517                        
26518                        if (sourceCol != null) {
26519                                DataFlowRelationship rel = modelFactory.createDataFlowRelation();
26520                                rel.setEffectType(EffectType.function);
26521                                rel.setTarget(new ResultColumnRelationshipElement(starColumn));
26522                                rel.addSource(new TableColumnRelationshipElement(sourceCol));
26523                        }
26524                        
26525                } else if (tableModel instanceof ResultSet) {
26526                        ResultSet sourceRS = (ResultSet) tableModel;
26527                        ResultColumn sourceCol = findOrCreateResultColumn(sourceRS, "*");
26528                        for (TObjectName colName : inferredColumns) {
26529                                sourceCol.bindStarLinkColumn(colName);
26530                        }
26531                        
26532                        if (sourceCol != null) {
26533                                DataFlowRelationship rel = modelFactory.createDataFlowRelation();
26534                                rel.setEffectType(EffectType.function);
26535                                rel.setTarget(new ResultColumnRelationshipElement(starColumn));
26536                                rel.addSource(new ResultColumnRelationshipElement(sourceCol));
26537                        }
26538                }
26539        }
26540
26541        /**
26542         * Builds the Function lineage model for a structured-dataflow descriptor.
26543         * Creates per-field FunctionResultColumns and links each to an exact
26544         * structured source TableColumn (e.g. {@code nodes[*].key}). Returns the
26545         * Function on success; returns {@code null} when the source column cannot
26546         * be resolved so the caller falls back to existing function lineage.
26547         */
26548        private Function createStructuredDataflowFunction(StructuredDataflowDescriptor desc) {
26549                TParseTreeNode syntaxNode = desc.getSyntaxNode();
26550                if (!(syntaxNode instanceof TFunctionCall)) {
26551                        return null;
26552                }
26553                TFunctionCall fnCall = (TFunctionCall) syntaxNode;
26554
26555                Function function = null;
26556                Object existing = modelManager.getModel(fnCall);
26557                if (existing instanceof Function) {
26558                        function = (Function) existing;
26559                }
26560                boolean alreadyBuilt = function != null
26561                                && function.getColumns() != null
26562                                && !function.getColumns().isEmpty();
26563                if (alreadyBuilt) {
26564                        return function;
26565                }
26566                if (function == null) {
26567                        function = modelFactory.createFunction(fnCall);
26568                }
26569
26570                StructuredValueSource src = desc.getSource();
26571                TObjectName sourceColumn = src.getSourceColumn();
26572                TTable srcTable = sourceColumn != null ? sourceColumn.getSourceTable() : null;
26573                Object srcTableModel = srcTable != null ? modelManager.getModel(srcTable) : null;
26574                if (!(srcTableModel instanceof Table)) {
26575                        return null;
26576                }
26577                Table sourceTable = (Table) srcTableModel;
26578
26579                for (StructuredFieldBinding fb : desc.getFieldBindings()) {
26580                        String pathDisplay = fb.getSourcePath().toDisplayString();
26581                        TableColumn pathColumn = findOrCreateStructuredPathColumn(sourceTable, pathDisplay);
26582                        if (pathColumn == null) {
26583                                continue;
26584                        }
26585
26586                        TObjectName fieldName = new TObjectName();
26587                        fieldName.setString(fb.getOutputFieldName());
26588                        ResultColumn fnResult = modelFactory.createFunctionResultColumn(function, fieldName);
26589
26590                        DataFlowRelationship rel = modelFactory.createDataFlowRelation();
26591                        rel.setEffectType(EffectType.function);
26592                        rel.setTarget(new ResultColumnRelationshipElement(fnResult));
26593                        rel.addSource(new TableColumnRelationshipElement(pathColumn));
26594                }
26595                return function;
26596        }
26597
26598        /**
26599         * Resolve a struct-field-style outer reference ({@code structAlias.field}
26600         * or {@code relAlias.structAlias.field}) against in-scope subquery
26601         * QueryTables that hold a struct-bearing result column. When found,
26602         * append a relation source pointing at the function's field result
26603         * column so the exact source path (e.g. {@code nodes[*].key}) is
26604         * preserved through the outer projection. Returns {@code true} when
26605         * the reference was bound and the caller should skip orphan-column
26606         * fallback.
26607         */
26608        private boolean tryAppendStructuredFieldRelation(DataFlowRelationship relation,
26609                        TObjectName columnName, TTableList tableList) {
26610                if (relation == null || columnName == null || tableList == null) {
26611                        return false;
26612                }
26613                String fullText = columnName.toString();
26614                if (fullText == null) return false;
26615                int firstDot = fullText.indexOf('.');
26616                if (firstDot < 0) return false;
26617
26618                String structAlias;
26619                String fieldName;
26620                int lastDot = fullText.lastIndexOf('.');
26621                if (firstDot == lastDot) {
26622                        structAlias = fullText.substring(0, firstDot);
26623                        fieldName = fullText.substring(firstDot + 1);
26624                } else {
26625                        structAlias = fullText.substring(firstDot + 1, lastDot);
26626                        fieldName = fullText.substring(lastDot + 1);
26627                }
26628                if (structAlias == null || structAlias.isEmpty()) return false;
26629                if (fieldName == null || fieldName.isEmpty()) return false;
26630
26631                for (int i = 0; i < tableList.size(); i++) {
26632                        TTable tt = tableList.getTable(i);
26633                        if (tt == null) continue;
26634                        Object tm = modelManager.getModel(tt);
26635                        if (!(tm instanceof QueryTable)) continue;
26636                        QueryTable qt = (QueryTable) tm;
26637                        if (qt.getColumns() == null) continue;
26638                        for (ResultColumn rc : qt.getColumns()) {
26639                                if (rc == null || rc.getName() == null) continue;
26640                                if (!SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotColumn, structAlias, rc.getName())) continue;
26641                                ResultColumn fieldColumn = findStructuredFieldColumn(rc, fieldName);
26642                                if (fieldColumn != null) {
26643                                        relation.addSource(new ResultColumnRelationshipElement(fieldColumn));
26644                                        return true;
26645                                }
26646                        }
26647                }
26648                return false;
26649        }
26650
26651        /**
26652         * Given a QueryTable result column that aliases a structured generator
26653         * (e.g. {@code nodes1} backed by Spark {@code explode(from_json(...))}),
26654         * locate the FunctionResultColumn named {@code fieldName}. The Function
26655         * model is discovered by walking the existing data-flow relations whose
26656         * target is the alias column.
26657         */
26658        private ResultColumn findStructuredFieldColumn(ResultColumn aliasColumn, String fieldName) {
26659                if (aliasColumn == null || fieldName == null) return null;
26660                Relationship[] rels = modelManager.getRelations();
26661                if (rels == null) return null;
26662                String normalField = fieldName.trim();
26663                for (Relationship r : rels) {
26664                        if (!(r instanceof DataFlowRelationship)) continue;
26665                        DataFlowRelationship dfr = (DataFlowRelationship) r;
26666                        if (dfr.getTarget() == null) continue;
26667                        if (!(dfr.getTarget().getElement() == aliasColumn)) continue;
26668                        if (dfr.getSources() == null) continue;
26669                        for (RelationshipElement<?> se : dfr.getSources()) {
26670                                Object srcElement = se.getElement();
26671                                if (!(srcElement instanceof ResultColumn)) continue;
26672                                ResultColumn src = (ResultColumn) srcElement;
26673                                if (SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotColumn, normalField, src.getName())) {
26674                                        return src;
26675                                }
26676                        }
26677                }
26678                return null;
26679        }
26680
26681        /**
26682         * Find-or-create a TableColumn carrying a structured display path
26683         * (e.g. {@code nodes[*].key}) on the given source table. Unlike the
26684         * standard {@link #findOrCreateTableColumn(Table, String)}, this does
26685         * not treat the dotted/bracketed text as a schema-qualified name;
26686         * the entire path is used as the column display, so existing flat
26687         * columns named after the leaf field (e.g. {@code key}) do not
26688         * collide with structured-path columns.
26689         */
26690        private TableColumn findOrCreateStructuredPathColumn(Table table, String pathDisplay) {
26691                if (table == null || pathDisplay == null || pathDisplay.isEmpty()) return null;
26692                for (TableColumn tc : table.getColumns()) {
26693                        if (pathDisplay.equalsIgnoreCase(tc.getName())) {
26694                                return tc;
26695                        }
26696                }
26697                return new TableColumn(table, pathDisplay);
26698        }
26699
26700        private TableColumn findOrCreateTableColumn(Table table, String colName) {
26701                String normalName = DlineageUtil.getIdentifierNormalColumnName(colName);
26702                for (TableColumn tc : table.getColumns()) {
26703                        if ("*".equals(colName) && "*".equals(tc.getName())) return tc;
26704                        if (DlineageUtil.compareColumnIdentifier(tc.getName(), normalName)) return tc;
26705                }
26706                TObjectName obj = new TObjectName();
26707                obj.setString(colName);
26708                return modelFactory.createTableColumn(table, obj, "*".equals(colName));
26709        }
26710
26711        private ResultColumn findOrCreateResultColumn(ResultSet resultSet, String colName) {
26712                String normalName = DlineageUtil.getIdentifierNormalColumnName(colName);
26713                for (ResultColumn rc : resultSet.getColumns()) {
26714                        if ("*".equals(colName) && "*".equals(rc.getName())) return rc;
26715                        if (DlineageUtil.compareColumnIdentifier(rc.getName(), normalName)) return rc;
26716                }
26717                TObjectName obj = new TObjectName();
26718                obj.setString(colName);
26719                return modelFactory.createResultColumn(resultSet, obj);
26720        }
26721
26722        private void getFunctionExpressions(List<TExpression> directExpressions, List<TExpression> indirectExpressions,
26723                        TFunctionCall functionCall) {
26724                if (functionCall.getArgs() != null) {
26725                        for (int k = 0; k < functionCall.getArgs().size(); k++) {
26726                                TExpression expr = functionCall.getArgs().getExpression(k);
26727                                if(FunctionUtility.isDirectRelation(option.getVendor(), functionCall.getFunctionName().toString(),  functionCall.getArgs().size(), k)) {
26728                                        directExpressions.add(expr);
26729                                }
26730                                if(FunctionUtility.isIndirectRelation(option.getVendor(), functionCall.getFunctionName().toString(),  functionCall.getArgs().size(), k)) {
26731                                        if("DECODE".equalsIgnoreCase(functionCall.getFunctionName().toString()) && option.getVendor() == EDbVendor.dbvoracle) {
26732                                                if(option.isShowCaseWhenAsDirect()) {
26733                                                        directExpressions.add(expr);
26734                                                        continue;
26735                                                }
26736                                        }
26737                                        indirectExpressions.add(expr);
26738                                }
26739                        }
26740                }
26741                collectOracleXmlFunctionExpressions(functionCall, directExpressions, indirectExpressions);
26742                collectArrayAggOrderByExpressions(functionCall, directExpressions, indirectExpressions);
26743                if (functionCall.getTrimArgument() != null) {
26744                        TTrimArgument args = functionCall.getTrimArgument();
26745                        TExpression expr = args.getStringExpression();
26746                        if (expr != null) {
26747                                directExpressions.add(expr);
26748                        }
26749                        expr = args.getTrimCharacter();
26750                        if (expr != null) {
26751                                directExpressions.add(expr);
26752                        }
26753                }
26754
26755                if (functionCall.getAgainstExpr() != null) {
26756                        directExpressions.add(functionCall.getAgainstExpr());
26757                }
26758//              if (functionCall.getBetweenExpr() != null) {
26759//                      directExpressions.add(functionCall.getBetweenExpr());
26760//              }
26761                if (functionCall.getExpr1() != null) {
26762                        directExpressions.add(functionCall.getExpr1());
26763                }
26764                if (functionCall.getExpr2() != null) {
26765                        directExpressions.add(functionCall.getExpr2());
26766                }
26767                if (functionCall.getExpr3() != null) {
26768                        directExpressions.add(functionCall.getExpr3());
26769                }
26770                if (functionCall.getParameter() != null) {
26771                        directExpressions.add(functionCall.getParameter());
26772                }
26773                if (functionCall.getWindowDef() != null && functionCall.getWindowDef().getPartitionClause() != null) {
26774                        TExpressionList args = functionCall.getWindowDef().getPartitionClause().getExpressionList();
26775                        if (args != null) {
26776                                for (int k = 0; k < args.size(); k++) {
26777                                        TExpression expr = args.getExpression(k);
26778                                        if (expr != null) {
26779                                                indirectExpressions.add(expr);
26780                                        }
26781                                }
26782                        }
26783                }
26784                if (functionCall.getWindowDef() != null && functionCall.getWindowDef().getOrderBy() != null) {
26785                        TOrderByItemList orderByList = functionCall.getWindowDef().getOrderBy().getItems();
26786                        for (int i = 0; i < orderByList.size(); i++) {
26787                                TOrderByItem element = orderByList.getOrderByItem(i);
26788                                TExpression expression = element.getSortKey();
26789                                indirectExpressions.add(expression);
26790                        }
26791                }
26792                if (functionCall.getWithinGroup() != null && functionCall.getWithinGroup().getOrderBy() != null) {
26793                        TOrderByItemList orderByList = functionCall.getWithinGroup().getOrderBy().getItems();
26794                        for (int i = 0; i < orderByList.size(); i++) {
26795                                TOrderByItem element = orderByList.getOrderByItem(i);
26796                                TExpression expression = element.getSortKey();
26797                                indirectExpressions.add(expression);
26798                        }
26799                }
26800                if (functionCall.getCallTarget() != null) {
26801                        directExpressions.add(functionCall.getCallTarget().getExpr());
26802                }
26803                if (functionCall.getFieldValues() != null) {
26804                        for (int k = 0; k < functionCall.getFieldValues().size(); k++) {
26805                                TExpression expr = functionCall.getFieldValues().getResultColumn(k).getExpr();
26806                                directExpressions.add(expr);
26807                        }
26808                }
26809                if (functionCall instanceof TJsonObjectFunction) {
26810                        TJsonObjectFunction jsonObject = (TJsonObjectFunction)functionCall;
26811                        for (int k = 0; k < jsonObject.getKeyValues().size(); k++) {
26812                                TExpression expr = jsonObject.getKeyValues().get(k).getValue();
26813                                directExpressions.add(expr);
26814                        }
26815                }
26816                if (functionCall.getGroupConcatParam() != null) {
26817                        for (int k = 0; k < functionCall.getGroupConcatParam().getExprList().size(); k++) {
26818                                TExpression expr = functionCall.getGroupConcatParam().getExprList().getExpression(k);
26819                                directExpressions.add(expr);
26820                        }
26821                }
26822        }
26823
26824        private void getFunctionExpressions(List<TExpression> directExpressions, List<TExpression> indirectExpressions,
26825                                                                                TFunctionCall functionCall, int argumentIndex) {
26826                if (functionCall.getArgs() != null && argumentIndex < functionCall.getArgs().size()) {
26827                        TExpression expr = functionCall.getArgs().getExpression(argumentIndex);
26828                        if (FunctionUtility.isDirectRelation(option.getVendor(), functionCall.getFunctionName().toString(), functionCall.getArgs().size(), argumentIndex)) {
26829                                directExpressions.add(expr);
26830                        }
26831                        if (FunctionUtility.isIndirectRelation(option.getVendor(), functionCall.getFunctionName().toString(), functionCall.getArgs().size(), argumentIndex)) {
26832                                indirectExpressions.add(expr);
26833                        }
26834                }
26835        }
26836        
26837        private void getFunctionExpressions(List<TExpression> directExpressions, List<TExpression> indirectExpressions,
26838                        TCallStatement functionCall, int argumentIndex) {
26839                if (functionCall.getArgs() != null && argumentIndex < functionCall.getArgs().size()) {
26840                        TExpression expr = functionCall.getArgs().getExpression(argumentIndex);
26841                        if (FunctionUtility.isDirectRelation(option.getVendor(), functionCall.getRoutineName().toString(),
26842                                        functionCall.getArgs().size(), argumentIndex)) {
26843                                directExpressions.add(expr);
26844                        }
26845                        if (FunctionUtility.isIndirectRelation(option.getVendor(), functionCall.getRoutineName().toString(),
26846                                        functionCall.getArgs().size(), argumentIndex)) {
26847                                indirectExpressions.add(expr);
26848                        }
26849                }
26850        }
26851
26852        private void getFunctionExpressions(List<TExpression> directExpressions, List<TExpression> indirectExpressions,
26853                        TDb2CallStmt functionCall, int argumentIndex) {
26854                if (functionCall.getParameters() != null && argumentIndex < functionCall.getParameters().size()) {
26855                        TExpression expr = functionCall.getParameters().getExpression(argumentIndex);
26856                        if (FunctionUtility.isDirectRelation(option.getVendor(), functionCall.getProcedureName().toString(),
26857                                        functionCall.getParameters().size(), argumentIndex)) {
26858                                directExpressions.add(expr);
26859                        }
26860                        if (FunctionUtility.isIndirectRelation(option.getVendor(), functionCall.getProcedureName().toString(),
26861                                        functionCall.getParameters().size(), argumentIndex)) {
26862                                indirectExpressions.add(expr);
26863                        }
26864                }
26865        }
26866
26867        private void getFunctionExpressions(List<TExpression> directExpressions, List<TExpression> indirectExpressions,
26868                                                                                TMssqlExecute functionCall, String argumentName, int argumentIndex) {
26869                if (functionCall.getParameters() != null) {
26870                        for (int i = 0; i < functionCall.getParameters().size(); i++) {
26871                                TExecParameter param = functionCall.getParameters().getExecParameter(i);
26872                                if (param.getParameterName() != null) {
26873                                        if (DlineageUtil.compareColumnIdentifier(param.getParameterName().toString(), argumentName)) {
26874                                                TExpression expr = param.getParameterValue();
26875                                                directExpressions.add(expr);
26876                                        }
26877                                } else if (i == argumentIndex) {
26878                                        TExpression expr = param.getParameterValue();
26879                                        directExpressions.add(expr);
26880                                }
26881                        }
26882                }
26883        }
26884
26885        private void analyzeJoin(TJoin join, EffectType effectType) {
26886                if (join.getJoinItems() != null) {
26887                        for (int j = 0; j < join.getJoinItems().size(); j++) {
26888                                TJoinItem joinItem = join.getJoinItems().getJoinItem(j);
26889                                TExpression expr = joinItem.getOnCondition();
26890                                if (expr != null) {
26891                                        analyzeFilterCondition(null, expr, joinItem.getJoinType(), JoinClauseType.on, effectType);
26892                                }
26893                        }
26894                }
26895
26896                if (join.getJoin() != null) {
26897                        analyzeJoin(join.getJoin(), effectType);
26898                }
26899        }
26900
26901        private TSelectSqlStatement getParentSetSelectStmt(TSelectSqlStatement stmt) {
26902                TCustomSqlStatement parent = stmt.getParentStmt();
26903                if (parent == null)
26904                        return null;
26905                if (parent.getStatements() != null) {
26906                        for (int i = 0; i < parent.getStatements().size(); i++) {
26907                                TCustomSqlStatement temp = parent.getStatements().get(i);
26908                                if (temp instanceof TSelectSqlStatement) {
26909                                        TSelectSqlStatement select = (TSelectSqlStatement) temp;
26910                                        if (select.getLeftStmt() == stmt || select.getRightStmt() == stmt)
26911                                                return select;
26912                                }
26913                        }
26914                }
26915                if (parent instanceof TSelectSqlStatement) {
26916                        TSelectSqlStatement select = (TSelectSqlStatement) parent;
26917                        if (select.getLeftStmt() == stmt || select.getRightStmt() == stmt)
26918                                return select;
26919                }
26920                return null;
26921        }
26922
26923        private void createSelectSetResultColumns(SelectSetResultSet resultSet, TSelectSqlStatement stmt) {
26924                if (stmt.getSetOperatorType() != ESetOperatorType.none) {
26925                        createSelectSetResultColumns(resultSet, stmt.getLeftStmt());
26926                } else {
26927                        TResultColumnList columnList = stmt.getResultColumnList();
26928                        ResultSet subqueryResultSet = (ResultSet) modelManager.getModel(columnList);
26929                        if(subqueryResultSet!=null && subqueryResultSet.isDetermined()) {
26930                                for (int j = 0; j < subqueryResultSet.getColumns().size(); j++) {
26931                                        ResultColumn tableColumn = subqueryResultSet.getColumns().get(j);
26932                                        String name = tableColumn.getRefColumnName() != null
26933                                                        ? tableColumn.getRefColumnName() : tableColumn.getName();
26934                                        boolean emptyName = SQLUtil.isEmpty(name);
26935                                        if (emptyName) {
26936                                                // Empty delimited identifiers have no normalized name. Keep
26937                                                // their set-column ordinal and recover the lexical spelling.
26938                                                name = tableColumn.getFullName();
26939                                        }
26940                                        TObjectName columnName = new TObjectName();
26941                                        columnName.setString(name == null ? "" : name);
26942                                        if (emptyName && tableColumn.getColumnObject() != null) {
26943                                                // Reuse the real span instead of publishing a synthetic 1:1
26944                                                // coordinate for the recovered spelling.
26945                                                columnName.setStartTokenDirectly(tableColumn.getColumnObject().getStartToken());
26946                                                columnName.setEndTokenDirectly(tableColumn.getColumnObject().getEndToken());
26947                                        }
26948                                        modelFactory.createDeterminedResultColumn(resultSet, columnName);
26949                                }
26950                                resultSet.setDetermined(true);
26951                                return;
26952                        }
26953                        
26954                        boolean isDetermined = true;
26955                        for (int i = 0; i < columnList.size(); i++) {
26956                                TResultColumn column = columnList.getResultColumn(i);
26957                                
26958                                if ("*".equals(column.getColumnNameOnly())) {
26959                                        TObjectName columnObject = column.getFieldAttr();
26960                                        TTable sourceTable = columnObject.getSourceTable();
26961                                        if (sourceTable != null) {
26962                                                Object tableModel = modelManager.getModel(sourceTable);
26963                                                if (tableModel instanceof Table && ((Table) tableModel).isCreateTable()) {
26964                                                        Table table = (Table) tableModel;
26965                                                        for (int j = 0; j < table.getColumns().size(); j++) {
26966                                                                TableColumn tableColumn = table.getColumns().get(j);
26967                                                                if (column.getExceptColumnList() != null) {
26968                                                                        boolean except = false;
26969                                                                        for (TObjectName objectName : column.getExceptColumnList()) {
26970                                                                                if (getColumnName(objectName.toString())
26971                                                                                                .equals(getColumnName(tableColumn.getName()))) {
26972                                                                                        except = true;
26973                                                                                        break;
26974                                                                                }
26975                                                                        }
26976                                                                        if (!except && tableColumn.isStruct()) {
26977                                                                                List<String> names = SQLUtil
26978                                                                                                .parseNames(tableColumn.getName());
26979                                                                                for (String name : names) {
26980                                                                                        for (TObjectName objectName : column
26981                                                                                                        .getExceptColumnList()) {
26982                                                                                                if (getColumnName(objectName.toString())
26983                                                                                                                .equals(getColumnName(name))) {
26984                                                                                                        except = true;
26985                                                                                                        break;
26986                                                                                                }
26987                                                                                        }
26988                                                                                        if (except) {
26989                                                                                                break;
26990                                                                                        }
26991                                                                                }
26992                                                                        }
26993                                                                        if (except) {
26994                                                                                continue;
26995                                                                        }
26996                                                                }
26997                                                                TObjectName columnName = new TObjectName();
26998                                                                columnName.setString(tableColumn.getName());
26999                                                                ResultColumn resultColumn = modelFactory.createResultColumn(
27000                                                                                resultSet, columnName);
27001                                                                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
27002                                                                relation.setEffectType(EffectType.select);
27003                                                                relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
27004                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
27005                                                        }
27006                                                        continue;
27007                                                } else if (tableModel instanceof ResultSet
27008                                                                && ((ResultSet) tableModel).isDetermined()) {
27009                                                        ResultSet table = (ResultSet) tableModel;
27010                                                        for (int j = 0; j < table.getColumns().size(); j++) {
27011                                                                ResultColumn tableColumn = table.getColumns().get(j);
27012                                                                if (column.getExceptColumnList() != null) {
27013                                                                        boolean except = false;
27014                                                                        for (TObjectName objectName : column.getExceptColumnList()) {
27015                                                                                if (getColumnName(objectName.toString())
27016                                                                                                .equals(getColumnName(tableColumn.getName()))) {
27017                                                                                        except = true;
27018                                                                                        break;
27019                                                                                }
27020                                                                        }
27021                                                                        if (!except && tableColumn.isStruct()) {
27022                                                                                List<String> names = SQLUtil
27023                                                                                                .parseNames(tableColumn.getName());
27024                                                                                for (String name : names) {
27025                                                                                        for (TObjectName objectName : column
27026                                                                                                        .getExceptColumnList()) {
27027                                                                                                if (getColumnName(objectName.toString())
27028                                                                                                                .equals(getColumnName(name))) {
27029                                                                                                        except = true;
27030                                                                                                        break;
27031                                                                                                }
27032                                                                                        }
27033                                                                                        if (except) {
27034                                                                                                break;
27035                                                                                        }
27036                                                                                }
27037                                                                        }
27038                                                                        if (except) {
27039                                                                                continue;
27040                                                                        }
27041                                                                }
27042                                                                if (tableColumn.getRefColumnName() != null) {
27043                                                                        TObjectName columnName = new TObjectName();
27044                                                                        columnName.setString(tableColumn.getRefColumnName());
27045                                                                        ResultColumn resultColumn = modelFactory.createResultColumn(
27046                                                                                        resultSet, columnName);
27047                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
27048                                                                        relation.setEffectType(EffectType.select);
27049                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
27050                                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn));
27051                                                                } else {
27052                                                                        TObjectName columnName = new TObjectName();
27053                                                                        columnName.setString(tableColumn.getName());
27054                                                                        ResultColumn resultColumn = modelFactory.createResultColumn(
27055                                                                                        resultSet, columnName);
27056                                                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
27057                                                                        relation.setEffectType(EffectType.select);
27058                                                                        relation.setTarget(new ResultColumnRelationshipElement(resultColumn));
27059                                                                        relation.addSource(new ResultColumnRelationshipElement(tableColumn));
27060                                                                }
27061                                                        }
27062                                                        continue;
27063                                                }
27064                                                else {
27065                                                        isDetermined = false;
27066                                                }
27067                                        }
27068                                }
27069                                        
27070                                ResultColumn resultColumn = modelFactory.createSelectSetResultColumn(resultSet, column, i);
27071
27072                                if (resultColumn.getColumnObject() instanceof TResultColumn) {
27073                                        TResultColumn columnObject = (TResultColumn) resultColumn.getColumnObject();
27074                                        if (columnObject.getFieldAttr() != null) {
27075                                                if ("*".equals(getColumnName(columnObject.getFieldAttr()))) {
27076                                                        TObjectName fieldAttr = columnObject.getFieldAttr();
27077                                                        TTable sourceTable = fieldAttr.getSourceTable();
27078                                                        if (fieldAttr.getTableToken() != null && sourceTable != null) {
27079                                                                TObjectName[] columns = modelManager.getTableColumns(sourceTable);
27080                                                                for (int j = 0; j < columns.length; j++) {
27081                                                                        TObjectName columnName = columns[j];
27082                                                                        if (columnName == null) {
27083                                                                                continue;
27084                                                                        }
27085                                                                        if ("*".equals(getColumnName(columnName))) {
27086                                                                                continue;
27087                                                                        }
27088                                                                        resultColumn.bindStarLinkColumn(columnName);
27089                                                                }
27090
27091                                                                if (modelManager.getModel(sourceTable) instanceof Table) {
27092                                                                        Table tableModel = (Table) modelManager.getModel(sourceTable);
27093                                                                        if (tableModel != null && !tableModel.getColumns().isEmpty()) {
27094                                                                                for (int z = 0; z < tableModel.getColumns().size(); z++) {
27095                                                                                        if ("*".equals(
27096                                                                                                        getColumnName(tableModel.getColumns().get(z).getColumnObject()))) {
27097                                                                                                continue;
27098                                                                                        }
27099                                                                                        resultColumn.bindStarLinkColumn(
27100                                                                                                        tableModel.getColumns().get(z).getColumnObject());
27101                                                                                }
27102                                                                        }
27103                                                                } else if (modelManager.getModel(sourceTable) instanceof QueryTable) {
27104                                                                        QueryTable tableModel = (QueryTable) modelManager.getModel(sourceTable);
27105                                                                        if (tableModel != null && !tableModel.getColumns().isEmpty()) {
27106                                                                                for (ResultColumn item : tableModel.getColumns()) {
27107                                                                                        if (item.hasStarLinkColumn()) {
27108                                                                                                for (TObjectName starLinkColumn : item.getStarLinkColumnList()) {
27109                                                                                                        if ("*".equals(getColumnName(starLinkColumn))) {
27110                                                                                                                continue;
27111                                                                                                        }
27112                                                                                                        resultColumn.bindStarLinkColumn(starLinkColumn);
27113                                                                                                }
27114                                                                                        } else if (item.getColumnObject() instanceof TObjectName) {
27115                                                                                                TObjectName starLinkColumn = (TObjectName) item.getColumnObject();
27116                                                                                                if ("*".equals(getColumnName(starLinkColumn))) {
27117                                                                                                        continue;
27118                                                                                                }
27119                                                                                                resultColumn.bindStarLinkColumn(starLinkColumn);
27120                                                                                        }
27121                                                                                }
27122                                                                        }
27123                                                                }
27124
27125                                                        } else {
27126                                                                TTableList tables = stmt.getTables();
27127                                                                for (int k = 0; k < tables.size(); k++) {
27128                                                                        TTable tableElement = tables.getTable(k);
27129                                                                        TObjectName[] columns = modelManager.getTableColumns(tableElement);
27130                                                                        for (int j = 0; j < columns.length; j++) {
27131                                                                                TObjectName columnName = columns[j];
27132                                                                                if (columnName == null) {
27133                                                                                        continue;
27134                                                                                }
27135                                                                                if ("*".equals(getColumnName(columnName))) {
27136                                                                                        if (modelManager.getModel(tableElement) instanceof Table) {
27137                                                                                                Table tableModel = (Table) modelManager.getModel(tableElement);
27138                                                                                                if (tableModel != null && !tableModel.getColumns().isEmpty()) {
27139                                                                                                        for (int z = 0; z < tableModel.getColumns().size(); z++) {
27140                                                                                                                resultColumn.bindStarLinkColumn(
27141                                                                                                                                tableModel.getColumns().get(z).getColumnObject());
27142                                                                                                        }
27143                                                                                                }
27144                                                                                        } else if (modelManager.getModel(tableElement) instanceof QueryTable) {
27145                                                                                                QueryTable tableModel = (QueryTable) modelManager
27146                                                                                                                .getModel(tableElement);
27147                                                                                                if (tableModel != null && !tableModel.getColumns().isEmpty()) {
27148                                                                                                        for (ResultColumn item : tableModel.getColumns()) {
27149                                                                                                                if (item.hasStarLinkColumn()) {
27150                                                                                                                        for (TObjectName starLinkColumn : item
27151                                                                                                                                        .getStarLinkColumnList()) {
27152                                                                                                                                resultColumn.bindStarLinkColumn(starLinkColumn);
27153                                                                                                                        }
27154                                                                                                                } else if (item.getColumnObject() instanceof TObjectName) {
27155                                                                                                                        resultColumn.bindStarLinkColumn(
27156                                                                                                                                        (TObjectName) item.getColumnObject());
27157                                                                                                                }
27158                                                                                                        }
27159                                                                                                }
27160                                                                                        }
27161                                                                                        continue;
27162                                                                                }
27163                                                                                resultColumn.bindStarLinkColumn(columnName);
27164                                                                        }
27165                                                                }
27166                                                        }
27167                                                }
27168                                        }
27169                                }
27170                                
27171                                resultSet.setDetermined(isDetermined);
27172                        }
27173                }
27174        }
27175
27176        private void analyzeResultColumn(TResultColumn column, EffectType effectType) {
27177                // A SELECT * star column whose source resolves to a determined result set
27178                // is bound to a LinkedHashMap of per-column ResultColumns (see
27179                // ModelFactory.createStarResultColumn), and its data-flow relationships
27180                // are already created inline during star expansion in analyzeSelectStmt().
27181                // Re-analyzing the raw "*" here is redundant; skip it. (analyzeDataFlowRelation
27182                // keeps the same guard as a defensive backstop.)
27183                if (modelManager.getModel(column) instanceof LinkedHashMap) {
27184                        return;
27185                }
27186                TExpression expression = column.getExpr();
27187                if (expression.getExpressionType() == EExpressionType.sqlserver_proprietary_column_alias_t) {
27188                        expression = expression.getRightOperand();
27189                }
27190
27191                if (expression.getExpressionType() == EExpressionType.array_t) {
27192                        if (expression.getExprList() != null) {
27193                                for (TExpression expr : expression.getExprList()) {
27194                                        columnsInExpr visitor = new columnsInExpr();
27195                                        expr.inOrderTraverse(visitor);
27196                                        List<TObjectName> objectNames = visitor.getObjectNames();
27197
27198                                        List<TParseTreeNode> functions = visitor.getFunctions();
27199
27200                                        if (functions != null && !functions.isEmpty()) {
27201                                                analyzeFunctionDataFlowRelation(column, functions, effectType);
27202                                        }
27203
27204                                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
27205                                        if (subquerys != null && !subquerys.isEmpty()) {
27206                                                analyzeSubqueryDataFlowRelation(column, subquerys, effectType);
27207                                        }
27208
27209                                        analyzeDataFlowRelation(column, objectNames, column.getExceptColumnList(), effectType, functions);
27210
27211                                        List<TParseTreeNode> constants = visitor.getConstants();
27212                                        Object columnObject = modelManager.getModel(column);
27213                                        analyzeConstantDataFlowRelation(columnObject, constants, effectType, functions);
27214
27215                                        analyzeRecordSetRelation(functions, effectType);
27216                                        // analyzeResultColumnImpact( column, effectType, functions);
27217                                }
27218                        }
27219                        else {
27220                                List<TParseTreeNode> constants = new ArrayList<TParseTreeNode>();
27221                                TConstant constant = new TConstant();
27222                                constant.setString(expression.toString());
27223                                constants.add(constant);
27224                                Object columnObject = modelManager.getModel(column);
27225                                analyzeConstantDataFlowRelation(columnObject, constants, effectType, null);
27226                        }
27227                } else {
27228                        columnsInExpr visitor = new columnsInExpr();
27229                        expression.inOrderTraverse(visitor);
27230                        List<TObjectName> objectNames = visitor.getObjectNames();
27231
27232                        List<TParseTreeNode> functions = visitor.getFunctions();
27233
27234                        if (functions != null && !functions.isEmpty()) {
27235                                analyzeFunctionDataFlowRelation(column, functions, effectType);
27236                        }
27237
27238                        List<TSelectSqlStatement> subquerys = visitor.getSubquerys();
27239                        if (subquerys != null && !subquerys.isEmpty()) {
27240                                analyzeSubqueryDataFlowRelation(column, subquerys, effectType);
27241                        }
27242
27243                        analyzeDataFlowRelation(column, objectNames, column.getExceptColumnList(), effectType, functions);
27244
27245                        List<TParseTreeNode> constants = visitor.getConstants();
27246                        Object columnObject = modelManager.getModel(column);
27247                        analyzeConstantDataFlowRelation(columnObject, constants, effectType, functions);
27248
27249                        analyzeRecordSetRelation(functions, effectType);
27250                        // analyzeResultColumnImpact( column, effectType, functions);
27251                }
27252        }
27253        
27254        
27255        private void analyzeValueColumn(Object object, TResultColumn column, EffectType effectType) {
27256                TExpression expression = column.getExpr();
27257                if (expression.getExpressionType() == EExpressionType.sqlserver_proprietary_column_alias_t) {
27258                        expression = expression.getRightOperand();
27259                }
27260
27261                if (expression.getExpressionType() == EExpressionType.array_t) {
27262                        if (expression.getExprList() != null) {
27263                                for (TExpression expr : expression.getExprList()) {
27264                                        columnsInExpr visitor = new columnsInExpr();
27265                                        expr.inOrderTraverse(visitor);
27266                                        List<TObjectName> objectNames = visitor.getObjectNames();
27267                                        analyzeDataFlowRelation(object, objectNames, column.getExceptColumnList(), effectType, null, null);
27268                                        List<TParseTreeNode> constants = visitor.getConstants();
27269                                        analyzeConstantDataFlowRelation(object, constants, effectType, null);
27270                                }
27271                        }
27272                        else {
27273                                List<TParseTreeNode> constants = new ArrayList<TParseTreeNode>();
27274                                TConstant constant = new TConstant();
27275                                constant.setString(expression.toString());
27276                                constants.add(constant);
27277                                Object columnObject = modelManager.getModel(column);
27278                                analyzeConstantDataFlowRelation(object, constants, effectType, null);
27279                        }
27280                } else {
27281                        columnsInExpr visitor = new columnsInExpr();
27282                        expression.inOrderTraverse(visitor);
27283                        List<TObjectName> objectNames = visitor.getObjectNames();
27284                        analyzeDataFlowRelation(object, objectNames, column.getExceptColumnList(), effectType, null, null);
27285                        List<TParseTreeNode> constants = visitor.getConstants();
27286                        analyzeConstantDataFlowRelation(object, constants, effectType, null);           }
27287        }
27288
27289        private void analyzeTableColumn(TableColumn tableColumn, TFunctionCall functionCall, EffectType effectType) {
27290                // ClickHouse external table functions are modelled as source TABLES
27291                // (modelClickhouseExternalTableFunction); running the generic
27292                // function-argument analysis on top fabricates argument-named columns
27293                // on that table (remote('h', mydb, mytable) invented a column "mydb")
27294                // and re-attaches the generic function node the modelling replaced.
27295                if (option.getVendor() == EDbVendor.dbvclickhouse && functionCall != null
27296                                && functionCall.getFunctionName() != null) {
27297                        String rawName = functionCall.getFunctionName().toString();
27298                        if (isClickhouseTableFunction(rawName, "remote")
27299                                        || isClickhouseTableFunction(rawName, "remoteSecure")
27300                                        || isClickhouseTableFunction(rawName, "file")
27301                                        || isClickhouseTableFunction(rawName, "s3")
27302                                        || isClickhouseTableFunction(rawName, "url")
27303                                        || isClickhouseTableFunction(rawName, "hdfs")
27304                                        || isClickhouseTableFunction(rawName, "fileCluster")
27305                                        || isClickhouseTableFunction(rawName, "s3Cluster")
27306                                        || isClickhouseTableFunction(rawName, "urlCluster")
27307                                        || isClickhouseTableFunction(rawName, "hdfsCluster")
27308                                        || isClickhouseTableFunction(rawName, "azureBlobStorage")
27309                                        || isClickhouseTableFunction(rawName, "azureBlobStorageCluster")) {
27310                                return;
27311                        }
27312                }
27313                List<TParseTreeNode> functions = new ArrayList<TParseTreeNode>();
27314                functions.add(functionCall);
27315
27316                if (functions != null && !functions.isEmpty()) {
27317                        analyzeFunctionDataFlowRelation(tableColumn, functions, effectType);
27318                }
27319
27320                analyzeRecordSetRelation(functions, effectType);
27321        }
27322
27323        private void analyzeRecordSetRelation(List<TParseTreeNode> functions, EffectType effectType) {
27324                if (functions == null || functions.size() == 0)
27325                        return;
27326
27327                List<TFunctionCall> aggregateFunctions = new ArrayList<TFunctionCall>();
27328                for (TParseTreeNode function : functions) {
27329                        if (function instanceof TFunctionCall && isAggregateFunction((TFunctionCall) function)) {
27330                                aggregateFunctions.add((TFunctionCall) function);
27331                        }
27332                }
27333
27334                if (aggregateFunctions.size() == 0)
27335                        return;
27336
27337                for (int i = 0; i < aggregateFunctions.size(); i++) {
27338                        TFunctionCall function = aggregateFunctions.get(i);
27339
27340                        TCustomSqlStatement stmt = stmtStack.peek();
27341                        if (stmt instanceof TSelectSqlStatement) {
27342                                TSelectSqlStatement select = (TSelectSqlStatement) stmt;
27343                                if (select.getGroupByClause() != null) {
27344                                        if (select.getGroupByClause().isAllModifier()) {
27345                                                // GROUP BY ALL: implicit grouping columns are the
27346                                                // non-aggregate expressions in the SELECT list.
27347                                                TResultColumnList resultColumns = select.getResultColumnList();
27348                                                if (resultColumns != null) {
27349                                                        for (int j = 0; j < resultColumns.size(); j++) {
27350                                                                TResultColumn column = resultColumns.getResultColumn(j);
27351                                                                TExpression expr = column.getExpr();
27352                                                                if (expr == null)
27353                                                                        continue;
27354                                                                columnsInExpr aggVisitor = new columnsInExpr();
27355                                                                expr.inOrderTraverse(aggVisitor);
27356                                                                boolean containsAggregate = false;
27357                                                                for (TParseTreeNode funcNode : aggVisitor.getFunctions()) {
27358                                                                        if (funcNode instanceof TFunctionCall
27359                                                                                        && isAggregateFunction((TFunctionCall) funcNode)) {
27360                                                                                containsAggregate = true;
27361                                                                                break;
27362                                                                        }
27363                                                                }
27364                                                                if (!containsAggregate) {
27365                                                                        analyzeAggregate(function, expr);
27366                                                                }
27367                                                        }
27368                                                }
27369                                        } else {
27370                                                TGroupByItemList groupByList = select.getGroupByClause().getItems();
27371                                                for (int j = 0; j < groupByList.size(); j++) {
27372                                                        TGroupByItem groupBy = groupByList.getGroupByItem(j);
27373                                                        TExpression expr = groupBy.getExpr();
27374                                                        analyzeAggregate(function, expr);
27375                                                }
27376                                        }
27377
27378                                        if (select.getGroupByClause().getHavingClause() != null) {
27379                                                analyzeAggregate(function, select.getGroupByClause().getHavingClause());
27380                                        }
27381                                        // if ("COUNT".equalsIgnoreCase(function.getFunctionName().toString()))
27382                                        {
27383                                                analyzeAggregate(function, null);
27384                                        }
27385                                } else {
27386                                        analyzeAggregate(function, null);
27387                                }
27388                        }
27389                }
27390        }
27391
27392        private void analyzeDataFlowRelation(TParseTreeNode gspObject, List<TObjectName> objectNames,
27393                        TObjectNameList exceptColumnList, EffectType effectType, List<TParseTreeNode> functions) {
27394                Object columnObject = modelManager.getModel(gspObject);
27395                analyzeDataFlowRelation(columnObject, objectNames, exceptColumnList, effectType, functions, null);
27396        }
27397
27398        private DataFlowRelationship analyzeDataFlowRelation(Object modelObject, List<TObjectName> objectNames, EffectType effectType,
27399                        List<TParseTreeNode> functions) {
27400                return analyzeDataFlowRelation(modelObject, objectNames, null, effectType, functions, null);
27401        }
27402        
27403        private DataFlowRelationship analyzeDataFlowRelation(Object modelObject, List<TObjectName> objectNames, EffectType effectType,
27404                        List<TParseTreeNode> functions, Process process) {
27405                return analyzeDataFlowRelation(modelObject, objectNames, null, effectType, functions, process);
27406        }
27407
27408        private DataFlowRelationship analyzeDataFlowRelation(Object modelObject, List<TObjectName> objectNames,
27409                        TObjectNameList exceptColumnList, EffectType effectType, List<TParseTreeNode> functions, Process process) {
27410                return analyzeDataFlowRelation(modelObject, objectNames, exceptColumnList, effectType, functions, process, null);
27411        }
27412        
27413        private DataFlowRelationship analyzeDataFlowRelation(Object modelObject, List<TObjectName> objectNames,
27414                        TObjectNameList exceptColumnList, EffectType effectType, List<TParseTreeNode> functions, Process process, Integer valueIndex) {
27415                if (objectNames == null || objectNames.size() == 0)
27416                        return null;
27417
27418                // Reject model objects this method cannot turn into a relationship BEFORE
27419                // creating one, since createDataFlowRelation() registers the relation
27420                // globally and an early return afterwards would leak an empty relation.
27421                //
27422                // - LinkedHashMap: a SELECT * star column whose source resolves to a
27423                //   determined result set is expanded into per-column ResultColumns held
27424                //   in a LinkedHashMap by ModelFactory.createStarResultColumn(), with its
27425                //   data-flow relationships created inline during star expansion in
27426                //   analyzeSelectStmt(). Re-analyzing the raw "*" here is a no-op.
27427                // - null: modelManager.getModel() found no binding for the gsp object.
27428                if (modelObject == null || modelObject instanceof LinkedHashMap) {
27429                        return null;
27430                }
27431                // Lineage is best-effort: an unexpected model type should not abort the
27432                // whole statement's lineage (this used to throw UnsupportedOperationException).
27433                // Log it for diagnosis and skip just this column instead.
27434                if (!(modelObject instanceof ResultColumn) && !(modelObject instanceof TableColumn)) {
27435                        logger.warn("analyzeDataFlowRelation: unhandled model type "
27436                                        + modelObject.getClass().getName() + ", effectType=" + effectType);
27437                        return null;
27438                }
27439
27440                boolean isStar = false;
27441                boolean showStar = false;
27442
27443                DataFlowRelationship relation = modelFactory.createDataFlowRelation();
27444                relation.setEffectType(effectType);
27445                relation.setProcess(process);
27446
27447                if (functions != null && !functions.isEmpty()) {
27448                        relation.setFunction(getFunctionName(functions.get(0)));
27449                }
27450
27451                int columnIndex = -1;
27452
27453                boolean isOut = false;
27454
27455                if (modelObject instanceof ResultColumn) {
27456                        relation.setTarget(new ResultColumnRelationshipElement((ResultColumn) modelObject));
27457
27458                        if ("*".equals(((ResultColumn) modelObject).getName())) {
27459                                isStar = true;
27460                                showStar = ((ResultColumn) modelObject).isShowStar();
27461                        }
27462
27463                        if (((ResultColumn) modelObject).getResultSet() != null) {
27464                                columnIndex = ((ResultColumn) modelObject).getResultSet().getColumns().indexOf(modelObject);
27465                        }
27466                } else if (modelObject instanceof TableColumn) {
27467                        Table table  = ((TableColumn) modelObject).getTable();
27468                        if(table.getSubType() == SubType.out && isNotInProcedure(table)){
27469                                isOut = true;
27470                                relation.addSource(new TableColumnRelationshipElement((TableColumn) modelObject));
27471                        }
27472                        else {
27473                                relation.setTarget(new TableColumnRelationshipElement((TableColumn) modelObject));
27474                        }
27475
27476                        if ("*".equals(((TableColumn) modelObject).getName())) {
27477                                isStar = true;
27478                        }
27479
27480                        if (((TableColumn) modelObject).getTable() != null) {
27481                                columnIndex = ((TableColumn) modelObject).getTable().getColumns().indexOf(modelObject);
27482                        }
27483                }
27484                // No trailing else: modelObject is guaranteed to be a ResultColumn or
27485                // TableColumn here (validated and logged above before relation creation).
27486
27487                for (int i = 0; i < objectNames.size(); i++) {
27488                        TObjectName columnName = objectNames.get(i);
27489                        if (columnName.toString().indexOf(".") == -1 && isConstant(columnName)) {
27490                                boolean isConstant = true;
27491                                if (columnName.getSourceTable() != null) {
27492                                        Table tableModel = modelManager.getTableByName(
27493                                                        DlineageUtil.getTableFullName(columnName.getSourceTable().getTableName().toString()));
27494                                        if (tableModel != null && tableModel.getColumns() != null) {
27495                                                for (int j = 0; j < tableModel.getColumns().size(); j++) {
27496                                                        if (DlineageUtil.compareColumnIdentifier(getColumnName(columnName),
27497                                                                        getColumnName(tableModel.getColumns().get(j).getName()))) {
27498                                                                isConstant = false;
27499                                                                break;
27500                                                        }
27501                                                }
27502                                        }
27503                                }
27504
27505                                if (isConstant) {
27506                                        if (option.isShowConstantTable()) {
27507                                                Table constantTable = modelFactory.createConstantsTable(stmtStack.peek());
27508                                                TableColumn tableColumn = modelFactory.createTableColumn(constantTable, columnName, false);
27509                                                if(tableColumn!=null) {
27510                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
27511                                                }
27512                                        }
27513                                        continue;
27514                                }
27515                        }
27516                        if (columnName.getDbObjectType() == EDbObjectType.variable) {
27517                                boolean find = false;
27518                                List<String> segments = SQLUtil.parseNames(columnName.toString());
27519                                if (segments.size() == 1) {
27520                                        Table variable = modelManager.getTableByName(DlineageUtil.getTableFullName(columnName.toString()));
27521                                        if (variable != null && !variable.getColumns().isEmpty()) {
27522                                                TableColumn columnModel = variable.getColumns().get(0);
27523                                                TableColumn matchColumn = matchColumn(variable.getColumns(), columnName);
27524                                                if(matchColumn!=null){
27525                                                        columnModel = matchColumn;
27526                                                } else if (variable.getColumns().size() > 1 && variable instanceof Variable) {
27527                                                        // A bare reference to a field-expanded record is the
27528                                                        // WHOLE record (#695): bind its star, never an
27529                                                        // arbitrary first field — citing one field asserts
27530                                                        // the value ignores the others.
27531                                                        columnModel = compositeBindingColumn((Variable) variable);
27532                                                }
27533                                                if(isOut){
27534                                                        relation.setTarget(new TableColumnRelationshipElement(columnModel));
27535                                                }
27536                                                else {
27537                                                        relation.addSource(new TableColumnRelationshipElement(columnModel));
27538                                                }
27539                                                find = true;
27540                                        } else {
27541                                                if (columnName.toString().matches("\\$\\d+")) {
27542                                                        TCustomSqlStatement stmt = stmtStack.peek();
27543                                                        Procedure procedure = modelManager
27544                                                                        .getProcedureByName(DlineageUtil.getTableFullName(getProcedureParentName(stmt)));
27545                                                        if(procedure!=null) {
27546                                                                Variable cursorVariable = modelFactory.createVariable(procedure.getArguments().get(Integer.valueOf(columnName.toString().replace("$", "")) - 1).getName());
27547                                                                if (isOut) {
27548                                                                        relation.setTarget(new TableColumnRelationshipElement(cursorVariable.getColumns().get(0)));
27549                                                                } else {
27550                                                                        relation.addSource(new TableColumnRelationshipElement(cursorVariable.getColumns().get(0)));
27551                                                                }
27552                                                        }
27553                                                } else {
27554                                                        Variable cursorVariable = modelFactory.createVariable(columnName);
27555                                                        cursorVariable.setCreateTable(true);
27556                                                        cursorVariable.setSubType(SubType.record);
27557                                                        TableColumn variableProperty = null;
27558                                                        if(cursorVariable.getColumns() == null || cursorVariable.getColumns().isEmpty()) {
27559                                variableProperty = modelFactory.createTableColumn(cursorVariable, columnName,
27560                                        true);
27561                            }
27562                                                        else{
27563                                                                variableProperty = compositeBindingColumn(cursorVariable);
27564                                                        }
27565                                                        if (isOut) {
27566                                                                relation.setTarget(new TableColumnRelationshipElement(variableProperty));
27567                                                        } else {
27568                                                                relation.addSource(new TableColumnRelationshipElement(variableProperty));
27569                                                        }
27570                                                }
27571                                                find = true;
27572                                        }
27573                                } else if (option.getVendor() == EDbVendor.dbvoracle && columnName.getTableToken()!=null) {
27574                                        Variable cursorVariable = modelFactory
27575                                                        .createVariable(columnName.getTableToken().toString());
27576
27577                                        TObjectName variableColumnName = new TObjectName();
27578                                        variableColumnName.setString(segments.get(segments.size() - 1));
27579
27580                                        if (cursorVariable.getColumns() != null) {
27581                                                for (int j = 0; j < cursorVariable.getColumns().size(); j++) {
27582                                                        if (getColumnName(variableColumnName)
27583                                                                        .equals(getColumnName(cursorVariable.getColumns().get(j).getColumnObject()))) {
27584                                                                TableColumn columnModel = cursorVariable.getColumns().get(j);
27585                                                                if (isOut) {
27586                                                                        relation.setTarget(new TableColumnRelationshipElement(columnModel));
27587                                                                } else {
27588                                                                        relation.addSource(new TableColumnRelationshipElement(columnModel));
27589                                                                }
27590                                                                find = true;
27591                                                        }
27592                                                }
27593                                        }
27594
27595                                        if (!find) {
27596                                                TableColumn variableColumn = new TableColumn(cursorVariable, variableColumnName);
27597                                                cursorVariable.addColumn(variableColumn);
27598                                                if (isOut) {
27599                                                        relation.setTarget(new TableColumnRelationshipElement(variableColumn));
27600                                                } else {
27601                                                        relation.addSource(new TableColumnRelationshipElement(variableColumn));
27602                                                }
27603                                        }
27604                                } else {
27605                                        Table variable = modelManager
27606                                                        .getTableByName(DlineageUtil.getTableFullName(segments.get(segments.size() - 2)));
27607                                        if (variable != null) {
27608                                                for (int j = 0; j < variable.getColumns().size(); j++) {
27609                                                        if (getColumnName(columnName)
27610                                                                        .equals(getColumnName(variable.getColumns().get(j).getColumnObject()))) {
27611                                                                TableColumn columnModel = variable.getColumns().get(j);
27612                                                                if (isOut) {
27613                                                                        relation.setTarget(new TableColumnRelationshipElement(columnModel));
27614                                                                } else {
27615                                                                        relation.addSource(new TableColumnRelationshipElement(columnModel));
27616                                                                }
27617                                                                find = true;
27618                                                        }
27619                                                }
27620                                        }
27621                                }
27622                                if (!find) {
27623                                        TCustomSqlStatement stmt = stmtStack.peek();
27624                                        if (getProcedureParentName(stmt) != null) {
27625                                                Procedure procedure = modelManager
27626                                                                .getProcedureByName(DlineageUtil.getTableFullName(getProcedureParentName(stmt)));
27627                                                if (procedure != null && procedure.getArguments() != null) {
27628                                                        for (Argument argument : procedure.getArguments()) {
27629                                                                if (DlineageUtil.getTableFullName(argument.getName())
27630                                                                                .equals(DlineageUtil.getTableFullName(columnName.toString()))) {
27631                                                                        relation.addSource(new ArgumentRelationshipElement(argument));
27632                                                                }
27633                                                        }
27634                                                }
27635                                        }
27636                                }
27637                                continue;
27638                        }
27639                        
27640                        // Handle sequence pseudocolumn syntax (sequence.NEXTVAL or sequence.CURRVAL)
27641                        // Used by Oracle, Snowflake, and accepted by other vendors for compatibility
27642                        if(("NEXTVAL".equalsIgnoreCase(columnName.getColumnNameOnly()) || "CURRVAL".equalsIgnoreCase(columnName.getColumnNameOnly()))){
27643                                List<String> segments = SQLUtil.parseNames(columnName.toString());
27644                                if (segments.size() > 1) {
27645                                        segments.remove(segments.size()-1);
27646                                        Table table = modelFactory.createTableByName(SQLUtil.mergeSegments(segments, 0), true);
27647                                        table.setSequence(true);
27648                                        TableColumn seqCursor = modelFactory.createTableColumn(table, columnName, true);
27649                                        relation.addSource(new TableColumnRelationshipElement(seqCursor));
27650                                        continue;
27651                                }
27652                        }
27653
27654                        {
27655                                if (columnName.getSourceTable() != null) {
27656
27657                                }
27658                                else {
27659                                        Table variable = modelManager.getTableByName(DlineageUtil.getTableFullName(columnName.toString()));
27660                                        if (variable == null) {
27661                                                variable = modelManager
27662                                                                .getTableByName(DlineageUtil.getTableFullName(columnName.getTableString()));
27663                                                if (variable != null && variable.isCursor()) {
27664                                                        TableColumn variableColumn = modelFactory.createInsertTableColumn(variable, columnName);
27665                                                        if (variableColumn != null) {
27666                                                                if(isOut){
27667                                                                        relation.setTarget(new TableColumnRelationshipElement(variableColumn));
27668                                                                }
27669                                                                else {
27670                                                                        relation.addSource(new TableColumnRelationshipElement(variableColumn));
27671                                                                }
27672                                                        } else {
27673                                                                TableColumn wholeValue = variable instanceof Variable
27674                                                                                ? compositeBindingColumn((Variable) variable)
27675                                                                                : variable.getColumns().get(0);
27676                                                                if(isOut){
27677                                                                        relation.setTarget(new TableColumnRelationshipElement(wholeValue));
27678                                                                }
27679                                                                else {
27680                                                                        relation.addSource(new TableColumnRelationshipElement(wholeValue));
27681                                                                }
27682                                                        }
27683                                                        continue;
27684                                                }
27685                                        } else if (variable.isVariable() || variable.isCursor()) {
27686                                                TableColumn columnModel = variable.getColumns().get(0);
27687                                                if (valueIndex != null) {
27688                                                        if(isOut){
27689                                                                relation.setTarget(new TableColumnRelationshipElement(columnModel, valueIndex));
27690                                                        }
27691                                                        else {
27692                                                                relation.addSource(new TableColumnRelationshipElement(columnModel, valueIndex));
27693                                                        }
27694                                                } else {
27695                                                        if(isOut){
27696                                                                relation.setTarget(new TableColumnRelationshipElement(columnModel));
27697                                                        }
27698                                                        else {
27699                                                                relation.addSource(new TableColumnRelationshipElement(columnModel));
27700                                                        }
27701                                                }
27702                                                continue;
27703                                        }
27704                                }
27705                        }
27706                        
27707                        if (columnName.getColumnNameOnly().startsWith("@")
27708                                        && (option.getVendor() == EDbVendor.dbvmssql || option.getVendor() == EDbVendor.dbvazuresql)) {
27709                                continue;
27710                        }
27711
27712                        if (columnName.getColumnNameOnly().startsWith(":")
27713                                        && (option.getVendor() == EDbVendor.dbvhana || option.getVendor() == EDbVendor.dbvteradata)) {
27714                                Table variable = modelManager
27715                                                .getTableByName(DlineageUtil.getTableFullName(columnName.getColumnNameOnly().replace(":", "")));
27716                                if (variable != null) {
27717                                        for (int j = 0; j < variable.getColumns().size(); j++) {
27718                                                if (getColumnName(columnName).replace(":", "")
27719                                                                .equals(getColumnName(variable.getColumns().get(j).getColumnObject()))) {
27720                                                        TableColumn columnModel = variable.getColumns().get(j);
27721                                                        relation.addSource(new TableColumnRelationshipElement(columnModel));
27722                                                }
27723                                        }
27724                                }
27725                                continue;
27726                        }
27727
27728                        boolean linkedFirstTable = false;
27729                        
27730                        TCustomSqlStatement stmt = stmtStack.peek();
27731                        TTableList tableList = stmt.tables;
27732                        if ((tableList == null || tableList.size() == 0) && (hiveFromTables != null && hiveFromTables.size() > 0)) {
27733                                tableList = hiveFromTables;
27734                        }
27735
27736                        List<TTable> tables = new ArrayList<TTable>();
27737                        {
27738                                TTable table = columnName.getSourceTable();
27739
27740                                // 针对CursorVariable特殊处理
27741                                if (columnName.getTableToken() != null) {
27742
27743                                        Table tableModel = null;
27744                                        if (table != null && modelManager.getModel(table) instanceof Table) {
27745                                                tableModel = (Table) modelManager.getModel(table);
27746                                        }
27747
27748                                        if (tableModel == null) {
27749                                                tableModel = modelManager
27750                                                                .getTableByName(DlineageUtil.getTableFullName(columnName.getTableToken().toString()));
27751                                        }
27752
27753                                        if (tableModel == null) {
27754                                                TCustomSqlStatement currentStmt = ModelBindingManager.getGlobalStmtStack().peek();
27755                                                String procedureName = DlineageUtil.getProcedureParentName(currentStmt);
27756                                                String variableString = columnName.getTableToken().toString();
27757                                                if (variableString.startsWith(":")) {
27758                                                        variableString = variableString.substring(variableString.indexOf(":") + 1);
27759                                                }
27760                                                if (!SQLUtil.isEmpty(procedureName)) {
27761                                                        variableString = procedureName + "." + SQLUtil.getIdentifierNormalTableName(variableString);
27762                                                }
27763
27764                                                if (modelManager
27765                                                                .getTableByName(DlineageUtil.getTableFullName(variableString)) instanceof Variable) {
27766                                                        tableModel = modelManager.getTableByName(DlineageUtil.getTableFullName(variableString));
27767                                                }
27768                                        }
27769
27770                                        if (tableModel != null) {
27771
27772                                                if (table == null) {
27773                                                        table = tableModel.getTableObject();
27774                                                }
27775
27776                                                if (tableModel.isVariable()) {
27777                                                        if (!isStar && "*".equals(getColumnName(columnName))) {
27778                                                                TObjectName[] columns = modelManager.getTableColumns(table);
27779                                                                for (int j = 0; j < columns.length; j++) {
27780                                                                        TObjectName objectName = columns[j];
27781                                                                        if (objectName == null || "*".equals(getColumnName(objectName))) {
27782                                                                                continue;
27783                                                                        }
27784                                                                        TableColumn columnModel = modelFactory.createTableColumn(tableModel, objectName,
27785                                                                                        false);
27786                                                                        relation.addSource(new TableColumnRelationshipElement(columnModel));
27787                                                                }
27788                                                        } else {
27789                                                                if ("*".equals(getColumnName(columnName)) && !tableModel.getColumns().isEmpty()) {
27790
27791                                                                        for (int j = 0; j < tableModel.getColumns().size(); j++) {
27792                                                                                TableColumn columnModel = tableModel.getColumns().get(j);
27793                                                                                if (exceptColumnList != null) {
27794                                                                                        boolean flag = false;
27795                                                                                        for (TObjectName objectName : exceptColumnList) {
27796                                                                                                if (getColumnName(objectName)
27797                                                                                                                .equals(getColumnName(columnModel.getColumnObject()))) {
27798                                                                                                        flag = true;
27799                                                                                                        break;
27800                                                                                                }
27801                                                                                        }
27802                                                                                        if (flag) {
27803                                                                                                continue;
27804                                                                                        }
27805                                                                                }
27806                                                                                relation.addSource(new TableColumnRelationshipElement(columnModel));
27807                                                                        }
27808
27809                                                                        if (isStar && showStar) {
27810                                                                                TableColumn columnModel = modelFactory.createTableColumn(tableModel, columnName,
27811                                                                                                false);
27812                                                                                if (columnModel == null) {
27813                                                                                        if (tableModel.isCreateTable()) {
27814                                                                                                for (TableColumn tableColumn : tableModel.getColumns()) {
27815                                                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
27816                                                                                                        relation.setShowStarRelation(true);
27817                                                                                                }
27818                                                                                        }
27819                                                                                } else {
27820                                                                                        relation.addSource(new TableColumnRelationshipElement(columnModel));
27821                                                                                        relation.setShowStarRelation(true);
27822                                                                                }
27823                                                                        }
27824                                                                } else {
27825                                                                        TableColumn columnModel = modelFactory.createTableColumn(tableModel, columnName,
27826                                                                                        false);
27827                                                                        
27828                                                                        if(columnModel == null && containStarColumn(tableModel.getColumns())){
27829                                                                                columnModel = getStarColumn(tableModel.getColumns());
27830                                                                                if (columnModel != null && tableModel.getSubType() == SubType.record_type) {
27831                                                                                        if(!"*".equals(getColumnName(columnName))){
27832                                                                                                columnModel.bindStarLinkColumn(columnName);
27833                                                                                        }
27834                                                                                }
27835                                                                        }
27836                                                                        
27837                                                                        if (columnModel == null) {
27838                                                                                continue;
27839                                                                        }
27840                                                                        if (columnModel.hasStarLinkColumn()) {
27841                                                                                relation.addSource(new TableColumnRelationshipElement(columnModel,
27842                                                                                                columnModel.getStarLinkColumnNames()
27843                                                                                                                .indexOf(DlineageUtil.getColumnName(columnName))));
27844                                                                        } else {
27845                                                                                if (isOut) {
27846                                                                                        relation.setTarget(new TableColumnRelationshipElement(columnModel));
27847                                                                                } else {
27848                                                                                        relation.addSource(new TableColumnRelationshipElement(columnModel));
27849                                                                                }
27850                                                                        }
27851                                                                        if (columnName.getSourceTable() != null
27852                                                                                        && columnName.getSourceTable().getTableType() == ETableSource.function) {
27853                                                                                analyzeTableColumn(columnModel, columnName.getSourceTable().getFuncCall(),
27854                                                                                                effectType);
27855                                                                        }
27856                                                                }
27857                                                        }
27858                                                        continue;
27859                                                }
27860                                        }
27861                                }
27862
27863                                if (table == null) {
27864                                        table = modelManager.getTable(stmt, columnName);
27865                                }
27866
27867                                if (table == null) {
27868                                        if (columnName.getTableToken() != null || !"*".equals(getColumnName(columnName))) {
27869                                                table = columnName.getSourceTable();
27870                                        }
27871                                        
27872                                        if (table == null && !SQLUtil.isEmpty(columnName.getTableString())) {
27873                                                table = modelManager.getTableFromColumn(columnName);
27874                                        }
27875                                }
27876
27877                                if (table == null) {
27878                                        if (tableList != null) {
27879                                                for (int j = 0; j < tableList.size(); j++) {
27880                                                        if (table != null)
27881                                                                break;
27882
27883                                                        TTable tTable = tableList.getTable(j);
27884                                                        if (tTable.getTableType().name().startsWith("open")) {
27885                                                                continue;
27886                                                        }
27887
27888                                                        if (getTableLinkedColumns(tTable) != null && getTableLinkedColumns(tTable).size() > 0) {
27889                                                                for (int z = 0; z < getTableLinkedColumns(tTable).size(); z++) {
27890                                                                        TObjectName refer = getTableLinkedColumns(tTable).getObjectName(z);
27891                                                                        if ("*".equals(getColumnName(refer)))
27892                                                                                continue;
27893                                                                        // For BigQuery struct field access, match base column name from FieldPath
27894                                                                        String structFullName = getStructFieldFullName(columnName);
27895                                                                        if (structFullName != null) {
27896                                                                                String baseName = getStructFieldBaseName(columnName);
27897                                                                                if (baseName != null && DlineageUtil.getIdentifierNormalColumnName(getColumnName(refer))
27898                                                                                                .equals(DlineageUtil.getIdentifierNormalColumnName(baseName))) {
27899                                                                                        table = tTable;
27900                                                                                        break;
27901                                                                                }
27902                                                                        }
27903                                                                        if (getColumnName(refer).equals(getColumnName(columnName))) {
27904                                                                                table = tTable;
27905                                                                                break;
27906                                                                        }
27907                                                                }
27908                                                        }
27909
27910                                                        if (tTable.getLinkTable() != null) {
27911                                                                tTable = tTable.getLinkTable();
27912                                                                for (int z = 0; z < getTableLinkedColumns(tTable).size(); z++) {
27913                                                                        TObjectName refer = getTableLinkedColumns(tTable).getObjectName(z);
27914                                                                        if ("*".equals(getColumnName(refer)))
27915                                                                                continue;
27916                                                                        if (getColumnName(refer).equals(getColumnName(columnName))) {
27917                                                                                table = tTable;
27918                                                                                break;
27919                                                                        }
27920                                                                }
27921                                                        }
27922
27923                                                        if (table != null)
27924                                                                break;
27925
27926                                                        if (columnName.getTableToken() != null && (SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotTable,
27927                                                                        columnName.getTableToken().getAstext(), tTable.getName())
27928                                                                        || SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotTable, columnName.getTableToken().getAstext(), tTable.getAliasName()))) {
27929                                                                table = tTable;
27930                                                                break;
27931                                                        }
27932                                                }
27933                                        }
27934
27935                                        if (table == null) {
27936                                                for (int j = 0; j < tableList.size(); j++) {
27937                                                        if (table != null)
27938                                                                break;
27939
27940                                                        TTable tTable = tableList.getTable(j);
27941                                                        Object model = ModelBindingManager.get().getModel(tTable);
27942                                                        if (model instanceof Table) {
27943                                                                Table tableModel = (Table) model;
27944                                                                for (int z = 0; tableModel.getColumns() != null
27945                                                                                && z < tableModel.getColumns().size(); z++) {
27946                                                                        TableColumn refer = tableModel.getColumns().get(z);
27947                                                                        if (getColumnName(refer.getName()).equals(getColumnName(columnName))) {
27948                                                                                table = tTable;
27949                                                                                break;
27950                                                                        }
27951                                                                        if (refer.hasStarLinkColumn()) {
27952                                                                                for (TObjectName linkColumn : refer.getStarLinkColumnList()) {
27953                                                                                        if (getColumnName(linkColumn).equals(getColumnName(columnName))) {
27954                                                                                                table = tTable;
27955                                                                                                break;
27956                                                                                        }
27957                                                                                }
27958                                                                        }
27959                                                                }
27960                                                        } else if (model instanceof QueryTable) {
27961                                                                QueryTable tableModel = (QueryTable) model;
27962                                                                for (int z = 0; tableModel.getColumns() != null
27963                                                                                && z < tableModel.getColumns().size(); z++) {
27964                                                                        ResultColumn refer = tableModel.getColumns().get(z);
27965                                                                        // Try FieldPath-based matching first
27966                                                                        String structFullName = getStructFieldFullName(columnName);
27967                                                                        if (structFullName != null) {
27968                                                                                String baseName = getStructFieldBaseName(columnName);
27969                                                                                if (baseName != null && DlineageUtil.getIdentifierNormalColumnName(refer.getName())
27970                                                                                                .equals(DlineageUtil.getIdentifierNormalColumnName(baseName))) {
27971                                                                                        table = tTable;
27972                                                                                        break;
27973                                                                                }
27974                                                                        }
27975                                                                        List<String> splits = SQLUtil.parseNames(columnName.toString());
27976                                                                        if (splits.size() > 1 && EDbVendor.dbvbigquery == getOption().getVendor()) {
27977                                                                                if (DlineageUtil.getIdentifierNormalColumnName(refer.getName())
27978                                                                                                .equals(DlineageUtil
27979                                                                                                                .getIdentifierNormalColumnName(getColumnName(splits.get(0))))) {
27980                                                                                        table = tTable;
27981                                                                                        break;
27982                                                                                }
27983                                                                        }
27984                                                                        else if (DlineageUtil.getIdentifierNormalColumnName(refer.getName()).equals(
27985                                                                                        DlineageUtil.getIdentifierNormalColumnName(getColumnName(columnName)))) {
27986                                                                                table = tTable;
27987                                                                                break;
27988                                                                        }
27989                                                                        if (refer.hasStarLinkColumn()) {
27990                                                                                for (TObjectName linkColumn : refer.getStarLinkColumnList()) {
27991                                                                                        if (getColumnName(linkColumn).equals(getColumnName(columnName))) {
27992                                                                                                table = tTable;
27993                                                                                                break;
27994                                                                                        }
27995                                                                                }
27996                                                                        }
27997                                                                }
27998                                                        }
27999                                                }
28000                                        }
28001                                }
28002
28003                                if (columnName.getTableToken() == null && "*".equals(getColumnName(columnName))) {
28004                                        if (!hasJoin(stmt)) {
28005                                                tables.add(table);
28006                                        } else {
28007                                                for (int j = 0; j < tableList.size(); j++) {
28008                                                        tables.add(tableList.getTable(j));
28009                                                }
28010                                        }
28011                                } else if (table != null) {
28012                                        tables.add(table);
28013                                }
28014
28015                                // 此处特殊处理,多表关联无法找到 column 所属的 Table, tTable.getLinkedColumns
28016                                // 也找不到,退而求其次采用第一个表
28017
28018                                if (stmt.getParentStmt() != null && isApplyJoin(stmt.getParentStmt())
28019                                                && (tableList != null && tableList.size() > 0)) {
28020                                        stmt = stmt.getParentStmt();
28021                                        TTable applyTable = tableList.getTable(0);
28022                                        if (modelManager.getModel(table) == null) {
28023                                                modelFactory.createTable(applyTable);
28024                                        }
28025                                }
28026
28027                                if (columnName.toString().indexOf(".")==-1 && isConstant(columnName)) {
28028                                        boolean isConstant = true;
28029                                        if (columnName.getSourceTable() != null) {
28030                                                Table tableModel = modelManager.getTableByName(
28031                                                                DlineageUtil.getTableFullName(columnName.getSourceTable().getTableName().toString()));
28032                                                if (tableModel != null && tableModel.getColumns() != null) {
28033                                                        for (int j = 0; j < tableModel.getColumns().size(); j++) {
28034                                                                if (DlineageUtil.compareColumnIdentifier(getColumnName(columnName),
28035                                                                                getColumnName(tableModel.getColumns().get(j).getName()))) {
28036                                                                        isConstant = false;
28037                                                                        break;
28038                                                                }
28039                                                        }
28040                                                }
28041                                        }
28042
28043                                        if (isConstant) {
28044                                                if (option.isShowConstantTable()) {
28045                                                        Table constantTable = modelFactory.createConstantsTable(stmtStack.peek());
28046                                                        TableColumn tableColumn = modelFactory.createTableColumn(constantTable, columnName, false);
28047                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
28048                                                }
28049                                                continue;
28050                                        }
28051                                }
28052
28053                                if (tableList != null && tableList.size() != 0 && tables.size() == 0
28054                                                && !(isBuiltInFunctionName(columnName) && isFromFunction(columnName))) {
28055                                        if (modelManager.getModel(stmt) instanceof ResultSet) {
28056                                                ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt);
28057                                                boolean find = false;
28058                                                for (ResultColumn resultColumn : resultSetModel.getColumns()) {
28059                                                        if(resultColumn.equals(modelObject)) {
28060                                                                continue;
28061                                                        }
28062                                                        if (!TSQLEnv.isAliasReferenceForbidden.get(option.getVendor())) {
28063                                                                if (getColumnName(columnName).equals(getColumnName(resultColumn.getName()))) {
28064                                                                        if (resultColumn.getColumnObject() != null) {
28065                                                                                int startToken = resultColumn.getColumnObject().getStartToken().posinlist;
28066                                                                                int endToken = resultColumn.getColumnObject().getEndToken().posinlist;
28067                                                                                if (columnName.getStartToken().posinlist >= startToken
28068                                                                                                && columnName.getEndToken().posinlist <= endToken) {
28069                                                                                        continue;
28070                                                                                }
28071                                                                        }
28072                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
28073                                                                        find = true;
28074                                                                        break;
28075                                                                }
28076                                                        }
28077                                                }
28078                                                if (find) {
28079                                                        continue;
28080                                                }
28081                                        }
28082
28083                                        // Structured-dataflow outer field resolution: an unresolved
28084                                        // reference like nodes1.key (or t.nodes1.key) may be a
28085                                        // struct-field projection produced by a structured generator
28086                                        // (e.g. Spark explode(from_json(...))) in a visible subquery.
28087                                        // Link directly to the matching FunctionResultColumn so the
28088                                        // exact source path (nodes[*].key) flows through.
28089                                        if (tryAppendStructuredFieldRelation(relation, columnName, tableList)) {
28090                                                continue;
28091                                        }
28092
28093                                        TObjectName pseudoTableName = new TObjectName();
28094                                        // Use qualified prefix from column name if available (e.g., sch.pk_constv2 from sch.pk_constv2.c_cdsl)
28095                                        // Otherwise fall back to default pseudo table name
28096                                        String qualifiedPrefix = getQualifiedPrefixFromColumn(columnName);
28097                                        pseudoTableName.setString(qualifiedPrefix != null ? qualifiedPrefix : "pseudo_table_include_orphan_column");
28098                                        Table pseudoTable = modelFactory.createTableByName(pseudoTableName);
28099                                        pseudoTable.setPseudo(true);
28100                                        TableColumn pseudoTableColumn = modelFactory.createTableColumn(pseudoTable, columnName, true);
28101
28102                                        // If not linking to first table and column has qualified prefix (3-part name like sch.pkg.col),
28103                                        // add the pseudo table column as source
28104                                        if (!isLinkOrphanColumnToFirstTable() && pseudoTableColumn != null && qualifiedPrefix != null) {
28105                                                if (isOut) {
28106                                                        relation.setTarget(new TableColumnRelationshipElement(pseudoTableColumn));
28107                                                } else {
28108                                                        relation.addSource(new TableColumnRelationshipElement(pseudoTableColumn));
28109                                                }
28110                                        }
28111
28112                                        if (isLinkOrphanColumnToFirstTable()) {
28113                                                TTable orphanTable = tableList.getTable(0);
28114                                                tables.add(orphanTable);
28115                                                Object tableModel = modelManager.getModel(orphanTable);
28116                                                if (tableModel == null) {
28117                                                        if(orphanTable.getSubquery()!=null) {
28118                                                                QueryTable queryTable = modelFactory.createQueryTable(orphanTable);
28119                                                                TSelectSqlStatement subquery = orphanTable.getSubquery();
28120                                                                analyzeSelectStmt(subquery);
28121                                                        }
28122                                                        else {
28123                                                                tableModel = modelFactory.createTable(orphanTable);
28124                                                        }
28125                                                }
28126                                                if (tableModel instanceof Table) {
28127                                                        TableColumn orphanColum = modelFactory.createTableColumn((Table) tableModel, columnName, false);
28128                                                        if(orphanColum!=null) {
28129                                                                ErrorInfo errorInfo = new ErrorInfo();
28130                                                                errorInfo.setErrorType(ErrorInfo.LINK_ORPHAN_COLUMN);
28131                                                                errorInfo.setErrorMessage("Link orphan column [" + columnName.toString()
28132                                                                                + "] to the first table [" + orphanTable.getFullNameWithAliasString() + "]");
28133                                                                errorInfo.setStartPosition(new Pair3<Long, Long, String>(columnName.getStartToken().lineNo,
28134                                                                                columnName.getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
28135                                                                errorInfo.setEndPosition(new Pair3<Long, Long, String>(columnName.getEndToken().lineNo,
28136                                                                                columnName.getEndToken().columnNo + columnName.getEndToken().getAstext().length(),
28137                                                                                ModelBindingManager.getGlobalHash()));
28138                                                                errorInfo.fillInfo(this);
28139                                                                errorInfos.add(errorInfo);
28140                                                        }
28141                                                        if (orphanTable.getSubquery() != null) {
28142                                                                TSelectSqlStatement subquery = orphanTable.getSubquery();
28143                                                                if (subquery.getResultColumnList().toString().endsWith("*") && subquery.getTables().size() == 1) {
28144                                                                        TTable subqueryTable = subquery.getTables().getTable(0);
28145                                                                        Object sourceTable = modelManager.getModel(subqueryTable);
28146                                                                        if(sourceTable instanceof Table) {
28147                                                                                modelFactory.createTableColumn((Table) sourceTable, columnName, false);
28148                                                                        }
28149                                                                        else if(sourceTable instanceof ResultSet) {
28150                                                                                modelFactory.createResultColumn((ResultSet) sourceTable, columnName, false);
28151                                                                        }
28152                                                                }
28153                                                        }
28154                                                        else if (orphanTable.getCTE()!=null && orphanTable.getCTE().getSubquery() != null) {
28155                                                                TSelectSqlStatement subquery = orphanTable.getCTE().getSubquery();
28156                                                                if (subquery.getResultColumnList().toString().endsWith("*") && subquery.getTables().size() == 1) {
28157                                                                        TTable subqueryTable = subquery.getTables().getTable(0);
28158                                                                        Object sourceTable = modelManager.getModel(subqueryTable);
28159                                                                        if(sourceTable instanceof Table) {
28160                                                                                modelFactory.createTableColumn((Table) sourceTable, columnName, false);
28161                                                                        }
28162                                                                        else if(sourceTable instanceof ResultSet) {
28163                                                                                modelFactory.createResultColumn((ResultSet) sourceTable, columnName, false);
28164                                                                        }
28165                                                                }
28166                                                        }
28167                                                }
28168                                                
28169                                                linkedFirstTable = true;
28170                                        }
28171                                }
28172                        }
28173
28174                        for (int k = 0; k < tables.size(); k++) {
28175                                TTable table = tables.get(k);
28176                                if (table != null) {
28177                                        Object object = modelManager.getModel(table);
28178                                        if(object instanceof PivotedTable) {
28179                                                TPivotClause clause = (TPivotClause)((PivotedTable)object).getGspObject();
28180                                                if(clause.getAliasClause()!=null) {
28181                                                        object = modelManager.getModel(clause.getAliasClause());
28182                                                }
28183                                        }
28184                                        if (object == null && table.getTableType() == ETableSource.objectname) {
28185                                                if (table.getCTE() != null) {
28186                                                        QueryTable queryTable = modelFactory.createQueryTable(table);
28187                                                        TSelectSqlStatement subquery = table.getCTE().getSubquery();
28188                                                        analyzeSelectStmt(subquery);
28189                                                        ResultSet resultSetModel = (ResultSet) modelManager.getModel(subquery);
28190
28191                                                        if (resultSetModel != null && resultSetModel != queryTable
28192                                                                        && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
28193                                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
28194                                                                impactRelation.setEffectType(EffectType.select);
28195                                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
28196                                                                                resultSetModel.getRelationRows()));
28197                                                                impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
28198                                                                                queryTable.getRelationRows()));
28199                                                        }
28200
28201                                                        if (resultSetModel != null && resultSetModel != queryTable) {
28202                                                                for (int j = 0; j < resultSetModel.getColumns().size(); j++) {
28203                                                                        ResultColumn sourceColumn = resultSetModel.getColumns().get(j);
28204                                                                        ResultColumn targetColumn = modelFactory.createSelectSetResultColumn(queryTable,
28205                                                                                        sourceColumn);
28206
28207                                                                        DataFlowRelationship queryRalation = modelFactory.createDataFlowRelation();
28208                                                                        queryRalation.setEffectType(EffectType.select);
28209                                                                        queryRalation.setTarget(new ResultColumnRelationshipElement(targetColumn));
28210                                                                        queryRalation.addSource(new ResultColumnRelationshipElement(sourceColumn));
28211                                                                }
28212                                                        }
28213                                                        
28214                                                        object = queryTable;
28215                                                        
28216                                                } else {
28217                                                        object = modelFactory.createTable(table);
28218                                                }
28219                                        }
28220                                        if (object instanceof Function) {
28221                                                relation.addSource(new ResultColumnRelationshipElement(((Function)object).getColumns().get(0)));
28222                                                continue;
28223                                        } else if (object instanceof ResultSet && !(object instanceof QueryTable)) {
28224                                                //Object tableModel = modelManager.getModel(columnName.getSourceTable());
28225                                                appendResultColumnRelationSource(modelObject, relation, columnIndex, columnName,
28226                                                                (ResultSet)object);
28227                                                if(table.getTableType() == ETableSource.function && !relation.getSources().isEmpty()) {
28228                                                        relation.getSources().stream().reduce((first, second) -> second).get().addTransform(Transform.FUNCTION, table);
28229                                                }
28230                                                continue;
28231                                        }
28232                                        else if (object instanceof Table) {
28233                                                Table tableModel = (Table) modelManager.getModel(table);
28234                                                if (tableModel != null) {
28235                                                        if (!isStar && "*".equals(getColumnName(columnName))) {
28236                                                                TObjectName[] columns = modelManager.getTableColumns(table);
28237                                                                for (int j = 0; j < columns.length; j++) {
28238                                                                        TObjectName objectName = columns[j];
28239                                                                        if (objectName == null || ("*".equals(getColumnName(objectName)) && tableModel.isDetermined())) {
28240                                                                                continue;
28241                                                                        }
28242                                                                        TableColumn columnModel = modelFactory.createTableColumn(tableModel, objectName,
28243                                                                                        false);
28244                                                                        if(columnModel == null) {
28245                                                                                continue;
28246                                                                        }
28247                                                                        relation.addSource(new TableColumnRelationshipElement(columnModel));
28248                                                                }
28249                                                        } else {
28250                                                                if ("*".equals(getColumnName(columnName)) && !tableModel.getColumns().isEmpty()) {
28251                                                                        Map<String, Pair<String, TExpression>> replaceAsIdentifierMap = new HashMap<String, Pair<String, TExpression>>();
28252                                                                        Map<String, TObjectName> replaceColumnMap = new HashMap<String, TObjectName>();
28253                                                                        if(modelObject instanceof ResultColumn && ((ResultColumn)modelObject).getColumnObject() instanceof TResultColumn) {
28254                                                                                TResultColumn resultColumn =  (TResultColumn )((ResultColumn)modelObject).getColumnObject();
28255                                                                                if(resultColumn.getReplaceExprAsIdentifiers()!=null && resultColumn.getReplaceExprAsIdentifiers().size()>0) {
28256                                                                                        for(TReplaceExprAsIdentifier replace: resultColumn.getReplaceExprAsIdentifiers()) {
28257                                                                                                replaceAsIdentifierMap.put(replace.getIdentifier().toString(), new Pair<String, TExpression>(resultColumn.getExpr().getExceptReplaceClause().toString(), replace.getExpr()));
28258                                                                                                replaceColumnMap.put(replace.getIdentifier().toString(), replace.getIdentifier());
28259                                                                                        }
28260                                                                                        ResultSet resultSet = ((ResultColumn)modelObject).getResultSet();
28261                                                                                        if(tableModel.isDetermined()) {
28262                                                                                                resultSet.getColumns().clear();
28263                                                                                        }
28264                                                                                }
28265                                                                        }
28266                                                                        
28267                                                                        for (int j = 0; j < tableModel.getColumns().size(); j++) {
28268                                                                                TableColumn columnModel = tableModel.getColumns().get(j);
28269                                                                                if (exceptColumnList != null) {
28270                                                                                        boolean flag = false;
28271                                                                                        for (TObjectName objectName : exceptColumnList) {
28272                                                                                                if (getColumnName(objectName)
28273                                                                                                                .equals(getColumnName(columnModel.getColumnObject()))) {
28274                                                                                                        flag = true;
28275                                                                                                        break;
28276                                                                                                }
28277                                                                                        }
28278                                                                                        if (flag) {
28279                                                                                                continue;
28280                                                                                        }
28281                                                                                }
28282                                                                                
28283                                                                                if (replaceAsIdentifierMap.containsKey(tableModel.getColumns().get(j).getName())) {
28284                                                                                        Pair<String, TExpression> expr = replaceAsIdentifierMap.get(tableModel.getColumns().get(j).getName());
28285                                                                                        ResultSet resultSet = ((ResultColumn)modelObject).getResultSet();
28286                                                                                        ResultColumn resultColumn = modelFactory.createResultColumn(resultSet, replaceColumnMap.get(tableModel.getColumns().get(j).getName()));
28287                                                                                        Transform transform = new Transform();
28288                                                                                        transform.setType(Transform.EXPRESSION);
28289                                                                                        TObjectName expression = new TObjectName();
28290                                                                                        expression.setString(expr.first);
28291                                                                                transform.setCode(expression);
28292                                                                                        resultColumn.setTransform(transform);
28293                                                                                        analyzeResultColumnExpressionRelation(resultColumn, expr.second);
28294                                                                                } else {
28295                                                                                        if(!replaceAsIdentifierMap.isEmpty()) {
28296                                                                                                ResultSet resultSet = ((ResultColumn) modelObject).getResultSet();
28297                                                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSet,
28298                                                                                                                columnModel.getColumnObject());
28299                                                                                                DataFlowRelationship relation1 = modelFactory.createDataFlowRelation();
28300                                                                                                relation1.setEffectType(effectType);
28301                                                                                                relation1.setProcess(process);
28302                                                                                                relation1.setTarget(new ResultColumnRelationshipElement(resultColumn));
28303                                                                                                relation1.addSource(new TableColumnRelationshipElement(columnModel));
28304                                                                                        }
28305                                                                                        else {
28306                                                                                                relation.addSource(
28307                                                                                                        new TableColumnRelationshipElement(columnModel));
28308                                                                                        }
28309                                                                                }
28310                                                                        }
28311
28312                                                                        if (isStar && showStar) {
28313                                                                                TableColumn columnModel = modelFactory.createTableColumn(tableModel, columnName,
28314                                                                                                false);
28315                                                                                if (columnModel == null) {
28316                                                                                        if(tableModel.isCreateTable()) {
28317                                                                                                for (TableColumn tableColumn : tableModel.getColumns()) {
28318                                                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
28319                                                                                                        relation.setShowStarRelation(true);
28320                                                                                                }
28321                                                                                        }
28322                                                                                } else {
28323                                                                                        relation.addSource(new TableColumnRelationshipElement(columnModel));
28324                                                                                        relation.setShowStarRelation(true);
28325                                                                                }
28326                                                                        }
28327                                                                } else {
28328                                                                        TableColumn columnModel = modelFactory.createTableColumn(tableModel, columnName,
28329                                                                                        false);
28330                                                                        if(columnModel == null) {
28331                                                                                if(tableModel.isCreateTable()) {
28332                                                                                        boolean flag = false;
28333                                                                                        // Try FieldPath-based matching first for BigQuery/Redshift struct fields
28334                                                                                        String structFullName = getStructFieldFullName(columnName);
28335                                                                                        if (structFullName != null && !flag) {
28336                                                                                                for (TableColumn tableColumn : tableModel.getColumns()) {
28337                                                                                                        if (tableColumn.isStruct()
28338                                                                                                                && DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName())
28339                                                                                                                        .equals(DlineageUtil.getIdentifierNormalColumnName(structFullName))) {
28340                                                                                                                relation.addSource(new TableColumnRelationshipElement(tableColumn));
28341                                                                                                                flag = true;
28342                                                                                                                if (modelObject instanceof ResultColumn) {
28343                                                                                                                        ((ResultColumn) modelObject).setStruct(true);
28344                                                                                                                } else if (modelObject instanceof TableColumn) {
28345                                                                                                                        ((TableColumn) modelObject).setStruct(true);
28346                                                                                                                }
28347                                                                                                                break;
28348                                                                                                        }
28349                                                                                                }
28350                                                                                        }
28351                                                                                        if (ModelBindingManager.getGlobalVendor() == EDbVendor.dbvbigquery || ModelBindingManager.getGlobalVendor() == EDbVendor.dbvredshift) {
28352                                                                                                for (TableColumn tableColumn : tableModel.getColumns()) {
28353                                                                                                        if(tableColumn.isStruct()) {
28354                                                                                                                List<String> names = SQLUtil.parseNames(tableColumn.getName());
28355                                                                                                                if (modelObject instanceof TableColumn) {
28356                                                                                                                        TableColumn targetColumn = (TableColumn) modelObject;
28357                                                                                                                        if (targetColumn.isStruct()) {
28358                                                                                                                                List<String> targetNames = SQLUtil
28359                                                                                                                                                .parseNames(targetColumn.getName());
28360                                                                                                                                if (!getColumnName(targetNames.get(0))
28361                                                                                                                                                .equals(getColumnName(names.get(0)))) {
28362                                                                                                                                        continue;
28363                                                                                                                                }
28364                                                                                                                        }
28365                                                                                                                }
28366                                                                                                                else if (modelObject instanceof ResultColumn) {
28367                                                                                                                        ResultColumn targetColumn = (ResultColumn) modelObject;
28368                                                                                                                        if (targetColumn.isStruct()) {
28369                                                                                                                                List<String> targetNames = SQLUtil
28370                                                                                                                                                .parseNames(targetColumn.getName());
28371                                                                                                                                if (!getColumnName(targetNames.get(0))
28372                                                                                                                                                .equals(getColumnName(names.get(0)))) {
28373                                                                                                                                        continue;
28374                                                                                                                                }
28375                                                                                                                        }
28376                                                                                                                }
28377                                                                                                                for(String name: names) {
28378                                                                                                                        if (getColumnName(name)
28379                                                                                                                                        .equals(getColumnName(modelObject.toString()))) {
28380                                                                                                                                relation.addSource(
28381                                                                                                                                                new TableColumnRelationshipElement(tableColumn));
28382                                                                                                                                flag = true;
28383                                                                                                                                if (modelObject instanceof ResultColumn) {
28384                                                                                                                                        ((ResultColumn)modelObject).setStruct(true);
28385                                                                                                                                }
28386                                                                                                                                else if (modelObject instanceof TableColumn) {
28387                                                                                                                                        ((TableColumn)modelObject).setStruct(true);
28388                                                                                                                                }
28389                                                                                                                                break;
28390                                                                                                                        }
28391                                                                                                                }
28392
28393                                                                                                                if (!flag && tableModel.getColumns().size() == 1 && tableModel
28394                                                                                                                                .getColumns().get(0).getSourceColumn() != null) {
28395                                                                                                                        TableColumn sourceColumn = tableModel
28396                                                                                                                                        .getColumns().get(0).getSourceColumn();
28397                                                                                                                        Table sourceTable = sourceColumn.getTable();
28398                                                                                                                        TObjectName sourceColumnName = new TObjectName();
28399                                                                                                                        sourceColumnName.setString(sourceColumn.getName() + "."
28400                                                                                                                                        + columnName.getColumnNameOnly());
28401                                                                                                                        TableColumn sourceTableColumn = modelFactory.createTableColumn(sourceTable, sourceColumnName, true);
28402                                                                                                                        relation.addSource(
28403                                                                                                                                        new TableColumnRelationshipElement(sourceTableColumn));
28404                                                                                                                        flag = true;
28405                                                                                                                        break;
28406                                                                                                                }
28407                                                                                                        }
28408                                                                                                        else if (getColumnName(tableColumn.getName())
28409                                                                                                                        .equals(getColumnName(modelObject.toString()))) {
28410                                                                                                                relation.addSource(
28411                                                                                                                                new TableColumnRelationshipElement(tableColumn));
28412                                                                                                                flag = true;
28413                                                                                                                break;
28414                                                                                                        }
28415                                                                                                }
28416                                                                                                if (modelObject instanceof TableColumn) {
28417                                                                                                        TableColumn column = (TableColumn) modelObject;
28418                                                                                                        for (TableColumn tableColumn : tableModel.getColumns()) {
28419                                                                                                                if (tableColumn.getColumnIndex() == null) {
28420                                                                                                                        continue;
28421                                                                                                                }
28422                                                                                                                if (tableColumn.getName().toLowerCase()
28423                                                                                                                                .indexOf(column.getName().toLowerCase()) == -1
28424                                                                                                                                && tableColumn.getName().toLowerCase()
28425                                                                                                                                                .indexOf(column.getColumnObject().toString()
28426                                                                                                                                                                .toLowerCase()) == -1) {
28427                                                                                                                        continue;
28428                                                                                                                }
28429                                                                                                                flag = true;
28430                                                                                                                relation.addSource(
28431                                                                                                                                new TableColumnRelationshipElement(tableColumn));
28432                                                                                                                relation.setShowStarRelation(true);
28433                                                                                                        }
28434                                                                                                        if (flag)
28435                                                                                                                break;
28436
28437                                                                                                } else if (modelObject instanceof ResultColumn) {
28438                                                                                                        ResultColumn column = (ResultColumn) modelObject;
28439                                                                                                        for (TableColumn tableColumn : tableModel.getColumns()) {
28440                                                                                                                if (tableColumn.getColumnIndex() == null) {
28441                                                                                                                        continue;
28442                                                                                                                }
28443                                                                                                                if (tableColumn.getName().toLowerCase()
28444                                                                                                                                .indexOf(column.getName().toLowerCase()) == -1
28445                                                                                                                                && tableColumn.getName().toLowerCase()
28446                                                                                                                                                .indexOf(column.getColumnObject().toString()
28447                                                                                                                                                                .toLowerCase()) == -1) {
28448                                                                                                                        continue;
28449                                                                                                                }
28450                                                                                                                flag = true;
28451                                                                                                                relation.addSource(
28452                                                                                                                                new TableColumnRelationshipElement(tableColumn));
28453                                                                                                                relation.setShowStarRelation(true);
28454                                                                                                        }
28455                                                                                                        if (flag)
28456                                                                                                                break;
28457
28458                                                                                                }
28459                                                                                        }
28460                                                                                        if (!flag 
28461                                                                                                        && (isStar 
28462                                                                                                                        || getColumnName(columnName).equals(getColumnName(tableModel.getName()))
28463                                                                                                                        || getColumnName(columnName).equals(getColumnName(tableModel.getAlias())))) {
28464                                                                                                for (TableColumn tableColumn : tableModel.getColumns()) {
28465                                                                                                        relation.addSource(new TableColumnRelationshipElement(tableColumn));
28466                                                                                                        relation.setShowStarRelation(true);
28467                                                                                                }
28468                                                                                        }
28469                                                                                }
28470                                                                        }
28471                                                                        else {
28472                                                                                if (linkedFirstTable || columnModel.getCandidateParents() != null) {
28473                                                                                        if (columnName.getCandidateTables() != null
28474                                                                                                        && columnName.getCandidateTables().size() > 1) {
28475                                                                                                List<Object> candidateParents = new ArrayList<Object>();
28476                                                                                                for(TTable tableItem: columnName.getCandidateTables()) {
28477                                                                                                        Object model = modelManager.getModel(tableItem);
28478                                                                                                        if(model!=null) {
28479                                                                                                                candidateParents.add(model);
28480                                                                                                        }
28481                                                                                                }
28482                                                                                                if (candidateParents.size() > 1) {
28483                                                                                                        columnModel.setCandidateParents(candidateParents);
28484                                                                                                }
28485                                                                                        }
28486                                                                                }
28487                                                                                relation.addSource(new TableColumnRelationshipElement(columnModel));
28488                                                                                relation.setShowStarRelation(true);
28489                                                                                if(modelObject instanceof TableColumn) {
28490                                                                                        TableColumn targetTableColumn = (TableColumn)modelObject;
28491                                                                                        if(targetTableColumn.getTable().getSubType() == SubType.unnest && targetTableColumn.getTable().getColumns().size() == 1) {
28492                                                                                                targetTableColumn.setSourceColumn(columnModel);
28493                                                                                                targetTableColumn.setStruct(true);
28494                                                                                        }
28495                                                                                }
28496                                                                                if (columnName.getSourceTable() != null
28497                                                                                                && columnName.getSourceTable().getTableType() == ETableSource.function) {
28498                                                                                        analyzeTableColumn(columnModel, columnName.getSourceTable().getFuncCall(),
28499                                                                                                        effectType);
28500                                                                                }
28501                                                                        }
28502                                                                }
28503                                                        }
28504                                                }
28505                                        } else if (modelManager.getModel(table) instanceof QueryTable) {
28506                                                QueryTable queryTable = (QueryTable) modelManager.getModel(table);
28507
28508                                                TObjectNameList cteColumns = null;
28509                                                TSelectSqlStatement subquery = null;
28510                                                if (queryTable.getTableObject().getCTE() != null) {
28511                                                        subquery = queryTable.getTableObject().getCTE().getSubquery();
28512                                                        cteColumns = queryTable.getTableObject().getCTE().getColumnList();
28513                                                } else if (queryTable.getTableObject().getAliasClause() != null
28514                                                                && queryTable.getTableObject().getAliasClause().getColumns() != null) {
28515
28516                                                } else if (queryTable.getTableObject().getTableExpr() != null
28517                                                                && queryTable.getTableObject().getTableExpr().getSubQuery() != null) {
28518                                                        subquery = queryTable.getTableObject().getTableExpr().getSubQuery();
28519                                                } else {
28520                                                        subquery = queryTable.getTableObject().getSubquery();
28521                                                }
28522
28523                                                if (cteColumns != null) {
28524                                                        for (int j = 0; j < cteColumns.size(); j++) {
28525                                                                modelFactory.createResultColumn(queryTable, cteColumns.getObjectName(j));
28526                                                        }
28527                                                }
28528
28529                                                if (subquery != null && subquery.isCombinedQuery()) {
28530                                                        SelectSetResultSet selectSetResultSetModel = (SelectSetResultSet) modelManager
28531                                                                        .getModel(subquery);
28532
28533                                                        if (selectSetResultSetModel != null
28534                                                                        && !selectSetResultSetModel.getRelationRows().getHoldRelations().isEmpty()) {
28535                                                                ImpactRelationship impactRelation = modelFactory.createImpactRelation();
28536                                                                impactRelation.setEffectType(EffectType.select);
28537                                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
28538                                                                                selectSetResultSetModel.getRelationRows()));
28539                                                                impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
28540                                                                                queryTable.getRelationRows()));
28541                                                        }
28542
28543                                                        if (selectSetResultSetModel != null) {
28544                                                                if (getColumnName(columnName).equals("*")) {
28545                                                                        Map<String, Pair<String, TExpression>> replaceAsIdentifierMap = new HashMap<String, Pair<String, TExpression>>();
28546                                                                        Map<String, TObjectName> replaceColumnMap = new HashMap<String, TObjectName>();
28547                                                                        if(modelObject instanceof ResultColumn && ((ResultColumn)modelObject).getColumnObject() instanceof TResultColumn) {
28548                                                                                TResultColumn resultColumn =  (TResultColumn )((ResultColumn)modelObject).getColumnObject();
28549                                                                                if(resultColumn.getReplaceExprAsIdentifiers()!=null && resultColumn.getReplaceExprAsIdentifiers().size()>0) {
28550                                                                                        for(TReplaceExprAsIdentifier replace: resultColumn.getReplaceExprAsIdentifiers()) {
28551                                                                                                replaceAsIdentifierMap.put(replace.getIdentifier().toString(), new Pair<String, TExpression>(resultColumn.getExpr().getExceptReplaceClause().toString(), replace.getExpr()));
28552                                                                                                replaceColumnMap.put(replace.getIdentifier().toString(), replace.getIdentifier());
28553                                                                                        }
28554                                                                                        ResultSet resultSet = ((ResultColumn)modelObject).getResultSet();
28555                                                                                        if(selectSetResultSetModel.isDetermined()) {
28556                                                                                                resultSet.getColumns().clear();
28557                                                                                        }
28558                                                                                }
28559                                                                        }
28560                                                                        
28561                                                                        
28562                                                                        for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) {
28563                                                                                ResultColumn sourceColumn = selectSetResultSetModel.getColumns().get(j);
28564                                                                                if (cteColumns != null) {
28565                                                                                        if (j < cteColumns.size()) {
28566                                                                                                ResultColumn targetColumn = queryTable.getColumns().get(j);
28567
28568                                                                                                if (exceptColumnList != null) {
28569                                                                                                        boolean flag = false;
28570                                                                                                        for (TObjectName objectName : exceptColumnList) {
28571                                                                                                                if (getColumnName(objectName)
28572                                                                                                                                .equals(getColumnName(targetColumn.getName()))) {
28573                                                                                                                        flag = true;
28574                                                                                                                        break;
28575                                                                                                                }
28576                                                                                                        }
28577                                                                                                        if (flag) {
28578                                                                                                                continue;
28579                                                                                                        }
28580                                                                                                }
28581
28582                                                                                                if (replaceAsIdentifierMap.containsKey(targetColumn.getName())) {
28583                                                                                                        Pair<String, TExpression> expr = replaceAsIdentifierMap.get(targetColumn.getName());
28584                                                                                                        ResultSet resultSet = ((ResultColumn)modelObject).getResultSet();
28585                                                                                                        ResultColumn resultColumn = modelFactory.createResultColumn(resultSet, replaceColumnMap.get(targetColumn.getName()));
28586                                                                                                        Transform transform = new Transform();
28587                                                                                                        transform.setType(Transform.EXPRESSION);
28588                                                                                                        TObjectName expression = new TObjectName();
28589                                                                                                        expression.setString(expr.first);
28590                                                                                                transform.setCode(expression);
28591                                                                                                        resultColumn.setTransform(transform);
28592                                                                                                        analyzeResultColumnExpressionRelation(resultColumn, expr.second);
28593                                                                                                } else {
28594                                                                                                        if(!replaceAsIdentifierMap.isEmpty()) {
28595                                                                                                                ResultSet resultSet = ((ResultColumn) modelObject).getResultSet();
28596                                                                                                                TObjectName resultColumnName = new TObjectName();
28597                                                                                                                resultColumnName.setString(targetColumn.getName());
28598                                                                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSet,
28599                                                                                                                                resultColumnName);
28600                                                                                                                DataFlowRelationship relation1 = modelFactory.createDataFlowRelation();
28601                                                                                                                relation1.setEffectType(effectType);
28602                                                                                                                relation1.setProcess(process);
28603                                                                                                                relation1.setTarget(new ResultColumnRelationshipElement(resultColumn));
28604                                                                                                                relation1.addSource(new ResultColumnRelationshipElement(targetColumn));
28605                                                                                                        }
28606                                                                                                        else {
28607                                                                                                                relation.addSource(
28608                                                                                                                        new ResultColumnRelationshipElement(targetColumn));
28609                                                                                                        }
28610                                                                                                }
28611                                                                                                
28612                                                                                        }
28613                                                                                } else {
28614                                                                                        ResultColumn targetColumn = modelFactory
28615                                                                                                        .createSelectSetResultColumn(queryTable, sourceColumn);
28616
28617                                                                                        DataFlowRelationship combinedQueryRelation = modelFactory
28618                                                                                                        .createDataFlowRelation();
28619                                                                                        combinedQueryRelation.setEffectType(effectType);
28620                                                                                        combinedQueryRelation
28621                                                                                                        .setTarget(new ResultColumnRelationshipElement(targetColumn));
28622                                                                                        combinedQueryRelation
28623                                                                                                        .addSource(new ResultColumnRelationshipElement(sourceColumn));
28624
28625                                                                                        relation.addSource(new ResultColumnRelationshipElement(targetColumn));
28626                                                                                }
28627                                                                        }
28628                                                                        break;
28629                                                                } else {
28630                                                                        boolean flag = false;
28631
28632                                                                        for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) {
28633                                                                                ResultColumn sourceColumn = selectSetResultSetModel
28634                                                                                                .getColumns().get(j);
28635                                                                                List<String> splits = SQLUtil.parseNames(columnName.toString());        
28636                                                                                if (splits.size() > 1 && EDbVendor.dbvbigquery == getOption().getVendor()) {
28637                                                                                        if (DlineageUtil.sameColumnSegment(sourceColumn.getName(), splits.get(0))
28638                                                                                                        || DlineageUtil.sameColumnSegment(sourceColumn.getName(), splits.get(1))) {
28639                                                                                                ResultColumn targetColumn = modelFactory
28640                                                                                                                .createSelectSetResultColumn(queryTable, sourceColumn);
28641
28642                                                                                                DataFlowRelationship combinedQueryRelation = modelFactory
28643                                                                                                                .createDataFlowRelation();
28644                                                                                                combinedQueryRelation.setEffectType(effectType);
28645                                                                                                combinedQueryRelation.setTarget(
28646                                                                                                                new ResultColumnRelationshipElement(targetColumn));
28647                                                                                                combinedQueryRelation.addSource(
28648                                                                                                                new ResultColumnRelationshipElement(sourceColumn));
28649
28650                                                                                                relation.addSource(
28651                                                                                                                new ResultColumnRelationshipElement(targetColumn));
28652                                                                                                flag = true;
28653                                                                                                break;
28654                                                                                        }
28655                                                                                }
28656                                                                                else if (DlineageUtil.sameColumnName(sourceColumn.getName(), columnName)) {
28657                                                                                        ResultColumn targetColumn = modelFactory
28658                                                                                                        .createSelectSetResultColumn(queryTable, sourceColumn);
28659
28660                                                                                        DataFlowRelationship combinedQueryRelation = modelFactory
28661                                                                                                        .createDataFlowRelation();
28662                                                                                        combinedQueryRelation.setEffectType(effectType);
28663                                                                                        combinedQueryRelation
28664                                                                                                        .setTarget(new ResultColumnRelationshipElement(targetColumn));
28665                                                                                        combinedQueryRelation
28666                                                                                                        .addSource(new ResultColumnRelationshipElement(sourceColumn));
28667
28668                                                                                        relation.addSource(new ResultColumnRelationshipElement(targetColumn));
28669                                                                                        flag = true;
28670                                                                                        break;
28671                                                                                }
28672                                                                                else if (sourceColumn instanceof SelectSetResultColumn && ((SelectSetResultColumn)sourceColumn).getAliasSet().size() > 1) {
28673                                                                                        for (String alias : ((SelectSetResultColumn)sourceColumn).getAliasSet()) {
28674                                                                                                if (DlineageUtil.sameColumnName(alias, columnName)) {
28675                                                                                                        ResultColumn targetColumn = modelFactory
28676                                                                                                                        .createSelectSetResultColumn(queryTable, sourceColumn);
28677
28678                                                                                                        DataFlowRelationship combinedQueryRelation = modelFactory
28679                                                                                                                        .createDataFlowRelation();
28680                                                                                                        combinedQueryRelation.setEffectType(effectType);
28681                                                                                                        combinedQueryRelation.setTarget(
28682                                                                                                                        new ResultColumnRelationshipElement(targetColumn));
28683                                                                                                        combinedQueryRelation.addSource(
28684                                                                                                                        new ResultColumnRelationshipElement(sourceColumn));
28685
28686                                                                                                        relation.addSource(
28687                                                                                                                        new ResultColumnRelationshipElement(targetColumn));
28688                                                                                                        flag = true;
28689                                                                                                        break;
28690                                                                                                }
28691                                                                                        }
28692                                                                                        if (flag) {
28693                                                                                                break;
28694                                                                                        }
28695                                                                                }
28696                                                                        }
28697
28698                                                                        if (flag) {
28699                                                                                break;
28700                                                                        } else if (columnIndex != -1) {
28701                                                                                for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) {
28702                                                                                        ResultColumn sourceColumn = selectSetResultSetModel.getColumns().get(j);
28703                                                                                        if (!sourceColumn.getStarLinkColumns().isEmpty()) {
28704                                                                                                if (cteColumns != null) {
28705                                                                                                        if (j < cteColumns.size()) {
28706                                                                                                                ResultColumn targetColumn = queryTable.getColumns().get(j);
28707                                                                                                                relation.addSource(
28708                                                                                                                                new ResultColumnRelationshipElement(targetColumn));
28709                                                                                                        }
28710                                                                                                } 
28711                                                                                                else {
28712                                                                                                        if(sourceColumn.hasStarLinkColumn()) {
28713                                                                                                                for (TObjectName linkColumn : sourceColumn.getStarLinkColumnList()) {
28714                                                                                                                        if (getColumnName(linkColumn).equals(getColumnName(columnName))) {
28715                                                                                                                                ResultColumn targetColumn = modelFactory
28716                                                                                                                                                .createSelectSetResultColumn(queryTable, sourceColumn);
28717
28718                                                                                                                                DataFlowRelationship combinedQueryRelation = modelFactory
28719                                                                                                                                                .createDataFlowRelation();
28720                                                                                                                                combinedQueryRelation.setEffectType(effectType);
28721                                                                                                                                combinedQueryRelation.setTarget(
28722                                                                                                                                                new ResultColumnRelationshipElement(targetColumn, linkColumn));
28723                                                                                                                                combinedQueryRelation.addSource(
28724                                                                                                                                                new ResultColumnRelationshipElement(sourceColumn));
28725
28726                                                                                                                                relation.addSource(
28727                                                                                                                                                new ResultColumnRelationshipElement(targetColumn, linkColumn));
28728                                                                                                                                flag = true;
28729                                                                                                                                break;
28730                                                                                                                        }
28731                                                                                                                }
28732                                                                                                        }
28733                                                                                                        
28734                                                                                                        if(!flag) {
28735                                                                                                                ResultColumn targetColumn = modelFactory
28736                                                                                                                                .createSelectSetResultColumn(queryTable, sourceColumn);
28737
28738                                                                                                                DataFlowRelationship combinedQueryRelation = modelFactory
28739                                                                                                                                .createDataFlowRelation();
28740                                                                                                                combinedQueryRelation.setEffectType(effectType);
28741                                                                                                                combinedQueryRelation.setTarget(
28742                                                                                                                                new ResultColumnRelationshipElement(targetColumn));
28743                                                                                                                combinedQueryRelation.addSource(
28744                                                                                                                                new ResultColumnRelationshipElement(sourceColumn));
28745
28746                                                                                                                relation.addSource(
28747                                                                                                                                new ResultColumnRelationshipElement(targetColumn));
28748                                                                                                        }
28749                                                                                                }
28750                                                                                                flag = true;
28751                                                                                                break;
28752                                                                                        }
28753                                                                                }
28754                                                                        }
28755
28756                                                                        if (flag) {
28757                                                                                break;
28758                                                                        } else if (columnIndex < selectSetResultSetModel.getColumns().size()
28759                                                                                        && columnIndex != -1) {
28760                                                                                ResultColumn sourceColumn = selectSetResultSetModel.getColumns()
28761                                                                                                .get(columnIndex);
28762                                                                                if (cteColumns != null) {
28763                                                                                        boolean flag1 = false;
28764                                                                                        for (ResultColumn targetColumn : queryTable.getColumns()) {
28765                                                                                                if (DlineageUtil.sameColumnName(targetColumn.getName(), columnName)) {
28766                                                                                                        relation.addSource(
28767                                                                                                                        new ResultColumnRelationshipElement(targetColumn));
28768                                                                                                        flag1 = true;
28769                                                                                                        break;
28770                                                                                                }
28771                                                                                        }
28772                                                                                        if (!flag1 && columnIndex < cteColumns.size()){
28773                                                                                                ResultColumn targetColumn = queryTable.getColumns()
28774                                                                                                                .get(columnIndex);
28775                                                                                                relation.addSource(
28776                                                                                                                new ResultColumnRelationshipElement(targetColumn));
28777                                                                                        }
28778                                                                                } else {
28779                                                                                        ResultColumn targetColumn = modelFactory
28780                                                                                                        .createSelectSetResultColumn(queryTable, sourceColumn);
28781
28782                                                                                        DataFlowRelationship combinedQueryRelation = modelFactory
28783                                                                                                        .createDataFlowRelation();
28784                                                                                        combinedQueryRelation.setEffectType(effectType);
28785                                                                                        combinedQueryRelation
28786                                                                                                        .setTarget(new ResultColumnRelationshipElement(targetColumn));
28787                                                                                        combinedQueryRelation
28788                                                                                                        .addSource(new ResultColumnRelationshipElement(sourceColumn));
28789
28790                                                                                        relation.addSource(new ResultColumnRelationshipElement(targetColumn));
28791                                                                                }
28792                                                                                flag = true;
28793                                                                                break;
28794                                                                        }
28795
28796                                                                        if (flag) {
28797                                                                                break;
28798                                                                        }
28799                                                                }
28800                                                        } else if (cteColumns != null) {
28801                                                                if (getColumnName(columnName).equals("*")) {
28802                                                                        Map<String, Pair<String, TExpression>> replaceAsIdentifierMap = new HashMap<String, Pair<String, TExpression>>();
28803                                                                        Map<String, TObjectName> replaceColumnMap = new HashMap<String, TObjectName>();
28804                                                                        if(modelObject instanceof ResultColumn && ((ResultColumn)modelObject).getColumnObject() instanceof TResultColumn) {
28805                                                                                TResultColumn resultColumn =  (TResultColumn )((ResultColumn)modelObject).getColumnObject();
28806                                                                                if(resultColumn.getReplaceExprAsIdentifiers()!=null && resultColumn.getReplaceExprAsIdentifiers().size()>0) {
28807                                                                                        for(TReplaceExprAsIdentifier replace: resultColumn.getReplaceExprAsIdentifiers()) {
28808                                                                                                replaceAsIdentifierMap.put(replace.getIdentifier().toString(), new Pair<String, TExpression>(resultColumn.getExpr().getExceptReplaceClause().toString(), replace.getExpr()));
28809                                                                                                replaceColumnMap.put(replace.getIdentifier().toString(), replace.getIdentifier());
28810                                                                                        }
28811                                                                                        ResultSet resultSet = ((ResultColumn)modelObject).getResultSet();
28812                                                                                        
28813                                                                                        if (columnName.getSourceColumn() != null) {
28814                                                                                                Object model = modelManager.getModel(columnName.getSourceColumn());
28815                                                                                                if (model instanceof ResultColumn && ((ResultColumn)model).getResultSet().isDetermined()) {
28816                                                                                                        resultSet.getColumns().clear();
28817                                                                                                }
28818                                                                                        } else if (columnName.getSourceTable() != null) {
28819                                                                                                Object tableModel = modelManager.getModel(columnName.getSourceTable());
28820                                                                                                if (tableModel instanceof Table && ((Table)tableModel).isDetermined()) {
28821                                                                                                        resultSet.getColumns().clear();
28822                                                                                                }
28823                                                                                        }
28824                                                                                }
28825                                                                        }
28826                                                                        
28827                                                                        for (int j = 0; j < cteColumns.size(); j++) {
28828                                                                                ResultColumn targetColumn = queryTable.getColumns().get(j);
28829
28830                                                                                if (exceptColumnList != null) {
28831                                                                                        boolean flag = false;
28832                                                                                        for (TObjectName objectName : exceptColumnList) {
28833                                                                                                if (getColumnName(objectName)
28834                                                                                                                .equals(getColumnName(targetColumn.getName()))) {
28835                                                                                                        flag = true;
28836                                                                                                        break;
28837                                                                                                }
28838                                                                                        }
28839                                                                                        if (flag) {
28840                                                                                                continue;
28841                                                                                        }
28842                                                                                }
28843
28844                                                                                if (replaceAsIdentifierMap.containsKey(targetColumn.getName())) {
28845                                                                                        Pair<String, TExpression> expr = replaceAsIdentifierMap.get(targetColumn.getName());
28846                                                                                        ResultSet resultSet = ((ResultColumn)modelObject).getResultSet();
28847                                                                                        ResultColumn resultColumn = modelFactory.createResultColumn(resultSet, replaceColumnMap.get(targetColumn.getName()));
28848                                                                                        Transform transform = new Transform();
28849                                                                                        transform.setType(Transform.EXPRESSION);
28850                                                                                        TObjectName expression = new TObjectName();
28851                                                                                        expression.setString(expr.first);
28852                                                                                transform.setCode(expression);
28853                                                                                        resultColumn.setTransform(transform);
28854                                                                                        analyzeResultColumnExpressionRelation(resultColumn, expr.second);
28855                                                                                } else {
28856                                                                                        if(!replaceAsIdentifierMap.isEmpty()) {
28857                                                                                                ResultSet resultSet = ((ResultColumn) modelObject).getResultSet();
28858                                                                                                TObjectName resultColumnName = new TObjectName();
28859                                                                                                resultColumnName.setString(targetColumn.getName());
28860                                                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSet,
28861                                                                                                                resultColumnName);
28862                                                                                                DataFlowRelationship relation1 = modelFactory.createDataFlowRelation();
28863                                                                                                relation1.setEffectType(effectType);
28864                                                                                                relation1.setProcess(process);
28865                                                                                                relation1.setTarget(new ResultColumnRelationshipElement(resultColumn));
28866                                                                                                relation1.addSource(new ResultColumnRelationshipElement(targetColumn));
28867                                                                                        }
28868                                                                                        else {
28869                                                                                                relation.addSource(
28870                                                                                                        new ResultColumnRelationshipElement(targetColumn));
28871                                                                                        }
28872                                                                                }
28873                                                                        }
28874                                                                        break;
28875                                                                } else {
28876                                                                        boolean flag = false;
28877
28878                                                                        for (int j = 0; j < cteColumns.size(); j++) {
28879                                                                                TObjectName sourceColumn = cteColumns.getObjectName(j);
28880
28881                                                                                if (DlineageUtil.sameColumnName(sourceColumn, columnName)) {
28882                                                                                        ResultColumn targetColumn = queryTable.getColumns().get(j);
28883
28884                                                                                        relation.addSource(new ResultColumnRelationshipElement(targetColumn));
28885                                                                                        flag = true;
28886                                                                                        break;
28887                                                                                }
28888                                                                        }
28889
28890                                                                        if (flag) {
28891                                                                                break;
28892                                                                        }
28893                                                                }
28894                                                        }
28895
28896                                                        if (columnName.getSourceColumn() != null) {
28897                                                                Object model = modelManager.getModel(columnName.getSourceColumn());
28898                                                                if (model instanceof ResultColumn) {
28899                                                                        ResultColumn resultColumn = (ResultColumn) model;
28900                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
28901                                                                }
28902                                                        } else if (columnName.getSourceTable() != null) {
28903                                                                Object tableModel = modelManager.getModel(columnName.getSourceTable());
28904                                                                if (tableModel instanceof Table) {
28905                                                                        Object model = modelManager
28906                                                                                        .getModel(new Pair<Table, TObjectName>((Table) tableModel, columnName));
28907                                                                        if (model instanceof TableColumn) {
28908                                                                                relation.addSource(new TableColumnRelationshipElement((TableColumn) model));
28909                                                                        }
28910                                                                }
28911                                                        }
28912                                                } else {
28913                                                        List<ResultColumn> columns = queryTable.getColumns();
28914                                                        if (getColumnName(columnName).equals("*")) {
28915                                                                Map<String, Pair<String, TExpression>> replaceAsIdentifierMap = new HashMap<String, Pair<String, TExpression>>();
28916                                                                Map<String, TObjectName> replaceColumnMap = new HashMap<String, TObjectName>();
28917                                                                if(modelObject instanceof ResultColumn && ((ResultColumn)modelObject).getColumnObject() instanceof TResultColumn) {
28918                                                                        TResultColumn resultColumn =  (TResultColumn )((ResultColumn)modelObject).getColumnObject();
28919                                                                        if(resultColumn.getReplaceExprAsIdentifiers()!=null && resultColumn.getReplaceExprAsIdentifiers().size()>0) {
28920                                                                                for(TReplaceExprAsIdentifier replace: resultColumn.getReplaceExprAsIdentifiers()) {
28921                                                                                        replaceAsIdentifierMap.put(replace.getIdentifier().toString(), new Pair<String, TExpression>(resultColumn.getExpr().getExceptReplaceClause().toString(), replace.getExpr()));
28922                                                                                        replaceColumnMap.put(replace.getIdentifier().toString(), replace.getIdentifier());
28923                                                                                }
28924                                                                                ResultSet resultSet = ((ResultColumn)modelObject).getResultSet();
28925                                                                                if(queryTable.isDetermined()) {
28926                                                                                        resultSet.getColumns().clear();
28927                                                                                }
28928                                                                        }
28929                                                                }
28930                                                                
28931                                                                int index = 0;
28932                                                                for (int j = 0; j < queryTable.getColumns().size(); j++) {
28933                                                                        ResultColumn targetColumn = queryTable.getColumns().get(j);
28934                                                                        if (exceptColumnList != null) {
28935                                                                                boolean flag = false;
28936                                                                                for (TObjectName objectName : exceptColumnList) {
28937                                                                                        if (getColumnName(objectName)
28938                                                                                                        .equals(getColumnName(targetColumn.getName()))) {
28939                                                                                                flag = true;
28940                                                                                                break;
28941                                                                                        }
28942                                                                                }
28943                                                                                if (flag) {
28944                                                                                        continue;
28945                                                                                }
28946                                                                        }
28947                                                                        
28948                                                                        if (replaceAsIdentifierMap.containsKey(targetColumn.getName())) {
28949                                                                                Pair<String, TExpression> expr = replaceAsIdentifierMap.get(targetColumn.getName());
28950                                                                                ResultSet resultSet = ((ResultColumn)modelObject).getResultSet();
28951                                                                                ResultColumn resultColumn = modelFactory.createResultColumn(resultSet, replaceColumnMap.get(targetColumn.getName()));
28952                                                                                Transform transform = new Transform();
28953                                                                                transform.setType(Transform.EXPRESSION);
28954                                                                                TObjectName expression = new TObjectName();
28955                                                                                expression.setString(expr.first);
28956                                                                        transform.setCode(expression);
28957                                                                                resultColumn.setTransform(transform);
28958                                                                                analyzeResultColumnExpressionRelation(resultColumn, expr.second);
28959                                                                        } else {
28960                                                                                if(!replaceAsIdentifierMap.isEmpty()) {
28961                                                                                        ResultSet resultSet = ((ResultColumn) modelObject).getResultSet();
28962                                                                                        TObjectName resultColumnName = new TObjectName();
28963                                                                                        resultColumnName.setString(targetColumn.getName());
28964                                                                                        ResultColumn resultColumn = modelFactory.createResultColumn(resultSet,
28965                                                                                                        resultColumnName);
28966                                                                                        DataFlowRelationship relation1 = modelFactory.createDataFlowRelation();
28967                                                                                        relation1.setEffectType(effectType);
28968                                                                                        relation1.setProcess(process);
28969                                                                                        relation1.setTarget(new ResultColumnRelationshipElement(resultColumn));
28970                                                                                        relation1.addSource(new ResultColumnRelationshipElement(targetColumn));
28971                                                                                }
28972                                                                                else {
28973                                                                                        relation.addSource(
28974                                                                                                new ResultColumnRelationshipElement(targetColumn));
28975                                                                                }
28976                                                                        }
28977                                                                        index++;
28978                                                                }
28979                                                        } else {
28980                                                                if (table.getCTE() != null) {
28981                                                                        
28982                                                                        if (modelObject instanceof TableColumn) {
28983                                                                                Table modelTable = ((TableColumn) modelObject).getTable();
28984                                                                                if (modelTable.getSubType() == SubType.unnest) {
28985                                                                                        boolean find = false;
28986                                                                                        for (k = 0; k < columns.size(); k++) {
28987                                                                                                ResultColumn column = columns.get(k);
28988                                                                                                if (column.isStruct()) {
28989                                                                                                        List<String> names = SQLUtil.parseNames(column.getName());
28990                                                                                                        for (String name : names) {
28991                                                                                                                if (getColumnName(name).equals(getColumnName(columnName))) {
28992                                                                                                                        DataFlowRelationship unnestRelation = modelFactory.createDataFlowRelation();
28993                                                                                                                        unnestRelation.setEffectType(effectType);
28994                                                                                                                        unnestRelation.setProcess(process);
28995                                                                                                                        unnestRelation.addSource(new ResultColumnRelationshipElement(
28996                                                                                                                                        column, columnName));
28997                                                                                                                        TObjectName unnestTableColumnName = new TObjectName();
28998                                                                                                                        unnestTableColumnName.setString(names.get(names.size()-1));
28999                                                                                                                        TableColumn unnestTableColumn = modelFactory.createTableColumn(modelTable, unnestTableColumnName, true);
29000                                                                                                                        unnestRelation.setTarget(new TableColumnRelationshipElement(unnestTableColumn));
29001                                                                                                                        find = true;
29002                                                                                                                }
29003                                                                                                        }
29004                                                                                                        List<String> names1 = SQLUtil.parseNames(column.getName());
29005                                                                                                        if (names.size() == 1 && names1.size() >= 1) {
29006                                                                                                                for (String name : names1) {
29007                                                                                                                        if (getColumnName(name)
29008                                                                                                                                        .equals(getColumnName(column.getName()))) {
29009                                                                                                                                DataFlowRelationship unnestRelation = modelFactory.createDataFlowRelation();
29010                                                                                                                                unnestRelation.setEffectType(effectType);
29011                                                                                                                                unnestRelation.setProcess(process);
29012                                                                                                                                unnestRelation.addSource(new ResultColumnRelationshipElement(
29013                                                                                                                                                column, columnName));
29014                                                                                                                                TObjectName unnestTableColumnName = new TObjectName();
29015                                                                                                                                unnestTableColumnName.setString(names1.get(names.size()-1));
29016                                                                                                                                TableColumn unnestTableColumn = modelFactory.createTableColumn(modelTable, unnestTableColumnName, true);
29017                                                                                                                                unnestRelation.setTarget(new TableColumnRelationshipElement(unnestTableColumn));
29018                                                                                                                                find = true;
29019                                                                                                                        }
29020                                                                                                                }
29021                                                                                                        }
29022                                                                                                }
29023                                                                                        }
29024                                                                                        if (find) {
29025                                                                                                modelTable.getColumns().remove(modelObject);
29026                                                                                                break;
29027                                                                                        }
29028                                                                                }
29029                                                                        }
29030                                                                        
29031                                                                        ColumnNameIndex nameIdx = getColumnNameIndex(columns);
29032                                                                        if (nameIdx.byNormalizedName != null) {
29033                                                                                // Fast path: simple column list (no star/struct). A direct index lookup
29034                                                                                // reproduces the linear scan below for such lists: first-match-wins,
29035                                                                                // skip when the match IS modelObject, and stop at the first match.
29036                                                                                ResultColumn match = nameIdx.byNormalizedName.get(getColumnName(columnName));
29037                                                                                if (match != null && !match.equals(modelObject)) {
29038                                                                                        relation.addSource(
29039                                                                                                        new ResultColumnRelationshipElement(match, columnName));
29040                                                                                }
29041                                                                        } else {
29042                                                                        String targetColName = getColumnName(columnName); // loop-invariant, hoisted
29043                                                                        for (k = 0; k < columns.size(); k++) {
29044                                                                                ResultColumn column = columns.get(k);
29045                                                                                if ("*".equals(column.getName())) {
29046                                                                                        if (!containsStarColumn(column, columnName)) {
29047                                                                                                column.bindStarLinkColumn(columnName);
29048                                                                                        }
29049                                                                                        relation.addSource(new ResultColumnRelationshipElement(column, columnName));
29050                                                                                } else if (DlineageUtil.compareColumnIdentifier(targetColName,
29051                                                                                                DlineageUtil.getIdentifierNormalColumnName(column.getName()))) {
29052                                                                                        if (!column.equals(modelObject)) {
29053                                                                                                relation.addSource(
29054                                                                                                                new ResultColumnRelationshipElement(column, columnName));
29055                                                                                        }
29056                                                                                        break;
29057                                                                                } else if(column.isStruct()) {
29058                                                                                        List<String> names = SQLUtil.parseNames(column.getName());
29059                                                                                        for(String name: names) {
29060                                                                                                if (getColumnName(name)
29061                                                                                                                .equals(getColumnName(columnName))) {
29062                                                                                                        relation.addSource(
29063                                                                                                                        new ResultColumnRelationshipElement(column, columnName));
29064                                                                                                }
29065                                                                                        }
29066                                                                                        List<String> names1 = SQLUtil.parseNames(column.getName());
29067                                                                                        if (names.size() == 1 && names1.size() >= 1) {
29068                                                                                                for(String name: names1) {
29069                                                                                                        if (getColumnName(name)
29070                                                                                                                        .equals(getColumnName(column.getName()))) {
29071                                                                                                                relation.addSource(
29072                                                                                                                                new ResultColumnRelationshipElement(column, columnName));
29073                                                                                                        }
29074                                                                                                }
29075                                                                                        }
29076                                                                                }
29077                                                                        }
29078                                                                        }
29079                                                                } else if (table.getAliasClause() != null
29080                                                                                && table.getAliasClause().getColumns() != null) {
29081                                                                        for (k = 0; k < columns.size(); k++) {
29082                                                                                ResultColumn column = columns.get(k);
29083                                                                                List<String> splits = SQLUtil.parseNames(columnName.toString());        
29084                                                                                if ("*".equals(column.getName())) {
29085                                                                                        if (!containsStarColumn(column, columnName)) {
29086                                                                                                column.bindStarLinkColumn(columnName);
29087                                                                                        }
29088                                                                                        relation.addSource(new ResultColumnRelationshipElement(column, columnName));
29089                                                                                } else if (splits.size() > 1 && EDbVendor.dbvbigquery == getOption().getVendor()) {
29090                                                                                        if (DlineageUtil.compareColumnIdentifier(getColumnName(splits.get(0)),
29091                                                                                                        DlineageUtil.getIdentifierNormalColumnName(column.getName()))) {
29092                                                                                                if (!column.equals(modelObject)) {
29093                                                                                                        relation.addSource(new ResultColumnRelationshipElement(column,
29094                                                                                                                        columnName));
29095                                                                                                }
29096                                                                                                break;
29097                                                                                        }
29098                                                                                } else if (DlineageUtil.compareColumnIdentifier(getColumnName(columnName),
29099                                                                                                DlineageUtil.getIdentifierNormalColumnName(column.getName()))) {
29100                                                                                        if (!column.equals(modelObject)) {
29101                                                                                                relation.addSource(
29102                                                                                                                new ResultColumnRelationshipElement(column, columnName));
29103                                                                                        }
29104                                                                                        break;
29105                                                                                }
29106                                                                        }
29107                                                                } else if (table.getSubquery() != null || (table.getTableExpr() != null
29108                                                                                && table.getTableExpr().getSubQuery() != null)) {
29109                                                                        TSelectSqlStatement select = table.getSubquery();
29110                                                                        if (select == null) {
29111                                                                                select = table.getTableExpr().getSubQuery();
29112                                                                        }
29113                                                                        if (columnName.getSourceTable() != null) {
29114                                                                                Object tableModel = modelManager.getModel(columnName.getSourceTable());
29115                                                                                appendResultColumnRelationSource(modelObject, relation, columnIndex, columnName,
29116                                                                                                tableModel);
29117                                                                        } else if (columnName.getObjectToken() != null
29118                                                                                        && !SQLUtil.isEmpty(table.getAliasName())) {
29119                                                                                if (DlineageUtil.compareTableIdentifier(columnName.getObjectToken().toString(),
29120                                                                                                table.getAliasName())) {
29121                                                                                        Object tableModel = modelManager.getModel(table);
29122                                                                                        appendResultColumnRelationSource(modelObject, relation, columnIndex,
29123                                                                                                        columnName, tableModel);
29124                                                                                }
29125                                                                        } else if(columns!=null) {
29126                                                                                for (k = 0; k < columns.size(); k++) {
29127                                                                                        ResultColumn column = columns.get(k);
29128                                                                                        List<String> splits = SQLUtil.parseNames(columnName.toString());        
29129                                                                                        if ("*".equals(column.getName())) {
29130                                                                                                if (!containsStarColumn(column, columnName)) {
29131                                                                                                        column.bindStarLinkColumn(columnName);
29132                                                                                                }
29133                                                                                                relation.addSource(new ResultColumnRelationshipElement(column, columnName));
29134                                                                                        } else if (splits.size() > 1 && EDbVendor.dbvbigquery == getOption().getVendor()) {
29135                                                                                                if (DlineageUtil.compareColumnIdentifier(getColumnName(splits.get(0)),
29136                                                                                                                DlineageUtil.getIdentifierNormalColumnName(column.getName()))) {
29137                                                                                                        if (!column.equals(modelObject)) {
29138                                                                                                                relation.addSource(new ResultColumnRelationshipElement(column,
29139                                                                                                                                columnName));
29140                                                                                                        }
29141                                                                                                        break;
29142                                                                                                }
29143                                                                                        } else if (DlineageUtil.compareColumnIdentifier(getColumnName(columnName),
29144                                                                                                        DlineageUtil.getIdentifierNormalColumnName(column.getName()))) {
29145                                                                                                if (!column.equals(modelObject)) {
29146                                                                                                        relation.addSource(
29147                                                                                                                        new ResultColumnRelationshipElement(column, columnName));
29148                                                                                                }
29149                                                                                                break;
29150                                                                                        }
29151                                                                                }
29152                                                                        }
29153                                                                } else if (table.getOutputMerge() != null) {
29154                                                                        if (columnName.getSourceColumn() != null) {
29155                                                                                Object model = modelManager.getModel(columnName.getSourceColumn());
29156                                                                                if (model instanceof ResultColumn) {
29157                                                                                        ResultColumn resultColumn = (ResultColumn) model;
29158                                                                                        if ("*".equals(resultColumn.getName())
29159                                                                                                        && !containsStarColumn(resultColumn, columnName)) {
29160                                                                                                resultColumn.bindStarLinkColumn(columnName);
29161                                                                                        }
29162                                                                                        relation.addSource(
29163                                                                                                        new ResultColumnRelationshipElement(resultColumn, columnName));
29164                                                                                }
29165                                                                        } else if (columnName.getSourceTable() != null) {
29166                                                                                Object tableModel = modelManager.getModel(columnName.getSourceTable());
29167                                                                                appendResultColumnRelationSource(modelObject, relation, columnIndex, columnName,
29168                                                                                                tableModel);
29169                                                                        } else if (columnName.getObjectToken() != null
29170                                                                                        && !SQLUtil.isEmpty(table.getAliasName())) {
29171                                                                                if (DlineageUtil.compareTableIdentifier(columnName.getObjectToken().toString(),
29172                                                                                                table.getAliasName())) {
29173                                                                                        Object tableModel = modelManager.getModel(table);
29174                                                                                        appendResultColumnRelationSource(modelObject, relation, columnIndex,
29175                                                                                                        columnName, tableModel);
29176                                                                                }
29177                                                                        }
29178                                                                }
29179                                                        }
29180                                                }
29181                                        }
29182                                }
29183                        }
29184                        if (relation.getSources().size() == 0 && isKeyword(columnName)) {
29185                                Table constantTable = modelFactory.createConstantsTable(stmtStack.peek());
29186                                TableColumn constantColumn = modelFactory.createTableColumn(constantTable, columnName, true);
29187                                relation.addSource(new ConstantRelationshipElement(constantColumn));
29188                        }
29189
29190                        if (relation.getSources().size() > 0) {
29191                                for (RelationshipElement<?> sourceItem: relation.getSources()) {
29192                                        Object source = sourceItem.getElement();
29193                                        ImpactRelationship impactRelation = null;
29194                                        if (source instanceof ResultColumn
29195                                                        && !((ResultColumn) source).getResultSet().getRelationRows().getHoldRelations().isEmpty()) {
29196                                                impactRelation = modelFactory.createImpactRelation();
29197                                                impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>(
29198                                                                ((ResultColumn) source).getResultSet().getRelationRows()));
29199                                                impactRelation.setEffectType(effectType);
29200                                        } else if (source instanceof TableColumn
29201                                                        && !((TableColumn) source).getTable().getRelationRows().getHoldRelations().isEmpty()) {
29202                                                impactRelation = modelFactory.createImpactRelation();
29203                                                impactRelation.addSource(new RelationRowsRelationshipElement<TableRelationRows>(
29204                                                                ((TableColumn) source).getTable().getRelationRows()));
29205                                                impactRelation.setEffectType(effectType);;
29206                                        }
29207
29208                                        if (impactRelation == null) {
29209                                                continue;
29210                                        }
29211
29212                                        if (relation.getTarget() != null) {
29213                                                Object target = relation.getTarget().getElement();
29214                                                if (target instanceof ResultColumn) {
29215                                                        impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
29216                                                                        ((ResultColumn) target).getResultSet().getRelationRows()));
29217                                                } else if (target instanceof TableColumn) {
29218                                                        impactRelation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(
29219                                                                        ((TableColumn) target).getTable().getRelationRows()));
29220                                                }
29221                                        }
29222
29223                                        if (impactRelation.getSources() != null && !impactRelation.getSources().isEmpty()
29224                                                        && impactRelation.getTarget() != null
29225                                                        && impactRelation.getSources().iterator().next().getElement() == impactRelation.getTarget().getElement()) {
29226                                                modelManager.removeRelation(impactRelation);
29227                                        }
29228                                }
29229                        }
29230                }
29231                return relation;
29232        }
29233
29234        private boolean isNotInProcedure(Table table) {
29235        TStoredProcedureSqlStatement stmt = getProcedureParent(stmtStack.peek());
29236                Procedure procedure = this.modelFactory.createProcedure(stmt);
29237                if(procedure!=null && procedure.getName().equals(table.getParent())){
29238                        return false;
29239                }
29240                return true;
29241        }
29242
29243        private boolean hasJoin(TCustomSqlStatement stmt) {
29244                if (stmt.getJoins() == null || stmt.getJoins().size() == 0)
29245                        return false;
29246                if (stmt.getJoins().size() > 1) {
29247                        return true;
29248                }
29249                TJoinItemList joinItems = stmt.getJoins().getJoin(0).getJoinItems();
29250                if (joinItems == null || joinItems.size() == 0) {
29251                        return false;
29252                }
29253                return true;
29254        }
29255
29256        private boolean isInQuery(TSelectSqlStatement query, TResultColumn column) {
29257                if(query == null)
29258                        return false;
29259                TResultColumnList columns = query.getResultColumnList();
29260                if (columns != null) {
29261                        for (int i = 0; i < columns.size(); i++) {
29262                                if (columns.getResultColumn(i).equals(column)) {
29263                                        return true;
29264                                }
29265                        }
29266                }
29267                return false;
29268        }
29269
29270        private void appendResultColumnRelationSource(Object modelObject, DataFlowRelationship relation, int columnIndex,
29271                        TObjectName columnName, Object tableModel) {
29272                if (tableModel instanceof Table) {
29273                        Object model = modelManager.getModel(new Pair<Table, TObjectName>((Table) tableModel, columnName));
29274                        if (model instanceof TableColumn) {
29275                                relation.addSource(new TableColumnRelationshipElement((TableColumn) model));
29276                        }
29277                } else if (tableModel instanceof ResultSet) {
29278                        List<ResultColumn> queryColumns = ((ResultSet) tableModel).getColumns();
29279                        boolean flag = false;
29280                        for (int l = 0; l < queryColumns.size(); l++) {
29281                                ResultColumn column = queryColumns.get(l);
29282                                if (DlineageUtil.compareColumnIdentifier(getColumnName(columnName),
29283                                                DlineageUtil.getIdentifierNormalColumnName(column.getName()))) {
29284                                        if (!column.equals(modelObject)) {
29285                                                relation.addSource(new ResultColumnRelationshipElement(column, columnName));
29286                                                flag = true;
29287                                        }
29288                                        break;
29289                                }
29290                        }
29291                        if (!flag) {
29292                                for (int l = 0; l < queryColumns.size(); l++) {
29293                                        ResultColumn column = queryColumns.get(l);
29294                                        if ("*".equals(column.getName())) {
29295                                                if (!containsStarColumn(column, columnName)) {
29296                                                        column.bindStarLinkColumn(columnName);
29297                                                }
29298                                                relation.addSource(new ResultColumnRelationshipElement(column, columnName));
29299                                                flag = true;
29300                                                break;
29301                                        }
29302                                }
29303                        }
29304                        if (!flag && columnIndex < queryColumns.size() && columnIndex != -1) {
29305                                relation.addSource(new ResultColumnRelationshipElement(queryColumns.get(columnIndex), columnName));
29306                        }
29307                }
29308        }
29309
29310        private boolean isApplyJoin(TCustomSqlStatement stmt) {
29311                if (stmt.getJoins() == null || stmt.getJoins().size() == 0)
29312                        return false;
29313                TJoinItemList joinItems = stmt.getJoins().getJoin(0).getJoinItems();
29314                if (joinItems == null || joinItems.size() == 0) {
29315                        return false;
29316                }
29317                if (joinItems.getJoinItem(0).getJoinType() == EJoinType.crossapply
29318                                || joinItems.getJoinItem(0).getJoinType() == EJoinType.outerapply)
29319                        return true;
29320                return false;
29321        }
29322
29323        private boolean containsStarColumn(ResultColumn resultColumn, TObjectName columnName) {
29324                String targetColumnName = getColumnName(columnName);
29325                if (resultColumn.hasStarLinkColumn()) {
29326                        return resultColumn.getStarLinkColumns().containsKey(targetColumnName);
29327                }
29328                return false;
29329        }
29330
29331        private void analyzeAggregate(TFunctionCall function, TExpression expr) {
29332                TCustomSqlStatement stmt = stmtStack.peek();
29333                ResultSet resultSet = (ResultSet) modelManager.getModel(stmt.getResultColumnList());
29334                if (resultSet == null) {
29335                        return;
29336                }
29337
29338                if (expr != null) {
29339                        columnsInExpr visitor = new columnsInExpr();
29340                        expr.inOrderTraverse(visitor);
29341                        List<TObjectName> objectNames = visitor.getObjectNames();
29342                        for (int j = 0; j < objectNames.size(); j++) {
29343                                TObjectName columnName = objectNames.get(j);
29344
29345                                if (columnName.getDbObjectType() == EDbObjectType.variable) {
29346                                        continue;
29347                                }
29348
29349                                if (columnName.getColumnNameOnly().startsWith("@")
29350                                                && (option.getVendor() == EDbVendor.dbvmssql || option.getVendor() == EDbVendor.dbvazuresql)) {
29351                                        continue;
29352                                }
29353
29354                                if (columnName.getColumnNameOnly().startsWith(":")
29355                                                && (option.getVendor() == EDbVendor.dbvhana || option.getVendor() == EDbVendor.dbvteradata)) {
29356                                        continue;
29357                                }
29358
29359                                Object targetModel0 = modelManager.getModel(function.getFunctionName());
29360                                if (targetModel0 == null && function.getFunctionType() == EFunctionType.array_agg_t && modelManager.getModel(function) instanceof Function) {
29361                                        Function functionModel = (Function) modelManager.getModel(function);
29362                                        if (functionModel.getColumns().size() == 1) {
29363                                                targetModel0 = ((Function) functionModel).getColumns().get(0);
29364                                        }
29365                                }
29366                                if (!(targetModel0 instanceof ResultColumn)) {
29367                                        continue;
29368                                }
29369                                ResultColumn targetResultColumn0 = (ResultColumn) targetModel0;
29370                                AbstractRelationship relation = modelFactory.createRecordSetRelation();
29371                                relation.setEffectType(EffectType.function);
29372                                relation.setFunction(function.getFunctionName().toString());
29373                                relation.setTarget(new ResultColumnRelationshipElement(targetResultColumn0));
29374                                TTable table = modelManager.getTable(stmt, columnName);
29375                                if (table != null) {
29376                                        if (modelManager.getModel(table) instanceof Table) {
29377                                                Table tableModel = (Table) modelManager.getModel(table);
29378                                                if (tableModel != null) {
29379                                                        TableColumn columnModel = modelFactory.createTableColumn(tableModel, columnName, false);
29380                                                        if (columnModel != null) {
29381                                                                relation.addSource(
29382                                                                                new TableColumnRelationshipElement(columnModel, columnName.getLocation(),
29383                                                                                                columnName));
29384                                                        }
29385                                                }
29386                                        } else if (modelManager.getModel(table) instanceof QueryTable) {
29387                                                Object model = modelManager.getModel(columnName.getSourceColumn());
29388                                                if (model instanceof ResultColumn) {
29389                                                        ResultColumn resultColumn = (ResultColumn) model;
29390                                                        if (resultColumn != null) {
29391                                                                relation.addSource(
29392                                                                                new ResultColumnRelationshipElement(resultColumn, columnName.getLocation(), columnName));
29393                                                        }
29394                                                }
29395                                        }
29396                                }
29397                        }
29398
29399                        List<TParseTreeNode> functions = visitor.getFunctions();
29400                        for (int j = 0; j < functions.size(); j++) {
29401                                TParseTreeNode functionObj = functions.get(j);
29402                                Object functionModel = modelManager.getModel(functionObj);
29403                                if (functionModel == null) {
29404                                        functionModel = createFunction(functionObj);
29405                                }
29406                                if (functionModel instanceof Function) {
29407                                        Object targetModel1 = modelManager.getModel(function.getFunctionName());
29408                                        if (targetModel1 == null && function.getFunctionType() == EFunctionType.array_agg_t && modelManager.getModel(function) instanceof Function) {
29409                                                Function functionModel1 = (Function) modelManager.getModel(function);
29410                                                if (functionModel1.getColumns().size() == 1) {
29411                                                        targetModel1 = ((Function) functionModel1).getColumns().get(0);
29412                                                }
29413                                        }
29414                                        if (!(targetModel1 instanceof ResultColumn)) {
29415                                                continue;
29416                                        }
29417                                        ResultColumn targetRC_func = (ResultColumn) targetModel1;
29418                                        AbstractRelationship relation;
29419                                        if ("COUNT".equalsIgnoreCase(function.getFunctionName().toString())) {
29420                                                // relation = modelFactory.createDataFlowRelation();
29421                                                relation = modelFactory.createRecordSetRelation();
29422                                        } else {
29423                                                relation = modelFactory.createRecordSetRelation();
29424                                        }
29425                                        relation.setEffectType(EffectType.function);
29426                                        relation.setFunction(function.getFunctionName().toString());
29427                                        relation.setTarget(new ResultColumnRelationshipElement(targetRC_func));
29428
29429                                        if (functionObj instanceof TFunctionCall) {
29430                                                ResultColumn resultColumn = (ResultColumn) modelManager
29431                                                                .getModel(((TFunctionCall) functionObj).getFunctionName());
29432                                                if (resultColumn != null) {
29433                                                        ResultColumnRelationshipElement element = new ResultColumnRelationshipElement(resultColumn,
29434                                                                        ((TFunctionCall) functionObj).getFunctionName().getLocation());
29435                                                        relation.addSource(element);
29436                                                }
29437                                        }
29438                                        if (functionObj instanceof TCaseExpression) {
29439                                                ResultColumn resultColumn = (ResultColumn) modelManager
29440                                                                .getModel(((TCaseExpression) functionObj).getWhenClauseItemList());
29441                                                if (resultColumn != null) {
29442                                                        ResultColumnRelationshipElement element = new ResultColumnRelationshipElement(resultColumn);
29443                                                        relation.addSource(element);
29444                                                }
29445                                        }
29446                                } else if (functionModel instanceof Table) {
29447                                        Object targetModel2 = modelManager.getModel(function.getFunctionName());
29448                                        if (targetModel2 == null && function.getFunctionType() == EFunctionType.array_agg_t && modelManager.getModel(function) instanceof Function) {
29449                                                Function functionModel1 = (Function) modelManager.getModel(function);
29450                                                if (functionModel1.getColumns().size() == 1) {
29451                                                        targetModel2 = ((Function) functionModel1).getColumns().get(0);
29452                                                }
29453                                        }
29454                                        if (!(targetModel2 instanceof ResultColumn)) {
29455                                                continue;
29456                                        }
29457                                        ResultColumn targetRC_table = (ResultColumn) targetModel2;
29458                                        TableColumn tableColumn = modelFactory.createTableColumn((Table) functionModel,
29459                                                        function.getFunctionName(), false);
29460                                        AbstractRelationship relation;
29461                                        if ("COUNT".equalsIgnoreCase(function.getFunctionName().toString())) {
29462                                                // relation = modelFactory.createDataFlowRelation();
29463                                                relation = modelFactory.createRecordSetRelation();
29464                                        } else {
29465                                                relation = modelFactory.createRecordSetRelation();
29466                                        }
29467                                        relation.setEffectType(EffectType.function);
29468                                        relation.setFunction(function.getFunctionName().toString());
29469                                        relation.setTarget(new ResultColumnRelationshipElement(targetRC_table));
29470                                        TableColumnRelationshipElement element = new TableColumnRelationshipElement(tableColumn);
29471                                        relation.addSource(element);
29472                                }
29473                        }
29474                }
29475
29476                if (expr == null || "COUNT".equalsIgnoreCase(function.getFunctionName().toString())) {
29477                        if ("COUNT".equalsIgnoreCase(function.getFunctionName().toString())
29478                                        // https://e.gitee.com/gudusoft/issues/list?issue=I4L5EO
29479                                        // 对于非 count() 函数,当有group by clause时, RelationRows 不参与indirect dataflow,
29480                                        // 让位与group by clause中的column.
29481                                        || ((TSelectSqlStatement) stmt).getGroupByClause() == null) {
29482                                TTableList tables = stmt.getTables();
29483                                if (tables != null) {
29484                                        for (int i = 0; i < tables.size(); i++) {
29485                                                TTable table = tables.getTable(i);
29486                                                if (modelManager.getModel(table) == null && table.getSubquery() == null) {
29487                                                        modelFactory.createTable(table);
29488                                                }
29489                                                if (modelManager.getModel(table) instanceof Table) {
29490                                                        Object targetModel3 = modelManager.getModel(function.getFunctionName());
29491                                                        if (targetModel3 == null && function.getFunctionType() == EFunctionType.array_agg_t && modelManager.getModel(function) instanceof Function) {
29492                                                                Function functionModel = (Function) modelManager.getModel(function);
29493                                                                if (functionModel.getColumns().size() == 1) {
29494                                                                        targetModel3 = functionModel.getColumns().get(0);
29495                                                                }
29496                                                        }
29497                                                        if (!(targetModel3 instanceof ResultColumn)) {
29498                                                                continue;
29499                                                        }
29500                                                        ResultColumn targetRC_tbl = (ResultColumn) targetModel3;
29501                                                        Table tableModel = (Table) modelManager.getModel(table);
29502                                                        AbstractRelationship relation;
29503                                                        if ("COUNT".equalsIgnoreCase(function.getFunctionName().toString())) {
29504                                                                // relation = modelFactory.createDataFlowRelation();
29505                                                                relation = modelFactory.createRecordSetRelation();
29506                                                        } else {
29507                                                                relation = modelFactory.createRecordSetRelation();
29508                                                        }
29509                                                        relation.setEffectType(EffectType.function);
29510                                                        relation.setFunction(function.getFunctionName().toString());
29511                                                        relation.setTarget(new ResultColumnRelationshipElement(targetRC_tbl));
29512
29513                                                        if (relation instanceof DataFlowRelationship && option.isShowCountTableColumn()) {
29514                                                                List<TExpression> expressions = new ArrayList<TExpression>();
29515                                                                getFunctionExpressions(expressions, new ArrayList<TExpression>(), function);
29516                                                                for (int j = 0; j < expressions.size(); j++) {
29517                                                                        columnsInExpr visitor = new columnsInExpr();
29518                                                                        expressions.get(j).inOrderTraverse(visitor);
29519                                                                        List<TObjectName> objectNames = visitor.getObjectNames();
29520                                                                        if (objectNames != null) {
29521                                                                                for (TObjectName columnName : objectNames) {
29522                                                                                        TTable tempTable = modelManager.getTable(stmt, columnName);
29523                                                                                        if (table.equals(tempTable)) {
29524                                                                                                TableColumn tableColumn = modelFactory.createTableColumn(tableModel,
29525                                                                                                                columnName, false);
29526                                                                                                TableColumnRelationshipElement element = new TableColumnRelationshipElement(
29527                                                                                                                tableColumn);
29528                                                                                                relation.addSource(element);
29529                                                                                        }
29530                                                                                }
29531                                                                        }
29532                                                                }
29533                                                        }
29534                                                        if (relation.getSources().size() == 0) {
29535                                                                RelationRowsRelationshipElement element = new RelationRowsRelationshipElement<TableRelationRows>(
29536                                                                                tableModel.getRelationRows());
29537                                                                relation.addSource(element);
29538                                                        }
29539                                                } else if (modelManager.getModel(table) instanceof QueryTable) {
29540                                                        Object targetModel4 = modelManager.getModel(function.getFunctionName());
29541                                                        if (targetModel4 == null && function.getFunctionType() == EFunctionType.array_agg_t && modelManager.getModel(function) instanceof Function) {
29542                                                                Function functionModel = (Function) modelManager.getModel(function);
29543                                                                targetModel4 = functionModel.getColumns().get(0);
29544                                                        }
29545                                                        if (!(targetModel4 instanceof ResultColumn)) {
29546                                                                continue;
29547                                                        }
29548                                                        ResultColumn targetRC_qt = (ResultColumn) targetModel4;
29549                                                        QueryTable tableModel = (QueryTable) modelManager.getModel(table);
29550                                                        AbstractRelationship relation;
29551                                                        if ("COUNT".equalsIgnoreCase(function.getFunctionName().toString())) {
29552                                                                // relation = modelFactory.createDataFlowRelation();
29553                                                                relation = modelFactory.createRecordSetRelation();
29554                                                        } else {
29555                                                                relation = modelFactory.createRecordSetRelation();
29556                                                        }
29557                                                        relation.setEffectType(EffectType.function);
29558                                                        relation.setFunction(function.getFunctionName().toString());
29559                                                        relation.setTarget(new ResultColumnRelationshipElement(targetRC_qt));
29560                                                        RelationRowsRelationshipElement element = new RelationRowsRelationshipElement<ResultSetRelationRows>(
29561                                                                        tableModel.getRelationRows());
29562                                                        relation.addSource(element);
29563                                                }
29564                                        }
29565                                }
29566                        }
29567
29568                        if (stmt.getWhereClause() == null || stmt.getWhereClause().getCondition() == null) {
29569                                return;
29570                        }
29571
29572                        columnsInExpr visitor = new columnsInExpr();
29573                        stmt.getWhereClause().getCondition().inOrderTraverse(visitor);
29574                        List<TObjectName> objectNames = visitor.getObjectNames();
29575                        for (int j = 0; j < objectNames.size(); j++) {
29576                                TObjectName columnName = objectNames.get(j);
29577                                if (columnName.getDbObjectType() == EDbObjectType.variable) {
29578                                        Variable tableModel;
29579                                        if (columnName.toString().indexOf(".") != -1) {
29580                                                List<String> splits = SQLUtil.parseNames(columnName.toString());
29581                                                tableModel = modelFactory.createVariable(splits.get(splits.size() - 2));
29582                                        } else {
29583                                                tableModel = modelFactory.createVariable(columnName);
29584                                        }
29585                                        tableModel.setCreateTable(true);
29586                                        tableModel.setSubType(SubType.record);
29587                                        TObjectName variableProperties = new TObjectName();
29588                                        variableProperties.setString("*");
29589                                        modelFactory.createTableColumn(tableModel, variableProperties, true);
29590                                }
29591
29592//                              if (columnName.getColumnNameOnly().startsWith("@")
29593//                                              && (option.getVendor() == EDbVendor.dbvmssql || option.getVendor() == EDbVendor.dbvazuresql)) {
29594//                                      continue;
29595//                              }
29596//
29597//                              if (columnName.getColumnNameOnly().startsWith(":") && (option.getVendor() == EDbVendor.dbvhana || option.getVendor() == EDbVendor.dbvteradata)) {
29598//                                      continue;
29599//                              }
29600
29601                                Object targetModel5 = modelManager.getModel(function.getFunctionName());
29602                                if (targetModel5 == null && function.getFunctionType() == EFunctionType.array_agg_t && modelManager.getModel(function) instanceof Function) {
29603                                        Function functionModel = (Function) modelManager.getModel(function);
29604                                        if (functionModel.getColumns().size() == 1) {
29605                                                targetModel5 = functionModel.getColumns().get(0);
29606                                        }
29607                                }
29608                                if (!(targetModel5 instanceof ResultColumn)) {
29609                                        continue;
29610                                }
29611                                AbstractRelationship relation = modelFactory.createRecordSetRelation();
29612                                relation.setEffectType(EffectType.function);
29613                                relation.setFunction(function.getFunctionName().toString());
29614                                relation.setTarget(new ResultColumnRelationshipElement((ResultColumn) targetModel5));
29615
29616                                TTable table = modelManager.getTable(stmt, columnName);
29617                                if (table != null) {
29618                                        if (modelManager.getModel(table) instanceof Table) {
29619                                                Table tableModel = (Table) modelManager.getModel(table);
29620                                                if (tableModel != null) {
29621                                                        TableColumn columnModel = modelFactory.createTableColumn(tableModel, columnName, false);
29622                                                        if(columnModel == null) {
29623                                                                continue;
29624                                                        }
29625                                                        relation.addSource(
29626                                                                        new TableColumnRelationshipElement(columnModel, columnName.getLocation(),
29627                                                                                        columnName));
29628                                                }
29629                                        } else if (modelManager.getModel(table) instanceof QueryTable) {
29630                                                Object model = modelManager.getModel(columnName.getSourceColumn());
29631                                                if (model instanceof ResultColumn) {
29632                                                        ResultColumn resultColumn = (ResultColumn) model;
29633                                                        if (resultColumn != null) {
29634                                                                relation.addSource(
29635                                                                                new ResultColumnRelationshipElement(resultColumn, columnName.getLocation(), columnName));
29636                                                        }
29637                                                }
29638                                        }
29639                                }
29640                        }
29641                }
29642        }
29643
29644        private void analyzeFilterCondition(Object modelObject, TExpression expr, EJoinType joinType,
29645                        JoinClauseType joinClauseType, EffectType effectType) {
29646                if (expr == null) {
29647                        return;
29648                }
29649
29650                TCustomSqlStatement stmt = stmtStack.peek();
29651
29652                columnsInExpr visitor = new columnsInExpr();
29653                expr.inOrderTraverse(visitor);
29654
29655                List<TObjectName> objectNames = visitor.getObjectNames();
29656                List<TParseTreeNode> functions = visitor.getFunctions();
29657                List<TResultColumn> resultColumns = visitor.getResultColumns();
29658                List<TParseTreeNode> constants = visitor.getConstants();
29659
29660                ImpactRelationship relation = modelFactory.createImpactRelation();
29661                relation.setEffectType(effectType);
29662                relation.setJoinClauseType(joinClauseType);
29663                if (modelObject instanceof ResultColumn) {
29664                        relation.setTarget(new ResultColumnRelationshipElement((ResultColumn) modelObject));
29665                } else if (modelObject instanceof Table) {
29666                        Table targetTable = (Table) modelObject;
29667                        relation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(
29668                                        targetTable.getRelationRows()));
29669                } else if (modelObject instanceof ResultSet) {
29670                        ResultSet targetResultSet = (ResultSet) modelObject;
29671                        relation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
29672                                        targetResultSet.getRelationRows()));
29673                } else {
29674                        ResultSet resultSet = (ResultSet) modelManager.getModel(stmt.getResultColumnList());
29675                        if (resultSet == null && stmt instanceof TUpdateSqlStatement) {
29676                                resultSet = (ResultSet) modelManager.getModel(stmt);
29677                        }
29678                        if (resultSet == null && stmt instanceof TMergeSqlStatement) {
29679                                TSelectSqlStatement subquery = ((TMergeSqlStatement) stmt).getUsingTable().getSubquery();
29680                                if (subquery != null) {
29681                                        resultSet = (ResultSet) modelManager.getModel(((TMergeSqlStatement) stmt).getUsingTable());
29682                                }
29683                                else {
29684                                        resultSet = modelFactory.createQueryTable(((TMergeSqlStatement) stmt).getUsingTable());
29685                                }
29686                        }
29687                        if (resultSet != null) {
29688                                relation.setTarget(
29689                                                new RelationRowsRelationshipElement<ResultSetRelationRows>(resultSet.getRelationRows()));
29690                        }
29691                        if (stmt instanceof TDeleteSqlStatement) {
29692                                Object deleteTarget = modelManager.getModel(((TDeleteSqlStatement) stmt).getTargetTable());
29693                                if (deleteTarget instanceof Table) {
29694                                        Table table = (Table) deleteTarget;
29695                                        relation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(table.getRelationRows()));
29696                                } else if (deleteTarget instanceof ResultSet) {
29697                                        ResultSet resultSetTarget = (ResultSet) deleteTarget;
29698                                        relation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
29699                                                        resultSetTarget.getRelationRows()));
29700                                }
29701                        }
29702                }
29703                if (relation.getTarget() != null) {
29704
29705                        if (constants != null && constants.size() > 0) {
29706                                if (option.isShowConstantTable()) {
29707                                        Table constantTable = modelFactory.createConstantsTable(stmtStack.peek());
29708                                        for (int i = 0; i < constants.size(); i++) {
29709                                                TParseTreeNode constant = constants.get(i);
29710                                                if (constant instanceof TConstant) {
29711                                                        TableColumn constantColumn = modelFactory.createTableColumn(constantTable,
29712                                                                        (TConstant) constant);
29713                                                        relation.addSource(new ConstantRelationshipElement(constantColumn));
29714                                                } else if (constant instanceof TObjectName) {
29715                                                        TableColumn constantColumn = modelFactory.createTableColumn(constantTable,
29716                                                                        (TObjectName) constant, false);
29717                                                        if(constantColumn == null) {
29718                                                                continue;
29719                                                        }
29720                                                        relation.addSource(new ConstantRelationshipElement(constantColumn));
29721                                                }
29722                                        }
29723                                }
29724                        }
29725
29726                        for (int j = 0; j < objectNames.size(); j++) {
29727                                TObjectName columnName = objectNames.get(j);
29728                                if (columnName.getDbObjectType() == EDbObjectType.variable) {
29729                                        Variable variable = modelFactory.createVariable(columnName);
29730                                        variable.setSubType(SubType.record);
29731                                        if (variable.getColumns().isEmpty()) {
29732                                                TObjectName variableProperties = new TObjectName();
29733                                                variableProperties.setString("*");
29734                                                modelFactory.createTableColumn(variable, variableProperties, true);
29735                                        }
29736                                        relation.addSource(new TableColumnRelationshipElement(variable.getColumns().get(0),
29737                                                        columnName.getLocation(), columnName));
29738                                        continue;
29739                                }
29740
29741                                if (columnName.getColumnNameOnly().startsWith("@")
29742                                                && (option.getVendor() == EDbVendor.dbvmssql || option.getVendor() == EDbVendor.dbvazuresql)) {
29743                                        continue;
29744                                }
29745
29746                                if (columnName.getColumnNameOnly().startsWith(":")
29747                                                && (option.getVendor() == EDbVendor.dbvhana || option.getVendor() == EDbVendor.dbvteradata)) {
29748                                        continue;
29749                                }
29750
29751                                TTable table = modelManager.getTable(stmt, columnName);
29752
29753                                if (table == null) {
29754                                        table = columnName.getSourceTable();
29755                                }
29756
29757                                if (table == null && stmt.tables != null) {
29758                                        for (int k = 0; k < stmt.tables.size(); k++) {
29759                                                if (table != null)
29760                                                        break;
29761
29762                                                TTable tTable = stmt.tables.getTable(k);
29763                                                if (tTable.getTableType().name().startsWith("open")) {
29764                                                        continue;
29765                                                } else if (getTableLinkedColumns(tTable) != null && getTableLinkedColumns(tTable).size() > 0) {
29766                                                        for (int z = 0; z < getTableLinkedColumns(tTable).size(); z++) {
29767                                                                TObjectName refer = getTableLinkedColumns(tTable).getObjectName(z);
29768                                                                if ("*".equals(getColumnName(refer)))
29769                                                                        continue;
29770                                                                if (getColumnName(refer).equals(getColumnName(columnName))) {
29771                                                                        table = tTable;
29772                                                                        break;
29773                                                                }
29774                                                        }
29775                                                } else if (columnName.getTableToken() != null
29776                                                                && (SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotTable, columnName.getTableToken().getAstext(), tTable.getName())
29777                                                                                || SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotTable, columnName.getTableToken().getAstext(), tTable.getAliasName()))) {
29778                                                        table = tTable;
29779                                                        break;
29780                                                }
29781                                        }
29782
29783                                        if (table == null) {
29784                                                for (int k = 0; k < stmt.tables.size(); k++) {
29785                                                        if (table != null)
29786                                                                break;
29787
29788                                                        TTable tTable = stmt.tables.getTable(k);
29789                                                        Object model = ModelBindingManager.get().getModel(tTable);
29790                                                        if (model instanceof Table) {
29791                                                                Table tableModel = (Table) model;
29792                                                                for (int z = 0; tableModel.getColumns() != null
29793                                                                                && z < tableModel.getColumns().size(); z++) {
29794                                                                        TableColumn refer = tableModel.getColumns().get(z);
29795                                                                        if (getColumnName(refer.getName()).equals(getColumnName(columnName))) {
29796                                                                                table = tTable;
29797                                                                                break;
29798                                                                        }
29799                                                                        if (refer.hasStarLinkColumn()) {
29800                                                                                for (TObjectName linkColumn : refer.getStarLinkColumnList()) {
29801                                                                                        if (getColumnName(linkColumn).equals(getColumnName(columnName))) {
29802                                                                                                table = tTable;
29803                                                                                                break;
29804                                                                                        }
29805                                                                                }
29806                                                                        }
29807                                                                }
29808                                                        } else if (model instanceof QueryTable) {
29809                                                                QueryTable tableModel = (QueryTable) model;
29810                                                                for (int z = 0; tableModel.getColumns() != null
29811                                                                                && z < tableModel.getColumns().size(); z++) {
29812                                                                        ResultColumn refer = tableModel.getColumns().get(z);
29813                                                                        if (DlineageUtil.getIdentifierNormalColumnName(refer.getName()).equals(
29814                                                                                        DlineageUtil.getIdentifierNormalColumnName(getColumnName(columnName)))) {
29815                                                                                table = tTable;
29816                                                                                break;
29817                                                                        }
29818                                                                        if (refer.hasStarLinkColumn()) {
29819                                                                                for (TObjectName linkColumn : refer.getStarLinkColumnList()) {
29820                                                                                        if (getColumnName(linkColumn).equals(getColumnName(columnName))) {
29821                                                                                                table = tTable;
29822                                                                                                break;
29823                                                                                        }
29824                                                                                }
29825                                                                        }
29826                                                                }
29827                                                        }
29828                                                }
29829                                        }
29830                                }
29831
29832                                if (table == null && stmt.tables != null && stmt.tables.size() != 0
29833                                                && !(isBuiltInFunctionName(columnName) && isFromFunction(columnName))) {
29834                                        
29835                                        if (modelManager.getModel(stmt) instanceof ResultSet) {
29836                                                ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt);
29837                                                boolean find = false;
29838                                                for (ResultColumn resultColumn : resultSetModel.getColumns()) {
29839                                                        if(resultColumn.equals(modelObject)) {
29840                                                                continue;
29841                                                        }
29842                                                        if (!TSQLEnv.isAliasReferenceForbidden.get(option.getVendor())) {
29843                                                                if (getColumnName(columnName).equals(getColumnName(resultColumn.getName()))) {
29844                                                                        if (resultColumn.getColumnObject() != null) {
29845                                                                                int startToken = resultColumn.getColumnObject().getStartToken().posinlist;
29846                                                                                int endToken = resultColumn.getColumnObject().getEndToken().posinlist;
29847                                                                                if (columnName.getStartToken().posinlist >= startToken
29848                                                                                                && columnName.getEndToken().posinlist <= endToken) {
29849                                                                                        continue;
29850                                                                                }
29851                                                                        }
29852                                                                        relation.addSource(new ResultColumnRelationshipElement(resultColumn));
29853                                                                        find = true;
29854                                                                        break;
29855                                                                }
29856                                                        }
29857                                                }
29858                                                if (find) {
29859                                                        continue;
29860                                                }
29861                                        }
29862
29863                                        TObjectName pseudoTableName = new TObjectName();
29864                                        // Use qualified prefix from column name if available (e.g., sch.pk_constv2 from sch.pk_constv2.c_cdsl)
29865                                        // Otherwise fall back to default pseudo table name
29866                                        String qualifiedPrefix = getQualifiedPrefixFromColumn(columnName);
29867                                        pseudoTableName.setString(qualifiedPrefix != null ? qualifiedPrefix : "pseudo_table_include_orphan_column");
29868                                        Table pseudoTable = modelFactory.createTableByName(pseudoTableName);
29869                                        pseudoTable.setPseudo(true);
29870                                        TableColumn pseudoTableColumn = modelFactory.createTableColumn(pseudoTable, columnName, true);
29871
29872                                        // If not linking to first table and column has qualified prefix (3-part name like sch.pkg.col),
29873                                        // add the pseudo table column as source
29874                                        if (!isLinkOrphanColumnToFirstTable() && pseudoTableColumn != null && qualifiedPrefix != null) {
29875                                                relation.addSource(new TableColumnRelationshipElement(pseudoTableColumn));
29876                                        }
29877
29878                                        if (isLinkOrphanColumnToFirstTable()) {
29879                                                TTable orphanTable = stmt.tables.getTable(0);
29880                                                table = stmt.tables.getTable(0);
29881                                                Object tableModel = modelManager.getModel(table);
29882                                                if (tableModel == null) {
29883                                                        tableModel = modelFactory.createTable(orphanTable);
29884                                                }
29885                                                if (tableModel instanceof Table) {
29886                                                        modelFactory.createTableColumn((Table) tableModel, columnName, false);
29887                                                        ErrorInfo errorInfo = new ErrorInfo();
29888                                                        errorInfo.setErrorType(ErrorInfo.LINK_ORPHAN_COLUMN);
29889                                                        errorInfo.setErrorMessage("Link orphan column [" + columnName.toString()
29890                                                                        + "] to the first table [" + orphanTable.getFullNameWithAliasString() + "]");
29891                                                        errorInfo.setStartPosition(new Pair3<Long, Long, String>(columnName.getStartToken().lineNo,
29892                                                                        columnName.getStartToken().columnNo, ModelBindingManager.getGlobalHash()));
29893                                                        errorInfo.setEndPosition(new Pair3<Long, Long, String>(columnName.getEndToken().lineNo,
29894                                                                        columnName.getEndToken().columnNo + columnName.getEndToken().getAstext().length(),
29895                                                                        ModelBindingManager.getGlobalHash()));
29896                                                        errorInfo.fillInfo(this);
29897                                                        errorInfos.add(errorInfo);
29898                                                }
29899                                        }
29900                                }
29901
29902                                if (table != null) {
29903                                        if (modelManager.getModel(table) instanceof Table) {
29904                                                Table tableModel = (Table) modelManager.getModel(table);
29905                                                if (tableModel != null) {
29906                                                        TableColumn columnModel = modelFactory.createTableColumn(tableModel, columnName, false);
29907                                                        if(columnModel!=null) {
29908                                                                // Carry the operand's own reference site: for created
29909                                                                // (e.g. #temp) tables the column model is shared with
29910                                                                // the definition site, whose coordinate is not where
29911                                                                // this ON/WHERE operand sits.
29912                                                                TableColumnRelationshipElement element = new TableColumnRelationshipElement(columnModel,
29913                                                                                columnName.getLocation(), columnName);
29914                                                                relation.addSource(element);
29915                                                        }
29916                                                }
29917                                        } else if (modelManager.getModel(table) instanceof QueryTable) {
29918                                                QueryTable tableModel = (QueryTable)modelManager.getModel(table);
29919                                                if (table.getSubquery() != null && table.getSubquery().isCombinedQuery()) {
29920                                                        TSelectSqlStatement subquery = table.getSubquery();
29921                                                        List<ResultSet> resultSets = new ArrayList<ResultSet>();
29922                                                        if (!subquery.getLeftStmt().isCombinedQuery()) {
29923                                                                ResultSet sourceResultSet = (ResultSet) modelManager
29924                                                                                .getModel(subquery.getLeftStmt().getResultColumnList());
29925                                                                resultSets.add(sourceResultSet);
29926                                                        } else {
29927                                                                ResultSet sourceResultSet = (ResultSet) modelManager.getModel(subquery.getLeftStmt());
29928                                                                resultSets.add(sourceResultSet);
29929                                                        }
29930
29931                                                        if (!subquery.getRightStmt().isCombinedQuery()) {
29932                                                                ResultSet sourceResultSet = (ResultSet) modelManager
29933                                                                                .getModel(subquery.getRightStmt().getResultColumnList());
29934                                                                resultSets.add(sourceResultSet);
29935                                                        } else {
29936                                                                ResultSet sourceResultSet = (ResultSet) modelManager.getModel(subquery.getRightStmt());
29937                                                                resultSets.add(sourceResultSet);
29938                                                        }
29939
29940                                                        for (ResultSet sourceResultSet : resultSets) {
29941                                                                if (sourceResultSet != null && columnName.getSourceColumn() != null) {
29942                                                                        for (int k = 0; k < sourceResultSet.getColumns().size(); k++) {
29943                                                                                if (getColumnName(sourceResultSet.getColumns().get(k).getName()).equals(
29944                                                                                                getColumnName(columnName.getSourceColumn().getColumnNameOnly()))) {
29945                                                                                        Set<TObjectName> starLinkColumnSet = sourceResultSet.getColumns().get(k)
29946                                                                                                        .getStarLinkColumns().get(getColumnName(columnName));
29947                                                                                        if (starLinkColumnSet != null && !starLinkColumnSet.isEmpty()) {
29948                                                                                                ResultColumn column = modelFactory.createResultColumn(sourceResultSet,
29949                                                                                                                starLinkColumnSet.iterator().next(), true);
29950                                                                                                ResultColumnRelationshipElement setopElement = new ResultColumnRelationshipElement(column);
29951                                                                                                setopElement.setReferencePosition(columnName);
29952                                                                                                relation.addSource(setopElement);
29953                                                                                        } else {
29954                                                                                                ResultColumnRelationshipElement setopElement = new ResultColumnRelationshipElement(
29955                                                                                                                sourceResultSet.getColumns().get(k));
29956                                                                                                setopElement.setReferencePosition(columnName);
29957                                                                                                relation.addSource(setopElement);
29958                                                                                        }
29959                                                                                }
29960                                                                        }
29961                                                                }
29962                                                        }
29963                                                } else {
29964                                                        Object model = modelManager.getModel(columnName.getSourceColumn());
29965                                                        if (model instanceof ResultColumn) {
29966                                                                ResultColumn resultColumn = (ResultColumn) model;
29967                                                                if (resultColumn != null) {
29968                                                                        if (resultColumn.hasStarLinkColumn()) {
29969                                                                                Set<TObjectName> starLinkColumnSet = resultColumn.getStarLinkColumns()
29970                                                                                                .get(getColumnName(columnName));
29971                                                                                if (starLinkColumnSet != null && !starLinkColumnSet.isEmpty()) {
29972                                                                                        ResultColumn column = modelFactory.createResultColumn(
29973                                                                                                        resultColumn.getResultSet(), starLinkColumnSet.iterator().next(), true);
29974                                                                                        ResultColumnRelationshipElement starElement = new ResultColumnRelationshipElement(column);
29975                                                                                        starElement.setReferencePosition(columnName);
29976                                                                                        relation.addSource(starElement);
29977                                                                                } else {
29978                                                                                        resultColumn.bindStarLinkColumn(columnName);
29979                                                                                        ResultColumn column = modelFactory
29980                                                                                                        .createResultColumn(resultColumn.getResultSet(), columnName, true);
29981                                                                                        ResultColumnRelationshipElement starElement = new ResultColumnRelationshipElement(column);
29982                                                                                        starElement.setReferencePosition(columnName);
29983                                                                                        relation.addSource(starElement);
29984                                                                                }
29985                                                                        } else {
29986                                                                                // Carry the operand's own reference site: the
29987                                                                                // derived table's / CTE's result column model
29988                                                                                // carries its select-list definition span, not
29989                                                                                // where this ON/WHERE operand sits.
29990                                                                                ResultColumnRelationshipElement element = new ResultColumnRelationshipElement(
29991                                                                                                resultColumn, columnName.getLocation(), columnName);
29992                                                                                relation.addSource(element);
29993                                                                        }
29994                                                                }
29995                                                        }
29996                                                        else{
29997                                                                boolean find = false;
29998                                                                for (int i = 0; i < tableModel.getColumns().size(); i++) {
29999                                                                        ResultColumn resultColumn = tableModel.getColumns().get(i);
30000                                                                        if (DlineageUtil.getIdentifierNormalColumnName(resultColumn.getName()).equals(
30001                                                                                        DlineageUtil.getIdentifierNormalColumnName(getColumnName(columnName)))) {
30002                                                                                // An explicit CTE column list resolves here;
30003                                                                                // the model span is the column-list entry, not
30004                                                                                // the operand.
30005                                                                                ResultColumnRelationshipElement element = new ResultColumnRelationshipElement(
30006                                                                                                resultColumn, columnName.getLocation(), columnName);
30007                                                                                relation.addSource(element);
30008                                                                                find = true;
30009                                                                                break;
30010                                                                        }
30011                                                                        else if (resultColumn.getName().endsWith("*")) {
30012                                                                                resultColumn.bindStarLinkColumn(columnName);
30013                                                                        }
30014                                                                }
30015                                                                if(!find){
30016                                                                        ResultColumn resultColumn = new ResultColumn(tableModel, columnName);
30017                                                                        ResultColumnRelationshipElement element = new ResultColumnRelationshipElement(
30018                                                                                                resultColumn, columnName.getLocation(), columnName);
30019                                                                                relation.addSource(element);
30020                                                                }
30021                                                        }
30022                                                }
30023                                        }
30024                                }
30025                        }
30026
30027                        for (int j = 0; j < functions.size(); j++) {
30028                                TParseTreeNode functionObj = functions.get(j);
30029                                Object functionModel = modelManager.getModel(functionObj);
30030                                if (functionModel == null) {
30031                                        functionModel = createFunction(functionObj);
30032                                }
30033                                if (functionModel instanceof Function) {
30034                                        if (functionObj instanceof TFunctionCall) {
30035                                                ResultColumn resultColumn = (ResultColumn) modelManager
30036                                                                .getModel(((TFunctionCall) functionObj).getFunctionName());
30037                                                if (resultColumn != null) {
30038                                                        ResultColumnRelationshipElement element = new ResultColumnRelationshipElement(resultColumn,
30039                                                                        ((TFunctionCall) functionObj).getFunctionName().getLocation());
30040                                                        relation.addSource(element);
30041                                                }
30042                                        }
30043                                        if (functionObj instanceof TCaseExpression) {
30044                                                ResultColumn resultColumn = (ResultColumn) modelManager
30045                                                                .getModel(((TCaseExpression) functionObj).getWhenClauseItemList());
30046                                                if (resultColumn != null) {
30047                                                        ResultColumnRelationshipElement element = new ResultColumnRelationshipElement(resultColumn);
30048                                                        relation.addSource(element);
30049                                                }
30050                                        }
30051                                } else if (functionModel instanceof Table && ((TFunctionCall) functionObj).getFunctionName()!=null) {
30052                                        TableColumn tableColumn = modelFactory.createTableColumn((Table) functionModel,
30053                                                        ((TFunctionCall) functionObj).getFunctionName(), false);
30054                                        if (tableColumn != null) {
30055                                                TableColumnRelationshipElement element = new TableColumnRelationshipElement(tableColumn);
30056                                                relation.addSource(element);
30057                                        }
30058                                }
30059                        }
30060
30061                        for (int j = 0; j < resultColumns.size(); j++) {
30062                                TResultColumn resultColumn = resultColumns.get(j);
30063                                if (modelManager.getModel(resultColumn) instanceof ResultColumn) {
30064                                        ResultColumn resultColumnModel = (ResultColumn) modelManager.getModel(resultColumn);
30065                                        relation.addSource(new ResultColumnRelationshipElement(resultColumnModel, ESqlClause.selectList));
30066                                }
30067                        }
30068                }
30069
30070                if (isShowJoin() && joinClauseType != null) {
30071                        joinInExpr joinVisitor = new joinInExpr(joinType, joinClauseType, effectType);
30072                        expr.inOrderTraverse(joinVisitor);
30073                }
30074        }
30075
30076        public void dispose() {
30077                authoritativeEvidenceCollector.reset();
30078                cteSelectRelationKeys.clear();
30079                accessedSubqueries.clear();
30080                accessedStatements.clear();
30081                stmtStack.clear();
30082                viewDDLMap.clear();
30083                procedureDDLMap.clear();
30084                structObjectMap.clear();
30085                appendResultSets.clear();
30086                appendStarColumns.clear();
30087                appendTableStarColumns.clear();
30088                columnNameIndexCache.clear();
30089                resultSetColumnLookupCache.clear();
30090                tableColumnLookupCache.clear();
30091                modelManager.DISPLAY_ID.clear();
30092                modelManager.DISPLAY_NAME.clear();
30093                tableIds.clear();
30094                ModelBindingManager.remove();
30095        }
30096
30097        class joinTreatColumnsInExpr implements IExpressionVisitor {
30098
30099                private List<TObjectName> objectNames = new ArrayList<TObjectName>();
30100                
30101                private TTable table;
30102
30103                public joinTreatColumnsInExpr(TTable table) {
30104                        this.table = table;
30105                }
30106
30107                public List<TObjectName> getObjectNames() {
30108                        return objectNames;
30109                }
30110
30111                boolean is_compare_condition(EExpressionType t) {
30112                        return t == EExpressionType.simple_comparison_t;
30113                }
30114
30115                @Override
30116                public boolean exprVisit(TParseTreeNode pNode, boolean isLeafNode) {
30117                        TExpression expr = (TExpression) pNode;
30118                        if (is_compare_condition(expr.getExpressionType())) {
30119                                TExpression leftExpr = expr.getLeftOperand();
30120                                columnsInExpr leftVisitor = new columnsInExpr();
30121                                leftExpr.inOrderTraverse(leftVisitor);
30122                                List<TObjectName> leftObjectNames = leftVisitor.getObjectNames();
30123
30124                                TExpression rightExpr = expr.getRightOperand();
30125                                columnsInExpr rightVisitor = new columnsInExpr();
30126                                rightExpr.inOrderTraverse(rightVisitor);
30127                                List<TObjectName> rightObjectNames = rightVisitor.getObjectNames();
30128
30129                                if (!leftObjectNames.isEmpty() && !rightObjectNames.isEmpty()) {
30130                                        for (TObjectName column : leftObjectNames) {
30131                                                if (column.getSourceTable() != null && column.getSourceTable().equals(table)) {
30132                                                        objectNames.add(column);
30133                                                        return false;
30134                                                }
30135                                        }
30136                                        for (TObjectName column : rightObjectNames) {
30137                                                if (column.getSourceTable() != null && column.getSourceTable().equals(table)) {
30138                                                        objectNames.add(column);
30139                                                        return false;
30140                                                }
30141                                        }
30142                                }
30143                                return false;
30144                        }
30145                        return true;
30146                }
30147        }
30148        
30149        class columnsInExpr implements IExpressionVisitor {
30150
30151                private List<TParseTreeNode> constants = new ArrayList<TParseTreeNode>();
30152                private List<TObjectName> objectNames = new ArrayList<TObjectName>();
30153                private List<TParseTreeNode> functions = new ArrayList<TParseTreeNode>();
30154                private List<TResultColumn> resultColumns = new ArrayList<TResultColumn>();
30155                private List<TSelectSqlStatement> subquerys = new ArrayList<TSelectSqlStatement>();
30156                private boolean skipFunction = false;
30157                // ClickHouse expression-CTE aliases already substituted during this
30158                // traversal — guards against cyclic alias definitions.
30159                private Set<TCTE> expandedExpressionCtes = new HashSet<TCTE>();
30160                // Alias-chain expansion is depth-bounded hybrid: direct recursion
30161                // preserves depth-first substitution order (source order of the
30162                // collected columns) up to CTE_EXPANSION_MAX_DEPTH; deeper chains
30163                // queue their definitions on an explicit deque drained by the
30164                // outermost frame — a 2000-alias chain must not overflow the -Xss2m
30165                // stack (same class as the repo's UNION/expression traversal rule).
30166                private static final int CTE_EXPANSION_MAX_DEPTH = 64;
30167                private Deque<TExpression> pendingCteExpressions = new ArrayDeque<TExpression>();
30168                private int cteExpansionDepth = 0;
30169
30170                /**
30171                 * Collects an expression operand as a column reference — unless it is a
30172                 * ClickHouse expression-CTE alias (WITH <expr> AS ident), in which case
30173                 * the CTE's definition is traversed instead so its real column
30174                 * dependencies flow to the consumer. Every branch of exprVisit that
30175                 * adds an operand's TObjectName directly must go through this method,
30176                 * or alias references in that expression shape leak out as columns.
30177                 */
30178                private void addColumnOrExpandCteAlias(TObjectName object) {
30179                        if (object.getExpressionCteRef() == null
30180                                        || object.getSourceTable() != null
30181                                        || object.getExpressionCteRef().getExpression() == null) {
30182                                objectNames.add(object);
30183                                return;
30184                        }
30185                        if (!expandedExpressionCtes.add(object.getExpressionCteRef())) {
30186                                return; // cyclic or already substituted
30187                        }
30188                        TExpression definition = object.getExpressionCteRef().getExpression();
30189                        if (cteExpansionDepth >= CTE_EXPANSION_MAX_DEPTH) {
30190                                pendingCteExpressions.add(definition);
30191                                return; // drained below when the outermost frame unwinds
30192                        }
30193                        cteExpansionDepth++;
30194                        try {
30195                                definition.inOrderTraverse(this);
30196                        } finally {
30197                                cteExpansionDepth--;
30198                        }
30199                        if (cteExpansionDepth == 0) {
30200                                while (!pendingCteExpressions.isEmpty()) {
30201                                        TExpression pending = pendingCteExpressions.poll();
30202                                        cteExpansionDepth++;
30203                                        try {
30204                                                pending.inOrderTraverse(this);
30205                                        } finally {
30206                                                cteExpansionDepth--;
30207                                        }
30208                                }
30209                        }
30210                }
30211
30212                public void setSkipFunction(boolean skipFunction) {
30213                        this.skipFunction = skipFunction;
30214                }
30215
30216                public List<TParseTreeNode> getFunctions() {
30217                        return functions;
30218                }
30219
30220                public List<TSelectSqlStatement> getSubquerys() {
30221                        return subquerys;
30222                }
30223
30224                public List<TParseTreeNode> getConstants() {
30225                        return constants;
30226                }
30227
30228                public List<TObjectName> getObjectNames() {
30229                        return objectNames;
30230                }
30231
30232                public List<TResultColumn> getResultColumns() {
30233                        return resultColumns;
30234                }
30235
30236                @Override
30237                public boolean exprVisit(TParseTreeNode pNode, boolean isLeafNode) {
30238                        TExpression lcexpr = (TExpression) pNode;
30239                        // Handle named argument expressions (e.g., "INPUT => value" in Snowflake FLATTEN)
30240                        // The left operand is the parameter name, NOT a column reference.
30241                        // Only traverse the right operand (the value).
30242                        if (lcexpr.getExpressionType() == EExpressionType.assignment_t) {
30243                                // Skip left operand (parameter name) - only traverse right operand (value)
30244                                if (lcexpr.getRightOperand() != null) {
30245                                        lcexpr.getRightOperand().inOrderTraverse(this);
30246                                }
30247                                return false; // Don't continue default traversal
30248                        }
30249                        if (lcexpr.getExpressionType() == EExpressionType.simple_constant_t) {
30250                                if (lcexpr.getConstantOperand() != null) {
30251                                        if(lcexpr.getConstantOperand().getInt64_expression()!=null 
30252                                                        && lcexpr.getConstantOperand().getInt64_expression().getExpressionType() == EExpressionType.function_t) {
30253                                                lcexpr.getConstantOperand().getInt64_expression().inOrderTraverse(this);
30254                                        }
30255                                        else {
30256                                                constants.add(lcexpr.getConstantOperand());
30257                                        }
30258                                }
30259                        } else if (lcexpr.getExpressionType() == EExpressionType.array_t) {
30260                                if(lcexpr.getObjectOperand()!=null) {
30261                                        TObjectName object = lcexpr.getObjectOperand();
30262                                        addColumnOrExpandCteAlias(object);
30263                                } else if (lcexpr.getExprList() != null) {
30264                                        for (int j = 0; j < lcexpr.getExprList().size(); j++) {
30265                                                TExpression expr = lcexpr.getExprList().getExpression(j);
30266                                                if (expr != null)
30267                                                        expr.inOrderTraverse(this);
30268                                        }
30269                                }
30270                        } else if (lcexpr.getExpressionType() == EExpressionType.simple_object_name_t) {
30271                                if (lcexpr.getObjectOperand() != null && !(isBuiltInFunctionName(lcexpr.getObjectOperand())
30272                                                && isFromFunction(lcexpr.getObjectOperand()))) {
30273                                        TObjectName object = lcexpr.getObjectOperand();
30274                                        // Skip named argument parameter names (e.g., INPUT in "INPUT => value")
30275                                        // These are function parameter names, NOT column references
30276                                        if (object.getObjectType() == TObjectName.ttobjNamedArgParameter) {
30277                                                // Skip - this is a named argument parameter name
30278                                        } else if (object.getDbObjectType() == EDbObjectType.column
30279                                                        || object.getDbObjectType() == EDbObjectType.column_alias
30280                                                        || object.getDbObjectType() == EDbObjectType.alias
30281                                                        || object.getDbObjectType() == EDbObjectType.unknown
30282                                                        || object.getDbObjectType() == EDbObjectType.variable
30283                                                        || isTableScopedPseudoColumn(object)) {
30284                                                addColumnOrExpandCteAlias(object);
30285                                        } else if (object.getDbObjectType() == EDbObjectType.notAColumn
30286                                                        || object.getDbObjectType() == EDbObjectType.date_time_part ) {
30287                                                constants.add(object);
30288                                        }
30289                                }
30290                        } else if (lcexpr.getExpressionType() == EExpressionType.between_t) {
30291                                if (lcexpr.getBetweenOperand() != null && lcexpr.getBetweenOperand().getObjectOperand() != null) {
30292                                        TObjectName object = lcexpr.getBetweenOperand().getObjectOperand();
30293                                        if (object.getDbObjectType() == EDbObjectType.column
30294                                                        || object.getDbObjectType() == EDbObjectType.column_alias
30295                                                        || object.getDbObjectType() == EDbObjectType.alias
30296                                                        || object.getDbObjectType() == EDbObjectType.unknown
30297                                                        || object.getDbObjectType() == EDbObjectType.variable) {
30298                                                addColumnOrExpandCteAlias(object);
30299                                        }
30300                                }
30301                        } else if (lcexpr.getExpressionType() == EExpressionType.object_access_t) {
30302                                if (lcexpr.getObjectAccess() != null) {
30303                                        TObjectNameList objects = lcexpr.getObjectAccess().getAttributes();
30304                                        TFunctionCall function = lcexpr.getObjectAccess().getObjectExpr().getFunctionCall();
30305                                        if (objects != null && function != null) {
30306                                                for (TObjectName object : objects) {
30307                                                        TGSqlParser sqlparser = createSqlParser(option.getVendor());
30308                                                        sqlparser.sqltext = "select " + function.getFunctionName().toString() + "."
30309                                                                        + object.getColumnNameOnly() + " from " + function.getFunctionName().toString();
30310                                                        if (sqlparser.parse() == 0) {
30311                                                                TObjectName objectName = sqlparser.sqlstatements.get(0).getResultColumnList()
30312                                                                                .getResultColumn(0).getFieldAttr();
30313                                                                objectNames.add(objectName);
30314                                                        }
30315                                                }
30316                                        }
30317                                }
30318                        } else if (lcexpr.getExpressionType() == EExpressionType.function_t || lcexpr.getExpressionType() == EExpressionType.fieldselection_t) {
30319                                TFunctionCall func = lcexpr.getFunctionCall();
30320                                if (func == null) {
30321                                        return true;
30322                                }
30323                                if (skipFunction) {
30324                                        if (func.getArgs() != null) {
30325                                                for (int k = 0; k < func.getArgs().size(); k++) {
30326                                                        TExpression expr = func.getArgs().getExpression(k);
30327                                                        if (expr != null)
30328                                                                expr.inOrderTraverse(this);
30329                                                }
30330                                        }
30331
30332                                        if (func.getTrimArgument() != null) {
30333                                                TTrimArgument args = func.getTrimArgument();
30334                                                TExpression expr = args.getStringExpression();
30335                                                if (expr != null) {
30336                                                        expr.inOrderTraverse(this);
30337                                                }
30338                                                expr = args.getTrimCharacter();
30339                                                if (expr != null) {
30340                                                        expr.inOrderTraverse(this);
30341                                                }
30342                                        }
30343
30344                                        if (func.getAgainstExpr() != null) {
30345                                                func.getAgainstExpr().inOrderTraverse(this);
30346                                        }
30347//                                      if (func.getBetweenExpr() != null) {
30348//                                              func.getBetweenExpr().inOrderTraverse(this);
30349//                                      }
30350                                        if (func.getExpr1() != null) {
30351                                                func.getExpr1().inOrderTraverse(this);
30352                                        }
30353                                        if (func.getExpr2() != null) {
30354                                                func.getExpr2().inOrderTraverse(this);
30355                                        }
30356                                        if (func.getExpr3() != null) {
30357                                                func.getExpr3().inOrderTraverse(this);
30358                                        }
30359                                        if (func.getParameter() != null) {
30360                                                func.getParameter().inOrderTraverse(this);
30361                                        }
30362                                } else {
30363                                        functions.add(func);
30364                                }
30365
30366                        } else if (lcexpr.getExpressionType() == EExpressionType.case_t) {
30367                                TCaseExpression expr = lcexpr.getCaseExpression();
30368                                if (skipFunction) {
30369                                        TExpression defaultExpr = expr.getElse_expr();
30370                                        if (defaultExpr != null) {
30371                                                defaultExpr.inOrderTraverse(this);
30372                                        }
30373                                        TWhenClauseItemList list = expr.getWhenClauseItemList();
30374                                        for (int i = 0; i < list.size(); i++) {
30375                                                TWhenClauseItem element = (TWhenClauseItem) list.getElement(i);
30376                                                (((TWhenClauseItem) element).getReturn_expr()).inOrderTraverse(this);
30377
30378                                        }
30379                                } else {
30380                                        functions.add(expr);
30381                                }
30382                        } else if (lcexpr.getSubQuery() != null) {
30383                                TSelectSqlStatement select = lcexpr.getSubQuery();
30384                                analyzeSelectStmt(select);
30385                                subquerys.add(select);
30386                                if (select.getResultColumnList() != null && select.getResultColumnList().size() > 0) {
30387                                        for (TResultColumn column : select.getResultColumnList()) {
30388                                                resultColumns.add(column);
30389                                        }
30390                                }
30391                        }
30392                        return true;
30393                }
30394        }
30395
30396        class joinInExpr implements IExpressionVisitor {
30397
30398                private EJoinType joinType;
30399                private JoinClauseType joinClauseType;
30400                private EffectType effectType;
30401
30402                public joinInExpr(EJoinType joinType, JoinClauseType joinClauseType, EffectType effectType) {
30403                        this.joinType = joinType;
30404                        this.joinClauseType = joinClauseType;
30405                        this.effectType = effectType;
30406                }
30407
30408                boolean is_compare_condition(EExpressionType t) {
30409                        return ((t == EExpressionType.simple_comparison_t) || (t == EExpressionType.group_comparison_t)
30410                                        || (t == EExpressionType.in_t) || (t == EExpressionType.pattern_matching_t)
30411                                        || (t == EExpressionType.left_join_t) || (t == EExpressionType.right_join_t));
30412                }
30413
30414                @Override
30415                public boolean exprVisit(TParseTreeNode pNode, boolean isLeafNode) {
30416                        TExpression expr = (TExpression) pNode;
30417                        if (is_compare_condition(expr.getExpressionType())) {
30418                                TExpression leftExpr = expr.getLeftOperand();
30419                                columnsInExpr leftVisitor = new columnsInExpr();
30420                                leftExpr.inOrderTraverse(leftVisitor);
30421                                List<TObjectName> leftObjectNames = leftVisitor.getObjectNames();
30422                                List<TParseTreeNode> leftObjects = leftVisitor.getFunctions();
30423                                leftObjects.addAll(leftObjectNames);
30424                                
30425                                TExpression rightExpr = expr.getRightOperand();
30426                                columnsInExpr rightVisitor = new columnsInExpr();
30427                                rightExpr.inOrderTraverse(rightVisitor);
30428                                List<TObjectName> rightObjectNames = rightVisitor.getObjectNames();
30429                                List<TParseTreeNode> rightObjects = rightVisitor.getFunctions();
30430                                rightObjects.addAll(rightObjectNames);
30431
30432                                if (!leftObjects.isEmpty() && !rightObjects.isEmpty()) {
30433                                        TCustomSqlStatement stmt = stmtStack.peek();
30434
30435                                        for (int i = 0; i < leftObjects.size(); i++) {
30436                                                TParseTreeNode leftObject = leftObjects.get(i);
30437                                                TTable leftTable = null;
30438                                                TFunctionCall leftFunction = null;
30439                                                TObjectName leftObjectName = null;
30440                                                if (leftObject instanceof TObjectName) {
30441                                                        leftObjectName = (TObjectName)leftObject;
30442
30443                                                        if (leftObjectName.getDbObjectType() == EDbObjectType.variable) {
30444                                                                continue;
30445                                                        }
30446
30447                                                        if (leftObjectName.getColumnNameOnly().startsWith("@")
30448                                                                        && (option.getVendor() == EDbVendor.dbvmssql
30449                                                                                        || option.getVendor() == EDbVendor.dbvazuresql)) {
30450                                                                continue;
30451                                                        }
30452
30453                                                        if (leftObjectName.getColumnNameOnly().startsWith(":")
30454                                                                        && (option.getVendor() == EDbVendor.dbvhana
30455                                                                                        || option.getVendor() == EDbVendor.dbvteradata)) {
30456                                                                continue;
30457                                                        }
30458
30459                                                        leftTable = modelManager.getTable(stmt, leftObjectName);
30460
30461                                                        if (leftTable == null) {
30462                                                                leftTable = leftObjectName.getSourceTable();
30463                                                        }
30464
30465                                                        if (leftTable == null) {
30466                                                                leftTable = modelManager.guessTable(stmt, leftObjectName);
30467                                                        }
30468                                                }
30469                                                else if(leftObject instanceof TFunctionCall){
30470                                                        leftFunction = (TFunctionCall)leftObject;
30471                                                }
30472
30473                                                if (leftTable != null || leftFunction != null) {
30474                                                        for (int j = 0; j < rightObjects.size(); j++) {
30475                                                                JoinRelationship joinRelation = modelFactory.createJoinRelation();
30476                                                                joinRelation.setEffectType(effectType);
30477                                                                if (joinType != null) {
30478                                                                        joinRelation.setJoinType(joinType);
30479                                                                } else {
30480                                                                        if (expr.getLeftOperand().isOracleOuterJoin()) {
30481                                                                                joinRelation.setJoinType(right);
30482                                                                        } else if (expr.getRightOperand().isOracleOuterJoin()) {
30483                                                                                joinRelation.setJoinType(EJoinType.left);
30484                                                                        } else if (expr.getExpressionType() == EExpressionType.left_join_t) {
30485                                                                                joinRelation.setJoinType(EJoinType.left);
30486                                                                        } else if (expr.getExpressionType() == EExpressionType.right_join_t) {
30487                                                                                joinRelation.setJoinType(right);
30488                                                                        } else {
30489                                                                                joinRelation.setJoinType(EJoinType.inner);
30490                                                                        }
30491                                                                }
30492
30493                                                                joinRelation.setJoinClauseType(joinClauseType);
30494                                                                joinRelation.setJoinCondition(expr.toString());
30495
30496
30497                                                                if (leftTable != null) {
30498                                                                        if (modelManager.getModel(leftTable) instanceof Table) {
30499                                                                                Table tableModel = (Table) modelManager.getModel(leftTable);
30500                                                                                if (tableModel != null) {
30501                                                                                        TableColumn columnModel = modelFactory.createTableColumn(tableModel,
30502                                                                                                        leftObjectName, false);
30503                                                                                        if (columnModel != null) {
30504                                                                                                joinRelation.addSource(new TableColumnRelationshipElement(columnModel));
30505                                                                                        }
30506                                                                                }
30507                                                                        } else if (modelManager.getModel(leftTable) instanceof QueryTable) {
30508                                                                                QueryTable table = (QueryTable) modelManager.getModel(leftTable);
30509                                                                                TSelectSqlStatement subquery = table.getTableObject().getSubquery();
30510                                                                                if (subquery != null && subquery.isCombinedQuery()) {
30511                                                                                        ResultColumn resultColumn = matchResultColumn(table.getColumns(),
30512                                                                                                        leftObjectName);
30513                                                                                        if (resultColumn != null) {
30514                                                                                                joinRelation
30515                                                                                                                .addSource(new ResultColumnRelationshipElement(resultColumn));
30516                                                                                        }
30517                                                                                } else if (leftObjectName.getSourceColumn() != null) {
30518                                                                                        Object model = modelManager.getModel(leftObjectName);
30519                                                                                        if (model == null) {
30520                                                                                                model = modelFactory.createResultColumn(table, leftObjectName);
30521                                                                                        }
30522                                                                                        if (model instanceof ResultColumn) {
30523                                                                                                ResultColumn resultColumn = (ResultColumn) model;
30524                                                                                                if (resultColumn != null) {
30525                                                                                                        joinRelation.addSource(
30526                                                                                                                        new ResultColumnRelationshipElement(resultColumn));
30527                                                                                                }
30528                                                                                        } else if (model instanceof LinkedHashMap) {
30529                                                                                                String columnName = getColumnNameOnly(leftObjectName.toString());
30530                                                                                                LinkedHashMap<String, ResultColumn> resultColumns = (LinkedHashMap<String, ResultColumn>) model;
30531                                                                                                if (resultColumns.containsKey(columnName)) {
30532                                                                                                        ResultColumn resultColumn = resultColumns.get(columnName);
30533                                                                                                        joinRelation.addSource(
30534                                                                                                                        new ResultColumnRelationshipElement(resultColumn));
30535                                                                                                }
30536                                                                                        }
30537                                                                                } else {
30538                                                                                        ResultColumn resultColumn = matchResultColumn(table.getColumns(),
30539                                                                                                        leftObjectName);
30540                                                                                        if (resultColumn != null) {
30541                                                                                                joinRelation
30542                                                                                                                .addSource(new ResultColumnRelationshipElement(resultColumn));
30543                                                                                        }
30544                                                                                }
30545                                                                        }
30546                                                                }
30547                                                                else if(leftFunction!=null) {
30548                                                                        Object functionObj = createFunction(leftFunction);
30549                                                                        if(functionObj instanceof Function) {
30550                                                                                Function function = (Function)functionObj;
30551                                                                                joinRelation.addSource(new ResultColumnRelationshipElement(function.getColumns().get(0)));
30552                                                                        }
30553                                                                }
30554
30555                                                                TParseTreeNode rightObject = rightObjects.get(j);
30556                                                                if(rightObject instanceof TObjectName) {
30557                                                                        TObjectName rightObjectName = (TObjectName)rightObject;
30558
30559                                                                        if (rightObjectName.getDbObjectType() == EDbObjectType.variable) {
30560                                                                                continue;
30561                                                                        }
30562
30563                                                                        if (rightObjectName.getColumnNameOnly().startsWith("@")
30564                                                                                        && (option.getVendor() == EDbVendor.dbvmssql
30565                                                                                                        || option.getVendor() == EDbVendor.dbvazuresql)) {
30566                                                                                continue;
30567                                                                        }
30568
30569                                                                        if (rightObjectName.getColumnNameOnly().startsWith(":")
30570                                                                                        && (option.getVendor() == EDbVendor.dbvhana
30571                                                                                                        || option.getVendor() == EDbVendor.dbvteradata)) {
30572                                                                                continue;
30573                                                                        }
30574
30575                                                                        TTable rightTable = modelManager.getTable(stmt, rightObjectName);
30576                                                                        if (rightTable == null) {
30577                                                                                rightTable = rightObjectName.getSourceTable();
30578                                                                        }
30579
30580                                                                        if (rightTable == null) {
30581                                                                                rightTable = modelManager.guessTable(stmt, rightObjectName);
30582                                                                        }
30583
30584                                                                        if (modelManager.getModel(rightTable) instanceof Table) {
30585                                                                                Table tableModel = (Table) modelManager.getModel(rightTable);
30586                                                                                if (tableModel != null) {
30587                                                                                        TableColumn columnModel = modelFactory.createTableColumn(tableModel,
30588                                                                                                        rightObjectName, false);
30589                                                                                        if(columnModel != null) {
30590                                                                                                joinRelation.setTarget(new TableColumnRelationshipElement(columnModel));
30591                                                                                        }
30592                                                                                }
30593                                                                        } else if (modelManager.getModel(rightTable) instanceof QueryTable) {
30594                                                                                QueryTable table = (QueryTable) modelManager.getModel(rightTable);
30595                                                                                TSelectSqlStatement subquery = table.getTableObject().getSubquery();
30596                                                                                if (subquery != null && subquery.isCombinedQuery()) {
30597                                                                                        ResultColumn resultColumn = matchResultColumn(table.getColumns(),
30598                                                                                                        rightObjectName);
30599                                                                                        if (resultColumn != null) {
30600                                                                                                joinRelation.setTarget(new ResultColumnRelationshipElement(resultColumn));
30601                                                                                        }
30602                                                                                } else if (rightObjectName.getSourceColumn() != null) {
30603                                                                                        Object model = modelManager.getModel(rightObjectName);
30604                                                                                        if (model == null) {
30605                                                                                                model = modelManager
30606                                                                                                                .getModel(rightObjectName.getSourceColumn());
30607                                                                                        }
30608                                                                                        if (model instanceof ResultColumn) {
30609                                                                                                joinRelation.setTarget(new ResultColumnRelationshipElement((ResultColumn)model));
30610                                                                                        }
30611                                                                                        else if (model instanceof LinkedHashMap) {
30612                                                                                                String columnName = getColumnNameOnly(rightObjectName.toString());
30613                                                                                                LinkedHashMap<String, ResultColumn> resultColumns = (LinkedHashMap<String, ResultColumn>)model;
30614                                                                                                if (resultColumns.containsKey(columnName)) {
30615                                                                                                        ResultColumn resultColumn = resultColumns.get(columnName);
30616                                                                                                        joinRelation.setTarget(new ResultColumnRelationshipElement(resultColumn));
30617                                                                                                }
30618                                                                                        }
30619                                                                                } else {
30620                                                                                        ResultColumn resultColumn = matchResultColumn(table.getColumns(),
30621                                                                                                        rightObjectName);
30622                                                                                        if (resultColumn != null) {
30623                                                                                                joinRelation.setTarget(new ResultColumnRelationshipElement(resultColumn));
30624                                                                                        }
30625                                                                                }
30626                                                                        }
30627                                                                }
30628                                                                else if(rightObject instanceof TFunctionCall) {
30629                                                                        Object functionObj = createFunction(rightObject);
30630                                                                        if(functionObj instanceof Function) {
30631                                                                                Function function = (Function)functionObj;
30632                                                                                joinRelation.setTarget(new ResultColumnRelationshipElement(function.getColumns().get(0)));
30633                                                                        }
30634                                                                }
30635                                                        }
30636                                                }
30637                                        }
30638                                }
30639                        }
30640                        return true;
30641                }
30642        }
30643
30644        @Deprecated
30645        public static Dataflow getSqlflowJSONModel(dataflow dataflow) {
30646                EDbVendor vendor = ModelBindingManager.getGlobalVendor();
30647                if(vendor == null) {
30648                        throw new IllegalArgumentException("getSqlflowJSONModel(dataflow dataflow) is deprecated, please call method getSqlflowJSONModel(dataflow dataflow, EDbVendor vendor).");
30649                }
30650                return getSqlflowJSONModel(vendor, dataflow, false);
30651        }
30652
30653        public static Dataflow getSqlflowJSONModel(dataflow dataflow, EDbVendor vendor) {
30654                return getSqlflowJSONModel(vendor, dataflow, false);
30655        }
30656
30657        public static Dataflow getSqlflowJSONModel(EDbVendor vendor, dataflow dataflow, boolean normalizeIdentifier) {
30658                Dataflow model = new Dataflow();
30659                
30660                if (dataflow.getErrors() != null && !dataflow.getErrors().isEmpty()) {
30661                        List<Error> errorList = new ArrayList<Error>();
30662                        for (error error : dataflow.getErrors()) {
30663                                Error err = new Error();
30664                                err.setErrorMessage(error.getErrorMessage());
30665                                err.setErrorType(error.getErrorType());
30666                                err.setCoordinates(Coordinate.parse(error.getCoordinate()));
30667                                err.setFile(err.getFile());
30668                                err.setOriginCoordinates(Coordinate.parse(error.getOriginCoordinate()));
30669                                errorList.add(err);
30670                        }
30671                        model.setErrors(errorList.toArray(new Error[0]));
30672                }
30673
30674                Sqlflow sqlflow = MetadataUtil.convertDataflowToMetadata(vendor, dataflow);
30675                sqlflow.setErrorMessages(null);
30676                model.setDbobjs(sqlflow);
30677                model.setOrientation(dataflow.getOrientation());
30678
30679
30680                List<gudusoft.gsqlparser.dlineage.dataflow.model.json.Process> processes = new ArrayList<gudusoft.gsqlparser.dlineage.dataflow.model.json.Process>();
30681                if(dataflow.getProcesses()!=null){
30682                        for(process process: dataflow.getProcesses()){
30683                                gudusoft.gsqlparser.dlineage.dataflow.model.json.Process processModel = new gudusoft.gsqlparser.dlineage.dataflow.model.json.Process();
30684                                processModel.setId(process.getId());
30685                                processModel.setName(process.getName());
30686                                processModel.setProcedureId(process.getProcedureId());
30687                                processModel.setProcedureName(process.getProcedureName());
30688                                processModel.setType(process.getType());
30689                                processModel.setCoordinate(process.getCoordinate());
30690                                processModel.setDatabase(process.getDatabase());
30691                                processModel.setSchema(process.getSchema());
30692                                processModel.setServer(process.getServer());
30693                                processModel.setQueryHashId(process.getQueryHashId());
30694                                if (process.getTransforms() != null && !process.getTransforms().isEmpty()) {
30695                                        List<gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform> transforms = new ArrayList<gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform>();
30696                                        for (transform transform : process.getTransforms()) {
30697                                                gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform item = new gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform();
30698                                                item.setCode(transform.getCode());
30699                                                item.setType(transform.getType());
30700                                                item.setCoordinate(transform.getCoordinate(true));
30701                                                transforms.add(item);
30702                                        }
30703                                        processModel.setTransforms(transforms
30704                                                        .toArray(new gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform[0]));
30705                                }
30706                                processes.add(processModel);
30707                        }
30708                }
30709                model.setProcesses(processes.toArray(new gudusoft.gsqlparser.dlineage.dataflow.model.json.Process[0]));
30710
30711                Map<String, String> tableIdToProcedureId = new HashMap<>();
30712                if (dataflow.getTables() != null) {
30713                        for (table t : dataflow.getTables()) {
30714                                if (t.getProcedureId() != null) {
30715                                        tableIdToProcedureId.put(t.getId(), t.getProcedureId());
30716                                }
30717                        }
30718                }
30719
30720                List<gudusoft.gsqlparser.dlineage.dataflow.model.json.Relationship> relations = new ArrayList<gudusoft.gsqlparser.dlineage.dataflow.model.json.Relationship>();
30721                if (dataflow.getRelationships() != null) {
30722                        for (relationship relation : dataflow.getRelationships()) {
30723                                gudusoft.gsqlparser.dlineage.dataflow.model.json.Relationship relationModel;
30724                                if (relation.getType().equals("join")) {
30725                                        gudusoft.gsqlparser.dlineage.dataflow.model.json.JoinRelationship joinRelationModel = new gudusoft.gsqlparser.dlineage.dataflow.model.json.JoinRelationship();
30726                                        joinRelationModel.setCondition(relation.getCondition());
30727                                        joinRelationModel.setJoinType(relation.getJoinType());
30728                                        joinRelationModel.setClause(relation.getClause());
30729                                        relationModel = joinRelationModel;
30730                                } else {
30731                                        relationModel = new gudusoft.gsqlparser.dlineage.dataflow.model.json.Relationship();
30732                                }
30733
30734                                relationModel.setId(relation.getId());
30735                                relationModel.setProcessId(relation.getProcessId());
30736                                relationModel.setProcessType(relation.getProcessType());
30737                                relationModel.setType(relation.getType());
30738                                relationModel.setEffectType(relation.getEffectType());
30739                                relationModel.setPartition(relation.getPartition());
30740                                relationModel.setFunction(relation.getFunction());
30741                                relationModel.setProcedureId(relation.getProcedureId());
30742                                relationModel.setSqlHash(relation.getSqlHash());
30743                                relationModel.setCondition(relation.getCondition());
30744                                relationModel.setSqlComment(relation.getSqlComment());
30745                                relationModel.setTimestampMax(relation.getTimestampMax());
30746                                relationModel.setTimestampMin(relation.getTimestampMin());
30747                                if (Boolean.TRUE.equals(relation.getBuiltIn())) {
30748                                        relationModel.setBuiltIn(relation.getBuiltIn());
30749                                }
30750                                relationModel.setCallStmt(relation.getCallStmt());
30751                                relationModel.setCallCoordinate(relation.getCallCoordinate());
30752                                
30753                                if (relation.getTarget() != null && relation.getSources() != null && !relation.getSources().isEmpty()) {
30754                                        {
30755                                                gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement targetModel = new gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement();
30756                                                targetColumn target = relation.getTarget();
30757                                                if (normalizeIdentifier) {
30758                                                        targetModel.setColumn(SQLUtil.getIdentifierNormalColumnName(vendor, target.getColumn()));
30759                                                        targetModel.setParentName(
30760                                                                        SQLUtil.getIdentifierNormalTableName(vendor, target.getParent_name()));
30761                                                        targetModel.setTargetName(
30762                                                                        SQLUtil.getIdentifierNormalColumnName(vendor, target.getTarget_name()));
30763                                                } else {
30764                                                        targetModel.setColumn(target.getColumn());
30765                                                        targetModel.setParentName(target.getParent_name());
30766                                                        targetModel.setTargetName(target.getTarget_name());
30767                                                }
30768                                                targetModel.setId(target.getId());
30769                                                targetModel.setTargetId(target.getTarget_id());
30770                                                targetModel.setParentId(target.getParent_id());
30771                                                targetModel.setCoordinates(Coordinate.parse(target.getCoordinate()));
30772                                                targetModel.setFunction(target.getFunction());
30773                                                targetModel.setType(target.getType());
30774                                                String targetProcId = tableIdToProcedureId.get(target.getParent_id());
30775                                                if (targetProcId != null) {
30776                                                        targetModel.setProcedureId(targetProcId);
30777                                                }
30778                                                relationModel.setTarget(targetModel);
30779                                        }
30780
30781                                        List<gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement> sourceModels = new ArrayList<gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement>();
30782                                        for (sourceColumn source : relation.getSources()) {
30783                                                gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement sourceModel = new gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement();
30784                                                if (normalizeIdentifier) {
30785                                                        sourceModel.setColumn(SQLUtil.getIdentifierNormalColumnName(vendor, source.getColumn()));
30786                                                        sourceModel.setParentName(
30787                                                                        SQLUtil.getIdentifierNormalTableName(vendor, source.getParent_name()));
30788                                                        sourceModel.setSourceName(
30789                                                                        SQLUtil.getIdentifierNormalColumnName(vendor, source.getSource_name()));
30790                                                } else {
30791                                                        sourceModel.setColumn(source.getColumn());
30792                                                        sourceModel.setParentName(source.getParent_name());
30793                                                        sourceModel.setSourceName(source.getSource_name());
30794                                                }
30795                                                sourceModel.setColumnType(source.getColumn_type());
30796                                                sourceModel.setId(source.getId());
30797                                                sourceModel.setParentId(source.getParent_id());
30798                                                sourceModel.setSourceId(source.getSource_id());
30799                                                sourceModel.setCoordinates(Coordinate.parse(source.getCoordinate()));
30800                                                sourceModel.setClauseType(source.getClauseType());
30801                                                sourceModel.setType(source.getType());
30802                                                String sourceProcId = tableIdToProcedureId.get(source.getParent_id());
30803                                                if (sourceProcId != null) {
30804                                                        sourceModel.setProcedureId(sourceProcId);
30805                                                }
30806                                                sourceModels.add(sourceModel);
30807                                                if (source.getTransforms() != null && !source.getTransforms().isEmpty()) {
30808                                                        List<gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform> transforms = new ArrayList<gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform>();
30809                                                        for (transform transform : source.getTransforms()) {
30810                                                                gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform item = new gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform();
30811                                                                item.setCode(transform.getCode());
30812                                                                item.setType(transform.getType());
30813                                                                item.setCoordinate(transform.getCoordinate(true));
30814                                                                transforms.add(item);
30815                                                        }
30816                                                        sourceModel.setTransforms(transforms
30817                                                                        .toArray(new gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform[0]));
30818                                                }
30819                                                
30820                                                if (source.getCandidateParents() != null && !source.getCandidateParents().isEmpty()) {
30821                                                        List<gudusoft.gsqlparser.dlineage.dataflow.model.json.CandidateTable> candidateParents = new ArrayList<gudusoft.gsqlparser.dlineage.dataflow.model.json.CandidateTable>();
30822                                                        for (candidateTable candidateTable : source.getCandidateParents()) {
30823                                                                gudusoft.gsqlparser.dlineage.dataflow.model.json.CandidateTable item = new gudusoft.gsqlparser.dlineage.dataflow.model.json.CandidateTable();
30824                                                                item.setId(candidateTable.getId());
30825                                                                if (normalizeIdentifier) {
30826                                                                        item.setName(
30827                                                                                        SQLUtil.getIdentifierNormalTableName(vendor, candidateTable.getName()));
30828                                                                } else {
30829                                                                        item.setName(candidateTable.getName());
30830                                                                }
30831                                                                candidateParents.add(item);
30832                                                        }
30833                                                        sourceModel.setCandidateParents(candidateParents
30834                                                                        .toArray(new gudusoft.gsqlparser.dlineage.dataflow.model.json.CandidateTable[0]));
30835                                                }
30836                                        }
30837                                        relationModel.setSources(sourceModels
30838                                                        .toArray(new gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement[0]));
30839                                        relations.add(relationModel);
30840                                } else if (relation.getCaller() != null && relation.getCallees() != null
30841                                                && !relation.getCallees().isEmpty()) {
30842                                        {
30843                                                gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement targetModel = new gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement();
30844                                                targetColumn target = relation.getCaller();
30845                                                if (normalizeIdentifier) {
30846                                                        targetModel.setName(SQLUtil.getIdentifierNormalColumnName(vendor, target.getName()));
30847                                                } else {
30848                                                        targetModel.setName(target.getName());
30849                                                }
30850                                                targetModel.setId(target.getId());
30851                                                targetModel.setCoordinates(Coordinate.parse(target.getCoordinate()));
30852                                                targetModel.setType(target.getType());
30853                                                relationModel.setCaller(targetModel);
30854                                        }
30855
30856                                        List<gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement> sourceModels = new ArrayList<gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement>();
30857                                        for (sourceColumn source : relation.getCallees()) {
30858                                                gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement sourceModel = new gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement();
30859                                                if (normalizeIdentifier) {
30860                                                        sourceModel.setName(SQLUtil.getIdentifierNormalColumnName(vendor, source.getName()));
30861                                                } else {
30862                                                        sourceModel.setName(source.getName());
30863                                                }
30864                                                sourceModel.setId(source.getId());
30865                                                sourceModel.setCoordinates(Coordinate.parse(source.getCoordinate()));
30866                                                sourceModel.setType(source.getType());
30867                                                sourceModels.add(sourceModel);
30868                                        }
30869                                        relationModel.setCallees(sourceModels
30870                                                        .toArray(new gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement[0]));
30871                                        relations.add(relationModel);
30872                                }
30873                        }
30874                }
30875                model.setRelationships(relations.toArray(new gudusoft.gsqlparser.dlineage.dataflow.model.json.Relationship[0]));
30876                return model;
30877        }
30878
30879        public static String getVersion() {
30880                // Return the authoritative product version instead of a stale hardcoded
30881                // constant so version-reporting stays in sync with each release (MantisBT
30882                // / QS-009). TBaseType.versionid is the single source of truth (also used
30883                // by the JAR manifest and release notes).
30884                return TBaseType.versionid;
30885        }
30886
30887        public static String getReleaseDate() {
30888                return TBaseType.releaseDate;
30889        }
30890
30891        public static void main(String[] args) {
30892                if (args.length < 1) {
30893                        System.out.println(
30894                                        "Usage: java DataFlowAnalyzer [/f <path_to_sql_file>] [/d <path_to_directory_includes_sql_files>] [/s [/text]] [/json] [/traceView] [/t <database type>] [/o <output file path>][/version]");
30895                        System.out.println("/f: Option, specify the sql file path to analyze fdd relation.");
30896                        System.out.println("/d: Option, specify the sql directory path to analyze fdd relation.");
30897                        System.out.println("/j: Option, analyze the join relation.");
30898                        System.out.println("/s: Option, simple output, ignore the intermediate results.");
30899                        System.out.println("/i: Option, ignore all result sets.");
30900                        System.out.println("/traceView: Option, analyze the source tables of views.");
30901                        System.out.println("/text: Option, print the plain text format output.");
30902                        System.out.println("/json: Option, print the json format output.");
30903                        System.out.println(
30904                                        "/t: Option, set the database type. Support oracle, mysql, mssql, db2, netezza, teradata, informix, sybase, postgresql, hive, greenplum and redshift, the default type is oracle");
30905                        System.out.println("/o: Option, write the output stream to the specified file.");
30906                        System.out.println("/log: Option, generate a dataflow.log file to log information.");
30907                        return;
30908                }
30909
30910                File sqlFiles = null;
30911
30912                List<String> argList = Arrays.asList(args);
30913
30914                if (argList.indexOf("/version") != -1) {
30915                        System.out.println("Version: " + DataFlowAnalyzer.getVersion());
30916                        System.out.println("Release Date: " + DataFlowAnalyzer.getReleaseDate());
30917                        return;
30918                }
30919
30920                if (argList.indexOf("/f") != -1 && argList.size() > argList.indexOf("/f") + 1) {
30921                        sqlFiles = new File(args[argList.indexOf("/f") + 1]);
30922                        if (!sqlFiles.exists() || !sqlFiles.isFile()) {
30923                                System.out.println(sqlFiles + " is not a valid file.");
30924                                return;
30925                        }
30926                } else if (argList.indexOf("/d") != -1 && argList.size() > argList.indexOf("/d") + 1) {
30927                        sqlFiles = new File(args[argList.indexOf("/d") + 1]);
30928                        if (!sqlFiles.exists() || !sqlFiles.isDirectory()) {
30929                                System.out.println(sqlFiles + " is not a valid directory.");
30930                                return;
30931                        }
30932                } else {
30933                        System.out.println("Please specify a sql file path or directory path to analyze dlineage.");
30934                        return;
30935                }
30936
30937                EDbVendor vendor = EDbVendor.dbvoracle;
30938
30939                int index = argList.indexOf("/t");
30940
30941                if (index != -1 && args.length > index + 1) {
30942                        vendor = TGSqlParser.getDBVendorByName(args[index + 1]);
30943                }
30944
30945                String outputFile = null;
30946
30947                index = argList.indexOf("/o");
30948
30949                if (index != -1 && args.length > index + 1) {
30950                        outputFile = args[index + 1];
30951                }
30952
30953                FileOutputStream writer = null;
30954                if (outputFile != null) {
30955                        try {
30956                                writer = new FileOutputStream(outputFile);
30957                                System.setOut(new PrintStream(writer));
30958                        } catch (FileNotFoundException e) {
30959                                logger.error("output file is not found.", e);
30960                        }
30961                }
30962
30963                boolean simple = argList.indexOf("/s") != -1;
30964                boolean ignoreResultSets = argList.indexOf("/i") != -1;
30965                boolean showJoin = argList.indexOf("/j") != -1;
30966                boolean textFormat = false;
30967                boolean jsonFormat = false;
30968                if (simple) {
30969                        textFormat = argList.indexOf("/text") != -1;
30970                }
30971
30972                boolean traceView = argList.indexOf("/traceView") != -1;
30973                if (traceView) {
30974                        simple = true;
30975                }
30976
30977                jsonFormat = argList.indexOf("/json") != -1;
30978
30979                DataFlowAnalyzer dlineage = new DataFlowAnalyzer(sqlFiles, vendor, simple);
30980
30981                dlineage.setShowJoin(showJoin);
30982                dlineage.setIgnoreRecordSet(ignoreResultSets);
30983                // dlineage.setShowImplicitSchema(true);
30984
30985                if (simple && !jsonFormat) {
30986                        dlineage.setTextFormat(textFormat);
30987                }
30988
30989                String result = dlineage.generateDataFlow();
30990
30991//              dataflow dataflow = ProcessUtility.generateTableLevelLineage(dlineage, dlineage.getDataFlow());
30992//              System.out.println(result);
30993
30994                if (jsonFormat) {
30995                        // Map jsonResult = new LinkedHashMap();
30996                        Dataflow model = getSqlflowJSONModel(vendor, dlineage.getDataFlow(), true);
30997                        // jsonResult.put("data", BeanUtils.bean2Map(model));
30998                        result = JSON.toJSONString(model);
30999                } else if (traceView) {
31000                        result = dlineage.traceView();
31001                }
31002
31003                if (result != null) {
31004                        System.out.println(result);
31005
31006                        if (writer != null && result.length() < 1024 * 1024) {
31007                                System.err.println(result);
31008                        }
31009                }
31010
31011                try {
31012                        if (writer != null) {
31013                                writer.close();
31014                        }
31015                } catch (IOException e) {
31016                        logger.error("close writer failed.", e);
31017                }
31018
31019                boolean log = argList.indexOf("/log") != -1;
31020
31021                PrintStream systemSteam = System.err;
31022                ByteArrayOutputStream sw = new ByteArrayOutputStream();
31023                PrintStream pw = new PrintStream(sw);
31024                System.setErr(pw);
31025
31026
31027                List<ErrorInfo> errors = dlineage.getErrorMessages();
31028                if (!errors.isEmpty()) {
31029                        System.err.println("Error log:\n");
31030                        for (int i = 0; i < errors.size(); i++) {
31031                                System.err.println(errors.get(i).getErrorMessage());
31032                        }
31033                }
31034
31035                if (sw != null) {
31036                        String errorMessage = sw.toString().trim();
31037                        if (errorMessage.length() > 0) {
31038                                if (log) {
31039                                        try {
31040                                                pw = new PrintStream(new File(".", "dataflow.log"));
31041                                                pw.print(errorMessage);
31042                                        } catch (FileNotFoundException e) {
31043                                                logger.error("error log file is not found.", e);
31044                                        }
31045                                }
31046
31047                                System.setErr(systemSteam);
31048                                System.err.println(errorMessage);
31049                        }
31050                }
31051        }
31052
31053        public List<ErrorInfo> getErrorMessages() {
31054                return errorInfos;
31055        }
31056
31057        /**
31058         * Every dynamic-SQL execution site (T-SQL {@code EXEC(...)} / {@code sp_executesql}) the analyzer
31059         * encountered, with whether it was statically resolved. Diagnostic only; not consumed by lineage.
31060         * Read-only snapshot; see {@link DynamicSqlSite}.
31061         */
31062        public List<DynamicSqlSite> getDynamicSqlSites() {
31063                return java.util.Collections.unmodifiableList(new ArrayList<DynamicSqlSite>(dynamicSqlSites));
31064        }
31065
31066        /**
31067         * Resolve dynamic-SQL object names by abstractly evaluating a stored procedure
31068         * with a set of parameter bindings.
31069         *
31070         * <p>Given a T-SQL {@code CREATE PROC} parse tree and the values its parameters
31071         * actually had at a call site (e.g. {@code @SourceDB='QSP'},
31072         * {@code @STG_AEG='STG_AEG'}), this evaluates the procedure's dynamic-SQL string
31073         * building (assignment + concatenation + name-building functions), materializes
31074         * the concrete SQL each {@code EXEC} / {@code sp_executesql} site runs, analyzes
31075         * it with ordinary dlineage, and returns the lineage edges tagged with dynamic
31076         * provenance ({@code origin=DYNAMIC_RESOLVED}, source proc, stable dynamic-site
31077         * id, binding hash). Sites that cannot be reduced are reported as honest
31078         * {@code UNRESOLVED} diagnostics — never guessed.
31079         *
31080         * <p>This is abstract string evaluation, NOT execution: no table reads, no
31081         * cursors, no data-dependent control flow. It is purely additive — it builds
31082         * throwaway analyzers for the materialized strings and does not change the
31083         * default dlineage output of this analyzer.
31084         *
31085         * @param procAst         the {@code CREATE PROC} parse tree (currently T-SQL)
31086         * @param vendor          the SQL dialect ({@code dbvmssql})
31087         * @param sqlEnv          catalog env for the resolved analysis (may be null)
31088         * @param currentDatabase the proc's database (resolves unqualified names)
31089         * @param defaultSchema   the proc's default schema (e.g. {@code dbo})
31090         * @param bindings        parameter name → concrete value
31091         * @param options         resource bounds (may be null for defaults)
31092         */
31093        public gudusoft.gsqlparser.dlineage.dynamicsql.DynamicLineageResult resolveDynamicSqlLineage(
31094                        TCustomSqlStatement procAst, EDbVendor vendor, gudusoft.gsqlparser.sqlenv.TSQLEnv sqlEnv,
31095                        String currentDatabase, String defaultSchema,
31096                        Map<String, gudusoft.gsqlparser.dlineage.dynamicsql.SqlValue> bindings,
31097                        gudusoft.gsqlparser.dlineage.dynamicsql.DynamicLineageOptions options) {
31098                return gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.resolve(
31099                                procAst, vendor, sqlEnv, currentDatabase, defaultSchema, bindings, options);
31100        }
31101
31102        public String traceView() {
31103                StringBuilder buffer = new StringBuilder();
31104                dataflow dataflow = this.getDataFlow();
31105                Map<table, Set<table>> traceViewMap = new LinkedHashMap<table, Set<table>>();
31106                if (dataflow != null && dataflow.getViews() != null) {
31107                        List<relationship> relations = dataflow.getRelationships();
31108                        Map<String, table> viewMap = new HashMap<String, table>();
31109                        Map<String, table> tableMap = new HashMap<String, table>();
31110                        for (table view : dataflow.getViews()) {
31111                                viewMap.put(view.getId(), view);
31112                                tableMap.put(view.getId(), view);
31113                        }
31114                        for (table table : dataflow.getTables()) {
31115                                tableMap.put(table.getId(), table);
31116                        }
31117                        for (relationship relation : relations) {
31118                                if (!RelationshipType.fdd.name().equals(relation.getType())) {
31119                                        continue;
31120                                }
31121                                String parentId = relation.getTarget().getParent_id();
31122                                if (viewMap.containsKey(parentId)) {
31123                                        if (!traceViewMap.containsKey(viewMap.get(parentId))) {
31124                                                traceViewMap.put(viewMap.get(parentId), new LinkedHashSet<table>());
31125                                        }
31126
31127                                        for (sourceColumn sourceColumn : relation.getSources()) {
31128                                                traceViewMap.get(viewMap.get(parentId)).add(tableMap.get(sourceColumn.getParent_id()));
31129                                        }
31130                                }
31131                        }
31132
31133                        Map<table, Set<table>> viewTableMap = new LinkedHashMap<table, Set<table>>();
31134                        for (table view : traceViewMap.keySet()) {
31135                                Set<table> tables = new LinkedHashSet<table>();
31136                                traverseViewSourceTables(tables, view, traceViewMap);
31137                                viewTableMap.put(view, tables);
31138                        }
31139
31140                        for (table view : viewTableMap.keySet()) {
31141                                buffer.append(view.getFullName());
31142                                for (table table : viewTableMap.get(view)) {
31143                                        buffer.append(",").append(table.getFullName());
31144                                }
31145                                buffer.append(System.getProperty("line.separator"));
31146                        }
31147                }
31148                return buffer.toString().trim();
31149        }
31150
31151        private void traverseViewSourceTables(Set<table> tables, table view, Map<table, Set<table>> traceViewMap) {
31152                Set<table> sourceTables = traceViewMap.get(view);
31153                for (table sourceTable : sourceTables) {
31154                        if (sourceTable.isTable()) {
31155                                tables.add(sourceTable);
31156                        } else if (sourceTable.isView()) {
31157                                traverseViewSourceTables(tables, sourceTable, traceViewMap);
31158                        }
31159                }
31160        }
31161
31162        protected List<SqlInfo> convertSQL(EDbVendor vendor, String json) {
31163                List<SqlInfo> sqlInfos = new ArrayList<SqlInfo>();
31164                List sqlContents = (List) JSON.parseObject(json);
31165                for (int j = 0; j < sqlContents.size(); j++) {
31166                        Map sqlContent = (Map) sqlContents.get(j);
31167                        String sql = (String) sqlContent.get("sql");
31168                        String fileName = (String) sqlContent.get("fileName");
31169                        String filePath = (String) sqlContent.get("filePath");
31170                        if (sql != null && sql.trim().startsWith("{")) {
31171                                if (sql.indexOf("createdBy") != -1) {
31172                                        String shardBaseDir = filePath != null ? new File(filePath).getParent() : null;
31173                                        // Route on the authoritative `format` first. A sqlflow-sharded
31174                                        // manifest carries the product brand (SQLdep/grabit) in createdBy
31175                                        // but has no single-file `queries` array — reading it through the
31176                                        // brand path silently dropped every source record.
31177                                        if (MetadataReader.isSqlflowSharded(sql)) {
31178                                                if (MetadataReader.isSupportedSqlflowSharded(sql)) {
31179                                                        appendShardedSourceSqlInfos(sql, shardBaseDir, fileName, sqlInfos);
31180                                                }
31181                                                // Unsupported version, or no base dir to locate shards:
31182                                                // nothing is added — never misrouted, never crashes.
31183                                        } else if (sql.toLowerCase().indexOf("sqldep") != -1
31184                                                        || sql.toLowerCase().indexOf("grabit") != -1) {
31185                                                Map queryObject = (Map) JSON.parseObject(sql);
31186                                                List querys = (List) queryObject.get("queries");
31187                                                if (querys != null) {
31188                                                        for (int i = 0; i < querys.size(); i++) {
31189                                                                Map object = (Map) querys.get(i);
31190                                                                SqlInfo info = new SqlInfo();
31191                                                                info.setSql(JSON.toJSONString(object));
31192                                                                info.setFileName(fileName);
31193                                                                info.setFilePath(filePath);
31194                                                                info.setOriginIndex(i);
31195                                                                sqlInfos.add(info);
31196                                                        }
31197                                                        queryObject.remove("queries");
31198                                                        SqlInfo info = new SqlInfo();
31199                                                        info.setSql(JSON.toJSONString(queryObject));
31200                                                        info.setFileName(fileName);
31201                                                        info.setFilePath(filePath);
31202                                                        info.setOriginIndex(querys.size());
31203                                                        sqlInfos.add(info);
31204                                                } else {
31205                                                        SqlInfo info = new SqlInfo();
31206                                                        info.setSql(JSON.toJSONString(queryObject));
31207                                                        info.setFileName(fileName);
31208                                                        info.setFilePath(filePath);
31209                                                        info.setOriginIndex(0);
31210                                                        sqlInfos.add(info);
31211                                                }
31212                                        } else if (sql.toLowerCase().indexOf("sqlflow") != -1) {
31213                                                Map sqlflow = (Map) JSON.parseObject(sql);
31214                                                List<Map> servers = (List<Map>) sqlflow.get("servers");
31215                                                if (servers != null) {
31216                                                        for (Map queryObject : servers) {
31217                                                                String name = (String) queryObject.get("name");
31218                                                                String dbVendor = (String) queryObject.get("dbVendor");
31219                                                                List querys = (List) queryObject.get("queries");
31220                                                                if (querys != null) {
31221                                                                        for (int i = 0; i < querys.size(); i++) {
31222                                                                                Map object = (Map) querys.get(i);
31223                                                                                SqlInfo info = new SqlInfo();
31224                                                                                info.setSql(JSON.toJSONString(object));
31225                                                                                info.setFileName(fileName);
31226                                                                                info.setFilePath(filePath);
31227                                                                                info.setOriginIndex(i);
31228                                                                                info.setDbVendor(dbVendor);
31229                                                                                info.setServer(name);
31230                                                                                sqlInfos.add(info);
31231                                                                        }
31232                                                                        queryObject.remove("queries");
31233                                                                        Map serverObject = new IndexedLinkedHashMap();
31234                                                                        serverObject.put("createdBy", sqlflow.get("createdBy"));
31235                                                                        serverObject.put("servers", Arrays.asList(queryObject));
31236                                                                        SqlInfo info = new SqlInfo();
31237                                                                        info.setSql(JSON.toJSONString(serverObject));
31238                                                                        info.setFileName(fileName);
31239                                                                        info.setFilePath(filePath);
31240                                                                        info.setOriginIndex(querys.size());
31241                                                                        info.setDbVendor(dbVendor);
31242                                                                        info.setServer(filePath);
31243                                                                        sqlInfos.add(info);
31244                                                                } else {
31245                                                                        SqlInfo info = new SqlInfo();
31246                                                                        info.setSql(JSON.toJSONString(queryObject));
31247                                                                        info.setFileName(fileName);
31248                                                                        info.setFilePath(filePath);
31249                                                                        info.setOriginIndex(0);
31250                                                                        sqlInfos.add(info);
31251                                                                }
31252                                                        }
31253                                                }
31254                                                
31255                                                List<Map> errorMessages =  (List<Map>) sqlflow.get("errorMessages");
31256                                                if(errorMessages!=null && !errorMessages.isEmpty()) {
31257                                                        for(Map error: errorMessages){
31258                                                                ErrorInfo errorInfo = new ErrorInfo();
31259                                                                errorInfo.setErrorType(ErrorInfo.METADATA_ERROR);
31260                                                                errorInfo.setErrorMessage((String)error.get("errorMessage"));
31261                                                                errorInfo.setFileName(fileName);
31262                                                                errorInfo.setFilePath(filePath);
31263                                                                errorInfo.setStartPosition(new Pair3<Long, Long, String>(-1L, -1L,
31264                                                                                ModelBindingManager.getGlobalHash()));
31265                                                                errorInfo.setEndPosition(new Pair3<Long, Long, String>(-1L, -1L,
31266                                                                                ModelBindingManager.getGlobalHash()));
31267                                                                errorInfo.setOriginStartPosition(new Pair<Long, Long>(-1L, -1L));
31268                                                                errorInfo.setOriginEndPosition(new Pair<Long, Long>(-1L, -1L));
31269                                                                metadataErrors.add(errorInfo);
31270                                                        }
31271                                                }
31272                                        }
31273                                }
31274                        } else if (sql != null) {
31275                                SqlInfo info = new SqlInfo();
31276                                info.setSql(sql);
31277                                info.setFileName(fileName);
31278                                info.setFilePath(filePath);
31279                                info.setOriginIndex(0);
31280                                sqlInfos.add(info);
31281                        } else if (filePath != null) {
31282                                SqlInfo info = new SqlInfo();
31283                                info.setFileName(fileName);
31284                                info.setFilePath(filePath);
31285                                info.setOriginIndex(0);
31286                                sqlInfos.add(info);
31287                        }
31288                }
31289                return sqlInfos;
31290        }
31291
31292        public void setTextFormat(boolean textFormat) {
31293                option.setTextFormat(textFormat);
31294        }
31295
31296        public boolean isBuiltInFunctionName(TObjectName object) {
31297                if (object == null || object.getGsqlparser() == null)
31298                        return false;
31299                try {
31300                        EDbVendor vendor = object.getGsqlparser().getDbVendor();
31301                        if (vendor == EDbVendor.dbvteradata) {
31302                                boolean result = TERADATA_BUILTIN_FUNCTIONS.contains(object.toString().toUpperCase());
31303                                if (result) {
31304                                        return true;
31305                                }
31306                        }
31307
31308                        List<String> versions = functionChecker.getAvailableDbVersions(vendor);
31309                        if (versions != null && versions.size() > 0) {
31310                                for (int i = 0; i < versions.size(); i++) {
31311                                        boolean result = functionChecker.isBuiltInFunction(object.toString(),
31312                                                        object.getGsqlparser().getDbVendor(), versions.get(i));
31313                                        if (result) {
31314                                                return result;
31315                                        }
31316                                }
31317
31318                                // boolean result =
31319                                // TERADATA_BUILTIN_FUNCTIONS.contains(object.toString());
31320                                // if (result) {
31321                                // return true;
31322                                // }
31323                        }
31324                } catch (Exception e) {
31325                }
31326
31327                return false;
31328        }
31329        
31330        public boolean isBuiltInFunctionName(String functionName) {
31331                if (functionName == null)
31332                        return false;
31333                try {
31334                        EDbVendor vendor = getOption().getVendor();
31335                        if (vendor == EDbVendor.dbvteradata) {
31336                                boolean result = TERADATA_BUILTIN_FUNCTIONS.contains(functionName.toUpperCase());
31337                                if (result) {
31338                                        return true;
31339                                }
31340                        }
31341
31342                        List<String> versions = functionChecker.getAvailableDbVersions(vendor);
31343                        if (versions != null && versions.size() > 0) {
31344                                for (int i = 0; i < versions.size(); i++) {
31345                                        // no caller-side folding: isBuiltInFunction folds with
31346                                        // Locale.ROOT itself; a default-locale toUpperCase here
31347                                        // breaks the lookup on Turkish JVMs
31348                                        boolean result = functionChecker.isBuiltInFunction(functionName,
31349                                                        vendor, versions.get(i));
31350                                        if (result) {
31351                                                return result;
31352                                        }
31353                                }
31354
31355                                // boolean result =
31356                                // TERADATA_BUILTIN_FUNCTIONS.contains(object.toString());
31357                                // if (result) {
31358                                // return true;
31359                                // }
31360                        }
31361                } catch (Exception e) {
31362                }
31363
31364                return false;
31365        }
31366
31367        public boolean isKeyword(TObjectName object) {
31368                if (object == null || object.getGsqlparser() == null)
31369                        return false;
31370                try {
31371                        EDbVendor vendor = object.getGsqlparser().getDbVendor();
31372
31373                        List<String> versions = keywordChecker.getAvailableDbVersions(vendor);
31374                        if (versions != null && versions.size() > 0) {
31375                                for (int i = 0; i < versions.size(); i++) {
31376                                        List<String> segments = SQLUtil.parseNames(object.toString());
31377                                        boolean result = keywordChecker.isKeyword(segments.get(segments.size() - 1),
31378                                                        object.getGsqlparser().getDbVendor(), versions.get(i), true);
31379                                        if (result) {
31380                                                return result;
31381                                        }
31382                                }
31383                        }
31384                } catch (Exception e) {
31385                }
31386
31387                return false;
31388        }
31389        
31390        public boolean isKeyword(String objectName) {
31391                if (objectName == null)
31392                        return false;
31393                try {
31394                        EDbVendor vendor = getOption().getVendor();
31395
31396                        List<String> versions = keywordChecker.getAvailableDbVersions(vendor);
31397                        if (versions != null && versions.size() > 0) {
31398                                for (int i = 0; i < versions.size(); i++) {
31399                                        List<String> segments = SQLUtil.parseNames(objectName);
31400                                        boolean result = keywordChecker.isKeyword(segments.get(segments.size() - 1),
31401                                                        vendor, versions.get(i), false);
31402                                        if (result) {
31403                                                return result;
31404                                        }
31405                                }
31406                        }
31407                } catch (Exception e) {
31408                }
31409
31410                return false;
31411        }
31412
31413        public boolean isAggregateFunction(TFunctionCall func) {
31414                if (func == null)
31415                        return false;
31416                return Arrays
31417                                .asList(new String[] { "AVG", "COUNT", "MAX", "MIN", "SUM", "COLLECT", "CORR", "COVAR_POP",
31418                                                "COVAR_SAMP", "CUME_DIST", "DENSE_RANK", "FIRST", "GROUP_ID", "GROUPING", "GROUPING_ID", "LAST",
31419                                                "LISTAGG", "MEDIAN", "PERCENT_RANK", "PERCENTILE_CONT", "PERCENTILE_DISC", "RANK",
31420                                                "STATS_BINOMIAL_TEST", "STATS_CROSSTAB", "STATS_F_TEST", "STATS_KS_TEST", "STATS_MODE",
31421                                                "STATS_MW_TEST", "STATS_ONE_WAY_ANOVA", "STATS_WSR_TEST", "STDDEV", "STDDEV_POP", "STDDEV_SAMP",
31422                                                "SYS_XMLAGG", "VAR_ POP", "VAR_ SAMP", "VARI ANCE", "XMLAGG", "ARRAY_AGG" })
31423                                .contains(func.getFunctionName().toString().toUpperCase());
31424        }
31425
31426        public boolean isConstant(TObjectName object) {
31427                if (object == null || object.getGsqlparser() == null)
31428                        return false;
31429                List<String> constants = Arrays.asList(new String[] { "NEXTVAL", "CURRVAL", "SYSDATE", "CENTURY", "YEAR",
31430                                "MONTH", "DAY", "HOUR", "MINUTE", "SECOND" });
31431                List<String> segments = SQLUtil.parseNames(object.toString());
31432                // sequence.NEXTVAL or sequence.CURRVAL is a sequence reference, not a constant
31433                // This syntax is used by Oracle, Snowflake, and accepted by other vendors for compatibility
31434                String columnNameOnly = object.getColumnNameOnly();
31435                if (segments.size() > 1 && ("NEXTVAL".equalsIgnoreCase(columnNameOnly) || "CURRVAL".equalsIgnoreCase(columnNameOnly))) {
31436                        return false;
31437                }
31438                boolean result = constants.indexOf(segments.get(segments.size() - 1).toUpperCase()) != -1;
31439                if (result) {
31440                        return result;
31441                }
31442                if (isKeyword(object)) {
31443                        return true;
31444                }
31445                return false;
31446        }
31447
31448        private Pair3<Long, Long, Integer> convertCoordinate(Pair3<Long, Long, String> position) {
31449//              if (ModelBindingManager.getGlobalOption()!=null && ModelBindingManager.getGlobalOption().isIgnoreCoordinate()) {
31450//                      return new Pair3<>(-1L, -1L, -1);
31451//              }
31452                return new Pair3<Long, Long, Integer>(position.first, position.second,
31453                                ModelBindingManager.getGlobalSqlInfo().getIndexOf(position.third));
31454        }
31455
31456        /**
31457         * Analyze MDX SELECT statement to generate data lineage.
31458         * Maps MDX cube as source table, measures/dimensions as columns,
31459         * and creates dataflow relationships to the result set.
31460         */
31461        private void analyzeMdxSelectStmt(gudusoft.gsqlparser.stmt.mdx.TMdxSelect stmt) {
31462                // Get cube name from FROM clause
31463                gudusoft.gsqlparser.nodes.mdx.TMdxIdentifierNode cube = stmt.getCube();
31464                if (cube == null) {
31465                        return;
31466                }
31467
31468                String cubeName = getMdxIdentifierName(cube);
31469                Table cubeTable = modelFactory.createTableByName(cubeName, false);
31470
31471                // Collect all MDX identifier references from axes and WHERE clause
31472                List<String> measureNames = new ArrayList<String>();
31473                List<String> dimensionNames = new ArrayList<String>();
31474
31475                // Process axes (COLUMNS, ROWS, etc.)
31476                if (stmt.getAxes() != null) {
31477                        for (int i = 0; i < stmt.getAxes().size(); i++) {
31478                                gudusoft.gsqlparser.nodes.mdx.TMdxAxisNode axis = stmt.getAxes().getElement(i);
31479                                if (axis.getExpNode() != null) {
31480                                        collectMdxReferences(axis.getExpNode(), measureNames, dimensionNames);
31481                                }
31482                        }
31483                }
31484
31485                // Process WHERE clause (slicer dimension)
31486                if (stmt.getWhere() != null && stmt.getWhere().getFilter() != null) {
31487                        collectMdxReferences(stmt.getWhere().getFilter(), measureNames, dimensionNames);
31488                }
31489
31490                // Process WITH MEMBER definitions
31491                if (stmt.getWiths() != null) {
31492                        for (int i = 0; i < stmt.getWiths().size(); i++) {
31493                                gudusoft.gsqlparser.nodes.mdx.TMdxWithNode withNode = stmt.getWiths().getElement(i);
31494                                if (withNode.getNameNode() != null) {
31495                                        String withName = getMdxIdentifierName(withNode.getNameNode());
31496                                        // Calculated members are treated as derived measures
31497                                        if (withName.toLowerCase().startsWith("[measures].")
31498                                                        || withName.toLowerCase().startsWith("measures.")) {
31499                                                measureNames.add(withName);
31500                                        }
31501                                }
31502                                // Also collect references used in the WITH expression
31503                                if (withNode.getExprNode() != null) {
31504                                        collectMdxReferences(withNode.getExprNode(), measureNames, dimensionNames);
31505                                }
31506                        }
31507                }
31508
31509                // Create columns on the cube table for all referenced measures and dimensions
31510                Set<String> addedColumns = new LinkedHashSet<String>();
31511                for (String measure : measureNames) {
31512                        if (addedColumns.add(measure)) {
31513                                modelFactory.createTableColumn(cubeTable, measure);
31514                        }
31515                }
31516                for (String dimension : dimensionNames) {
31517                        if (addedColumns.add(dimension)) {
31518                                modelFactory.createTableColumn(cubeTable, dimension);
31519                        }
31520                }
31521
31522                // Create result set with all referenced columns
31523                ResultSet resultSet = modelFactory.createResultSet(stmt, false);
31524                if (resultSet != null) {
31525                        for (String colName : addedColumns) {
31526                                TableColumn sourceCol = findTableColumnByName(cubeTable, colName);
31527                                if (sourceCol != null) {
31528                                        DataFlowRelationship relation = modelFactory.createDataFlowRelation();
31529                                        relation.setEffectType(EffectType.select);
31530                                        relation.addSource(new TableColumnRelationshipElement(sourceCol));
31531                                        relation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>(
31532                                                        resultSet.getRelationRows()));
31533                                }
31534                        }
31535                }
31536        }
31537
31538        private TableColumn findTableColumnByName(Table table, String name) {
31539                for (TableColumn col : table.getColumns()) {
31540                        if (col.getName().equals(name)) {
31541                                return col;
31542                        }
31543                }
31544                return null;
31545        }
31546
31547        /**
31548         * Extract the display name from an MDX identifier node.
31549         * E.g., [Measures].[Departures NEAT] -> [Measures].[Departures NEAT]
31550         */
31551        private String getMdxIdentifierName(gudusoft.gsqlparser.nodes.mdx.TMdxIdentifierNode idNode) {
31552                StringBuilder sb = new StringBuilder();
31553                for (int i = 0; i < idNode.getSegmentList().size(); i++) {
31554                        if (i > 0) sb.append(".");
31555                        gudusoft.gsqlparser.nodes.mdx.IMdxIdentifierSegment seg = idNode.getSegmentList().getElement(i);
31556                        if (seg.getQuoting() == gudusoft.gsqlparser.nodes.mdx.EMdxQuoting.QUOTED) {
31557                                sb.append("[").append(seg.getName()).append("]");
31558                        } else {
31559                                sb.append(seg.getName());
31560                        }
31561                }
31562                return sb.toString();
31563        }
31564
31565        /**
31566         * Recursively collect measure and dimension references from MDX expression tree.
31567         * Uses iterative DFS to avoid StackOverflow on deeply nested expressions.
31568         */
31569        private void collectMdxReferences(gudusoft.gsqlparser.nodes.mdx.TMdxExpNode expr,
31570                        List<String> measures, List<String> dimensions) {
31571                Deque<gudusoft.gsqlparser.nodes.mdx.TMdxExpNode> stack = new ArrayDeque<gudusoft.gsqlparser.nodes.mdx.TMdxExpNode>();
31572                stack.push(expr);
31573
31574                while (!stack.isEmpty()) {
31575                        gudusoft.gsqlparser.nodes.mdx.TMdxExpNode current = stack.pop();
31576                        if (current == null) continue;
31577
31578                        if (current instanceof gudusoft.gsqlparser.nodes.mdx.TMdxIdentifierNode) {
31579                                gudusoft.gsqlparser.nodes.mdx.TMdxIdentifierNode idNode =
31580                                                (gudusoft.gsqlparser.nodes.mdx.TMdxIdentifierNode) current;
31581                                String name = getMdxIdentifierName(idNode);
31582                                if (name.toLowerCase().startsWith("[measures].")
31583                                                || name.toLowerCase().startsWith("measures.")) {
31584                                        measures.add(name);
31585                                } else if (idNode.getSegmentList().size() > 1) {
31586                                        // Multi-segment identifiers that aren't measures are dimensions
31587                                        dimensions.add(name);
31588                                }
31589                        } else if (current instanceof gudusoft.gsqlparser.nodes.mdx.TMdxSetNode) {
31590                                gudusoft.gsqlparser.nodes.mdx.TMdxSetNode setNode =
31591                                                (gudusoft.gsqlparser.nodes.mdx.TMdxSetNode) current;
31592                                if (setNode.getTupleList() != null) {
31593                                        for (int i = 0; i < setNode.getTupleList().size(); i++) {
31594                                                stack.push(setNode.getTupleList().getElement(i));
31595                                        }
31596                                }
31597                        } else if (current instanceof gudusoft.gsqlparser.nodes.mdx.TMdxTupleNode) {
31598                                gudusoft.gsqlparser.nodes.mdx.TMdxTupleNode tupleNode =
31599                                                (gudusoft.gsqlparser.nodes.mdx.TMdxTupleNode) current;
31600                                if (tupleNode.getExprList() != null) {
31601                                        for (int i = 0; i < tupleNode.getExprList().size(); i++) {
31602                                                stack.push(tupleNode.getExprList().getElement(i));
31603                                        }
31604                                }
31605                        } else if (current instanceof gudusoft.gsqlparser.nodes.mdx.TMdxBinOpNode) {
31606                                gudusoft.gsqlparser.nodes.mdx.TMdxBinOpNode binOp =
31607                                                (gudusoft.gsqlparser.nodes.mdx.TMdxBinOpNode) current;
31608                                if (binOp.getRightExprNode() != null) stack.push(binOp.getRightExprNode());
31609                                if (binOp.getLeftExprNode() != null) stack.push(binOp.getLeftExprNode());
31610                        } else if (current instanceof gudusoft.gsqlparser.nodes.mdx.TMdxFunctionNode) {
31611                                gudusoft.gsqlparser.nodes.mdx.TMdxFunctionNode funcNode =
31612                                                (gudusoft.gsqlparser.nodes.mdx.TMdxFunctionNode) current;
31613                                if (funcNode.getArguments() != null) {
31614                                        for (int i = 0; i < funcNode.getArguments().size(); i++) {
31615                                                stack.push(funcNode.getArguments().getElement(i));
31616                                        }
31617                                }
31618                        }
31619                }
31620        }
31621
31622        /**
31623         * Analyze a Power Query M-language document for data lineage.
31624         *
31625         * Delegates to TPowerQueryAnalyzer to extract navigation chains and
31626         * NativeQuery embedded SQL, then feeds the results back through the
31627         * standard SQL analysis pipeline so the dataflow model contains
31628         * regular table/column lineage.
31629         */
31630        private void analyzePowerQueryDocumentStmt(TPowerQueryDocumentStmt pqStmt) {
31631                TPowerQueryAnalyzer pqAnalyzer = new TPowerQueryAnalyzer(pqStmt);
31632
31633                if (option.getPowerQueryInnerVendor() != null) {
31634                        pqAnalyzer.withExplicitInnerVendor(option.getPowerQueryInnerVendor());
31635                }
31636
31637                PowerQueryLineageResult result = pqAnalyzer.analyze();
31638
31639                if (result.isEmpty()) {
31640                        for (String w : result.getWarnings()) {
31641                                logger.warn("Power Query: " + w);
31642                        }
31643                        return;
31644                }
31645
31646                for (PowerQueryLineageResult.NativeQueryRef nq : result.getNativeQueryReferences()) {
31647                        if (nq.innerParser != null && nq.innerParseReturnCode == 0) {
31648                                for (int i = 0; i < nq.innerParser.sqlstatements.size(); i++) {
31649                                        analyzeCustomSqlStmt(nq.innerParser.sqlstatements.get(i));
31650                                }
31651                        }
31652                }
31653
31654                for (PowerQueryLineageResult.NavigationRef nav : result.getNavigationReferences()) {
31655                        if (nav.syntheticSelect != null && nav.resolvedVendor != null) {
31656                                TGSqlParser synParser = createSqlParser(nav.resolvedVendor);
31657                                synParser.sqltext = nav.syntheticSelect;
31658                                int rc = synParser.parse();
31659                                if (rc == 0) {
31660                                        for (int i = 0; i < synParser.sqlstatements.size(); i++) {
31661                                                analyzeCustomSqlStmt(synParser.sqlstatements.get(i));
31662                                        }
31663                                }
31664                        }
31665                }
31666
31667                for (String w : result.getWarnings()) {
31668                        logger.warn("Power Query: " + w);
31669                }
31670        }
31671
31672
31673        /**
31674         * True for an Oracle pseudocolumn that is scoped to one table, i.e. ROWID.
31675         *
31676         * <p>These report {@link EDbObjectType#notAColumn} so that nothing resolves
31677         * them against a catalog that will never contain them, but they still carry
31678         * a real source table: {@code SELECT a.ROWID AS rn FROM t a} genuinely does
31679         * derive {@code rn} from a row of {@code t}. Letting the notAColumn branch
31680         * claim them would route them to constants and delete that edge
31681         * (Mantis #4675).</p>
31682         */
31683        private boolean isTableScopedPseudoColumn(TObjectName object) {
31684                if (object == null) return false;
31685                if (object.getDbObjectType() != EDbObjectType.notAColumn) return false;
31686                if (object.getSourceTable() == null) return false;
31687                return gudusoft.gsqlparser.util.OraclePseudoColumnUtil
31688                                .isTableScopedPseudoColumn(object.getColumnNameOnly());
31689        }
31690
31691}