001 002package gudusoft.gsqlparser.dlineage; 003 004import gudusoft.gsqlparser.*; 005import gudusoft.gsqlparser.common.structured.StructuredAdapterContext; 006import gudusoft.gsqlparser.common.structured.StructuredDataflowDescriptor; 007import gudusoft.gsqlparser.common.structured.StructuredDataflowRegistry; 008import gudusoft.gsqlparser.common.structured.StructuredFieldBinding; 009import gudusoft.gsqlparser.common.structured.StructuredValueSource; 010import gudusoft.gsqlparser.dlineage.dataflow.listener.DataFlowHandleListener; 011import gudusoft.gsqlparser.dlineage.dataflow.metadata.MetadataReader; 012import gudusoft.gsqlparser.dlineage.dataflow.metadata.grabit.GrabitMetadataAnalyzer; 013import gudusoft.gsqlparser.dlineage.dataflow.metadata.sqldep.SQLDepMetadataAnalyzer; 014import gudusoft.gsqlparser.dlineage.dataflow.metadata.sqlflow.SqlflowMetadataAnalyzer; 015import gudusoft.gsqlparser.dlineage.dataflow.metadata.sqlflow.sharded.SqlflowShardedMetadataAnalyzer; 016import gudusoft.gsqlparser.dlineage.impl.powerquery.PowerQueryLineageResult; 017import gudusoft.gsqlparser.dlineage.impl.powerquery.TPowerQueryAnalyzer; 018import gudusoft.gsqlparser.dlineage.dataflow.model.*; 019import gudusoft.gsqlparser.dlineage.dataflow.model.Process; 020import gudusoft.gsqlparser.dlineage.dataflow.model.JoinRelationship.JoinClauseType; 021import gudusoft.gsqlparser.dlineage.dataflow.model.json.Coordinate; 022import gudusoft.gsqlparser.dlineage.dataflow.model.json.Dataflow; 023import gudusoft.gsqlparser.dlineage.dataflow.model.json.Error; 024import gudusoft.gsqlparser.dlineage.dataflow.model.xml.*; 025import gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlTrustMode; 026import gudusoft.gsqlparser.dlineage.dataflow.sqlenv.SQLEnvParser; 027import gudusoft.gsqlparser.dlineage.metadata.MetadataUtil; 028import gudusoft.gsqlparser.dlineage.metadata.Sqlflow; 029import gudusoft.gsqlparser.dlineage.util.*; 030import gudusoft.gsqlparser.nodes.TTable; 031import gudusoft.gsqlparser.nodes.*; 032import gudusoft.gsqlparser.nodes.couchbase.TObjectConstruct; 033import gudusoft.gsqlparser.nodes.couchbase.TPair; 034import gudusoft.gsqlparser.nodes.functions.TJsonObjectFunction; 035import gudusoft.gsqlparser.nodes.hive.THiveTransformClause; 036import gudusoft.gsqlparser.nodes.hive.THiveTransformClause.ETransformType; 037import gudusoft.gsqlparser.resolver2.ScopeBuildResult; 038import gudusoft.gsqlparser.resolver2.TSQLResolver2; 039import gudusoft.gsqlparser.sqlenv.*; 040import gudusoft.gsqlparser.sqlenv.parser.TJSONSQLEnvParser; 041import gudusoft.gsqlparser.stmt.*; 042import gudusoft.gsqlparser.stmt.db2.TDb2CallStmt; 043import gudusoft.gsqlparser.stmt.db2.TDb2ReturnStmt; 044import gudusoft.gsqlparser.stmt.db2.TDb2SqlVariableDeclaration; 045import gudusoft.gsqlparser.stmt.hive.THiveLoad; 046import gudusoft.gsqlparser.stmt.mssql.*; 047import gudusoft.gsqlparser.stmt.mysql.TLoadDataStmt; 048import gudusoft.gsqlparser.stmt.oracle.*; 049import gudusoft.gsqlparser.stmt.powerquery.TPowerQueryDocumentStmt; 050import gudusoft.gsqlparser.stmt.redshift.TRedshiftCopy; 051import gudusoft.gsqlparser.stmt.redshift.TRedshiftDeclare; 052import gudusoft.gsqlparser.stmt.snowflake.*; 053import gudusoft.gsqlparser.stmt.teradata.TTeradataCreateProcedure; 054import gudusoft.gsqlparser.util.*; 055import gudusoft.gsqlparser.util.json.JSON; 056 057import javax.script.ScriptEngine; 058import javax.script.ScriptEngineManager; 059import javax.script.ScriptException; 060import java.io.*; 061import java.util.zip.GZIPInputStream; 062import java.util.*; 063import java.util.concurrent.atomic.AtomicInteger; 064import java.util.regex.Matcher; 065import java.util.regex.Pattern; 066import java.util.stream.Collectors; 067 068import static gudusoft.gsqlparser.EJoinType.right; 069 070@SuppressWarnings("rawtypes") 071/** 072 * Single-threaded data flow analyzer that performs lineage analysis on SQL scripts. 073 * 074 * <h2>Avoiding OOM with Large Files</h2> 075 * <p> 076 * Analyzing very large SQL files or manifests (e.g. sqlflow-sharded exports with many 077 * shards) in a single JVM can easily exhaust heap memory and cause {@code OutOfMemoryError}. 078 * To avoid OOM, enable automatic large-file detection and delegation: 079 * 080 * <pre>{@code 081 * Option option = new Option(); 082 * option.setAutoDetectLargeFile(true); // enable auto-detection 083 * option.setLargeSqlInfoCountThreshold(1000); // max SqlInfo entries before delegating 084 * option.setLargeSqlTotalSizeThreshold(25 * 1024 * 1024); // max total SQL size (25 MB) before delegating 085 * option.setLargeQueryCountThreshold(1000); // max manifest queries before delegating 086 * option.setLargeShardCountThreshold(10); // max sharded source files before delegating 087 * option.setLargeFileSplitSizeMB(5); // split files larger than 5 MB 088 * option.setEstimatedMemoryPerTaskMB(2560); // estimated memory per parallel task (2.5 GB) 089 * option.setParallel(0); // 0 = auto-calculate from CPU cores 090 * 091 * DataFlowAnalyzer analyzer = new DataFlowAnalyzer(sqlFiles, option); 092 * analyzer.generateDataFlow(); 093 * }</pre> 094 * 095 * <p> 096 * When {@code autoDetectLargeFile} is enabled, {@link #generateDataFlow(boolean)} invokes 097 * {@link gudusoft.gsqlparser.dlineage.util.LargeFileDetector} to check the input scale against 098 * the configured thresholds. If any threshold is exceeded, the analyzer automatically delegates 099 * to {@link gudusoft.gsqlparser.dlineage.ParallelDataFlowAnalyzer}, which: 100 * <ul> 101 * <li>Splits large files into smaller chunks via {@link gudusoft.gsqlparser.util.FileSplitter}</li> 102 * <li>Processes each chunk in a separate {@code DataFlowAnalyzer} instance within a bounded 103 * thread pool (size capped by {@code estimatedMemoryPerTaskMB})</li> 104 * <li>For sharded manifests, distributes individual shard source files across parallel tasks</li> 105 * <li>Merges results from all parallel tasks into a single dataflow output</li> 106 * </ul> 107 * 108 * <p> 109 * If delegation fails for any reason, the analyzer falls back to single-threaded processing 110 * with a warning log, so the analysis still completes (though it may OOM on very large inputs). 111 * 112 * @see gudusoft.gsqlparser.dlineage.util.LargeFileDetector 113 * @see gudusoft.gsqlparser.dlineage.ParallelDataFlowAnalyzer 114 * @see gudusoft.gsqlparser.dlineage.dataflow.model.Option#setAutoDetectLargeFile(boolean) 115 */ 116public class DataFlowAnalyzer implements IDataFlowAnalyzer { 117 118 private static final Logger logger = LoggerFactory.getLogger(DataFlowAnalyzer.class); 119 120 private static final List<String> TERADATA_BUILTIN_FUNCTIONS = Arrays 121 .asList(new String[] { "ACCOUNT", "CURRENT_DATE", "CURRENT_ROLE", "CURRENT_TIME", "CURRENT_TIMESTAMP", 122 "CURRENT_USER", "DATABASE", "DATE", "PROFILE", "ROLE", "SESSION", "TIME", "USER", "SYSDATE", }); 123 124 private static final List<String> CONSTANT_BUILTIN_FUNCTIONS = Arrays.asList(new String[] { "ACCOUNT", 125 "CURRENT_DATE", "CURRENT_ROLE", "CURRENT_TIME", "CURRENT_TIMESTAMP", "CURRENT_USER", "DATABASE", "DATE", 126 "PROFILE", "ROLE", "SESSION", "TIME", "USER", "SYSDATE", "GETDATE" }); 127 128 private Stack<TCustomSqlStatement> stmtStack = new Stack<TCustomSqlStatement>(); 129 private List<ResultSet> appendResultSets = new ArrayList<ResultSet>(); 130 private Set<TCustomSqlStatement> accessedStatements = new HashSet<TCustomSqlStatement>(); 131 private Set<TSelectSqlStatement> accessedSubqueries = new HashSet<TSelectSqlStatement>(); 132 private Map<String, TCustomSqlStatement> viewDDLMap = new HashMap<String, TCustomSqlStatement>(); 133 private Map<String, TCustomSqlStatement> procedureDDLMap = new HashMap<String, TCustomSqlStatement>(); 134 private Map<TTable, TObjectNameList> structObjectMap = new HashMap<TTable, TObjectNameList>(); 135 private final Map<String, String> normalizedColumnNameCache = new HashMap<String, String>(); 136 private final Set<TParseTreeNode> processingFunctions = new HashSet<TParseTreeNode>(); 137 138 // perf (10x, perf/column-lineage-10x): normalized-column-name -> first ResultColumn index over a 139 // resultset's column list. analyzeDataFlowRelation otherwise LINEAR-scans queryTable.getColumns() 140 // once per target column => O(W^2) for wide resultsets (now the dominant dlineage hotspot after the 141 // CTENamespace memo). Keyed by the getColumns() list identity, which is the stable ResultSet.columns 142 // field. Only built for "simple" column lists (no star, no struct) because the linear scan has ordered 143 // side effects on star/struct columns; other lists fall back to the scan. Rebuilt when size changes. 144 // The fast path is used only when it is byte-identical to the scan (verified over the full corpus). 145 private final java.util.IdentityHashMap<List<ResultColumn>, ColumnNameIndex> columnNameIndexCache = 146 new java.util.IdentityHashMap<List<ResultColumn>, ColumnNameIndex>(); 147 148 /** Cached normalized-name -> first ResultColumn map for a simple resultset column list. */ 149 private static final class ColumnNameIndex { 150 final int size; 151 /** null when the column list is NOT simple (has a star or struct column) — use the linear scan. */ 152 final Map<String, ResultColumn> byNormalizedName; 153 ColumnNameIndex(int size, Map<String, ResultColumn> byNormalizedName) { 154 this.size = size; 155 this.byNormalizedName = byNormalizedName; 156 } 157 } 158 159 /** 160 * Return the normalized-name index for {@code columns}, building (and caching) it on first use and 161 * rebuilding when the list size changes. Returns an index whose {@code byNormalizedName} is null when 162 * the list contains any star ("*") or struct column — signalling the caller to use the linear scan. 163 */ 164 private ColumnNameIndex getColumnNameIndex(List<ResultColumn> columns) { 165 ColumnNameIndex idx = columnNameIndexCache.get(columns); 166 if (idx != null && idx.size == columns.size()) { 167 return idx; 168 } 169 Map<String, ResultColumn> map = new HashMap<String, ResultColumn>(); 170 boolean simple = true; 171 for (int i = 0; i < columns.size(); i++) { 172 ResultColumn c = columns.get(i); 173 String nm = c.getName(); 174 if ("*".equals(nm) || c.isStruct()) { 175 simple = false; 176 break; 177 } 178 // putIfAbsent semantics: keep the FIRST column for a given normalized name, matching the 179 // linear scan's first-match-wins + break. 180 String key = DlineageUtil.getIdentifierNormalColumnName(nm); 181 if (!map.containsKey(key)) { 182 map.put(key, c); 183 } 184 } 185 idx = new ColumnNameIndex(columns.size(), simple ? map : null); 186 columnNameIndexCache.put(columns, idx); 187 return idx; 188 } 189 190 // perf: normalized-column-name -> ResultColumn map for a resultset's column list, used by 191 // appendStarRelation to avoid O(S*C) linear scans per star column. Unlike ColumnNameIndex, 192 // this cache does NOT require "simple" column lists — it works for any list including those 193 // with star/struct columns, because appendStarRelation only needs O(1) lookup, not the 194 // ordered side effects that the linear scan in analyzeDataFlowRelation provides. 195 private final java.util.IdentityHashMap<List<ResultColumn>, Map<String, ResultColumn>> resultSetColumnLookupCache = 196 new java.util.IdentityHashMap<List<ResultColumn>, Map<String, ResultColumn>>(); 197 198 private Map<String, ResultColumn> getResultSetColumnLookup(List<ResultColumn> columns) { 199 Map<String, ResultColumn> map = resultSetColumnLookupCache.get(columns); 200 if (map != null) { 201 return map; 202 } 203 map = new HashMap<String, ResultColumn>(columns.size() * 2); 204 for (ResultColumn c : columns) { 205 String key = DlineageUtil.getIdentifierNormalColumnName(c.getName()); 206 if (!map.containsKey(key)) { 207 map.put(key, c); 208 } 209 } 210 resultSetColumnLookupCache.put(columns, map); 211 return map; 212 } 213 214 // perf: normalized-column-name -> TableColumn map for a table's column list, used by 215 // appendStarRelation to avoid O(C) linear scans via searchTableColumn. 216 private final java.util.IdentityHashMap<List<TableColumn>, Map<String, TableColumn>> tableColumnLookupCache = 217 new java.util.IdentityHashMap<List<TableColumn>, Map<String, TableColumn>>(); 218 219 private Map<String, TableColumn> getTableColumnLookup(List<TableColumn> columns) { 220 Map<String, TableColumn> map = tableColumnLookupCache.get(columns); 221 if (map != null) { 222 return map; 223 } 224 map = new HashMap<String, TableColumn>(columns.size() * 2); 225 for (TableColumn c : columns) { 226 String key = DlineageUtil.getIdentifierNormalColumnName(c.getName()); 227 if (!map.containsKey(key)) { 228 map.put(key, c); 229 } 230 } 231 tableColumnLookupCache.put(columns, map); 232 return map; 233 } 234 235 private SqlInfo[] sqlInfos; 236 private File[] originalFiles; 237 private List<ErrorInfo> errorInfos = new ArrayList<ErrorInfo>(); 238 private final List<DynamicSqlSite> dynamicSqlSites = new ArrayList<DynamicSqlSite>(); 239 private final Set<CteSelectRelationKey> cteSelectRelationKeys = new HashSet<CteSelectRelationKey>(); 240 /** 241 * Per-procedure evaluation variants for the default dynamic-SQL path (identity keys). 242 * Index 0 is the no-bindings evaluation; further entries are one per literal call-site 243 * binding set discovered in the same file (capped). 244 */ 245 private final Map<TCustomSqlStatement, List<Map<TCustomSqlStatement, gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite>>> dynamicEvalCache = new IdentityHashMap<TCustomSqlStatement, List<Map<TCustomSqlStatement, gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite>>>(); 246 /** 247 * Multi-file mssql analyses only (plan Phase 5): the raw text of every analysis 248 * unit, so parameter-dependent dynamic SQL in one file can pick up literal EXEC 249 * bindings from another. Null in single-file analyses — the cross-file discovery 250 * path is then never entered and per-file output is byte-identical to before. 251 */ 252 private final Map<String, TStatementList> crossFileParseCache = new HashMap<String, TStatementList>(); 253 /** Shared routine/call-site index over all units (plan §5.6); null in single-file analyses. */ 254 private gudusoft.gsqlparser.dlineage.dynamicsql.RoutineCatalog routineCatalog; 255 /** Oracle files (by unit text) already run through the PL/SQL interpreter this analysis. */ 256 private final Set<String> plsqlEvaluatedTexts = new HashSet<String>(); 257 private IndexedLinkedHashMap<String, List<SqlInfo>> sqlInfoMap = new IndexedLinkedHashMap<String, List<SqlInfo>>(); 258 private TSQLEnv sqlenv = null; 259 private ModelBindingManager modelManager = new ModelBindingManager(); 260 private ModelFactory modelFactory = new ModelFactory(modelManager); 261 private PipelinedFunctionAnalyzer pipelinedAnalyzer; 262 private List<Long> tableIds = new ArrayList<Long>(); 263 private TTableList hiveFromTables; 264 private dataflow dataflow; 265 private String dataflowString; 266 private final AuthoritativeLineageEvidenceCollector authoritativeEvidenceCollector = 267 new AuthoritativeLineageEvidenceCollector(); 268 269 /** 270 * Whether this analyzer is still holding authoritative-evidence DRAFT state. 271 * 272 * <p>Drafts are per-analysis working state; the lifecycle invariant is that 273 * none are retained once a generation completes or the analyzer is disposed. 274 * Always false in normal use — a true here means evidence accumulated across 275 * analyses, which would leak one analysis's findings into the next.</p> 276 * 277 * <p>This deliberately exposes the INVARIANT, not the collector: the 278 * collector class and its fields stay package-private, so they remain free to 279 * change, and nothing internal is frozen into the public API. It is public so 280 * the assertion can be made against the shipped (ProGuard-obfuscated) jar, 281 * where package-private members are renamed and unreachable.</p> 282 * 283 * @return true if any evidence draft state is still held 284 * @since 4.2.6 285 */ 286 public boolean hasRetainedEvidenceDrafts() { 287 return authoritativeEvidenceCollector.hasRetainedDrafts(); 288 } 289 private AuthoritativeLineageEvidence authoritativeLineageEvidence = 290 AuthoritativeLineageEvidence.empty(); 291 private Option option = new Option(); 292 293 { 294 modelManager.TABLE_COLUMN_ID = option.getStartId(); 295 modelManager.RELATION_ID = option.getStartId(); 296 ModelBindingManager.set(modelManager); 297 ModelBindingManager.setGlobalStmtStack(stmtStack); 298 ModelBindingManager.setGlobalOption(option); 299 ModelBindingManager.setGlobalSqlInfo(sqlInfoMap); 300 } 301 302 private void initPipelinedFunctionAnalyzer() { 303 // Only initialize pipelinedAnalyzer for Oracle 304 if (option.getVendor() == EDbVendor.dbvoracle) 305 { 306 pipelinedAnalyzer = new PipelinedFunctionAnalyzer(modelManager, modelFactory, option); 307 } 308 else{ 309 pipelinedAnalyzer = null; 310 } 311 } 312 313 public DataFlowAnalyzer(Sqlflow sqlflow, Option option) { 314 this.option = option; 315 ModelBindingManager.setGlobalOption(this.option); 316 317 SqlInfo info = new SqlInfo(); 318 info.setOriginIndex(0); 319 320 File tempFile = LargeFileDetector.writeSqlflowToTempFile(sqlflow); 321 322 if (option.isAutoDetectLargeFile() && tempFile.length() > option.getLargeSqlThresholdBytes()) { 323 info.setFilePath(tempFile.getAbsolutePath()); 324 info.setFileName(tempFile.getName()); 325 this.sqlInfos = new SqlInfo[] { info }; 326 return; 327 } 328 329 String sql = SQLUtil.getFileContent(tempFile); 330 tempFile.delete(); 331 info.setSql(sql); 332 333 this.sqlInfos = convertSQL(option.getVendor(), JSON.toJSONString(new SqlInfo[] { info })).toArray(new SqlInfo[0]); 334 } 335 336 public DataFlowAnalyzer(String sqlContent, Option option) { 337 SqlInfo[] sqlInfos = new SqlInfo[1]; 338 SqlInfo info = new SqlInfo(); 339 info.setSql(sqlContent); 340 info.setOriginIndex(0); 341 sqlInfos[0] = info; 342 this.option = option; 343 ModelBindingManager.setGlobalOption(this.option); 344 if (option.isLargeSql(sqlContent)) { 345 LargeFileDetector.persistSqlToTempFile(info, sqlContent); 346 this.sqlInfos = sqlInfos; 347 } 348 else { 349 this.sqlInfos = convertSQL(option.getVendor(), JSON.toJSONString(sqlInfos)).toArray(new SqlInfo[0]); 350 } 351 } 352 353 public DataFlowAnalyzer(String sqlContent, EDbVendor dbVendor, boolean simpleOutput, String defaultServer, 354 String defaultDatabase, String defaltSchema) { 355 SqlInfo[] sqlInfos = new SqlInfo[1]; 356 SqlInfo info = new SqlInfo(); 357 info.setSql(sqlContent); 358 info.setOriginIndex(0); 359 sqlInfos[0] = info; 360 option.setVendor(dbVendor); 361 option.setSimpleOutput(simpleOutput); 362 option.setDefaultServer(defaultServer); 363 option.setDefaultDatabase(defaultDatabase); 364 option.setDefaultSchema(defaltSchema); 365 this.sqlInfos = convertSQL(dbVendor, JSON.toJSONString(sqlInfos)).toArray(new SqlInfo[0]); 366 } 367 368 public DataFlowAnalyzer(String sqlContent, EDbVendor dbVendor, boolean simpleOutput) { 369 SqlInfo[] sqlInfos = new SqlInfo[1]; 370 SqlInfo info = new SqlInfo(); 371 info.setSql(sqlContent); 372 info.setOriginIndex(0); 373 sqlInfos[0] = info; 374 option.setVendor(dbVendor); 375 option.setSimpleOutput(simpleOutput); 376 this.sqlInfos = convertSQL(dbVendor, JSON.toJSONString(sqlInfos)).toArray(new SqlInfo[0]); 377 } 378 379 public DataFlowAnalyzer(String[] sqlContents, Option option) { 380 SqlInfo[] sqlInfos = new SqlInfo[sqlContents.length]; 381 for (int i = 0; i < sqlContents.length; i++) { 382 SqlInfo info = new SqlInfo(); 383 info.setSql(sqlContents[i]); 384 info.setOriginIndex(0); 385 sqlInfos[i] = info; 386 } 387 this.option = option; 388 ModelBindingManager.setGlobalOption(this.option); 389 this.sqlInfos = convertSQL(option.getVendor(), JSON.toJSONString(sqlInfos)).toArray(new SqlInfo[0]); 390 } 391 392 public DataFlowAnalyzer(String[] sqlContents, EDbVendor dbVendor, boolean simpleOutput, String defaultServer, 393 String defaultDatabase, String defaltSchema) { 394 SqlInfo[] sqlInfos = new SqlInfo[sqlContents.length]; 395 for (int i = 0; i < sqlContents.length; i++) { 396 SqlInfo info = new SqlInfo(); 397 info.setSql(sqlContents[i]); 398 info.setOriginIndex(0); 399 sqlInfos[i] = info; 400 } 401 option.setVendor(dbVendor); 402 option.setSimpleOutput(simpleOutput); 403 option.setDefaultServer(defaultServer); 404 option.setDefaultDatabase(defaultDatabase); 405 option.setDefaultSchema(defaltSchema); 406 this.sqlInfos = convertSQL(dbVendor, JSON.toJSONString(sqlInfos)).toArray(new SqlInfo[0]); 407 } 408 409 public DataFlowAnalyzer(String[] sqlContents, EDbVendor dbVendor, boolean simpleOutput) { 410 SqlInfo[] sqlInfos = new SqlInfo[sqlContents.length]; 411 for (int i = 0; i < sqlContents.length; i++) { 412 SqlInfo info = new SqlInfo(); 413 info.setSql(sqlContents[i]); 414 info.setOriginIndex(0); 415 sqlInfos[i] = info; 416 } 417 option.setVendor(dbVendor); 418 option.setSimpleOutput(simpleOutput); 419 this.sqlInfos = convertSQL(dbVendor, JSON.toJSONString(sqlInfos)).toArray(new SqlInfo[0]); 420 } 421 422 public DataFlowAnalyzer(SqlInfo[] sqlInfos, Option option) { 423 this.sqlInfos = sqlInfos; 424 this.option = option; 425 ModelBindingManager.setGlobalOption(this.option); 426 } 427 428 public DataFlowAnalyzer(SqlInfo[] sqlInfos, EDbVendor dbVendor, boolean simpleOutput) { 429 this.sqlInfos = sqlInfos; 430 option.setVendor(dbVendor); 431 option.setSimpleOutput(simpleOutput); 432 } 433 434 public DataFlowAnalyzer(File[] sqlFiles, Option option) { 435 this.originalFiles = sqlFiles; 436 SqlInfo[] sqlInfos = new SqlInfo[sqlFiles.length]; 437 for (int i = 0; i < sqlFiles.length; i++) { 438 SqlInfo info = new SqlInfo(); 439 info.setFileName(sqlFiles[i].getName()); 440 info.setFilePath(sqlFiles[i].getAbsolutePath()); 441 info.setOriginIndex(0); 442 sqlInfos[i] = info; 443 } 444 this.sqlInfos = sqlInfos; 445 this.option = option; 446 ModelBindingManager.setGlobalOption(this.option); 447 } 448 449 public DataFlowAnalyzer(File[] sqlFiles, EDbVendor dbVendor, boolean simpleOutput) { 450 this.originalFiles = sqlFiles; 451 SqlInfo[] sqlInfos = new SqlInfo[sqlFiles.length]; 452 for (int i = 0; i < sqlFiles.length; i++) { 453 SqlInfo info = new SqlInfo(); 454 info.setFileName(sqlFiles[i].getName()); 455 info.setFilePath(sqlFiles[i].getAbsolutePath()); 456 info.setOriginIndex(0); 457 sqlInfos[i] = info; 458 } 459 this.sqlInfos = sqlInfos; 460 option.setVendor(dbVendor); 461 option.setSimpleOutput(simpleOutput); 462 } 463 464 public DataFlowAnalyzer(File sqlFile, Option option) { 465 File[] children = SQLUtil.listFiles(sqlFile); 466 this.originalFiles = children; 467 SqlInfo[] sqlInfos = new SqlInfo[children.length]; 468 for (int i = 0; i < children.length; i++) { 469 SqlInfo info = new SqlInfo(); 470 info.setFileName(children[i].getName()); 471 info.setFilePath(children[i].getAbsolutePath()); 472 info.setOriginIndex(0); 473 sqlInfos[i] = info; 474 } 475 this.sqlInfos = sqlInfos; 476 this.option = option; 477 ModelBindingManager.setGlobalOption(this.option); 478 } 479 480 public DataFlowAnalyzer(File sqlFile, EDbVendor dbVendor, boolean simpleOutput) { 481 File[] children = SQLUtil.listFiles(sqlFile); 482 this.originalFiles = children; 483 List<SqlInfo> sqlInfos = new ArrayList<>(); 484 for (int i = 0; i < children.length; i++) { 485 SqlInfo info = new SqlInfo(); 486 info.setSql(SQLUtil.getFileContent(children[i])); 487 if(children[i].getName().toLowerCase().endsWith(".csv")){ 488 if(!MetadataReader.isMetadata(info.getSql())){ 489 continue; 490 } 491 } 492 else if(children[i].getName().toLowerCase().endsWith(".json")){ 493 if (!(MetadataReader.isGrabit(info.getSql()) || MetadataReader.isSqlflow(info.getSql()) || MetadataReader.isSqlflowSharded(info.getSql()))){ 494 continue; 495 } 496 } 497 info.setFileName(children[i].getName()); 498 info.setFilePath(children[i].getAbsolutePath()); 499 info.setOriginIndex(0); 500 sqlInfos.add(info); 501 } 502 this.sqlInfos = sqlInfos.toArray(new SqlInfo[0]); 503 option.setVendor(dbVendor); 504 option.setSimpleOutput(simpleOutput); 505 } 506 507 public boolean isIgnoreRecordSet() { 508 return option.isIgnoreRecordSet(); 509 } 510 511 public void setIgnoreRecordSet(boolean ignoreRecordSet) { 512 option.setIgnoreRecordSet(ignoreRecordSet); 513 } 514 515 public boolean isSimpleShowTopSelectResultSet() { 516 return option.isSimpleShowTopSelectResultSet(); 517 } 518 519 public void setSimpleShowTopSelectResultSet(boolean simpleShowTopSelectResultSet) { 520 option.setSimpleShowTopSelectResultSet(simpleShowTopSelectResultSet); 521 } 522 523 public boolean isSimpleShowFunction() { 524 return option.isSimpleShowFunction(); 525 } 526 527 public void setSimpleShowFunction(boolean simpleShowFunction) { 528 option.setSimpleShowFunction(simpleShowFunction); 529 } 530 531 public boolean isShowJoin() { 532 return option.isShowJoin(); 533 } 534 535 public void setShowJoin(boolean showJoin) { 536 option.setShowJoin(showJoin); 537 } 538 539 public void setShowCallRelation(boolean showCallRelation) { 540 option.setShowCallRelation(showCallRelation); 541 } 542 543 public boolean isShowCallRelation() { 544 return option.isShowCallRelation(); 545 } 546 547 public boolean isShowImplicitSchema() { 548 return option.isShowImplicitSchema(); 549 } 550 551 public void setShowImplicitSchema(boolean showImplicitSchema) { 552 option.setShowImplicitSchema(showImplicitSchema); 553 } 554 555 public boolean isShowConstantTable() { 556 return option.isShowConstantTable(); 557 } 558 559 public void setShowConstantTable(boolean showConstantTable) { 560 option.setShowConstantTable(showConstantTable); 561 } 562 563 public boolean isShowCountTableColumn() { 564 return option.isShowCountTableColumn(); 565 } 566 567 public void setShowCountTableColumn(boolean showCountTableColumn) { 568 option.setShowCountTableColumn(showCountTableColumn); 569 } 570 571 public boolean isTransform() { 572 return option.isTransform(); 573 } 574 575 public void setTransform(boolean transform) { 576 option.setTransform(transform); 577 if (option.isTransformCoordinate()) { 578 option.setTransform(true); 579 } 580 } 581 582 public boolean isTransformCoordinate() { 583 return option.isTransformCoordinate(); 584 } 585 586 public void setTransformCoordinate(boolean transformCoordinate) { 587 option.setTransformCoordinate(transformCoordinate); 588 if (transformCoordinate) { 589 option.setTransform(true); 590 } 591 } 592 593 public boolean isLinkOrphanColumnToFirstTable() { 594 return option.isLinkOrphanColumnToFirstTable(); 595 } 596 597 public void setLinkOrphanColumnToFirstTable(boolean linkOrphanColumnToFirstTable) { 598 option.setLinkOrphanColumnToFirstTable(linkOrphanColumnToFirstTable); 599 } 600 601 public boolean isIgnoreTemporaryTable() { 602 return option.isIgnoreTemporaryTable(); 603 } 604 605 public void setIgnoreTemporaryTable(boolean ignoreTemporaryTable) { 606 option.setIgnoreTemporaryTable(ignoreTemporaryTable); 607 } 608 609 public boolean isIgnoreCoordinate() { 610 return option.isIgnoreCoordinate(); 611 } 612 613 public void setIgnoreCoordinate(boolean ignoreCoordinate) { 614 option.setIgnoreCoordinate(ignoreCoordinate); 615 } 616 617 public void setHandleListener(DataFlowHandleListener listener) { 618 option.setHandleListener(listener); 619 } 620 621 public void setSqlEnv(TSQLEnv sqlenv) { 622 this.sqlenv = sqlenv; 623 } 624 625 public void setOption(Option option) { 626 this.option = option; 627 ModelBindingManager.setGlobalOption(this.option); 628 } 629 630 public Option getOption() { 631 return option; 632 } 633 634 /** 635 * Create a parser carrying the syntax compatibility options owned by this 636 * analyzer. All primary and nested SQL parsing paths must use this method. 637 */ 638 private TGSqlParser createSqlParser(EDbVendor vendor) { 639 TGSqlParser sqlparser = new TGSqlParser(vendor); 640 sqlparser.setEnableMssqlColonBindVariables( 641 option != null && option.isMssqlColonBindVariablesEnabled()); 642 return sqlparser; 643 } 644 645 public synchronized String chechSyntax() { 646 StringBuilder builder = new StringBuilder(); 647 if (sqlInfos != null) { 648 for (SqlInfo sqlInfo : sqlInfos) { 649 String content = sqlInfo.getSql(); 650 if (content != null && content.indexOf("<dlineage") != -1) { 651 try { 652 XML2Model.loadXML(dataflow.class, content); 653 continue; 654 } catch (Exception e) { 655 builder.append("Parsing dataflow ").append("occurs errors.\n").append(e.getMessage()) 656 .append("\n"); 657 } 658 } 659 if (content != null && content.trim().startsWith("{")) { 660 Map queryObject = (Map) JSON.parseObject(content); 661 if (isSourceUnavailable(queryObject)) { 662 continue; // DDL body not retrieved; its text is not the definition 663 } 664 content = (String) queryObject.get("sourceCode"); 665 } 666 if (MetadataReader.isMetadata(content)) { 667 continue; 668 } 669 TGSqlParser sqlparser = createSqlParser(option.getVendor()); 670 sqlparser.sqltext = content; 671 int result = sqlparser.parse(); 672 if (result != 0) { 673 builder.append("Parsing sql ").append("occurs errors.\n").append(sqlparser.getErrormessage()) 674 .append("\n"); 675 } 676 } 677 } 678 return builder.toString(); 679 } 680 681 public synchronized String generateDataFlow(boolean withExtraInfo) { 682 // Gated here as well as in TGSqlParser.parse(), which this reaches 683 // transitively: refusing at the API boundary gives the caller a clear 684 // failure instead of one surfacing from deep inside the analysis. 685 gudusoft.gsqlparser.runtime.GspRuntime.requireActivated(); 686 authoritativeLineageEvidence = AuthoritativeLineageEvidence.empty(); 687 if (option.isAutoDetectLargeFile()) { 688 LargeFileDetector.LargeFileDetectionResult result = LargeFileDetector.detect(sqlInfos, originalFiles, option); 689 if (result.shouldDelegate && delegateToParallel(result)) { 690 return finishGenerateDataFlow(withExtraInfo); 691 } 692 } 693 694 if (ModelBindingManager.get() == null) { 695 ModelBindingManager.set(modelManager); 696 } 697 698 initPipelinedFunctionAnalyzer(); 699 700 dataflow = analyzeSqlScript(); 701 702 return finishGenerateDataFlow(withExtraInfo); 703 } 704 705 private boolean delegateToParallel(LargeFileDetector.LargeFileDetectionResult result) { 706 ParallelDataFlowAnalyzer parallelAnalyzer = null; 707 //autogenerate 委托路径产生的拆分目录、gsp-input 临时文件及拆分产物统一在此清理 708 File splitDir = new File(System.getProperty("java.io.tmpdir"), "gsp-split-" + UUID.randomUUID().toString()); 709 if (!splitDir.exists() && !splitDir.mkdirs()) { 710 logger.error("Failed to create split directory: " + splitDir.getAbsolutePath()); 711 return false; 712 } 713 try { 714 logger.info("Delegating to ParallelDataFlowAnalyzer: " + result.reason); 715 716 parallelAnalyzer = buildParallelAnalyzer(splitDir); 717 parallelAnalyzer.setSqlEnv(sqlenv); 718 719 parallelAnalyzer.generateDataFlow(false, true); 720 721 this.dataflow = parallelAnalyzer.getDataFlow(); 722 this.dataflowString = null; 723 this.errorInfos.clear(); 724 this.errorInfos.addAll(parallelAnalyzer.getErrorMessages()); 725 this.sqlInfoMap = (IndexedLinkedHashMap<String, List<SqlInfo>>) parallelAnalyzer.getSqlInfos(); 726 727 return true; 728 } catch (Exception e) { 729 logger.error("Delegation to ParallelDataFlowAnalyzer failed, falling back to single-threaded", e); 730 return false; 731 } finally { 732 //autogenerate dispose 异常不能阻塞 splitDir 清理,避免临时产物残留 733 if (parallelAnalyzer != null) { 734 try { 735 parallelAnalyzer.dispose(); 736 } catch (Exception ex) { 737 logger.error("Failed to dispose parallel analyzer", ex); 738 } 739 } 740 //autogenerate 清理委托路径产生的拆分目录、gsp-input 临时文件及拆分产物,成功或失败均必须执行 741 SQLUtil.deltree(splitDir); 742 } 743 } 744 745 private ParallelDataFlowAnalyzer buildParallelAnalyzer(File splitDir) { 746 if (originalFiles != null) { 747 return new ParallelDataFlowAnalyzer(originalFiles, option, 748 option.getLargeFileSplitSizeMB(), splitDir); 749 } 750 // SqlInfo[] constructor path: collect files from all entries. Entries 751 // without a filePath (e.g., String/String[] constructors, or large 752 // raw SQL strings) are written to temp files so they can be split. 753 // All paths must go through split, otherwise ParallelDataFlowAnalyzer 754 // would create DataFlowAnalyzer instances internally which would 755 // detect the same large input and delegate again → infinite recursion. 756 if (sqlInfos != null && sqlInfos.length > 0) { 757 List<File> files = new ArrayList<>(); 758 for (SqlInfo info : sqlInfos) { 759 if (info == null) continue; 760 File f = null; 761 if (info.getFilePath() != null) { 762 f = new File(info.getFilePath()); 763 } 764 if (f != null && f.exists()) { 765 if (!files.contains(f)) { 766 files.add(f); 767 } 768 } else if (info.getSql() != null) { 769 try { 770 f = File.createTempFile("gsp-input-", ".tmp", splitDir); 771 try (FileWriter fw = new FileWriter(f)) { 772 fw.write(info.getSql()); 773 } 774 files.add(f); 775 } catch (Exception e) { 776 logger.error("Failed to create temp file for split", e); 777 } 778 } 779 } 780 if (!files.isEmpty()) { 781 return new ParallelDataFlowAnalyzer(files.toArray(new File[0]), option, 782 option.getLargeFileSplitSizeMB(), splitDir); 783 } 784 } 785 return new ParallelDataFlowAnalyzer(sqlInfos, option); 786 } 787 788 private String finishGenerateDataFlow(boolean withExtraInfo) { 789 if (dataflow != null && !withExtraInfo && dataflow.getResultsets() != null) { 790 for (table t : dataflow.getResultsets()) { 791 t.setIsTarget(null); 792 t.setProcedureId(null); 793 if (t.getColumns() != null) { 794 for (column t1 : t.getColumns()) { 795 t1.setIsFunction(null); 796 } 797 } 798 } 799 } 800 801 if (dataflow != null && !withExtraInfo && dataflow.getVariables() != null) { 802 for (table t : dataflow.getVariables()) { 803 t.setIsTarget(null); 804 t.setProcedureId(null); 805 } 806 } 807 808 if(dataflow!=null && dataflow.getRelationships()!=null && option.getFilterRelationTypes()!=null && !option.getFilterRelationTypes().isEmpty()) { 809 List<relationship> relationships = new ArrayList<relationship>(); 810 for(relationship relationship: dataflow.getRelationships()) { 811 if(option.getFilterRelationTypes().contains(relationship.getType())) { 812 relationships.add(relationship); 813 } 814 } 815 dataflow.setRelationships(relationships); 816 } 817 818 if (option.getHandleListener() != null) { 819 option.getHandleListener().endAnalyze(dataflow); 820 } 821 822 if (option.isOutput()) { 823 if (option.getHandleListener() != null) { 824 option.getHandleListener().startOutputDataFlowXML(); 825 } 826 if (dataflow != null) { 827 if (option.isTextFormat()) { 828 dataflowString = getTextOutput(dataflow); 829 } else { 830 try { 831 dataflowString = XML2Model.saveXML(dataflow); 832 }catch (Exception e){ 833 logger.error("Output dataflow to xml failed.", e); 834 dataflowString = null; 835 } 836 } 837 } 838 if (option.getHandleListener() != null) { 839 option.getHandleListener().endOutputDataFlowXML(dataflowString == null ? 0 : dataflowString.length()); 840 } 841 } 842 843 return dataflowString; 844 } 845 846 private dataflow removeDuplicateColumns(dataflow dataflow) { 847 List<table> tables = new ArrayList<table>(); 848 if (dataflow.getTables() != null) { 849 tables.addAll(dataflow.getTables()); 850 } 851 if (dataflow.getViews() != null) { 852 tables.addAll(dataflow.getViews()); 853 } 854 if (dataflow.getStages() != null) { 855 tables.addAll(dataflow.getStages()); 856 } 857 if (dataflow.getDatasources() != null) { 858 tables.addAll(dataflow.getDatasources()); 859 } 860 if (dataflow.getStreams() != null) { 861 tables.addAll(dataflow.getStreams()); 862 } 863 if (dataflow.getPaths() != null) { 864 tables.addAll(dataflow.getPaths()); 865 } 866 if (dataflow.getVariables() != null) { 867 tables.addAll(dataflow.getVariables()); 868 } 869 if (dataflow.getResultsets() != null) { 870 tables.addAll(dataflow.getResultsets()); 871 } 872 for (table table : tables) { 873 if (table.getColumns() == null) { 874 continue; 875 } 876 Set<String> columnIds = new HashSet<String>(); 877 Iterator<column> iter = table.getColumns().iterator(); 878 while(iter.hasNext()) { 879 column column = iter.next(); 880 String id = column.getId(); 881 if (columnIds.contains(id)) { 882 iter.remove(); 883 } else { 884 columnIds.add(id); 885 } 886 } 887 } 888 return dataflow; 889 } 890 891 public synchronized String generateDataFlow() { 892 return generateDataFlow(false); 893 } 894 895 public synchronized String generateSqlInfos() { 896 return JSON.toJSONString(sqlInfoMap); 897 } 898 899 public Map<String, List<SqlInfo>> getSqlInfos() { 900 return sqlInfoMap; 901 } 902 903 public Map getHashSQLMap() { 904 return modelManager.getHashSQLMap(); 905 } 906 907 public Map getDynamicSQLMap() { 908 return modelManager.getDynamicSQLMap(); 909 } 910 911 /** 912 * @deprecated please use SqlInfoHelper.getSelectedDbObjectInfo 913 */ 914 public DbObjectPosition getSelectedDbObjectInfo(Coordinate start, Coordinate end) { 915 if (start == null || end == null) { 916 throw new IllegalArgumentException("Coordinate can't be null."); 917 } 918 919 String hashCode = start.getHashCode(); 920 921 if (hashCode == null) { 922 throw new IllegalArgumentException("Coordinate hashcode can't be null."); 923 } 924 925 int dbObjectStartLine = (int) start.getX() - 1; 926 int dbObjectStarColumn = (int) start.getY() - 1; 927 int dbObjectEndLine = (int) end.getX() - 1; 928 int dbObjectEndColumn = (int) end.getY() - 1; 929 List<SqlInfo> sqlInfoList; 930 if (hashCode.matches("\\d+")) { 931 sqlInfoList = sqlInfoMap.getValueAtIndex(Integer.valueOf(hashCode)); 932 } else { 933 sqlInfoList = sqlInfoMap.get(hashCode); 934 } 935 for (int j = 0; j < sqlInfoList.size(); j++) { 936 SqlInfo sqlInfo = sqlInfoList.get(j); 937 int startLine = sqlInfo.getLineStart(); 938 int endLine = sqlInfo.getLineEnd(); 939 if (dbObjectStartLine >= startLine && dbObjectStartLine <= endLine) { 940 DbObjectPosition position = new DbObjectPosition(); 941 position.setFile(sqlInfo.getFileName()); 942 position.setFilePath(sqlInfo.getFilePath()); 943 position.setSql(sqlInfo.getSql()); 944 position.setIndex(sqlInfo.getOriginIndex()); 945 List<Pair<Integer, Integer>> positions = position.getPositions(); 946 positions.add(new Pair<Integer, Integer>( 947 dbObjectStartLine - startLine + sqlInfo.getOriginLineStart() + 1, dbObjectStarColumn + 1)); 948 positions.add(new Pair<Integer, Integer>(dbObjectEndLine - startLine + sqlInfo.getOriginLineStart() + 1, 949 dbObjectEndColumn + 1)); 950 return position; 951 } 952 } 953 return null; 954 } 955 956 /** 957 * Whether a serialized coordinate ("[line,col,offset],[line,col,offset]...") 958 * points at a real reference site. A position synthesized for a column that 959 * has no source token renders as "[1,1,0],[1,<n>,0]" (start-of-module, 960 * width = name length) or carries -1 markers; a genuine column reference can 961 * never start at line 1, column 1, because the statement head keyword 962 * occupies that position. 963 */ 964 private static boolean hasUsableCoordinate(String coordinate) { 965 return DlineageUtil.hasUsableCoordinate(coordinate); 966 } 967 968 public static dataflow mergeTables(dataflow dataflow, Long startId) { 969 return mergeTables(dataflow, startId, new Option()); 970 } 971 972 public static dataflow mergeTables(dataflow dataflow, Long startId, Option option) { 973 List<table> tableCopy = new ArrayList<table>(); 974 List<table> viewCopy = new ArrayList<table>(); 975 List<table> databaseCopy = new ArrayList<table>(); 976 List<table> schemaCopy = new ArrayList<table>(); 977 List<table> stageCopy = new ArrayList<table>(); 978 List<table> dataSourceCopy = new ArrayList<table>(); 979 List<table> streamCopy = new ArrayList<table>(); 980 List<table> fileCopy = new ArrayList<table>(); 981 List<table> variableCopy = new ArrayList<table>(); 982 List<table> cursorCopy = new ArrayList<table>(); 983 List<table> resultSetCopy = new ArrayList<table>(); 984 if (dataflow.getTables() != null) { 985 tableCopy.addAll(dataflow.getTables()); 986 } 987 dataflow.setTables(tableCopy); 988 if (dataflow.getViews() != null) { 989 viewCopy.addAll(dataflow.getViews()); 990 } 991 dataflow.setViews(viewCopy); 992 if (dataflow.getDatabases() != null) { 993 databaseCopy.addAll(dataflow.getDatabases()); 994 } 995 dataflow.setDatabases(databaseCopy); 996 if (dataflow.getSchemas() != null) { 997 schemaCopy.addAll(dataflow.getSchemas()); 998 } 999 dataflow.setSchemas(schemaCopy); 1000 if (dataflow.getStages() != null) { 1001 stageCopy.addAll(dataflow.getStages()); 1002 } 1003 dataflow.setStages(stageCopy); 1004 if (dataflow.getDatasources() != null) { 1005 dataSourceCopy.addAll(dataflow.getDatasources()); 1006 } 1007 dataflow.setDatasources(dataSourceCopy); 1008 if (dataflow.getStreams() != null) { 1009 streamCopy.addAll(dataflow.getStreams()); 1010 } 1011 dataflow.setStreams(streamCopy); 1012 if (dataflow.getPaths() != null) { 1013 fileCopy.addAll(dataflow.getPaths()); 1014 } 1015 dataflow.setPaths(fileCopy); 1016 if (dataflow.getVariables() != null) { 1017 variableCopy.addAll(dataflow.getVariables()); 1018 } 1019 dataflow.setVariables(variableCopy); 1020 if (dataflow.getResultsets() != null) { 1021 resultSetCopy.addAll(dataflow.getResultsets()); 1022 } 1023 dataflow.setResultsets(resultSetCopy); 1024 1025 Map<String, List<table>> tableMap = new HashMap<String, List<table>>(); 1026 Map<String, String> tableTypeMap = new HashMap<String, String>(); 1027 Map<String, TMssqlCreateType> mssqlTypeMap = new HashMap<String, TMssqlCreateType>(); 1028 Map<String, String> tableIdMap = new HashMap<String, String>(); 1029 1030 Map<String, List<column>> columnMap = new HashMap<String, List<column>>(); 1031 Map<String, Set<String>> tableColumnMap = new HashMap<String, Set<String>>(); 1032 Map<String, String> columnIdMap = new HashMap<String, String>(); 1033 Map<String, column> columnMergeIdMap = new HashMap<String, column>(); 1034 1035 List<table> tables = new ArrayList<table>(); 1036 tables.addAll(dataflow.getTables()); 1037 tables.addAll(dataflow.getViews()); 1038 tables.addAll(dataflow.getDatabases()); 1039 tables.addAll(dataflow.getSchemas()); 1040 tables.addAll(dataflow.getStages()); 1041 tables.addAll(dataflow.getDatasources()); 1042 tables.addAll(dataflow.getStreams()); 1043 tables.addAll(dataflow.getPaths()); 1044 tables.addAll(dataflow.getResultsets()); 1045 tables.addAll(dataflow.getVariables()); 1046 1047 Set<String> columnIds = new HashSet<String>(); 1048 1049 for (table table : tables) { 1050 String qualifiedTableName = DlineageUtil.getQualifiedTableName(table); 1051 String tableFullName = DlineageUtil.getIdentifierNormalTableName(qualifiedTableName); 1052 if ("variable".endsWith(table.getType()) && !SQLUtil.isEmpty(table.getParent())) { 1053 tableFullName = table.getParent() + "." + tableFullName; 1054 } 1055 1056 if (!tableMap.containsKey(tableFullName)) { 1057 tableMap.put(tableFullName, new ArrayList<table>()); 1058 } 1059 1060 tableMap.get(tableFullName).add(table); 1061 1062 if (!tableTypeMap.containsKey(tableFullName)) { 1063 tableTypeMap.put(tableFullName, table.getType()); 1064 } else if ("view".equals(table.getSubType())) { 1065 tableTypeMap.put(tableFullName, table.getType()); 1066 } else if ("database".equals(table.getSubType())) { 1067 tableTypeMap.put(tableFullName, table.getType()); 1068 } else if ("schema".equals(table.getSubType())) { 1069 tableTypeMap.put(tableFullName, table.getType()); 1070 } else if ("stage".equals(table.getSubType())) { 1071 tableTypeMap.put(tableFullName, table.getType()); 1072 } else if ("sequence".equals(table.getSubType())) { 1073 tableTypeMap.put(tableFullName, table.getType()); 1074 } else if ("datasource".equals(table.getSubType())) { 1075 tableTypeMap.put(tableFullName, table.getType()); 1076 } else if ("stream".equals(table.getSubType())) { 1077 tableTypeMap.put(tableFullName, table.getType()); 1078 } else if ("file".equals(table.getSubType())) { 1079 tableTypeMap.put(tableFullName, table.getType()); 1080 } else if ("table".equals(tableTypeMap.get(tableFullName))) { 1081 tableTypeMap.put(tableFullName, table.getType()); 1082 } else if ("variable".equals(tableTypeMap.get(tableFullName))) { 1083 tableTypeMap.put(tableFullName, table.getType()); 1084 } 1085 1086 if (table.getColumns() != null) { 1087 if (!tableColumnMap.containsKey(tableFullName)) { 1088 tableColumnMap.put(tableFullName, new LinkedHashSet<String>()); 1089 } 1090 for (column column : table.getColumns()) { 1091 String columnFullName = tableFullName + "." 1092 + (column.getQualifiedTable() != null 1093 ? (DlineageUtil.getIdentifierNormalTableName(column.getQualifiedTable()) + ".") 1094 : "") 1095 + ("false".equals(table.getIsTarget()) ? DlineageUtil.normalizeColumnName(column.getName()) 1096 : DlineageUtil.getIdentifierNormalColumnName(column.getName())); 1097 1098 if (!columnMap.containsKey(columnFullName)) { 1099 columnMap.put(columnFullName, new ArrayList<column>()); 1100 tableColumnMap.get(tableFullName).add(columnFullName); 1101 } 1102 1103 columnMap.get(columnFullName).add(column); 1104 columnIds.add(column.getId()); 1105 } 1106 } 1107 } 1108 1109 Set<String> relationParentIds = new HashSet<String>(); 1110 if (dataflow.getRelationships() != null) { 1111 for (relationship rel : dataflow.getRelationships()) { 1112 if (rel.getSources() != null) { 1113 for (sourceColumn src : rel.getSources()) { 1114 if (src.getParent_id() != null) { 1115 relationParentIds.add(src.getParent_id()); 1116 } 1117 } 1118 } 1119 if (rel.getTarget() != null) { 1120 if (rel.getTarget().getParent_id() != null) { 1121 relationParentIds.add(rel.getTarget().getParent_id()); 1122 } 1123 } 1124 } 1125 } 1126 1127 Iterator<String> tableNameIter = tableMap.keySet().iterator(); 1128 while (tableNameIter.hasNext()) { 1129 String tableName = tableNameIter.next(); 1130 List<table> tableList = tableMap.get(tableName); 1131 table table; 1132 if (tableList.size() > 1) { 1133 table standardTable = tableList.get(0); 1134 // Function允许重名,不做合并处理 1135 if (standardTable.isFunction()) { 1136 continue; 1137 } 1138 1139 // Variable允许重名,不做合并处理 1140 if (standardTable.isVariable()) { 1141 continue; 1142 } 1143 1144 // 临时表不做合并处理 1145 if(SQLUtil.isTempTable(standardTable)) { 1146 continue; 1147 } 1148 1149 String type = tableTypeMap.get(tableName); 1150 table = new table(); 1151 table.setId(String.valueOf(++startId)); 1152 table.setServer(standardTable.getServer()); 1153 table.setDatabase(standardTable.getDatabase()); 1154 table.setSchema(standardTable.getSchema()); 1155 table.setName(standardTable.getName()); 1156 table.setDisplayName(standardTable.getDisplayName()); 1157 table.setParent(standardTable.getParent()); 1158 table.setMore(standardTable.getMore()); 1159 if (standardTable.getCandidateTables() != null && !standardTable.getCandidateTables().isEmpty()) { 1160 table.setCandidateTables(new ArrayList<String>(standardTable.getCandidateTables())); 1161 } 1162 table.setColumns(new ArrayList<column>()); 1163 String subType = null; 1164 for(table item: tableList){ 1165 if (item.getSubType() != null) { 1166 subType = item.getSubType(); 1167 break; 1168 } 1169 } 1170 if (subType != null) { 1171 table.setSubType(subType); 1172 } else { 1173 table.setSubType(standardTable.getSubType()); 1174 } 1175 Set<String> processIds = new LinkedHashSet<String>(); 1176 for (int k = 0; k < tableList.size(); k++) { 1177 if (tableList.get(k).getProcessIds() != null) { 1178 processIds.addAll(tableList.get(k).getProcessIds()); 1179 } 1180 } 1181 if (!processIds.isEmpty()) { 1182 table.setProcessIds(new ArrayList<String>(processIds)); 1183 } 1184 Set<String> alias = new LinkedHashSet<String>(); 1185 for (int k = 0; k < tableList.size(); k++) { 1186 if (tableList.get(k).getAlias() != null) { 1187 alias.addAll(Arrays.asList(tableList.get(k).getAlias().split("\\s*,\\s*"))); 1188 } 1189 } 1190 if (!alias.isEmpty()) { 1191 String aliasString = Arrays.toString(alias.toArray(new String[0])); 1192 table.setAlias(aliasString.substring(1, aliasString.length() - 1)); 1193 } 1194 table.setType(type); 1195 for (table item : tableList) { 1196 if (!SQLUtil.isEmpty(table.getCoordinate()) && !SQLUtil.isEmpty(item.getCoordinate())) { 1197 if (table.getCoordinate().indexOf(item.getCoordinate()) == -1) { 1198 table.appendCoordinate(item.getCoordinate()); 1199 } 1200 } else if (!SQLUtil.isEmpty(item.getCoordinate())) { 1201 table.setCoordinate(item.getCoordinate()); 1202 } 1203 1204 if (item.getStarStmt() != null) { 1205 table.setStarStmt(item.getStarStmt()); 1206 } 1207 1208 tableIdMap.put(item.getId(), table.getId()); 1209 1210 if (item.isView()) { 1211 dataflow.getViews().remove(item); 1212 } else if (item.isDatabaseType()) { 1213 dataflow.getDatabases().remove(item); 1214 } else if (item.isSchemaType()) { 1215 dataflow.getSchemas().remove(item); 1216 } else if (item.isStage()) { 1217 dataflow.getStages().remove(item); 1218 } else if (item.isDataSource()) { 1219 dataflow.getDatasources().remove(item); 1220 } else if (item.isStream()) { 1221 dataflow.getStreams().remove(item); 1222 } else if (item.isFile()) { 1223 dataflow.getPaths().remove(item); 1224 } else if (item.isVariable()) { 1225 dataflow.getVariables().remove(item); 1226 } else if (item.isTable()) { 1227 dataflow.getTables().remove(item); 1228 } else if (item.isResultSet()) { 1229 dataflow.getResultsets().remove(item); 1230 } 1231 } 1232 1233 if (table.isView()) { 1234 dataflow.getViews().add(table); 1235 } else if (table.isDatabaseType()) { 1236 dataflow.getDatabases().add(table); 1237 } else if (table.isSchemaType()) { 1238 dataflow.getSchemas().add(table); 1239 } else if (table.isStage()) { 1240 dataflow.getStages().add(table); 1241 } else if (table.isDataSource()) { 1242 dataflow.getDatasources().add(table); 1243 } else if (table.isStream()) { 1244 dataflow.getStreams().add(table); 1245 } else if (table.isFile()) { 1246 dataflow.getPaths().add(table); 1247 } else if (table.isVariable()) { 1248 dataflow.getVariables().add(table); 1249 } else if (table.isResultSet()) { 1250 dataflow.getResultsets().add(table); 1251 } else { 1252 dataflow.getTables().add(table); 1253 } 1254 } else { 1255 table = tableList.get(0); 1256 if(Boolean.TRUE.toString().equals(table.getIsDetermined())){ 1257 continue; 1258 } 1259 1260 if (option.isIgnoreUnusedSynonym() && SubType.synonym.name().equals(table.getSubType())) { 1261 boolean hasSourceRelation = relationParentIds.contains(table.getId()); 1262 if (!hasSourceRelation) { 1263 dataflow.getTables().remove(table); 1264 tableColumnMap.get(tableName).clear(); 1265 continue; 1266 } 1267 } 1268 } 1269 1270 Set<String> columns = tableColumnMap.get(tableName); 1271 Iterator<String> columnIter = columns.iterator(); 1272 List<column> mergeColumns = new ArrayList<column>(); 1273 while (columnIter.hasNext()) { 1274 String columnName = columnIter.next(); 1275 List<column> columnList = columnMap.get(columnName); 1276 List<column> functions = new ArrayList<column>(); 1277 for (column t : columnList) { 1278 if (Boolean.TRUE.toString().equals(t.getIsFunction())) { 1279 functions.add(t); 1280 } 1281 } 1282 if (functions != null && !functions.isEmpty()) { 1283 for (column function : functions) { 1284 mergeColumns.add(function); 1285 columnIdMap.put(function.getId(), function.getId()); 1286 columnMergeIdMap.put(function.getId(), function); 1287 } 1288 1289 columnList.removeAll(functions); 1290 } 1291 if (!columnList.isEmpty()) { 1292 column firstColumn = columnList.iterator().next(); 1293 if (columnList.size() > 1) { 1294 column mergeColumn = new column(); 1295 mergeColumn.setId(String.valueOf(++startId)); 1296 mergeColumn.setName(firstColumn.getName()); 1297 mergeColumn.setDisplayName(firstColumn.getDisplayName()); 1298 mergeColumn.setSource(firstColumn.getSource()); 1299 mergeColumn.setQualifiedTable(firstColumn.getQualifiedTable()); 1300 mergeColumn.setDataType(firstColumn.getDataType()); 1301 mergeColumn.setForeignKey(firstColumn.isForeignKey()); 1302 mergeColumn.setPrimaryKey(firstColumn.isPrimaryKey()); 1303 mergeColumn.setUnqiueKey(firstColumn.isUnqiueKey()); 1304 mergeColumn.setIndexKey(firstColumn.isIndexKey()); 1305 mergeColumns.add(mergeColumn); 1306 for (column item : columnList) { 1307 mergeColumn.appendCoordinate(item.getCoordinate()); 1308 columnIdMap.put(item.getId(), mergeColumn.getId()); 1309 } 1310 columnMergeIdMap.put(mergeColumn.getId(), mergeColumn); 1311 columnIds.add(mergeColumn.getId()); 1312 } else { 1313 mergeColumns.add(firstColumn); 1314 columnIdMap.put(firstColumn.getId(), firstColumn.getId()); 1315 columnMergeIdMap.put(firstColumn.getId(), firstColumn); 1316 } 1317 } 1318 } 1319 table.setColumns(mergeColumns); 1320 } 1321 1322 if (dataflow.getRelationships() != null) { 1323 Map<Long, relationship> mergeRelations = new LinkedHashMap<Long, relationship>(); 1324 for (int i = 0; i < dataflow.getRelationships().size(); i++) { 1325 relationship relation = dataflow.getRelationships().get(i); 1326 1327 if("crud".equals(relation.getType()) && option.getAnalyzeMode() == AnalyzeMode.crud) { 1328 long key = relation.toDedupHash(); 1329 if (!mergeRelations.containsKey(key)) { 1330 mergeRelations.put(key, relation); 1331 } 1332 continue; 1333 } 1334 1335 targetColumn target = relation.getTarget(); 1336 if ("call".equals(relation.getType())) { 1337 target = relation.getCaller(); 1338 } 1339 if (target != null && tableIdMap.containsKey(target.getParent_id())) { 1340 target.setParent_id(tableIdMap.get(target.getParent_id())); 1341 } 1342 1343 if (columnIdMap.containsKey(target.getId())) { 1344 target.setId(columnIdMap.get(target.getId())); 1345 target.setCoordinate(columnMergeIdMap.get(target.getId()).getCoordinate()); 1346 } 1347 else if(option.isIgnoreUnusedSynonym() && EffectType.synonym.name().equals(relation.getEffectType())){ 1348 continue; 1349 } 1350 1351 if (!"call".equals(relation.getType()) && !columnIds.contains(target.getId())) { 1352 continue; 1353 } 1354 1355 List<sourceColumn> sources = relation.getSources(); 1356 if ("call".equals(relation.getType())) { 1357 sources = relation.getCallees(); 1358 } 1359 // Sources of merged columns keep their own reference-site 1360 // coordinate (the ON/WHERE operand span) instead of the merged 1361 // column's stamped definition coordinate. Source and relation 1362 // dedup, however, must still collapse EXACTLY as they did under 1363 // the stamped coordinates — sourceColumn.equals and 1364 // relationship.toDedupHash both include the coordinate — so the 1365 // stamped coordinate is applied for the dedup steps and the 1366 // reference-site coordinate is restored on the survivors below. 1367 Map<sourceColumn, String> referenceCoordinates = new IdentityHashMap<sourceColumn, String>(); 1368 Set<sourceColumn> sourceSet = new LinkedHashSet<sourceColumn>(); 1369 if (sources != null) { 1370 for (sourceColumn source : sources) { 1371 // Reference-published source of an UNMERGED column: stamp 1372 // the definition span it carried before reference-site 1373 // publishing, so dedup keeps its legacy behavior. 1374 String dedupCoordinate = source.getDedupCoordinate(); 1375 if (dedupCoordinate != null && !dedupCoordinate.equals(source.getCoordinate())) { 1376 referenceCoordinates.put(source, source.getCoordinate()); 1377 source.setCoordinate(dedupCoordinate); 1378 } 1379 if (!"call".equals(relation.getType()) && !columnIds.contains(source.getId())) { 1380 continue; 1381 } 1382 if (tableIdMap.containsKey(source.getParent_id())) { 1383 source.setParent_id(tableIdMap.get(source.getParent_id())); 1384 } 1385 if (tableIdMap.containsKey(source.getSource_id())) { 1386 source.setSource_id(tableIdMap.get(source.getSource_id())); 1387 } 1388 if (columnIdMap.containsKey(source.getId())) { 1389 String ownCoordinate = referenceCoordinates.containsKey(source) 1390 ? referenceCoordinates.get(source) 1391 : source.getCoordinate(); 1392 source.setId(columnIdMap.get(source.getId())); 1393 source.setCoordinate(columnMergeIdMap.get(source.getId()).getCoordinate()); 1394 if (hasUsableCoordinate(ownCoordinate)) { 1395 // The operand's own coordinate survives the merge; 1396 // only a synthetic/absent one is replaced by the 1397 // merged column's definition coordinate for good. 1398 referenceCoordinates.put(source, ownCoordinate); 1399 } else { 1400 referenceCoordinates.remove(source); 1401 } 1402 } 1403 } 1404 1405 sourceSet.addAll(sources); 1406 if ("call".equals(relation.getType())) { 1407 relation.setCallees(new ArrayList<sourceColumn>(sourceSet)); 1408 } else { 1409 relation.setSources(new ArrayList<sourceColumn>(sourceSet)); 1410 } 1411 } 1412 1413 long key = relation.toDedupHash(); 1414 if (!mergeRelations.containsKey(key)) { 1415 mergeRelations.put(key, relation); 1416 // Restore the reference-site coordinates on the surviving 1417 // relation's sources, now that dedup ran under the stamped 1418 // legacy coordinates. 1419 for (Map.Entry<sourceColumn, String> entry : referenceCoordinates.entrySet()) { 1420 entry.getKey().setCoordinate(entry.getValue()); 1421 } 1422 } 1423 } 1424 1425 dataflow.setRelationships(new ArrayList<relationship>(mergeRelations.values())); 1426 } 1427 1428 tableMap.clear(); 1429 tableTypeMap.clear(); 1430 tableIdMap.clear(); 1431 columnMap.clear(); 1432 tableColumnMap.clear(); 1433 columnIdMap.clear(); 1434 columnMergeIdMap.clear(); 1435 tables.clear(); 1436 return dataflow; 1437 } 1438 1439 public synchronized dataflow getDataFlow() { 1440 if (dataflow != null) { 1441 return dataflow; 1442 } else if (dataflowString != null) { 1443 return XML2Model.loadXML(dataflow.class, dataflowString); 1444 } 1445 return null; 1446 } 1447 1448 /** 1449 * Returns immutable AST/catalog evidence captured during this analyzer run. 1450 * The evidence is correlated to {@link #getDataFlow()} by document-local IDs 1451 * and is intentionally absent from the legacy serialized contract. 1452 */ 1453 @Override 1454 public synchronized AuthoritativeLineageEvidence getAuthoritativeLineageEvidence() { 1455 return authoritativeLineageEvidence; 1456 } 1457 1458 List<ErrorInfo> metadataErrors = new ArrayList<>(); 1459 1460 private synchronized dataflow analyzeSqlScript() { 1461 init(); 1462 1463 try { 1464 dataflow dataflow = new dataflow(); 1465 1466 if (sqlInfos != null) { 1467 if (option.getHandleListener() != null) { 1468 if (sqlInfos.length == 1) { 1469 int sqlLength = sqlInfos[0].getSql() != null ? sqlInfos[0].getSql().length() : 0; 1470 option.getHandleListener().startAnalyze(null, sqlLength, false); 1471 } else { 1472 option.getHandleListener().startAnalyze(null, sqlInfos.length, true); 1473 } 1474 } 1475 1476 if (sqlenv == null) { 1477 if (option.getHandleListener() != null) { 1478 option.getHandleListener().startParseSQLEnv(); 1479 } 1480 TSQLEnv[] sqlenvs = new SQLEnvParser(option.getDefaultServer(), option.getDefaultDatabase(), 1481 option.getDefaultSchema()).parseSQLEnv(option.getVendor(), sqlInfos); 1482 if (sqlenvs != null && sqlenvs.length > 0) { 1483 sqlenv = sqlenvs[0]; 1484 } 1485 if (option.getHandleListener() != null) { 1486 option.getHandleListener().endParseSQLEnv(); 1487 } 1488 } 1489 TGSqlParser sqlparser = createSqlParser(option.getVendor()); 1490 Map<String, Pair3<StringBuilder, AtomicInteger, String>> databaseMap = new LinkedHashMap<String, Pair3<StringBuilder, AtomicInteger, String>>(); 1491 for (int i = 0; i < sqlInfos.length; i++) { 1492 SqlInfo sqlInfo = sqlInfos[i]; 1493 if (sqlInfo == null) { 1494 sqlInfoMap.put(String.valueOf(i), new ArrayList<SqlInfo>()); 1495 continue; 1496 } 1497 String sql = sqlInfo.getSql(); 1498 if (SQLUtil.isEmpty(sql) && sqlInfo.getFileName() != null 1499 && new File(sqlInfo.getFileName()).exists()) { 1500 sql = SQLUtil.getFileContent(sqlInfo.getFileName()); 1501 } 1502 if (SQLUtil.isEmpty(sql) && sqlInfo.getFilePath() != null 1503 && new File(sqlInfo.getFilePath()).exists()) { 1504 sql = SQLUtil.getFileContent(sqlInfo.getFilePath()); 1505 } 1506 String sqlTrim = null; 1507 if (sql != null) { 1508 sqlTrim = sql.substring(0, Math.min(sql.length(), 512)).trim(); 1509 } 1510 if(sql!=null && sqlTrim.startsWith("<") && sqlTrim.indexOf("<dlineage")!=-1){ 1511 dataflow temp = XML2Model.loadXML(dataflow.class, sql); 1512 if(sqlInfos.length == 1){ 1513 dataflow = temp; 1514 } 1515 else { 1516 if (temp.getTables() != null) { 1517 dataflow.getTables().addAll(temp.getTables()); 1518 } 1519 if (temp.getViews() != null) { 1520 dataflow.getViews().addAll(temp.getViews()); 1521 } 1522 if (temp.getResultsets() != null) { 1523 dataflow.getResultsets().addAll(temp.getResultsets()); 1524 } 1525 if (temp.getRelationships() != null) { 1526 dataflow.getRelationships().addAll(temp.getRelationships()); 1527 } 1528 if (temp.getErrors() != null) { 1529 dataflow.getErrors().addAll(temp.getErrors()); 1530 } 1531 } 1532 } 1533 else if (sql != null && sqlTrim.startsWith("{")) { 1534 EDbVendor vendor = SQLUtil.isEmpty(sqlInfo.getDbVendor()) ? option.getVendor() 1535 : EDbVendor.valueOf(sqlInfo.getDbVendor()); 1536 // Pass the manifest's directory so a sharded manifest supplied 1537 // as inline content can still resolve its relative catalog 1538 // paths (otherwise baseDir is null and the catalog is not read). 1539 String envBaseDir = sqlInfo.getFilePath() != null 1540 ? new File(sqlInfo.getFilePath()).getParent() : null; 1541 TSQLEnv[] sqlenvs = new TJSONSQLEnvParser(option.getDefaultServer(), 1542 option.getDefaultDatabase(), option.getDefaultSchema()).parseSQLEnv(vendor, sql, envBaseDir); 1543 if (sqlenvs != null && sqlenvs.length > 0) { 1544 if (sqlenv == null) { 1545 sqlenv = sqlenvs[0]; 1546 } else { 1547 sqlenv = SQLEnvParser.mergeSQLEnv(Arrays.asList(sqlenv, sqlenvs[0])); 1548 } 1549 } 1550 if (sqlenv != null) { 1551 if (MetadataReader.isGrabit(sql) || MetadataReader.isSqlflow(sql) || MetadataReader.isSqlflowSharded(sql)) { 1552 String hash = SHA256.getMd5(sql); 1553 String fileHash = SHA256.getMd5(hash); 1554 if (!sqlInfoMap.containsKey(fileHash)) { 1555 sqlInfoMap.put(fileHash, new ArrayList<SqlInfo>()); 1556 sqlInfoMap.get(fileHash).add(sqlInfo); 1557 } 1558 ModelBindingManager.setGlobalHash(fileHash); 1559 dataflow temp = null; 1560 1561 // Check the authoritative sharded `format` before the 1562 // grabit brand: a grabit-branded sharded manifest must go 1563 // to the sharded reader, not GrabitMetadataAnalyzer. 1564 if (MetadataReader.isSqlflowSharded(sql)) { 1565 String baseDir = null; 1566 if (sqlInfo.getFilePath() != null) { 1567 baseDir = new File(sqlInfo.getFilePath()).getParent(); 1568 } 1569 temp = new SqlflowShardedMetadataAnalyzer(baseDir).analyzeMetadata(option.getVendor(), sql); 1570 } else if (MetadataReader.isGrabit(sql)) { 1571 temp = new GrabitMetadataAnalyzer().analyzeMetadata(option.getVendor(), sql); 1572 } else { 1573 temp = new SqlflowMetadataAnalyzer(sqlenv).analyzeMetadata(option.getVendor(), sql); 1574 } 1575// if (temp.getPackages() != null) { 1576// dataflow.getPackages().addAll(temp.getPackages()); 1577// } 1578// if (temp.getProcedures() != null) { 1579// dataflow.getProcedures().addAll(temp.getProcedures()); 1580// } 1581 if (temp.getTables() != null) { 1582 dataflow.getTables().addAll(temp.getTables()); 1583 } 1584 if (temp.getViews() != null) { 1585 dataflow.getViews().addAll(temp.getViews()); 1586 } 1587 if (temp.getResultsets() != null) { 1588 dataflow.getResultsets().addAll(temp.getResultsets()); 1589 } 1590 if (temp.getRelationships() != null) { 1591 dataflow.getRelationships().addAll(temp.getRelationships()); 1592 } 1593 if (temp.getErrors() != null) { 1594 dataflow.getErrors().addAll(temp.getErrors()); 1595 } 1596 if (sql.indexOf("createdBy") != -1) { 1597 // Route on the authoritative `format` field, not the 1598 // `createdBy` product brand. A sqlflow-sharded manifest 1599 // carries the exporter brand (SQLdep/sqlflow/grabit) in 1600 // createdBy; a plain substring check on the brand hijacked 1601 // SQLdep-branded sharded exports into the single-file 1602 // `queries` path, which finds no `queries` array and read 1603 // nothing. Exclude sharded manifests from the brand branch 1604 // so they fall through to the sharded reader below. 1605 if ((sql.toLowerCase().indexOf("sqldep") != -1 1606 || sql.toLowerCase().indexOf("grabit") != -1) 1607 && !MetadataReader.isSqlflowSharded(sql)) { 1608 Map jsonObject = (Map) JSON.parseObject(sql); 1609 List<Map> queries = (List<Map>) jsonObject.get("queries"); 1610 if (queries != null) { 1611 for (int j = 0; j < queries.size(); j++) { 1612 Map queryObject = queries.get(j); 1613 appendSqlInfo(databaseMap, j, sqlInfo, queryObject); 1614 } 1615 } 1616 } else if (sql.toLowerCase().indexOf("sqlflow") != -1) { 1617 Map sqlflow = (Map) JSON.parseObject(sql); 1618 if ("sqlflow-sharded".equals(sqlflow.get("format")) 1619 && !MetadataReader.isSupportedSqlflowSharded(sql)) { 1620 ErrorInfo errorInfo = new ErrorInfo(); 1621 errorInfo.setErrorType(ErrorInfo.METADATA_ERROR); 1622 errorInfo.setErrorMessage("Unsupported sqlflow-sharded formatVersion " 1623 + MetadataReader.shardedFormatVersion(sql) 1624 + "; this build supports up to " 1625 + MetadataReader.SUPPORTED_SHARDED_FORMAT_VERSION 1626 + ". Source SQL not loaded."); 1627 errorInfo.setFileName(sqlInfo.getFileName()); 1628 errorInfo.setFilePath(sqlInfo.getFilePath()); 1629 errorInfo.setStartPosition(new Pair3<Long, Long, String>(-1L, -1L, 1630 ModelBindingManager.getGlobalHash())); 1631 errorInfo.setEndPosition(new Pair3<Long, Long, String>(-1L, -1L, 1632 ModelBindingManager.getGlobalHash())); 1633 errorInfo.setOriginStartPosition(new Pair<Long, Long>(-1L, -1L)); 1634 errorInfo.setOriginEndPosition(new Pair<Long, Long>(-1L, -1L)); 1635 metadataErrors.add(errorInfo); 1636 } else if ("sqlflow-sharded".equals(sqlflow.get("format"))) { 1637 String baseDir = null; 1638 if (sqlInfo.getFilePath() != null) { 1639 baseDir = new File(sqlInfo.getFilePath()).getParent(); 1640 } 1641 String sourceCompression = (String) sqlflow.get("sourceCompression"); 1642 List<Map> servers = (List<Map>) sqlflow.get("servers"); 1643 if (servers != null) { 1644 for (Map serverObject : servers) { 1645 List<Map> databases = (List<Map>) serverObject.get("databases"); 1646 // Source shards live under databases[] (catalog topology) or 1647 // schemas[] (schema topology, the natural Oracle shape). Both 1648 // carry a per-shard `source`; iterate whichever is present. 1649 // Reading only databases[] silently dropped every source record 1650 // of a schema-topology manifest -> empty lineage. 1651 List<Map> schemaShards = (List<Map>) serverObject.get("schemas"); 1652 if (schemaShards != null && !schemaShards.isEmpty()) { 1653 databases = schemaShards; 1654 } 1655 if (databases != null) { 1656 for (Map database : databases) { 1657 Map source = (Map) database.get("source"); 1658 if (source != null) { 1659 String sourcePath = (String) source.get("path"); 1660 if (sourcePath != null && baseDir != null) { 1661 File sourceFile = new File(baseDir, sourcePath); 1662 String fullPath = sourceFile.getAbsolutePath(); 1663 if ("block".equals(sourceCompression)) { 1664 readGzipBlockSource(fullPath, databaseMap, sqlInfo); 1665 } else { 1666 String sourceContent = SQLUtil.getFileContent(fullPath); 1667 if (sourceContent != null) { 1668 String[] lines = sourceContent.split("\\r?\\n"); 1669 for (int j = 0; j < lines.length; j++) { 1670 String line = lines[j].trim(); 1671 if (line.isEmpty()) { 1672 continue; 1673 } 1674 try { 1675 Map sourceObject = (Map) JSON.parseObject(line); 1676 String sourceCode = (String) sourceObject.get("sourceCode"); 1677 if (sourceCode != null && !sourceCode.isEmpty()) { 1678 SqlInfo sourceSqlInfo = new SqlInfo(); 1679 sourceSqlInfo.setFileName(sourceFile.getName()); 1680 sourceSqlInfo.setFilePath(sourceFile.getAbsolutePath()); 1681 sourceSqlInfo.setSql(sourceCode); 1682 sourceSqlInfo.setOriginIndex(j); 1683 appendSqlInfo(databaseMap, j, sourceSqlInfo, sourceObject); 1684 } 1685 } catch (Exception e) { 1686 logger.warn("Parse source jsonl line failed.", e); 1687 } 1688 } 1689 } 1690 } 1691 } 1692 } 1693 } 1694 } 1695 } 1696 } 1697 } else { 1698 List<Map> servers = (List<Map>) sqlflow.get("servers"); 1699 if (servers != null) { 1700 for (Map serverObject : servers) { 1701 List<Map> queries = (List<Map>) serverObject.get("queries"); 1702 if (queries != null) { 1703 for (int j = 0; j < queries.size(); j++) { 1704 Map queryObject = queries.get(j); 1705 appendSqlInfo(databaseMap, j, sqlInfo, queryObject); 1706 } 1707 } 1708 } 1709 } 1710 } 1711 List<Map> errorMessages = (List<Map>) sqlflow.get("errorMessages"); 1712 if(errorMessages!=null && !errorMessages.isEmpty()) { 1713 for(Map error: errorMessages){ 1714 ErrorInfo errorInfo = new ErrorInfo(); 1715 errorInfo.setErrorType(ErrorInfo.METADATA_ERROR); 1716 errorInfo.setErrorMessage((String)error.get("errorMessage")); 1717 errorInfo.setFileName(sqlInfo.getFileName()); 1718 errorInfo.setFilePath(sqlInfo.getFilePath()); 1719 errorInfo.setStartPosition(new Pair3<Long, Long, String>(-1L, -1L, 1720 ModelBindingManager.getGlobalHash())); 1721 errorInfo.setEndPosition(new Pair3<Long, Long, String>(-1L, -1L, 1722 ModelBindingManager.getGlobalHash())); 1723 errorInfo.setOriginStartPosition(new Pair<Long, Long>(-1L, -1L)); 1724 errorInfo.setOriginEndPosition(new Pair<Long, Long>(-1L, -1L)); 1725 metadataErrors.add(errorInfo); 1726 } 1727 } 1728 } 1729 } 1730 } else { 1731 Map queryObject = (Map) JSON.parseObject(sql); 1732 appendSqlInfo(databaseMap, i, sqlInfo, queryObject); 1733 } 1734 } else { 1735 Map queryObject = (Map) JSON.parseObject(sql); 1736 appendSqlInfo(databaseMap, i, sqlInfo, queryObject); 1737 } 1738 } else { 1739 ModelBindingManager.removeGlobalDatabase(); 1740 ModelBindingManager.removeGlobalSchema(); 1741 ModelBindingManager.removeGlobalHash(); 1742 1743 String content = sql; 1744 1745 if (content == null) { 1746 continue; 1747 } 1748 1749 String delimiterChar = String.valueOf(sqlparser.getDelimiterChar()); 1750 1751 if (sqlInfos.length > 1) { 1752 String endTrim = SQLUtil.endTrim(content); 1753 if (endTrim.endsWith(delimiterChar) || endTrim.endsWith(";")) { 1754 content += "\n"; 1755 } else if (option.getVendor() == EDbVendor.dbvredshift 1756 || option.getVendor() == EDbVendor.dbvgaussdb 1757 || option.getVendor() == EDbVendor.dbvedb 1758 || option.getVendor() == EDbVendor.dbvpostgresql 1759 || option.getVendor() == EDbVendor.dbvmysql 1760 || option.getVendor() == EDbVendor.dbvteradata) { 1761 content += ("\n\n-- " + TBaseType.sqlflow_stmt_delimiter_str + "\n\n"); 1762 } else { 1763 content = endTrim + ";" + "\n"; 1764 } 1765 } 1766 1767 sqlInfo.setSql(content); 1768 1769 if (MetadataReader.isMetadata(content)) { 1770 String hash = SHA256.getMd5(content); 1771 ModelBindingManager.setGlobalHash(hash); 1772 dataflow temp = new SQLDepMetadataAnalyzer().analyzeMetadata(option.getVendor(), content); 1773 if (temp.getProcedures() != null) { 1774 dataflow.getProcedures().addAll(temp.getProcedures()); 1775 } 1776 if (temp.getTables() != null) { 1777 dataflow.getTables().addAll(temp.getTables()); 1778 } 1779 if (temp.getViews() != null) { 1780 dataflow.getViews().addAll(temp.getViews()); 1781 } 1782 if (temp.getResultsets() != null) { 1783 dataflow.getResultsets().addAll(temp.getResultsets()); 1784 } 1785 if (temp.getRelationships() != null) { 1786 dataflow.getRelationships().addAll(temp.getRelationships()); 1787 } 1788 if (temp.getErrors() != null) { 1789 dataflow.getErrors().addAll(temp.getErrors()); 1790 } 1791 String fileHash = SHA256.getMd5(hash); 1792 if (!sqlInfoMap.containsKey(fileHash)) { 1793 sqlInfoMap.put(fileHash, new ArrayList<SqlInfo>()); 1794 sqlInfoMap.get(fileHash).add(sqlInfo); 1795 } 1796 } else { 1797 String sqlHash = SHA256.getMd5(content); 1798 String fileHash = SHA256.getMd5(sqlHash); 1799 if (!sqlInfoMap.containsKey(fileHash)) { 1800 sqlInfoMap.put(fileHash, new ArrayList<SqlInfo>()); 1801 } 1802 1803 String database = TSQLEnv.DEFAULT_DB_NAME; 1804 String schema = TSQLEnv.DEFAULT_SCHEMA_NAME; 1805 if (sqlenv != null) { 1806 // Prefer the per-analysis Option default database over the attached 1807 // env's default catalog, so a shared multi-catalog TSQLEnv can be 1808 // reused across analyses with different default databases without 1809 // the env's default catalog deciding name resolution. 1810 if (!SQLUtil.isEmpty(option.getDefaultDatabase()) 1811 && !TSQLEnv.DEFAULT_DB_NAME.equals(option.getDefaultDatabase())) { 1812 database = option.getDefaultDatabase(); 1813 } else { 1814 database = sqlenv.getDefaultCatalogName(); 1815 } 1816 if (database == null) { 1817 database = TSQLEnv.DEFAULT_DB_NAME; 1818 } 1819 schema = sqlenv.getDefaultSchemaName(); 1820 if (schema == null) { 1821 schema = TSQLEnv.DEFAULT_SCHEMA_NAME; 1822 } 1823 } 1824 1825 boolean supportCatalog = TSQLEnv.supportCatalog(option.getVendor()); 1826 boolean supportSchema = TSQLEnv.supportSchema(option.getVendor()); 1827 String group; 1828 if (sqlInfo.getGroup() != null && !sqlInfo.getGroup().isEmpty()) { 1829 group = sqlInfo.getGroup(); 1830 } else { 1831 StringBuilder builder = new StringBuilder(); 1832 if (supportCatalog) { 1833 builder.append(database); 1834 } 1835 if (supportSchema) { 1836 if (builder.length() > 0) { 1837 builder.append("."); 1838 } 1839 builder.append(schema); 1840 } 1841 group = builder.toString(); 1842 } 1843 SqlInfo sqlInfoItem = new SqlInfo(); 1844 sqlInfoItem.setFileName(sqlInfo.getFileName()); 1845 sqlInfoItem.setFilePath(sqlInfo.getFilePath()); 1846 sqlInfoItem.setSql(sqlInfo.getSql()); 1847 sqlInfoItem.setOriginIndex(0); 1848 sqlInfoItem.setOriginLineStart(0); 1849 int lineEnd = sqlInfo.getSql().split("\n").length - 1; 1850 sqlInfoItem.setOriginLineEnd(lineEnd); 1851 sqlInfoItem.setIndex(0); 1852 sqlInfoItem.setLineStart(0); 1853 sqlInfoItem.setLineEnd(lineEnd); 1854 sqlInfoItem.setHash(SHA256.getMd5(sqlHash)); 1855 sqlInfoItem.setGroup(group); 1856 sqlInfoMap.get(fileHash).add(sqlInfoItem); 1857 if (!databaseMap.containsKey(sqlHash)) { 1858 databaseMap.put(sqlHash, new Pair3<StringBuilder, AtomicInteger, String>( 1859 new StringBuilder(), new AtomicInteger(), group)); 1860 } 1861 databaseMap.get(sqlHash).first.append(sqlInfoItem.getSql()); 1862 databaseMap.get(sqlHash).second.incrementAndGet(); 1863 } 1864 } 1865 } 1866 1867 boolean supportCatalog = TSQLEnv.supportCatalog(option.getVendor()); 1868 boolean supportSchema = TSQLEnv.supportSchema(option.getVendor()); 1869 1870 // Cross-file call-site binding discovery (plan Phase 5) needs every 1871 // unit's text before the first unit is analyzed: units are parsed and 1872 // analyzed strictly one at a time, so a caller in a later file would 1873 // otherwise be invisible while an earlier file's procedure is analyzed. 1874 if ((option.getVendor() == EDbVendor.dbvmssql || option.getVendor() == EDbVendor.dbvoracle) 1875 && databaseMap.size() > 1) { 1876 List<String> unitTexts = new ArrayList<String>(); 1877 for (Pair3<StringBuilder, AtomicInteger, String> unit : databaseMap.values()) { 1878 unitTexts.add(unit.first.toString()); 1879 } 1880 routineCatalog = new gudusoft.gsqlparser.dlineage.dynamicsql.RoutineCatalog( 1881 option.getVendor(), unitTexts, crossFileParseCache); 1882 } 1883 1884 Iterator<String> schemaIter = databaseMap.keySet().iterator(); 1885 while (schemaIter.hasNext()) { 1886 if (option.getHandleListener() != null && option.getHandleListener().isCanceled()) { 1887 break; 1888 } 1889 String key = schemaIter.next(); 1890 String group = databaseMap.get(key).third; 1891 String[] split = SQLUtil.parseNames(group).toArray(new String[0]); 1892 1893 ModelBindingManager.removeGlobalDatabase(); 1894 ModelBindingManager.removeGlobalSchema(); 1895 ModelBindingManager.removeGlobalSQLEnv(); 1896 ModelBindingManager.removeGlobalHash(); 1897 1898 if (sqlenv == null) { 1899 sqlenv = new TSQLEnv(option.getVendor()) { 1900 1901 @Override 1902 public void initSQLEnv() { 1903 // TODO Auto-generated method stub 1904 1905 } 1906 }; 1907 } 1908 1909 String defaultDatabase = null; 1910 String defaultSchema = null; 1911 if(sqlenv!=null){ 1912 defaultDatabase = sqlenv.getDefaultCatalogName(); 1913 defaultSchema = sqlenv.getDefaultSchemaName(); 1914 } 1915 if (supportCatalog && supportSchema) { 1916 if (split.length >= 2) { 1917 if (!TSQLEnv.DEFAULT_DB_NAME.equals(split[split.length - 2])) { 1918 ModelBindingManager.setGlobalDatabase(split[split.length - 2]); 1919 sqlenv.setDefaultCatalogName(ModelBindingManager.getGlobalDatabase()); 1920 } 1921 } 1922 if (split.length >= 1) { 1923 if (!TSQLEnv.DEFAULT_SCHEMA_NAME.equals(split[split.length - 1])) { 1924 ModelBindingManager.setGlobalSchema(split[split.length - 1]); 1925 sqlenv.setDefaultSchemaName(ModelBindingManager.getGlobalSchema()); 1926 } 1927 } 1928 } else if (supportCatalog) { 1929 if (!TSQLEnv.DEFAULT_DB_NAME.equals(split[split.length - 1])) { 1930 ModelBindingManager.setGlobalDatabase(split[split.length - 1]); 1931 sqlenv.setDefaultCatalogName(ModelBindingManager.getGlobalDatabase()); 1932 } 1933 } else if (supportSchema) { 1934 if (!TSQLEnv.DEFAULT_SCHEMA_NAME.equals(split[split.length - 1])) { 1935 ModelBindingManager.setGlobalSchema(split[split.length - 1]); 1936 sqlenv.setDefaultSchemaName(ModelBindingManager.getGlobalSchema()); 1937 } 1938 } 1939 if (option.getHandleListener() != null) { 1940 option.getHandleListener().startParse(null, databaseMap.get(key).first.toString()); 1941 } 1942 1943 ModelBindingManager.setGlobalSQLEnv(sqlenv); 1944 sqlparser.sqltext = databaseMap.get(key).first.toString(); 1945 ModelBindingManager.setGlobalHash(SHA256.getMd5(key)); 1946 analyzeAndOutputResult(sqlparser); 1947 if(sqlenv!=null){ 1948 sqlenv.setDefaultCatalogName(defaultDatabase); 1949 sqlenv.setDefaultSchemaName(defaultSchema); 1950 } 1951 } 1952 1953 materializeSqlEnvSynonyms(); 1954 1955 appendProcesses(dataflow); 1956 appendOraclePackages(dataflow); 1957 appendProcedures(dataflow); 1958 appendTables(dataflow); 1959 appendViews(dataflow); 1960 appendResultSets(dataflow); 1961 appendRelations(dataflow); 1962 appendErrors(dataflow); 1963 } 1964 1965 dataflow = handleDataflowExecProcedure(dataflow); 1966 1967 if (dataflow != null && option.getAnalyzeMode() != AnalyzeMode.crud) { 1968 if (!isShowJoin()) { 1969 dataflow = mergeTables(dataflow, modelManager.TABLE_COLUMN_ID, option); 1970 } else { 1971 dataflow = removeDuplicateColumns(dataflow); 1972 } 1973 } 1974 1975 if (dataflow != null && option.isNormalizeOutput()) { 1976 dataflow = getNormalizeDataflow(dataflow); 1977 } 1978 1979 if (dataflow != null && option.isIgnoreCoordinate()) { 1980 dataflow = filterDataflowCoordinate(dataflow); 1981 } 1982 1983 if (option.isSimpleOutput() || option.isIgnoreRecordSet()) { 1984 List<String> showTypes = new ArrayList<>(); 1985 if(option.getSimpleShowRelationTypes()!=null) { 1986 showTypes.addAll(option.getSimpleShowRelationTypes()); 1987 } 1988 if(showTypes.isEmpty()) { 1989 showTypes.add("fdd"); 1990 } 1991 if(option.isShowCallRelation()) { 1992 showTypes.add("call"); 1993 } 1994 if(option.isShowERDiagram()) { 1995 showTypes.add("er"); 1996 } 1997 dataflow simpleDataflow = getSimpleDataflow(dataflow, option.isSimpleOutput(), showTypes); 1998 if (simpleDataflow.getResultsets() != null) { 1999 for (table t : simpleDataflow.getResultsets()) { 2000 t.setIsTarget(null); 2001 } 2002 } 2003 return finalizeAuthoritativeLineageEvidence(simpleDataflow); 2004 } else { 2005 return finalizeAuthoritativeLineageEvidence(dataflow); 2006 } 2007 2008 } catch (Exception e) { 2009 logger.error("analyze sql failed.", e); 2010 ErrorInfo errorInfo = new ErrorInfo(); 2011 errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR); 2012 if (e.getMessage() == null) { 2013 if (e.getStackTrace() != null && e.getStackTrace().length > 0) { 2014 errorInfo.setErrorMessage(e.getClass().getSimpleName() + ": " + e.getStackTrace()[0].toString()); 2015 } else { 2016 errorInfo.setErrorMessage(e.getClass().getSimpleName()); 2017 } 2018 } else { 2019 errorInfo.setErrorMessage(e.getClass().getSimpleName() + ": " + e.getMessage()); 2020 } 2021 errorInfo.fillInfo(this); 2022 errorInfos.add(errorInfo); 2023 } 2024 2025 modelManager.reset(); 2026 return null; 2027 } 2028 2029 private dataflow finalizeAuthoritativeLineageEvidence(dataflow completed) { 2030 if (!shouldCollectAuthoritativeLineageEvidence()) { 2031 authoritativeEvidenceCollector.reset(); 2032 authoritativeLineageEvidence = AuthoritativeLineageEvidence.empty(); 2033 return completed; 2034 } 2035 try { 2036 authoritativeLineageEvidence = authoritativeEvidenceCollector.build(modelManager, completed); 2037 } catch (ThreadDeath fatal) { 2038 throw fatal; 2039 } catch (VirtualMachineError fatal) { 2040 throw fatal; 2041 } catch (Throwable evidenceFailure) { 2042 // Additive evidence must never make the established legacy analyzer fail. 2043 logger.error("Capture authoritative lineage evidence failed; legacy output is preserved.", 2044 evidenceFailure); 2045 authoritativeLineageEvidence = AuthoritativeLineageEvidence.empty(); 2046 } finally { 2047 // Drafts retain AST/model objects. The published bundle contains only 2048 // immutable scalar values, so release all per-run references immediately. 2049 authoritativeEvidenceCollector.reset(); 2050 } 2051 return completed; 2052 } 2053 2054 private boolean shouldCollectAuthoritativeLineageEvidence() { 2055 return option != null && option.isCollectAuthoritativeLineageEvidence(); 2056 } 2057 2058 private dataflow handleDataflowExecProcedure(dataflow dataflow) { 2059 Set<String> procedures = new HashSet<String>(); 2060 for (procedure procedure : dataflow.getProcedures()) { 2061 procedures.add(DlineageUtil.getIdentifierNormalTableName(procedure.getName())); 2062 } 2063 List<table> removeResultSets = new ArrayList<table>(); 2064 2065 Map<String, table> allMap = new HashMap<String, table>(); 2066 2067 if (dataflow.getResultsets() != null) { 2068 for (table t : dataflow.getResultsets()) { 2069 allMap.put(t.getId().toLowerCase(), t); 2070 } 2071 } 2072 2073 if (dataflow.getTables() != null) { 2074 for (table t : dataflow.getTables()) { 2075 allMap.put(t.getId().toLowerCase(), t); 2076 } 2077 } 2078 2079 if (dataflow.getViews() != null) { 2080 for (table t : dataflow.getViews()) { 2081 allMap.put(t.getId().toLowerCase(), t); 2082 } 2083 } 2084 2085 if (dataflow.getVariables() != null) { 2086 for (table t : dataflow.getVariables()) { 2087 allMap.put(t.getId().toLowerCase(), t); 2088 } 2089 } 2090 2091 2092 for (table resultSet : dataflow.getResultsets()) { 2093 String resultSetName = DlineageUtil.getIdentifierNormalTableName(resultSet.getName()); 2094 if (!(ResultSetType.function.name().equals(resultSet.getType()) 2095 && modelManager.getFunctionTable(resultSetName) != null && procedures.contains(resultSetName))) { 2096 continue; 2097 } 2098 2099 Set<Object> functionTables = modelManager 2100 .getFunctionTable(DlineageUtil.getIdentifierNormalTableName(resultSet.getName())); 2101 List<ResultSet> functionResults = functionTables.stream().filter(t -> t instanceof ResultSet) 2102 .map(t -> (ResultSet) t).collect(Collectors.toList()); 2103 if (!(resultSet.getColumns().size() == 1 2104 && DlineageUtil.getIdentifierNormalTableName(resultSet.getColumns().get(0).getName()).equals(resultSetName)) 2105 || functionResults.isEmpty()) { 2106 continue; 2107 } 2108 2109 column column = resultSet.getColumns().get(0); 2110 String sourceColumnId = column.getId(); 2111 boolean reserveResultSet = false; 2112 List<relationship> appendRelations = new ArrayList<relationship>(); 2113 Set<relationship> removeRelations = new HashSet<relationship>(); 2114 2115 for (relationship relationship : dataflow.getRelationships()) { 2116 targetColumn targetColumn = relationship.getTarget(); 2117 List<sourceColumn> sourceColumns = relationship.getSources(); 2118 List<sourceColumn> newSourceColumns = new ArrayList<sourceColumn>(); 2119 for (sourceColumn sourceColumn : sourceColumns) { 2120 if (!sourceColumn.getId().equals(sourceColumnId)) { 2121 continue; 2122 } 2123 for (ResultSet sourceResultSet : functionResults) { 2124 for (ResultColumn resultColumn : sourceResultSet.getColumns()) { 2125 if (!DlineageUtil.compareColumnIdentifier(resultColumn.getName(), 2126 targetColumn.getColumn())) { 2127 if (targetColumn.getColumn().endsWith("*")) { 2128 table parent = allMap.get(targetColumn.getParent_id()); 2129 if (parent != null && parent.getColumns().size() > 1) { 2130 for (column item : parent.getColumns()) { 2131 if (DlineageUtil.compareColumnIdentifier(resultColumn.getName(), 2132 item.getName())) { 2133 relationship newRelation = appendStarRelation(dataflow, relationship, item, resultColumn); 2134 appendRelations.add(newRelation); 2135 removeRelations.add(relationship); 2136 newSourceColumns.add(newRelation.getSources().get(0)); 2137 if (resultColumn.getResultSet() != null && resultColumn.getResultSet().isTarget()) { 2138 dataflow.getResultsets().stream().filter( 2139 t -> t.getId().equals(String.valueOf(resultColumn.getResultSet().getId()))) 2140 .forEach(t -> t.setIsTarget(Boolean.FALSE.toString())); 2141 } 2142 } 2143 } 2144 } 2145 } 2146 continue; 2147 } 2148 2149 sourceColumn sourceColumn1 = new sourceColumn(); 2150 sourceColumn1.setId(String.valueOf(resultColumn.getId())); 2151 sourceColumn1.setColumn(resultColumn.getName()); 2152 sourceColumn1.setParent_id(String.valueOf(resultColumn.getResultSet().getId())); 2153 sourceColumn1.setParent_name(getResultSetName(resultColumn.getResultSet())); 2154 if (resultColumn.getStartPosition() != null 2155 && resultColumn.getEndPosition() != null) { 2156 sourceColumn1.setCoordinate(convertCoordinate(resultColumn.getStartPosition()) 2157 + "," + convertCoordinate(resultColumn.getEndPosition())); 2158 } 2159 newSourceColumns.add(sourceColumn1); 2160 2161 if (resultColumn.getResultSet() != null && resultColumn.getResultSet().isTarget()) { 2162 dataflow.getResultsets().stream().filter( 2163 t -> t.getId().equals(String.valueOf(resultColumn.getResultSet().getId()))) 2164 .forEach(t -> t.setIsTarget(Boolean.FALSE.toString())); 2165 } 2166 } 2167 } 2168 2169 if (!newSourceColumns.isEmpty()) { 2170 if (removeRelations.isEmpty()) { 2171 relationship.setSources(newSourceColumns); 2172 } 2173 } 2174 else { 2175 reserveResultSet = true; 2176 } 2177 } 2178 } 2179 2180 dataflow.getRelationships().addAll(appendRelations); 2181 dataflow.getRelationships().removeAll(removeRelations); 2182 2183 if(!reserveResultSet) { 2184 removeResultSets.add(resultSet); 2185 } 2186 } 2187 dataflow.getResultsets().removeAll(removeResultSets); 2188 return dataflow; 2189 } 2190 2191 private relationship appendStarRelation(dataflow dataflow, 2192 relationship relationship, column item, ResultColumn resultColumn) { 2193 relationship relationElement = new relationship(); 2194 relationElement.setType(relationship.getType()); 2195 relationElement.setEffectType(relationship.getEffectType()); 2196 relationElement.setSqlHash(relationship.getSqlHash()); 2197 relationElement.setSqlComment(relationship.getSqlComment()); 2198 relationElement.setProcedureId(relationship.getProcedureId()); 2199 relationElement.setId(String.valueOf(++ModelBindingManager.get().RELATION_ID)); 2200 relationElement.setProcessId(relationship.getProcessId()); 2201 relationElement.setProcessType(relationship.getProcessType()); 2202 2203 targetColumn targetColumn1 = new targetColumn(); 2204 targetColumn1.setId(String.valueOf(item.getId())); 2205 targetColumn1.setColumn(item.getName()); 2206 targetColumn1.setParent_id(relationship.getTarget().getParent_id()); 2207 targetColumn1.setParent_name(relationship.getTarget().getParent_name()); 2208 targetColumn1.setParent_alias(relationship.getTarget().getParent_alias()); 2209 if (relationship.getTarget().getCoordinate() != null) { 2210 targetColumn1.setCoordinate(item.getCoordinate()); 2211 } 2212 relationElement.setTarget(targetColumn1); 2213 2214 sourceColumn sourceColumn1 = new sourceColumn(); 2215 sourceColumn1.setId(String.valueOf(resultColumn.getId())); 2216 sourceColumn1.setColumn(resultColumn.getName()); 2217 sourceColumn1.setParent_id(String.valueOf(resultColumn.getResultSet().getId())); 2218 sourceColumn1.setParent_name(getResultSetName(resultColumn.getResultSet())); 2219 if (resultColumn.getStartPosition() != null 2220 && resultColumn.getEndPosition() != null) { 2221 sourceColumn1.setCoordinate(convertCoordinate(resultColumn.getStartPosition()) 2222 + "," + convertCoordinate(resultColumn.getEndPosition())); 2223 } 2224 relationElement.addSource(sourceColumn1); 2225 return relationElement; 2226 } 2227 2228 private dataflow getNormalizeDataflow(dataflow instance) { 2229 List<table> tables = new ArrayList<>(); 2230 if (instance.getResultsets() != null) { 2231 for (table t : instance.getResultsets()) { 2232 tables.add(t); 2233 } 2234 } 2235 2236 if (instance.getTables() != null) { 2237 for (table t : instance.getTables()) { 2238 tables.add(t); 2239 } 2240 } 2241 2242 if (instance.getViews() != null) { 2243 for (table t : instance.getViews()) { 2244 tables.add(t); 2245 } 2246 } 2247 2248 if (instance.getPaths() != null) { 2249 for (table t : instance.getPaths()) { 2250 tables.add(t); 2251 } 2252 } 2253 2254 if (instance.getStages() != null) { 2255 for (table t : instance.getStages()) { 2256 tables.add(t); 2257 } 2258 } 2259 2260 if (instance.getSequences() != null) { 2261 for (table t : instance.getSequences()) { 2262 tables.add(t); 2263 } 2264 } 2265 2266 if (instance.getDatasources() != null) { 2267 for (table t : instance.getDatasources()) { 2268 tables.add(t); 2269 } 2270 } 2271 2272 if (instance.getDatabases() != null) { 2273 for (table t : instance.getDatabases()) { 2274 tables.add(t); 2275 } 2276 } 2277 2278 if (instance.getSchemas() != null) { 2279 for (table t : instance.getSchemas()) { 2280 tables.add(t); 2281 } 2282 } 2283 2284 if (instance.getStreams() != null) { 2285 for (table t : instance.getStreams()) { 2286 tables.add(t); 2287 } 2288 } 2289 2290 if (instance.getVariables() != null) { 2291 for (table t : instance.getVariables()) { 2292 tables.add(t); 2293 } 2294 } 2295 2296 Map<String, String> idNameMap = new HashMap(); 2297 2298 for(table table: tables){ 2299 if(table.getDatabase()!=null) { 2300 table.setDatabase(DlineageUtil.getIdentifierNormalName(table.getDatabase(), ESQLDataObjectType.dotCatalog)); 2301 } 2302 if(table.getSchema()!=null) { 2303 table.setSchema(DlineageUtil.getIdentifierNormalName(table.getSchema(), ESQLDataObjectType.dotSchema)); 2304 } 2305 if(table.getName()!=null) { 2306 table.setName(DlineageUtil.getIdentifierNormalName(table.getName(), ESQLDataObjectType.dotTable)); 2307 idNameMap.put(table.getId(), table.getName()); 2308 } 2309 if(table.getColumns()!=null){ 2310 for(column column: table.getColumns()){ 2311 column.setName(DlineageUtil.getIdentifierNormalName(column.getName(), ESQLDataObjectType.dotColumn)); 2312 idNameMap.put(column.getId(), column.getName()); 2313 } 2314 } 2315 } 2316 if(instance.getPackages()!=null){ 2317 for(oraclePackage oraclePackage: instance.getPackages()){ 2318 if(oraclePackage.getDatabase()!=null) { 2319 oraclePackage.setDatabase(DlineageUtil.getIdentifierNormalName(oraclePackage.getDatabase(), ESQLDataObjectType.dotCatalog)); 2320 } 2321 if(oraclePackage.getSchema()!=null) { 2322 oraclePackage.setSchema(DlineageUtil.getIdentifierNormalName(oraclePackage.getSchema(), ESQLDataObjectType.dotSchema)); 2323 } 2324 if(oraclePackage.getName()!=null) { 2325 oraclePackage.setName(DlineageUtil.getIdentifierNormalName(oraclePackage.getName(), ESQLDataObjectType.dotTable)); 2326 idNameMap.put(oraclePackage.getId(), oraclePackage.getName()); 2327 } 2328 if(oraclePackage.getArguments()!=null){ 2329 for(argument argument: oraclePackage.getArguments()){ 2330 argument.setName(DlineageUtil.getIdentifierNormalName(argument.getName(), ESQLDataObjectType.dotColumn)); 2331 idNameMap.put(argument.getId(), argument.getName()); 2332 } 2333 } 2334 if(oraclePackage.getProcedures()!=null){ 2335 for(procedure procedure: oraclePackage.getProcedures()){ 2336 if(procedure.getDatabase()!=null) { 2337 procedure.setDatabase(DlineageUtil.getIdentifierNormalName(procedure.getDatabase(), ESQLDataObjectType.dotCatalog)); 2338 } 2339 if(procedure.getSchema()!=null) { 2340 procedure.setSchema(DlineageUtil.getIdentifierNormalName(procedure.getSchema(), ESQLDataObjectType.dotSchema)); 2341 } 2342 if(procedure.getName()!=null) { 2343 procedure.setName(DlineageUtil.getIdentifierNormalName(procedure.getName(), ESQLDataObjectType.dotTable)); 2344 idNameMap.put(procedure.getId(), procedure.getName()); 2345 } 2346 for(argument argument: procedure.getArguments()){ 2347 argument.setName(DlineageUtil.getIdentifierNormalName(argument.getName(), ESQLDataObjectType.dotColumn)); 2348 idNameMap.put(argument.getId(), argument.getName()); 2349 } 2350 } 2351 } 2352 } 2353 } 2354 if(instance.getProcedures()!=null){ 2355 for(procedure procedure: instance.getProcedures()){ 2356 if(procedure.getDatabase()!=null) { 2357 procedure.setDatabase(DlineageUtil.getIdentifierNormalName(procedure.getDatabase(), ESQLDataObjectType.dotCatalog)); 2358 } 2359 if(procedure.getSchema()!=null) { 2360 procedure.setSchema(DlineageUtil.getIdentifierNormalName(procedure.getSchema(), ESQLDataObjectType.dotSchema)); 2361 } 2362 if(procedure.getName()!=null) { 2363 procedure.setName(DlineageUtil.getIdentifierNormalName(procedure.getName(), ESQLDataObjectType.dotTable)); 2364 idNameMap.put(procedure.getId(), procedure.getName()); 2365 } 2366 for(argument argument: procedure.getArguments()){ 2367 argument.setName(DlineageUtil.getIdentifierNormalName(argument.getName(), ESQLDataObjectType.dotColumn)); 2368 idNameMap.put(argument.getId(), argument.getName()); 2369 } 2370 } 2371 } 2372 if (instance.getProcesses() != null) { 2373 for (process process : instance.getProcesses()) { 2374 if(process.getDatabase()!=null) { 2375 process.setDatabase(DlineageUtil.getIdentifierNormalName(process.getDatabase(), ESQLDataObjectType.dotCatalog)); 2376 } 2377 if(process.getSchema()!=null) { 2378 process.setSchema(DlineageUtil.getIdentifierNormalName(process.getSchema(), ESQLDataObjectType.dotSchema)); 2379 } 2380 if (process.getProcedureId() != null) { 2381 process.setProcedureName(DlineageUtil.getIdentifierNormalName(process.getProcedureName(), ESQLDataObjectType.dotTable)); 2382 idNameMap.put(process.getId(), process.getName()); 2383 } 2384 } 2385 } 2386 2387 if(instance.getRelationships()!=null){ 2388 for(relationship relation: instance.getRelationships()){ 2389 targetColumn targetColumn = relation.getTarget(); 2390 if(targetColumn != null) { 2391 targetColumn.setColumn(idNameMap.get(targetColumn.getId())); 2392 targetColumn.setTarget_name(idNameMap.get(targetColumn.getTarget_id())); 2393 targetColumn.setParent_name(idNameMap.get(targetColumn.getParent_id())); 2394 } 2395 List<sourceColumn> sourceColumns = relation.getSources(); 2396 if(sourceColumns!=null){ 2397 for(sourceColumn sourceColumn: sourceColumns){ 2398 sourceColumn.setColumn(idNameMap.get(sourceColumn.getId())); 2399 sourceColumn.setSource_name(idNameMap.get(sourceColumn.getSource_id())); 2400 sourceColumn.setParent_name(idNameMap.get(sourceColumn.getParent_id())); 2401 } 2402 } 2403 targetColumn = relation.getCaller(); 2404 if(targetColumn != null) { 2405 targetColumn.setName(idNameMap.get(targetColumn.getId())); 2406 } 2407 sourceColumns = relation.getCallees(); 2408 if(sourceColumns!=null){ 2409 for(sourceColumn sourceColumn: sourceColumns){ 2410 sourceColumn.setName(idNameMap.get(sourceColumn.getId())); 2411 } 2412 } 2413 } 2414 } 2415 2416 2417 return instance; 2418 } 2419 2420 private dataflow filterDataflowCoordinate(dataflow instance) { 2421 List<table> tables = new ArrayList<>(); 2422 if (instance.getResultsets() != null) { 2423 for (table t : instance.getResultsets()) { 2424 tables.add(t); 2425 } 2426 } 2427 2428 if (instance.getTables() != null) { 2429 for (table t : instance.getTables()) { 2430 tables.add(t); 2431 } 2432 } 2433 2434 if (instance.getViews() != null) { 2435 for (table t : instance.getViews()) { 2436 tables.add(t); 2437 } 2438 } 2439 2440 if (instance.getPaths() != null) { 2441 for (table t : instance.getPaths()) { 2442 tables.add(t); 2443 } 2444 } 2445 2446 if (instance.getStages() != null) { 2447 for (table t : instance.getStages()) { 2448 tables.add(t); 2449 } 2450 } 2451 2452 if (instance.getSequences() != null) { 2453 for (table t : instance.getSequences()) { 2454 tables.add(t); 2455 } 2456 } 2457 2458 if (instance.getDatasources() != null) { 2459 for (table t : instance.getDatasources()) { 2460 tables.add(t); 2461 } 2462 } 2463 2464 if (instance.getDatabases() != null) { 2465 for (table t : instance.getDatabases()) { 2466 tables.add(t); 2467 } 2468 } 2469 2470 if (instance.getSchemas() != null) { 2471 for (table t : instance.getSchemas()) { 2472 tables.add(t); 2473 } 2474 } 2475 2476 if (instance.getStreams() != null) { 2477 for (table t : instance.getStreams()) { 2478 tables.add(t); 2479 } 2480 } 2481 2482 if (instance.getVariables() != null) { 2483 for (table t : instance.getVariables()) { 2484 tables.add(t); 2485 } 2486 } 2487 2488 for(table table: tables){ 2489 table.clearCoordinate(); 2490 if(table.getColumns()!=null){ 2491 for(column column: table.getColumns()){ 2492 column.clearCoordinate(); 2493 } 2494 } 2495 } 2496 if(instance.getPackages()!=null){ 2497 for(oraclePackage oraclePackage: instance.getPackages()){ 2498 oraclePackage.setCoordinate(null); 2499 if(oraclePackage.getProcedures()!=null){ 2500 for(procedure procedure: oraclePackage.getProcedures()){ 2501 procedure.setCoordinate(null); 2502 for(argument argument: procedure.getArguments()){ 2503 argument.setCoordinate(null); 2504 } 2505 } 2506 } 2507 } 2508 } 2509 if(instance.getProcedures()!=null){ 2510 for(procedure procedure: instance.getProcedures()){ 2511 procedure.setCoordinate(null); 2512 for(argument argument: procedure.getArguments()){ 2513 argument.setCoordinate(null); 2514 } 2515 } 2516 } 2517 if (instance.getProcesses() != null) { 2518 for (process process : instance.getProcesses()) { 2519 process.setCoordinate(null); 2520 } 2521 } 2522 2523 if(instance.getRelationships()!=null){ 2524 for(relationship relation: instance.getRelationships()){ 2525 targetColumn targetColumn = relation.getTarget(); 2526 if(targetColumn != null) { 2527 targetColumn.setCoordinate(null); 2528 } 2529 List<sourceColumn> sourceColumns = relation.getSources(); 2530 if(sourceColumns!=null){ 2531 for(sourceColumn sourceColumn: sourceColumns){ 2532 sourceColumn.setCoordinate(null); 2533 } 2534 } 2535 targetColumn = relation.getCaller(); 2536 if(targetColumn != null) { 2537 targetColumn.setCoordinate(null); 2538 } 2539 sourceColumns = relation.getCallees(); 2540 if(sourceColumns!=null){ 2541 for(sourceColumn sourceColumn: sourceColumns){ 2542 sourceColumn.setCoordinate(null); 2543 } 2544 } 2545 } 2546 } 2547 2548 return instance; 2549 } 2550 2551 /** 2552 * Read a sqlflow-sharded manifest's source records into {@link SqlInfo}s (for the 2553 * convertSQL / text-constructor path). Relative source paths resolve against 2554 * {@code baseDir}; without a base directory the shard files cannot be located, so 2555 * nothing is added (a manifest passed as inline text with no file location simply 2556 * cannot be followed to its shards). Unavailable records are skipped. Block-compressed 2557 * sources are handled on the generateDataFlow path, not here. 2558 */ 2559 private void appendShardedSourceSqlInfos(String manifestSql, String baseDir, String fileName, 2560 List<SqlInfo> sqlInfos) { 2561 if (baseDir == null) { 2562 return; 2563 } 2564 Map sqlflow = (Map) JSON.parseObject(manifestSql); 2565 boolean block = "block".equals(sqlflow.get("sourceCompression")); 2566 List<Map> servers = (List<Map>) sqlflow.get("servers"); 2567 if (servers == null) { 2568 return; 2569 } 2570 for (Map server : servers) { 2571 String dbVendor = (String) server.get("dbVendor"); 2572 String serverName = (String) server.get("name"); 2573 // A sharded export is organized either as catalog shards 2574 // (servers[].databases[]) or as schema shards 2575 // (servers[].schemas[], the natural shape for schema-topology 2576 // vendors like Oracle). Both carry a per-shard `source` pointer; 2577 // prefer schema shards when present, treating an empty list as 2578 // absent so a manifest with "databases":[] alongside schemas[] 2579 // still loads. Handling only `databases` silently dropped every 2580 // source record of a schema-topology manifest -> empty lineage. 2581 List<Map> shards = (List<Map>) server.get("schemas"); 2582 if (shards == null || shards.isEmpty()) { 2583 shards = (List<Map>) server.get("databases"); 2584 } 2585 if (shards == null || shards.isEmpty()) { 2586 continue; 2587 } 2588 for (Map shard : shards) { 2589 appendShardedSourceRecords((Map) shard.get("source"), baseDir, block, 2590 dbVendor, serverName, sqlInfos); 2591 } 2592 } 2593 } 2594 2595 /** 2596 * Read one shard's source jsonl (catalog- or schema-topology) into SqlInfos. 2597 * Shared by the {@code databases[]} and {@code schemas[]} shard shapes so a 2598 * schema-topology manifest loads its source SQL identically to a 2599 * catalog-topology one. 2600 */ 2601 private void appendShardedSourceRecords(Map source, String baseDir, boolean block, 2602 String dbVendor, String serverName, List<SqlInfo> sqlInfos) { 2603 if (source == null) { 2604 return; 2605 } 2606 String sourcePath = (String) source.get("path"); 2607 if (sourcePath == null) { 2608 return; 2609 } 2610 File sourceFile = new File(baseDir, sourcePath); 2611 String content = block ? readGzipContent(sourceFile.getAbsolutePath()) 2612 : SQLUtil.getFileContent(sourceFile.getAbsolutePath()); 2613 if (content == null) { 2614 return; 2615 } 2616 String[] lines = content.split("\\r?\\n"); 2617 EDbVendor vendor = EDbVendor.valueOf(dbVendor); 2618 boolean supportCatalog = TSQLEnv.supportCatalog(vendor); 2619 boolean supportSchema = TSQLEnv.supportSchema(vendor); 2620 2621 for (int j = 0; j < lines.length; j++) { 2622 String line = lines[j].trim(); 2623 if (line.isEmpty()) { 2624 continue; 2625 } 2626 try { 2627 Map rec = (Map) JSON.parseObject(line); 2628 String sc = (String) rec.get("sourceCode"); 2629 if (sc != null && !sc.isEmpty() && !isSourceUnavailable(rec)) { 2630 SqlInfo info = new SqlInfo(); 2631 info.setSql(sc); 2632 info.setFileName(sourceFile.getName()); 2633 info.setFilePath(sourceFile.getAbsolutePath()); 2634 info.setOriginIndex(j); 2635 info.setDbVendor(dbVendor); 2636 info.setServer(serverName); 2637 2638 String database = (String) rec.get("database"); 2639 String schema = (String) rec.get("schema"); 2640 StringBuilder groupBuilder = new StringBuilder(); 2641 if (supportCatalog) { 2642 if (database.indexOf(".") != -1) { 2643 database = SQLUtil.quoteDottedName(vendor, ESQLDataObjectType.dotCatalog, database); 2644 } 2645 groupBuilder.append(database); 2646 } 2647 if (supportSchema) { 2648 if (schema.indexOf(".") != -1) { 2649 schema = SQLUtil.quoteDottedName(vendor, ESQLDataObjectType.dotSchema, schema); 2650 } 2651 if (groupBuilder.length() > 0) { 2652 groupBuilder.append("."); 2653 } 2654 groupBuilder.append(schema); 2655 } 2656 info.setGroup(groupBuilder.toString()); 2657 2658 sqlInfos.add(info); 2659 } 2660 } catch (Exception e) { 2661 logger.warn("Parse sharded source jsonl line failed.", e); 2662 } 2663 } 2664 } 2665 2666 /** Decompress a gzip/block-compressed source shard to its text, or null on failure. */ 2667 private static String readGzipContent(String path) { 2668 File f = new File(path); 2669 if (!f.exists()) { 2670 return null; 2671 } 2672 StringBuilder sb = new StringBuilder(); 2673 try (java.io.BufferedReader reader = new java.io.BufferedReader(new java.io.InputStreamReader( 2674 new java.util.zip.GZIPInputStream(new java.io.FileInputStream(f)), 2675 java.nio.charset.StandardCharsets.UTF_8))) { 2676 String line; 2677 while ((line = reader.readLine()) != null) { 2678 sb.append(line).append('\n'); 2679 } 2680 } catch (IOException e) { 2681 logger.warn("Failed to read gzip source shard: " + path, e); 2682 return null; 2683 } 2684 return sb.toString(); 2685 } 2686 2687 /** 2688 * True when a sqlflow source record declares its DDL body unavailable 2689 * ({@code sourceUnavailable:true}). Robust to the JSON value arriving as a 2690 * boolean or a string. A record with no such field is available. 2691 */ 2692 static boolean isSourceUnavailable(Map record) { 2693 if (record == null) { 2694 return false; 2695 } 2696 Object v = record.get("sourceUnavailable"); 2697 if (v instanceof Boolean) { 2698 return (Boolean) v; 2699 } 2700 return v != null && "true".equalsIgnoreCase(v.toString().trim()); // non-identifier-compare: JSON boolean literal, not a DB name 2701 } 2702 2703 private void appendSqlInfo(Map<String, Pair3<StringBuilder, AtomicInteger, String>> databaseMap, int index, 2704 SqlInfo sqlInfo, Map queryObject) { 2705 if (queryObject == null) { 2706 return; 2707 } 2708 EDbVendor vendor = option.getVendor(); 2709 if (!SQLUtil.isEmpty(sqlInfo.getDbVendor())) { 2710 vendor = EDbVendor.valueOf(sqlInfo.getDbVendor()); 2711 } 2712 2713 boolean supportCatalog = TSQLEnv.supportCatalog(vendor); 2714 boolean supportSchema = TSQLEnv.supportSchema(vendor); 2715 2716 String groupName = (String) queryObject.get("groupName"); 2717 if (DlineageUtil.isProcedureExcluded(groupName)) { 2718 return; 2719 } 2720 2721 // sourceUnavailable: the DDL body was not retrieved (encrypted proc, 2722 // permission denied). Any sourceCode present is a placeholder or a 2723 // reason string, not the real definition, so it must NOT be parsed as 2724 // live SQL — doing so fabricates lineage from text that is not the 2725 // object's body. 2726 if (isSourceUnavailable(queryObject)) { 2727 return; 2728 } 2729 2730 String content = (String) queryObject.get("sourceCode"); 2731 if (SQLUtil.isEmpty(content)) { 2732 return; 2733 } 2734 StringBuilder builder = new StringBuilder(); 2735 if (supportCatalog) { 2736 String database = (String) queryObject.get("database"); 2737 if (database.indexOf(".") != -1) { 2738 database = SQLUtil.quoteDottedName(vendor, ESQLDataObjectType.dotCatalog, database); 2739 } 2740 builder.append(database); 2741 } 2742 if (supportSchema) { 2743 String schema = (String) queryObject.get("schema"); 2744 if (schema.indexOf(".") != -1) { 2745 schema = SQLUtil.quoteDottedName(vendor, ESQLDataObjectType.dotSchema, schema); 2746 } 2747 if (builder.length() > 0) { 2748 builder.append("."); 2749 } 2750 builder.append(schema); 2751 } 2752 String group = builder.toString(); 2753 String sqlHash = SHA256.getMd5(content); 2754 String hash = SHA256.getMd5(sqlHash); 2755 if (!databaseMap.containsKey(sqlHash)) { 2756 databaseMap.put(sqlHash, 2757 new Pair3<StringBuilder, AtomicInteger, String>(new StringBuilder(), new AtomicInteger(), group)); 2758 } 2759 String delimiterChar = String.valueOf(TGSqlParser.getDelimiterChar(option.getVendor())); 2760 StringBuilder buffer = new StringBuilder(content); 2761 if (content.trim().endsWith(delimiterChar) || content.trim().endsWith(";")) { 2762 buffer.append("\n"); 2763 } else if(vendor == EDbVendor.dbvredshift 2764 || vendor == EDbVendor.dbvgaussdb 2765 || vendor == EDbVendor.dbvedb 2766 || vendor == EDbVendor.dbvpostgresql 2767 || vendor == EDbVendor.dbvmysql 2768 || vendor == EDbVendor.dbvteradata){ 2769 buffer.append("\n\n-- " + TBaseType.sqlflow_stmt_delimiter_str + "\n\n"); 2770 } else{ 2771 SQLUtil.endTrim(buffer); 2772 buffer.append(";").append("\n"); 2773 } 2774 2775 int lineStart = databaseMap.get(sqlHash).first.toString().split("\n", -1).length - 1; 2776 if (databaseMap.get(sqlHash).first.toString().length() == 0) { 2777 lineStart = 0; 2778 } 2779 databaseMap.get(sqlHash).first.append(buffer.toString()); 2780 SqlInfo sqlInfoItem = new SqlInfo(); 2781 sqlInfoItem.setServer(sqlInfo.getServer()); 2782 sqlInfoItem.setDbVendor(sqlInfo.getDbVendor()); 2783 sqlInfoItem.setFileName(sqlInfo.getFileName()); 2784 sqlInfoItem.setFilePath(sqlInfo.getFilePath()); 2785 sqlInfoItem.setSql(buffer.toString()); 2786 sqlInfoItem.setOriginIndex(index); 2787 sqlInfoItem.setOriginLineStart(0); 2788 sqlInfoItem.setOriginLineEnd(buffer.toString().split("\n", -1).length - 1); 2789 sqlInfoItem.setIndex(databaseMap.get(sqlHash).second.getAndIncrement()); 2790 sqlInfoItem.setLineStart(lineStart); 2791 sqlInfoItem.setLineEnd(databaseMap.get(sqlHash).first.toString().split("\n", -1).length - 1); 2792 sqlInfoItem.setGroup(group); 2793 sqlInfoItem.setHash(hash); 2794 2795 if (!sqlInfoMap.containsKey(hash)) { 2796 sqlInfoMap.put(hash, new ArrayList<SqlInfo>()); 2797 } 2798 sqlInfoMap.get(hash).add(sqlInfoItem); 2799 } 2800 2801 private void readGzipBlockSource(String fullPath, Map<String, Pair3<StringBuilder, AtomicInteger, String>> databaseMap, SqlInfo sqlInfo) { 2802 try (FileInputStream fis = new FileInputStream(fullPath); 2803 GZIPInputStream gzis = new GZIPInputStream(fis); 2804 BufferedReader reader = new BufferedReader(new InputStreamReader(gzis, "UTF-8"))) { 2805 String line; 2806 int j = 0; 2807 while ((line = reader.readLine()) != null) { 2808 line = line.trim(); 2809 if (line.isEmpty()) { 2810 continue; 2811 } 2812 try { 2813 Map sourceObject = (Map) JSON.parseObject(line); 2814 String sourceCode = (String) sourceObject.get("sourceCode"); 2815 if (sourceCode != null && !sourceCode.isEmpty() 2816 && !isSourceUnavailable(sourceObject)) { 2817 SqlInfo sourceSqlInfo = new SqlInfo(); 2818 sourceSqlInfo.setFileName(sqlInfo.getFileName()); 2819 sourceSqlInfo.setFilePath(sqlInfo.getFilePath()); 2820 sourceSqlInfo.setSql(sourceCode); 2821 sourceSqlInfo.setOriginIndex(j); 2822 // Pass the record (not null): appendSqlInfo dereferences it 2823 // for groupName/database/schema; null NPE'd and the catch 2824 // silently discarded every source record on this path. 2825 appendSqlInfo(databaseMap, j, sourceSqlInfo, sourceObject); 2826 } 2827 } catch (Exception e) { 2828 logger.warn("Parse gzip source jsonl line failed.", e); 2829 } 2830 j++; 2831 } 2832 } catch (Exception e) { 2833 logger.warn("Read gzip source file failed: " + fullPath, e); 2834 } 2835 } 2836 2837 static String getTextOutput(dataflow dataflow) { 2838 StringBuffer buffer = new StringBuffer(); 2839 List<relationship> relations = dataflow.getRelationships(); 2840 if (relations != null) { 2841 for (int i = 0; i < relations.size(); i++) { 2842 relationship relation = relations.get(i); 2843 targetColumn target = relation.getTarget(); 2844 List<sourceColumn> sources = relation.getSources(); 2845 if (target != null && sources != null && sources.size() > 0) { 2846 buffer.append(target.getColumn()).append(" depends on: "); 2847 Set<String> columnSet = new LinkedHashSet<String>(); 2848 for (int j = 0; j < sources.size(); j++) { 2849 sourceColumn sourceColumn = sources.get(j); 2850 String columnName = sourceColumn.getColumn(); 2851 if (sourceColumn.getParent_name() != null && sourceColumn.getParent_name().length() > 0) { 2852 columnName = sourceColumn.getParent_name() + "." + columnName; 2853 } 2854 columnSet.add(columnName); 2855 } 2856 String[] columns = columnSet.toArray(new String[0]); 2857 for (int j = 0; j < columns.length; j++) { 2858 buffer.append(columns[j]); 2859 if (j == columns.length - 1) { 2860 buffer.append("\n"); 2861 } else 2862 buffer.append(", "); 2863 } 2864 } 2865 } 2866 } 2867 return buffer.toString(); 2868 } 2869 2870 private String mergeRelationType(List<Pair<sourceColumn, List<String>>> typePaths) { 2871 RelationshipType relationType = RelationshipType.join; 2872 for (int i = 0; i < typePaths.size(); i++) { 2873 List<String> path = typePaths.get(i).second; 2874 RelationshipType type = RelationshipType.valueOf(getRelationType(path)); 2875 if (type.ordinal() < relationType.ordinal()) { 2876 relationType = type; 2877 } 2878 } 2879 return relationType.name(); 2880 } 2881 2882 private String getRelationType(List<String> typePaths) { 2883 if (typePaths.contains("join")) 2884 return "join"; 2885 if (typePaths.contains("fdr")) 2886 return "fdr"; 2887 if (typePaths.contains("frd")) 2888 return "frd"; 2889 if (typePaths.contains("fddi")) 2890 return "fddi"; 2891 return "fdd"; 2892 } 2893 2894 public dataflow getSimpleDataflow(dataflow instance, boolean simpleOutput) throws Exception { 2895 return getSimpleDataflow(instance, simpleOutput, Arrays.asList("fdd")); 2896 } 2897 2898 public dataflow getSimpleDataflow(dataflow instance, boolean simpleOutput, List<String> types) throws Exception { 2899 ModelBindingManager.setGlobalVendor(option.getVendor()); 2900 allMap.clear(); 2901 targetTables.clear(); 2902 resultSetMap.clear(); 2903 tableMap.clear(); 2904 viewMap.clear(); 2905 cursorMap.clear(); 2906 variableMap.clear(); 2907 fileMap.clear(); 2908 stageMap.clear(); 2909 sequenceMap.clear(); 2910 dataSourceMap.clear(); 2911 databaseMap.clear(); 2912 schemaMap.clear(); 2913 streamMap.clear(); 2914 dataflow simple = new dataflow(); 2915 List<relationship> simpleRelations = new ArrayList<relationship>(); 2916 List<relationship> relations = instance.getRelationships(); 2917 if (instance.getResultsets() != null) { 2918 for (table t : instance.getResultsets()) { 2919 resultSetMap.put(t.getId().toLowerCase(), t); 2920 allMap.put(t.getId().toLowerCase(), t); 2921 } 2922 if (option.isSimpleRetainIntermediate()) { 2923 for (table t : instance.getResultsets()) { 2924 if (t.isFunction() || t.getProcedureId() != null) { 2925 t.setIsTarget("true"); 2926 } 2927 } 2928 } 2929 } 2930 2931 if (instance.getTables() != null) { 2932 for (table t : instance.getTables()) { 2933 tableMap.put(t.getId().toLowerCase(), t); 2934 allMap.put(t.getId().toLowerCase(), t); 2935 } 2936 } 2937 2938 if (instance.getViews() != null) { 2939 for (table t : instance.getViews()) { 2940 viewMap.put(t.getId().toLowerCase(), t); 2941 allMap.put(t.getId().toLowerCase(), t); 2942 } 2943 } 2944 2945 if (instance.getPaths() != null) { 2946 for (table t : instance.getPaths()) { 2947 fileMap.put(t.getId().toLowerCase(), t); 2948 allMap.put(t.getId().toLowerCase(), t); 2949 } 2950 } 2951 2952 if (instance.getStages() != null) { 2953 for (table t : instance.getStages()) { 2954 stageMap.put(t.getId().toLowerCase(), t); 2955 allMap.put(t.getId().toLowerCase(), t); 2956 } 2957 } 2958 2959 if (instance.getSequences() != null) { 2960 for (table t : instance.getSequences()) { 2961 sequenceMap.put(t.getId().toLowerCase(), t); 2962 allMap.put(t.getId().toLowerCase(), t); 2963 } 2964 } 2965 2966 if (instance.getDatasources() != null) { 2967 for (table t : instance.getDatasources()) { 2968 dataSourceMap.put(t.getId().toLowerCase(), t); 2969 allMap.put(t.getId().toLowerCase(), t); 2970 } 2971 } 2972 2973 if (instance.getDatabases() != null) { 2974 for (table t : instance.getDatabases()) { 2975 databaseMap.put(t.getId().toLowerCase(), t); 2976 allMap.put(t.getId().toLowerCase(), t); 2977 } 2978 } 2979 2980 if (instance.getSchemas() != null) { 2981 for (table t : instance.getSchemas()) { 2982 schemaMap.put(t.getId().toLowerCase(), t); 2983 allMap.put(t.getId().toLowerCase(), t); 2984 } 2985 } 2986 2987 if (instance.getStreams() != null) { 2988 for (table t : instance.getStreams()) { 2989 streamMap.put(t.getId().toLowerCase(), t); 2990 allMap.put(t.getId().toLowerCase(), t); 2991 } 2992 } 2993 2994 if (instance.getVariables() != null) { 2995 for (table t : instance.getVariables()) { 2996 if(SubType.cursor.name().equals(t.getSubType())){ 2997 cursorMap.put(t.getId().toLowerCase(), t); 2998 } 2999 else { 3000 variableMap.put(t.getId().toLowerCase(), t); 3001 } 3002 allMap.put(t.getId().toLowerCase(), t); 3003 } 3004 } 3005 3006 if (relations != null) { 3007 3008 List<relationship> filterRelations = new ArrayList<>(); 3009 for (relationship relationElem : relations) { 3010 if (!types.contains(relationElem.getType())) { 3011 if (option.isSimpleRetainIntermediate() 3012 && RelationshipType.call.name().equals(relationElem.getType())) { 3013 filterRelations.add(relationElem); 3014 } 3015 continue; 3016 } 3017 else { 3018 filterRelations.add(relationElem); 3019 } 3020 } 3021 3022 relations = filterRelations; 3023 3024 Map<String, Set<relationship>> targetIdRelationMap = new LinkedHashMap<String, Set<relationship>>(); 3025 for (relationship relation : relations) { 3026 if (relation.getTarget() != null) { 3027 String key = relation.getTarget().getParent_id() + "." + relation.getTarget().getId(); 3028 if (!targetIdRelationMap.containsKey(key)) { 3029 targetIdRelationMap.put(key, new TreeSet<relationship>(new Comparator<relationship>() { 3030 @Override 3031 public int compare(relationship o1, relationship o2) { 3032 return o1.getId().compareTo(o2.getId()); 3033 } 3034 })); 3035 } 3036 targetIdRelationMap.get(key).add(relation); 3037 } 3038 } 3039 3040 Iterator<String> keys = targetIdRelationMap.keySet().iterator(); 3041 while (keys.hasNext()) { 3042 String key = keys.next(); 3043 if (targetIdRelationMap.get(key).size() > 500) { 3044 keys.remove(); 3045 } 3046 } 3047 3048 for (relationship relationElem : relations) { 3049 if (RelationshipType.call.name().equals(relationElem.getType())) { 3050 continue; 3051 } 3052 if (RelationshipType.er.name().equals(relationElem.getType())) { 3053 continue; 3054 } 3055 targetColumn target = relationElem.getTarget(); 3056 String targetParent = target.getParent_id(); 3057 if (isTarget(instance, targetParent, simpleOutput)) { 3058 List<Pair<sourceColumn, List<String>>> relationSources = new ArrayList<Pair<sourceColumn, List<String>>>(); 3059 findSourceRelations(target, instance, targetIdRelationMap, relationElem, relationSources, 3060 new String[] { relationElem.getType() }, simpleOutput); 3061 if (relationSources.size() > 0) { 3062 Map<sourceColumn, List<Pair<sourceColumn, List<String>>>> columnMap = new LinkedHashMap<sourceColumn, List<Pair<sourceColumn, List<String>>>>(); 3063 for (Pair<sourceColumn, List<String>> t : relationSources) { 3064 sourceColumn key = ((Pair<sourceColumn, List<String>>) t).first; 3065 if (!columnMap.containsKey(key)) { 3066 columnMap.put(key, new ArrayList<Pair<sourceColumn, List<String>>>()); 3067 } 3068 columnMap.get(key).add(t); 3069 } 3070 Iterator<sourceColumn> iter = columnMap.keySet().iterator(); 3071 Map<String, List<sourceColumn>> relationSourceMap = new LinkedHashMap<String, List<sourceColumn>>(); 3072 Set<sourceColumn> passthroughColumns = new HashSet<sourceColumn>(); 3073 while (iter.hasNext()) { 3074 sourceColumn column = iter.next(); 3075 String relationType = mergeRelationType(columnMap.get(column)); 3076 if (!relationSourceMap.containsKey(relationType)) { 3077 relationSourceMap.put(relationType, new ArrayList<sourceColumn>()); 3078 } 3079 relationSourceMap.get(relationType).add(column); 3080 // Mark columns that reached this target through an assumed passthrough edge. 3081 for (Pair<sourceColumn, List<String>> typePath : columnMap.get(column)) { 3082 if (typePath.second != null 3083 && typePath.second.contains(EffectType.external_script_passthrough.name())) { 3084 passthroughColumns.add(column); 3085 break; 3086 } 3087 } 3088 } 3089 3090 Iterator<String> sourceIter = relationSourceMap.keySet().iterator(); 3091 while (sourceIter.hasNext()) { 3092 String relationType = sourceIter.next(); 3093 relationship simpleRelation = (relationship) relationElem.clone(); 3094 List<sourceColumn> groupSources = relationSourceMap.get(relationType); 3095 simpleRelation.setSources(groupSources); 3096 simpleRelation.setType(relationType); 3097 simpleRelation.setId(String.valueOf(++ModelBindingManager.get().RELATION_ID)); 3098 // If any source in this group reached the target through an assumed external- 3099 // script passthrough edge, tag the flattened edge so simple output does not 3100 // present assumed lineage as proven. 3101 for (sourceColumn groupSource : groupSources) { 3102 if (passthroughColumns.contains(groupSource)) { 3103 simpleRelation.setEffectType(EffectType.external_script_passthrough.name()); 3104 break; 3105 } 3106 } 3107 simpleRelations.add(simpleRelation); 3108 } 3109 } 3110 } 3111 } 3112 } 3113 3114 simple.setProcedures(instance.getProcedures()); 3115 simple.setPackages(instance.getPackages()); 3116 simple.setProcesses(instance.getProcesses()); 3117 simple.setErrors(instance.getErrors()); 3118 3119 List<table> tables = new ArrayList<table>(); 3120 for (table t : instance.getTables()) { 3121 if (!SQLUtil.isTempTable(t)) { 3122 tables.add(t); 3123 } 3124 else { 3125 if (option.isIgnoreTemporaryTable()) { 3126 continue; 3127 } 3128 else { 3129 tables.add(t); 3130 } 3131 } 3132 } 3133 simple.setStages(instance.getStages()); 3134 simple.setSequences(instance.getSequences()); 3135 simple.setDatasources(instance.getDatasources()); 3136 simple.setStreams(instance.getStreams()); 3137 simple.setPaths(instance.getPaths()); 3138 simple.setTables(tables); 3139 simple.setViews(instance.getViews()); 3140 if(option.isSimpleShowVariable()) { 3141 simple.setVariables(instance.getVariables()); 3142 } 3143 else if(option.isSimpleShowCursor()) { 3144 simple.setVariables(instance.getVariables().stream().filter(t->SubType.cursor.name().equals(t.getSubType())).collect(Collectors.toList())); 3145 } 3146 else{ 3147 simple.setVariables(new ArrayList()); 3148 } 3149 3150 if (option.isSimpleRetainIntermediate() && instance.getVariables() != null) { 3151 for (table var : instance.getVariables()) { 3152 if (var.isTarget()) { 3153 simple.getVariables().add(var); 3154 } 3155 } 3156 } 3157 if (instance.getResultsets() != null) { 3158 List<table> resultSets = new ArrayList<table>(); 3159 for (int i = 0; i < instance.getResultsets().size(); i++) { 3160 table resultSet = instance.getResultsets().get(i); 3161 if (isTargetResultSet(instance, resultSet.getId(), simpleOutput)) { 3162 // special handle function #524 #296 3163 resultSets.add(resultSet); 3164 } 3165 else if (option.isSimpleRetainIntermediate() && (resultSet.isFunction() || resultSet.getProcedureId() != null)) { 3166 resultSet.setIsTarget("true"); 3167 resultSets.add(resultSet); 3168 } 3169 } 3170 simple.setResultsets(resultSets); 3171 } 3172 3173 List<table> functions = new ArrayList<table>(); 3174 if (option.isShowCallRelation()) { 3175 for (int i = 0; i < relations.size(); i++) { 3176 relationship relationElem = relations.get(i); 3177 if (!RelationshipType.call.name().equals(relationElem.getType())) { 3178 continue; 3179 } 3180 simpleRelations.add(relationElem); 3181 for (sourceColumn callee : relationElem.getCallees()) { 3182 String calleeId = callee.getId(); 3183 if (resultSetMap.containsKey(calleeId)) { 3184 table function = resultSetMap.get(calleeId); 3185 function.setIsTarget("true"); 3186 functions.add(function); 3187 } 3188 } 3189 } 3190 } 3191 3192 if (option.isShowERDiagram()) { 3193 for (int i = 0; i < relations.size(); i++) { 3194 relationship relationElem = relations.get(i); 3195 if (!RelationshipType.er.name().equals(relationElem.getType())) { 3196 continue; 3197 } 3198 simpleRelations.add(relationElem); 3199 for (sourceColumn callee : relationElem.getCallees()) { 3200 String calleeId = callee.getId(); 3201 if (resultSetMap.containsKey(calleeId)) { 3202 table function = resultSetMap.get(calleeId); 3203 function.setIsTarget("true"); 3204 functions.add(function); 3205 } 3206 } 3207 } 3208 } 3209 3210 if (option.isSimpleRetainIntermediate() && !option.isShowCallRelation()) { 3211 for (int i = 0; i < relations.size(); i++) { 3212 relationship relationElem = relations.get(i); 3213 if (RelationshipType.call.name().equals(relationElem.getType())) { 3214 simpleRelations.add(relationElem); 3215 } 3216 } 3217 } 3218 3219 if (!functions.isEmpty()) { 3220 if (simple.getResultsets() == null) { 3221 simple.setResultsets(functions); 3222 } else { 3223 simple.getResultsets().addAll(functions); 3224 } 3225 } 3226 3227 simple.setRelationships(simpleRelations); 3228 simple.setOrientation(instance.getOrientation()); 3229 3230 targetTables.clear(); 3231 resultSetMap.clear(); 3232 tableMap.clear(); 3233 viewMap.clear(); 3234 cursorMap.clear(); 3235 variableMap.clear(); 3236 fileMap.clear(); 3237 stageMap.clear(); 3238 dataSourceMap.clear(); 3239 databaseMap.clear(); 3240 schemaMap.clear(); 3241 streamMap.clear(); 3242 return simple; 3243 } 3244 3245 private void findSourceRelations(targetColumn target, dataflow instance, Map<String, Set<relationship>> sourceIdRelationMap, 3246 relationship targetRelation, List<Pair<sourceColumn, List<String>>> relationSources, String[] pathTypes, boolean simpleOutput) { 3247 findStarSourceRelations(target, instance, null, sourceIdRelationMap, targetRelation, relationSources, pathTypes, 3248 new HashSet<String>(), new LinkedHashSet<transform>(), new LinkedHashSet<candidateTable>(), 0, simpleOutput); 3249 } 3250 3251 private void findStarSourceRelations(targetColumn target, dataflow instance, targetColumn starRelationTarget, 3252 Map<String, Set<relationship>> sourceIdRelationMap, relationship targetRelation, 3253 List<Pair<sourceColumn, List<String>>> relationSources, String[] pathTypes, Set<String> paths, 3254 Set<transform> transforms, Set<candidateTable> candidateTables, int level, boolean simpleOutput) { 3255 if (targetRelation != null && targetRelation.getSources() != null) { 3256 3257 //获取source为*的Column Parent 3258 String starParentId = null; 3259 for (int i = 0; i < targetRelation.getSources().size(); i++) { 3260 sourceColumn source = targetRelation.getSources().get(i); 3261 if (starRelationTarget != null && "*".equals(source.getColumn())) { 3262 starParentId = source.getParent_id(); 3263 } 3264 } 3265 3266 for (int i = 0; i < targetRelation.getSources().size(); i++) { 3267 sourceColumn source = targetRelation.getSources().get(i); 3268 if (starRelationTarget != null && !"*".equals(source.getColumn()) 3269 && !DlineageUtil.getIdentifierNormalColumnName(starRelationTarget.getColumn()) 3270 .equals(DlineageUtil.getIdentifierNormalColumnName(source.getColumn()))) { 3271 table parent = allMap.get(source.getParent_id()); 3272 if (parent != null && isFunction(parent)) { 3273 // function返回值未知,不对星号做处理 3274 } 3275 else if (parent == null) { 3276 continue; 3277 } else if(starParentId!=null && starParentId.equals(parent.getId())){ 3278 //如果source和 * column的parent相同,则跳过 3279 continue; 3280 } 3281 } 3282 3283 String sourceColumnId = source.getId(); 3284 String sourceParentId = source.getParent_id(); 3285 if (sourceParentId == null || sourceColumnId == null) { 3286 continue; 3287 } 3288 if (isTarget(instance, sourceParentId, simpleOutput)) { 3289 List<transform> transforms2 = new ArrayList<transform>(transforms.size()); 3290 transforms2.addAll(transforms); 3291 Collections.reverse(transforms2); 3292 3293 List<candidateTable> candidateTables2 = new ArrayList<candidateTable>(candidateTables.size()); 3294 candidateTables2.addAll(candidateTables); 3295 3296 sourceColumn sourceColumnCopy = DlineageUtil.copySourceColumn(source); 3297 for (transform t : transforms2) { 3298 sourceColumnCopy.addTransform(t); 3299 } 3300 3301 for (candidateTable t : candidateTables2) { 3302 sourceColumnCopy.addCandidateParent(t); 3303 } 3304 3305 if(Boolean.TRUE.equals(target.isStruct()) && Boolean.TRUE.equals(source.isStruct())) { 3306 List<String> targetColumns = SQLUtil.parseNames(target.getColumn()); 3307 List<String> sourceColumns = SQLUtil.parseNames(source.getColumn()); 3308 if(!DlineageUtil.getIdentifierNormalColumnName(targetColumns.get(targetColumns.size()-1)) 3309 .equals(DlineageUtil.getIdentifierNormalColumnName(sourceColumns.get(sourceColumns.size()-1)))) { 3310 continue; 3311 } 3312 } 3313 relationSources.add(new Pair<sourceColumn, List<String>>(sourceColumnCopy, Arrays.asList(pathTypes))); 3314 } else { 3315 Set<relationship> sourceRelations = sourceIdRelationMap 3316 .get(source.getParent_id() + "." + source.getId()); 3317 if (sourceRelations != null) { 3318 if (paths.contains(source.getParent_id() + "." + source.getId())) { 3319 continue; 3320 } else { 3321 paths.add(source.getParent_id() + "." + source.getId()); 3322 if (source.getTransforms() != null) { 3323 transforms.addAll(source.getTransforms()); 3324 } 3325 if (source.getCandidateParents() != null) { 3326 candidateTables.addAll(source.getCandidateParents()); 3327 } 3328 } 3329 for (relationship relation : sourceRelations) { 3330 LinkedHashSet<transform> transforms2 = new LinkedHashSet<transform>(transforms.size()); 3331 transforms2.addAll(transforms); 3332 LinkedHashSet<candidateTable> candidateTables2 = new LinkedHashSet<candidateTable>(candidateTables.size()); 3333 candidateTables2.addAll(candidateTables); 3334 // Carry an assumed external-script passthrough marker down the path 3335 // (alongside the relationship type) so a flattened simple-output edge 3336 // that traversed it can be tagged and never look proven. The marker is 3337 // ignored by getRelationType, so type computation is unaffected. 3338 boolean passthroughHop = EffectType.external_script_passthrough.name() 3339 .equals(relation.getEffectType()); 3340 String[] types = new String[pathTypes.length + (passthroughHop ? 2 : 1)]; 3341 int typeOffset = 0; 3342 types[typeOffset++] = relation.getType(); 3343 if (passthroughHop) { 3344 types[typeOffset++] = EffectType.external_script_passthrough.name(); 3345 } 3346 System.arraycopy(pathTypes, 0, types, typeOffset, pathTypes.length); 3347 if (!"*".equals(source.getColumn())) { 3348 findStarSourceRelations(target, instance, null, sourceIdRelationMap, relation, relationSources, 3349 types, paths, transforms2, candidateTables2, level + 1, simpleOutput); 3350 } else { 3351 findStarSourceRelations(target, instance, 3352 starRelationTarget == null ? targetRelation.getTarget() : starRelationTarget, 3353 sourceIdRelationMap, relation, relationSources, types, paths, transforms, candidateTables2, 3354 level + 1, simpleOutput); 3355 } 3356 } 3357 } 3358 } 3359 } 3360 } 3361 } 3362 3363 private Map<String, Boolean> targetTables = new HashMap<String, Boolean>(); 3364 private Map<String, table> resultSetMap = new HashMap<String, table>(); 3365 private Map<String, table> tableMap = new HashMap<String, table>(); 3366 private Map<String, table> viewMap = new HashMap<String, table>(); 3367 private Map<String, table> cursorMap = new HashMap<String, table>(); 3368 private Map<String, table> variableMap = new HashMap<String, table>(); 3369 private Map<String, table> fileMap = new HashMap<String, table>(); 3370 private Map<String, table> stageMap = new HashMap<String, table>(); 3371 private Map<String, table> sequenceMap = new HashMap<String, table>(); 3372 private Map<String, table> dataSourceMap = new HashMap<String, table>(); 3373 private Map<String, table> databaseMap = new HashMap<String, table>(); 3374 private Map<String, table> schemaMap = new HashMap<String, table>(); 3375 private Map<String, table> streamMap = new HashMap<String, table>(); 3376 private Map<String, table> allMap = new HashMap<String, table>(); 3377 3378 private boolean isTarget(dataflow instance, String targetParentId, boolean simpleOutput) { 3379 if (targetTables.containsKey(targetParentId)) 3380 return targetTables.get(targetParentId); 3381 if (isTable(instance, targetParentId)) { 3382 targetTables.put(targetParentId, true); 3383 return true; 3384 } else if (isView(instance, targetParentId)) { 3385 targetTables.put(targetParentId, true); 3386 return true; 3387 } else if (isFile(instance, targetParentId)) { 3388 targetTables.put(targetParentId, true); 3389 return true; 3390 } else if (isDatabase(instance, targetParentId)) { 3391 targetTables.put(targetParentId, true); 3392 return true; 3393 } else if (isSchema(instance, targetParentId)) { 3394 targetTables.put(targetParentId, true); 3395 return true; 3396 } else if (isStage(instance, targetParentId)) { 3397 targetTables.put(targetParentId, true); 3398 return true; 3399 } else if (isSequence(instance, targetParentId)) { 3400 targetTables.put(targetParentId, true); 3401 return true; 3402 } else if (isDataSource(instance, targetParentId)) { 3403 targetTables.put(targetParentId, true); 3404 return true; 3405 } else if (isStream(instance, targetParentId)) { 3406 targetTables.put(targetParentId, true); 3407 return true; 3408 } else if (isCursor(instance, targetParentId) && option.isSimpleShowCursor()) { 3409 targetTables.put(targetParentId, true); 3410 return true; 3411 } else if ((isVariable(instance, targetParentId) || isCursor(instance, targetParentId)) && option.isSimpleShowVariable()) { 3412 targetTables.put(targetParentId, true); 3413 return true; 3414 } else if ((isVariable(instance, targetParentId) || isCursor(instance, targetParentId)) && option.isSimpleRetainIntermediate()) { 3415 table var = variableMap.get(targetParentId.toLowerCase()); 3416 if (var != null && var.isTarget()) { 3417 targetTables.put(targetParentId, true); 3418 return true; 3419 } 3420 } else if (isTargetResultSet(instance, targetParentId, simpleOutput)) { 3421 targetTables.put(targetParentId, true); 3422 return true; 3423 } 3424 targetTables.put(targetParentId, false); 3425 return false; 3426 } 3427 3428 private boolean isTargetResultSet(dataflow instance, String targetParent, boolean simpleOutput) { 3429 if (resultSetMap.containsKey(targetParent.toLowerCase())) { 3430 table result = resultSetMap.get(targetParent.toLowerCase()); 3431 boolean isTarget = result.isTarget(); 3432 Option option = ModelBindingManager.getGlobalOption(); 3433 if (option != null && option.isSqlflowIgnoreFunction() && isFunction(result)) { 3434 return false; 3435 } 3436 if (isTarget && simpleOutput) { 3437 if (option != null && option.isSimpleShowFunction() && isFunction(result)) { 3438 return true; 3439 3440 } else if (option != null && option.isSimpleShowTopSelectResultSet()) { 3441 return true; 3442 } 3443 if (option != null && option.isSimpleRetainIntermediate() && (isFunction(result) || result.getProcedureId() != null)) { 3444 return true; 3445 } 3446 if (ResultSetType.of(result.getType()) != null && option.containsResultSetType(ResultSetType.of(result.getType()))) { 3447 return true; 3448 } 3449 } else 3450 return isTarget; 3451 } 3452 return false; 3453 } 3454 3455 private boolean isFunction(table resultSet) { 3456 if("function".equals(resultSet.getType())){ 3457 return true; 3458 } 3459 else if("resultset".equals(resultSet.getType()) && "function".equals(resultSet.getSubType())){ 3460 return true; 3461 } 3462 return false; 3463 } 3464 3465 private boolean isView(dataflow instance, String targetParent) { 3466 if (viewMap.containsKey(targetParent.toLowerCase())) { 3467 return true; 3468 } 3469 return false; 3470 } 3471 3472 private boolean isCursor(dataflow instance, String targetParent) { 3473 if (cursorMap.containsKey(targetParent.toLowerCase())) { 3474 return true; 3475 } 3476 return false; 3477 } 3478 3479 private boolean isVariable(dataflow instance, String targetParent) { 3480 if (variableMap.containsKey(targetParent.toLowerCase())) { 3481 return true; 3482 } 3483 return false; 3484 } 3485 3486 private boolean isFile(dataflow instance, String targetParent) { 3487 if (fileMap.containsKey(targetParent.toLowerCase())) { 3488 return true; 3489 } 3490 return false; 3491 } 3492 3493 private boolean isStage(dataflow instance, String targetParent) { 3494 if (stageMap.containsKey(targetParent.toLowerCase())) { 3495 return true; 3496 } 3497 return false; 3498 } 3499 3500 private boolean isSequence(dataflow instance, String targetParent) { 3501 if (sequenceMap.containsKey(targetParent.toLowerCase())) { 3502 return true; 3503 } 3504 return false; 3505 } 3506 3507 private boolean isDataSource(dataflow instance, String targetParent) { 3508 if (dataSourceMap.containsKey(targetParent.toLowerCase())) { 3509 return true; 3510 } 3511 return false; 3512 } 3513 3514 private boolean isDatabase(dataflow instance, String targetParent) { 3515 if (databaseMap.containsKey(targetParent.toLowerCase())) { 3516 return true; 3517 } 3518 return false; 3519 } 3520 3521 private boolean isSchema(dataflow instance, String targetParent) { 3522 if (schemaMap.containsKey(targetParent.toLowerCase())) { 3523 return true; 3524 } 3525 return false; 3526 } 3527 3528 private boolean isStream(dataflow instance, String targetParent) { 3529 if (streamMap.containsKey(targetParent.toLowerCase())) { 3530 return true; 3531 } 3532 return false; 3533 } 3534 3535 private boolean isTable(dataflow instance, String targetParent) { 3536 if (tableMap.containsKey(targetParent.toLowerCase())) { 3537 if (SQLUtil.isTempTable(tableMap.get(targetParent))) { 3538 if (option.isIgnoreTemporaryTable()) { 3539 return false; 3540 } 3541 } 3542 if (tableMap.get(targetParent).isFunction()) { 3543 if (option != null && option.isSimpleShowFunction()) { 3544 return true; 3545 } else { 3546 return false; 3547 } 3548 } 3549 if (SubType.synonym.name().equals(tableMap.get(targetParent).getSubType())) { 3550 if (option != null && option.isSimpleShowSynonym()) { 3551 return true; 3552 } else { 3553 return false; 3554 } 3555 } 3556 return true; 3557 } else { 3558 return false; 3559 } 3560 } 3561 3562 private void init() { 3563 metadataErrors.clear(); 3564 sqlInfoMap.clear(); 3565 errorInfos.clear(); 3566 dynamicSqlSites.clear(); 3567 cteSelectRelationKeys.clear(); 3568 authoritativeEvidenceCollector.reset(); 3569 authoritativeLineageEvidence = AuthoritativeLineageEvidence.empty(); 3570 dynamicEvalCache.clear(); 3571 routineCatalog = null; 3572 crossFileParseCache.clear(); 3573 plsqlEvaluatedTexts.clear(); 3574 dataflow = null; 3575 dataflowString = null; 3576 ModelBindingManager.removeGlobalDatabase(); 3577 ModelBindingManager.removeGlobalSchema(); 3578 ModelBindingManager.removeGlobalVendor(); 3579 ModelBindingManager.removeGlobalSQLEnv(); 3580 ModelBindingManager.removeGlobalHash(); 3581 appendResultSets.clear(); 3582 appendStarColumns.clear(); 3583 appendTableStarColumns.clear(); 3584 normalizedColumnNameCache.clear(); 3585 columnNameIndexCache.clear(); 3586 resultSetColumnLookupCache.clear(); 3587 tableColumnLookupCache.clear(); 3588 modelManager.TABLE_COLUMN_ID = option.getStartId(); 3589 modelManager.RELATION_ID = option.getStartId(); 3590 modelManager.DISPLAY_ID.clear(); 3591 modelManager.DISPLAY_NAME.clear(); 3592 tableIds.clear(); 3593 clickhouseExternalSourceModels.clear(); 3594 ModelBindingManager.setGlobalVendor(option.getVendor()); 3595 modelManager.reset(); 3596 } 3597 3598 private String getErrorMessage(TSyntaxError error, String errorType) { 3599 String s = "", hint = "Syntax error"; 3600 if (ErrorInfo.SYNTAX_HINT.equals(errorType)) { 3601 hint = "Syntax hint"; 3602 } 3603 if (error.hint.length() > 0) 3604 hint = error.hint; 3605 s = s + hint + "(" + error.errorno + ") near: " + error.tokentext; 3606 s = s + "(" + error.lineNo; 3607 s = s + "," + error.columnNo + ")"; 3608 return s; 3609 } 3610 3611// boolean OLD_ENABLE_RESOLVER = TBaseType.isEnableResolver(); 3612 private void analyzeAndOutputResult(TGSqlParser sqlparser) { 3613 try { 3614 accessedSubqueries.clear(); 3615 accessedStatements.clear(); 3616 stmtStack.clear(); 3617 viewDDLMap.clear(); 3618 procedureDDLMap.clear(); 3619 structObjectMap.clear(); 3620 try { 3621 if(sqlenv!=null) { 3622 sqlparser.setSqlEnv(sqlenv); 3623 } 3624 int result = sqlparser.parse(); 3625 if (result != 0) { 3626 ArrayList<TSyntaxError> errors = sqlparser.getSyntaxErrors(); 3627 if ((errors == null || errors.isEmpty()) 3628 && sqlparser.getErrormessage() != null 3629 && !sqlparser.getErrormessage().isEmpty()) { 3630 // A refusal with no syntax errors -- the trial size cap 3631 // is the one that behaves this way. Recording nothing 3632 // here turned the rejection into an ordinary EMPTY 3633 // lineage result, which reads as "this SQL has no 3634 // dependencies" rather than "this SQL was not analyzed". 3635 ErrorInfo errorInfo = new ErrorInfo(); 3636 errorInfo.setErrorType(ErrorInfo.SYNTAX_ERROR); 3637 errorInfo.setErrorMessage(sqlparser.getErrormessage()); 3638 errorInfo.setStartPosition(new Pair3<Long, Long, String>(1L, 1L, 3639 ModelBindingManager.getGlobalHash())); 3640 errorInfo.setEndPosition(new Pair3<Long, Long, String>(1L, 1L, 3641 ModelBindingManager.getGlobalHash())); 3642 errorInfo.fillInfo(this); 3643 errorInfos.add(errorInfo); 3644 } 3645 if (errors != null && !errors.isEmpty()) { 3646 for (int i = 0; i < errors.size(); i++) { 3647 TSyntaxError error = errors.get(i); 3648 ErrorInfo errorInfo = new ErrorInfo(); 3649 errorInfo.setErrorType(ErrorInfo.SYNTAX_ERROR); 3650 errorInfo.setErrorMessage(getErrorMessage(error, ErrorInfo.SYNTAX_ERROR)); 3651 errorInfo.setStartPosition(new Pair3<Long, Long, String>(error.lineNo, error.columnNo, 3652 ModelBindingManager.getGlobalHash())); 3653 String[] segments = error.tokentext.split("\n"); 3654 if (segments.length <= 1) { 3655 errorInfo.setEndPosition(new Pair3<Long, Long, String>(error.lineNo, 3656 error.columnNo + error.tokentext.length(), 3657 ModelBindingManager.getGlobalHash())); 3658 } else { 3659 errorInfo.setEndPosition( 3660 new Pair3<Long, Long, String>(error.lineNo + segments.length - 1, 3661 (long) segments[segments.length - 1].length() + 1, 3662 ModelBindingManager.getGlobalHash())); 3663 } 3664 ; 3665 errorInfo.fillInfo(this); 3666 errorInfos.add(errorInfo); 3667 } 3668 } 3669 } 3670 3671 if (option.getHandleListener() != null) { 3672 option.getHandleListener().endParse(result == 0); 3673 } 3674 } catch (Exception e) { 3675 logger.error("analyze sql failed.", e); 3676 if (option.getHandleListener() != null) { 3677 option.getHandleListener().endParse(false); 3678 } 3679 ErrorInfo errorInfo = new ErrorInfo(); 3680 errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR); 3681 if (e.getMessage() == null) { 3682 if (e.getStackTrace() != null && e.getStackTrace().length > 0) { 3683 errorInfo 3684 .setErrorMessage(e.getClass().getSimpleName() + ": " + e.getStackTrace()[0].toString()); 3685 } else { 3686 errorInfo.setErrorMessage(e.getClass().getSimpleName()); 3687 } 3688 } else { 3689 errorInfo.setErrorMessage(e.getClass().getSimpleName() + ": " + e.getMessage()); 3690 } 3691 errorInfo.fillInfo(this); 3692 errorInfos.add(errorInfo); 3693 return; 3694 } 3695 3696 3697 TSQLResolver2 resolver = sqlparser.getResolver2(); 3698 if (resolver != null && option.getVendor() == EDbVendor.dbvbigquery) { 3699 ScopeBuildResult buildResult = resolver.getScopeBuildResult(); 3700 List<TObjectName> columns = buildResult.getAllColumnReferences(); 3701 for (TObjectName col : columns) { 3702 structObjectMap.putIfAbsent(col.getSourceTable(), new TObjectNameList()); 3703 structObjectMap.get(col.getSourceTable()).addObjectName(col); 3704 } 3705 } 3706 3707 if (option.getHandleListener() != null) { 3708 option.getHandleListener().startAnalyzeDataFlow(sqlparser); 3709 } 3710 3711 ModelBindingManager.setSchemaUnsetStatements(collectSchemaUnsetStatements(sqlparser)); 3712 3713 for (int i = 0; i < sqlparser.getSqlstatements().size(); i++) { 3714 if (option.getHandleListener() != null && option.getHandleListener().isCanceled()) { 3715 break; 3716 } 3717 3718 TCustomSqlStatement stmt = sqlparser.getSqlstatements().get(i); 3719 if (stmt.getErrorCount() == 0) { 3720 if (stmt.getParentStmt() == null) { 3721 modelManager.collectSqlHash(stmt); 3722 if (stmt instanceof TUseDatabase || stmt instanceof TUseSchema 3723 || stmt instanceof TCreateTableSqlStatement 3724 || stmt instanceof TCreateExternalDataSourceStmt || stmt instanceof TCreateStageStmt 3725 || stmt instanceof TMssqlCreateType 3726 || stmt instanceof TMssqlDeclare 3727 || stmt instanceof TPlsqlCreateType_Placeholder 3728 || stmt instanceof TPlsqlCreateType 3729 || stmt instanceof TPlsqlTableTypeDefStmt 3730 || (stmt instanceof TCreateFunctionStmt && hasDb2ReturnStmt((TCreateFunctionStmt)stmt)) 3731 || (stmt instanceof TMssqlCreateFunction 3732 && (((TMssqlCreateFunction) stmt).getReturnTableDefinitions() != null 3733 || ((TMssqlCreateFunction) stmt).getReturnStmt() != null))) { 3734 boolean listen = false; 3735 if (option.getHandleListener() != null && !accessedStatements.contains(stmt)) { 3736 option.getHandleListener().startAnalyzeStatment(stmt); 3737 listen = true; 3738 } 3739 analyzeCustomSqlStmt(stmt); 3740 if (listen && option.getHandleListener() != null) { 3741 option.getHandleListener().endAnalyzeStatment(stmt); 3742 } 3743 } 3744 } 3745 } 3746 } 3747 3748 for (int i = 0; i < sqlparser.sqlstatements.size(); i++) { 3749 if (option.getHandleListener() != null && option.getHandleListener().isCanceled()) { 3750 break; 3751 } 3752 3753 TCustomSqlStatement stmt = sqlparser.getSqlstatements().get(i); 3754 if (stmt.getErrorCount() == 0) { 3755 if (stmt.getParentStmt() == null) { 3756 if (stmt instanceof TUseDatabase 3757 || stmt instanceof TUseSchema 3758 || stmt instanceof TCreateViewSqlStatement 3759 || stmt instanceof TCreateSynonymStmt 3760 || stmt instanceof TStoredProcedureSqlStatement) { 3761 boolean listen = false; 3762 if (option.getHandleListener() != null && !accessedStatements.contains(stmt)) { 3763 option.getHandleListener().startAnalyzeStatment(stmt); 3764 listen = true; 3765 } 3766 if (stmt instanceof TUseDatabase || stmt instanceof TUseSchema) { 3767 analyzeCustomSqlStmt(stmt); 3768 } else if (stmt instanceof TCreateViewSqlStatement) { 3769 TCreateViewSqlStatement view = (TCreateViewSqlStatement) stmt; 3770 if(view.getViewName()!=null) { 3771 viewDDLMap.put(DlineageUtil.getTableFullName(view.getViewName().toString()), view); 3772 } 3773 } else if (stmt instanceof TCreateSynonymStmt) { 3774 TCreateSynonymStmt synonym = (TCreateSynonymStmt) stmt; 3775 if(synonym.getSynonymName()!=null) { 3776 viewDDLMap.put(DlineageUtil.getTableFullName(synonym.getSynonymName().toString()), synonym); 3777 } 3778 } else if (stmt instanceof TStoredProcedureSqlStatement) { 3779 TStoredProcedureSqlStatement procedure = (TStoredProcedureSqlStatement) stmt; 3780 if(procedure.getStoredProcedureName() == null) { 3781 continue; 3782 } 3783 procedureDDLMap.put(DlineageUtil.getProcedureNameWithArgNum(procedure), procedure); 3784 } 3785 if (listen && option.getHandleListener() != null) { 3786 option.getHandleListener().endAnalyzeStatment(stmt); 3787 } 3788 } 3789 } 3790 } 3791 } 3792 3793 for (int i = 0; i < sqlparser.sqlstatements.size(); i++) { 3794 if (option.getHandleListener() != null && option.getHandleListener().isCanceled()) { 3795 break; 3796 } 3797 3798 TCustomSqlStatement stmt = sqlparser.getSqlstatements().get(i); 3799 if (stmt.getErrorCount() == 0) { 3800 if (stmt.getParentStmt() == null) { 3801 if (stmt instanceof TUseDatabase 3802 || stmt instanceof TUseSchema 3803 || stmt instanceof TCreateViewSqlStatement 3804 || stmt instanceof TCreateSynonymStmt) { 3805 boolean listen = false; 3806 if (option.getHandleListener() != null && !accessedStatements.contains(stmt)) { 3807 option.getHandleListener().startAnalyzeStatment(stmt); 3808 listen = true; 3809 } 3810 analyzeCustomSqlStmt(stmt); 3811 if (listen && option.getHandleListener() != null) { 3812 option.getHandleListener().endAnalyzeStatment(stmt); 3813 } 3814 } 3815 } 3816 } 3817 } 3818 3819 for (int i = 0; i < sqlparser.sqlstatements.size(); i++) { 3820 if (option.getHandleListener() != null && option.getHandleListener().isCanceled()) { 3821 break; 3822 } 3823 3824 TCustomSqlStatement stmt = sqlparser.getSqlstatements().get(i); 3825 if (stmt.getErrorCount() == 0) { 3826 if (stmt.getParentStmt() == null) { 3827 if (stmt instanceof TUseDatabase || stmt instanceof TUseSchema 3828 || stmt instanceof TStoredProcedureSqlStatement) { 3829 boolean listen = false; 3830 if (option.getHandleListener() != null && !accessedStatements.contains(stmt)) { 3831 option.getHandleListener().startAnalyzeStatment(stmt); 3832 listen = true; 3833 } 3834 if (stmt instanceof TPlsqlCreateTrigger) 3835 continue; 3836 analyzeCustomSqlStmt(stmt); 3837 if (listen && option.getHandleListener() != null) { 3838 option.getHandleListener().endAnalyzeStatment(stmt); 3839 } 3840 } 3841 } 3842 } 3843 } 3844 3845 for (int i = 0; i < sqlparser.sqlstatements.size(); i++) { 3846 if (option.getHandleListener() != null && option.getHandleListener().isCanceled()) { 3847 break; 3848 } 3849 3850 TCustomSqlStatement stmt = sqlparser.getSqlstatements().get(i); 3851 3852 if (option.isIgnoreTopSelect()) { 3853 if ((stmt instanceof TSelectSqlStatement && ((TSelectSqlStatement)stmt).getIntoClause() == null && ((TSelectSqlStatement)stmt).getIntoTableClause() == null ) || stmt instanceof TRedshiftDeclare 3854 || stmt instanceof TRedshiftDeclare) { 3855 continue; 3856 } 3857 } 3858 3859 if (stmt.getErrorCount() == 0) { 3860 if (stmt.getParentStmt() == null) { 3861 if (!(stmt instanceof TCreateViewSqlStatement) && !(stmt instanceof TCreateStageStmt) 3862 && !(stmt instanceof TCreateExternalDataSourceStmt) 3863 && !(stmt instanceof TCreateViewSqlStatement) && !(stmt instanceof TMssqlDeclare) 3864 && !(stmt instanceof TMssqlCreateFunction 3865 && ((TMssqlCreateFunction) stmt).getReturnTableDefinitions() != null)) { 3866 boolean listen = false; 3867 if (option.getHandleListener() != null && !accessedStatements.contains(stmt)) { 3868 option.getHandleListener().startAnalyzeStatment(stmt); 3869 listen = true; 3870 } 3871 analyzeCustomSqlStmt(stmt); 3872 if (listen && option.getHandleListener() != null) { 3873 option.getHandleListener().endAnalyzeStatment(stmt); 3874 } 3875 } 3876 } 3877 } 3878 } 3879 3880 if (option.getHandleListener() != null) { 3881 option.getHandleListener().endAnalyzeDataFlow(sqlparser); 3882 } 3883 3884 // Finalize pipelined function stitching after all passes 3885 if (!modelManager.getPendingPipelinedCallSites().isEmpty() && sqlparser.getSqlstatements().size() > 0 && pipelinedAnalyzer != null) { 3886 TCustomSqlStatement lastStmt = sqlparser.getSqlstatements().get(sqlparser.getSqlstatements().size() - 1); 3887 stmtStack.push(lastStmt); 3888 try { 3889 pipelinedAnalyzer.stitchPendingCallSites(); 3890 } catch (Exception ex) { 3891 // Don't let pipelined stitching failure break main flow 3892 } finally { 3893 stmtStack.pop(); 3894 } 3895 } 3896 } catch (Throwable e) { 3897 logger.error("analyze sql failed.", e); 3898 ErrorInfo errorInfo = new ErrorInfo(); 3899 errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR); 3900 if (e.getMessage() == null) { 3901 if (e.getStackTrace() != null && e.getStackTrace().length > 0) { 3902 errorInfo.setErrorMessage(e.getClass().getSimpleName() + ": " + e.getStackTrace()[0].toString()); 3903 } else { 3904 errorInfo.setErrorMessage(e.getClass().getSimpleName()); 3905 } 3906 } else { 3907 errorInfo.setErrorMessage(e.getClass().getSimpleName() + ": " + e.getMessage()); 3908 } 3909 errorInfo.fillInfo(this); 3910 errorInfos.add(errorInfo); 3911 } 3912 3913 } 3914 3915 private boolean hasDb2ReturnStmt(TCreateFunctionStmt stmt) { 3916 if (stmt.getReturnStmt() != null) { 3917 return true; 3918 } 3919 if (stmt.getBodyStatements() != null) { 3920 for (int i = 0; i < stmt.getBodyStatements().size(); i++) { 3921 if(stmt.getBodyStatements().get(i) instanceof TDb2ReturnStmt) { 3922 return true; 3923 } 3924 } 3925 } 3926 return false; 3927 } 3928 3929 3930 /** 3931 * The statements that a Snowflake {@code USE <database>} left with no current 3932 * schema: everything after such a switch, up to the next schema switch. 3933 * 3934 * <p>Computed once, in statement order, because the analysis below walks the 3935 * list several times and replays the {@code USE} statements in every pass. A 3936 * running flag would still be set when a later pass reached a statement that 3937 * PRECEDES the switch, and would strip that statement's already-exact 3938 * qualification. Keyed by statement, the answer does not depend on the pass. 3939 * 3940 * <p>Only Snowflake can populate this, so no other vendor's output moves. 3941 * See GitHub #715 / MantisBT 4677. 3942 */ 3943 private Set<TCustomSqlStatement> collectSchemaUnsetStatements(TGSqlParser sqlparser) { 3944 Set<TCustomSqlStatement> unset = new HashSet<TCustomSqlStatement>(); 3945 if (sqlparser == null || sqlparser.getSqlstatements() == null) { 3946 return unset; 3947 } 3948 boolean schemaUnset = false; 3949 for (int i = 0; i < sqlparser.getSqlstatements().size(); i++) { 3950 TCustomSqlStatement stmt = sqlparser.getSqlstatements().get(i); 3951 if (stmt == null) { 3952 continue; 3953 } 3954 if (schemaUnset) { 3955 unset.add(stmt); 3956 } 3957 if (stmt.dbvendor != EDbVendor.dbvsnowflake) { 3958 continue; 3959 } 3960 if (stmt instanceof TUseSchema || stmt.sqlstatementtype == ESqlStatementType.sstSetSchema) { 3961 schemaUnset = false; 3962 } else if (stmt instanceof TUseDatabase) { 3963 schemaUnset = !((TUseDatabase) stmt).isSchema(); 3964 } 3965 } 3966 return unset; 3967 } 3968 3969 private void analyzeCustomSqlStmt(TCustomSqlStatement stmt) { 3970 if (stmt != null && dynamicFoldDepth == 0 && anyDynamicTemplateMarked) { 3971 // A STATIC statement analyzed after a fold marked something: its references 3972 // withdraw marks on shared nodes (see the field javadoc). 3973 applyStaticTemplateVetoes(stmt); 3974 } 3975 if (!accessedStatements.contains(stmt)) { 3976 accessedStatements.add(stmt); 3977 } else if (!(stmt instanceof TUseDatabase || stmt instanceof TUseSchema)) { 3978 return; 3979 } 3980 3981 ArrayList<TSyntaxError> errors = stmt.getSyntaxHints(); 3982 if (errors != null && !errors.isEmpty()) { 3983 for (int i = 0; i < errors.size(); i++) { 3984 TSyntaxError error = errors.get(i); 3985 ErrorInfo errorInfo = new ErrorInfo(); 3986 errorInfo.setErrorType(ErrorInfo.SYNTAX_HINT); 3987 errorInfo.setErrorMessage(getErrorMessage(error, ErrorInfo.SYNTAX_HINT)); 3988 errorInfo.setStartPosition(new Pair3<Long, Long, String>(error.lineNo, error.columnNo, 3989 ModelBindingManager.getGlobalHash())); 3990 String[] segments = error.tokentext.split("\n"); 3991 if (segments.length == 1) { 3992 errorInfo.setEndPosition(new Pair3<Long, Long, String>(error.lineNo, 3993 error.columnNo + error.tokentext.length(), ModelBindingManager.getGlobalHash())); 3994 } else { 3995 errorInfo.setEndPosition(new Pair3<Long, Long, String>(error.lineNo + segments.length - 1, 3996 (long) segments[segments.length - 1].length() + 1, ModelBindingManager.getGlobalHash())); 3997 } 3998 errorInfo.fillInfo(this); 3999 errorInfos.add(errorInfo); 4000 } 4001 } 4002 4003 if (option.getAnalyzeMode() == AnalyzeMode.dynamic) { 4004 if (!(stmt instanceof TStoredProcedureSqlStatement 4005 || stmt instanceof TExecuteSqlStatement 4006 || stmt instanceof TMssqlExecute 4007 || stmt instanceof TExecImmeStmt)) { 4008 return; 4009 } 4010 } 4011 4012 if(DlineageUtil.getTopStmt(stmt) == stmt){ 4013 modelManager.collectSqlHash(stmt); 4014 } 4015 4016 try { 4017 if (stmt instanceof TUseDatabase) { 4018 if (((TUseDatabase) stmt).getDatabaseName() != null) { 4019 // Snowflake: a database switch invalidates the schema carried 4020 // over from the PREVIOUS database. Leaving it attached makes 4021 // later unqualified names resolve as <newdb>.<oldschema>, a 4022 // pair that need not exist. Clear rather than assign PUBLIC: 4023 // that the old schema is invalid is provable from the 4024 // statement, that PUBLIC exists in the new database is not. 4025 // Matches TDDLSQLEnv.analyzeUseDatabase and 4026 // DatabaseContextTracker.processUseDatabase. See GitHub #715. 4027 // 4028 // The clearing happens HERE, in the statement-by-statement 4029 // analysis, and NOT in TUseDatabase.doParseStatement: the whole 4030 // batch is parsed before any of it is analyzed and one TSQLEnv 4031 // is shared across it, so a parse-time clear would also apply 4032 // to the statements BEFORE the switch and would downgrade their 4033 // already-exact names to unqualified ones. The env fallback is 4034 // handled per statement instead - see 4035 // ModelBindingManager.setSchemaUnsetStatements. 4036 if (stmt.dbvendor == EDbVendor.dbvsnowflake 4037 && !((TUseDatabase) stmt).isSchema()) { 4038 ModelBindingManager.removeGlobalSchema(); 4039 } 4040 ModelBindingManager.setGlobalDatabase(((TUseDatabase) stmt).getDatabaseName().toString()); 4041 } 4042 } else if (stmt instanceof TUseSchema) { 4043 if (((TUseSchema) stmt).getSchemaName() != null) { 4044 String schemaName = ((TUseSchema) stmt).getSchemaName().toString(); 4045 List<String> splits = SQLUtil.parseNames(schemaName); 4046 if (splits.size() == 1) { 4047 ModelBindingManager.setGlobalSchema(schemaName); 4048 } else if (splits.size() > 1) { 4049 ModelBindingManager.setGlobalSchema(splits.get(splits.size() - 1)); 4050 ModelBindingManager.setGlobalDatabase(splits.get(splits.size() - 2)); 4051 } 4052 } 4053 } else if (stmt instanceof TPlsqlRecordTypeDefStmt) { 4054 this.stmtStack.push(stmt); 4055 this.analyzePlsqlRecordTypeDefStmt((TPlsqlRecordTypeDefStmt) stmt); 4056 this.stmtStack.pop(); 4057 } else if (stmt instanceof TPlsqlCreateType_Placeholder) { 4058 this.stmtStack.push(stmt); 4059 TPlsqlCreateType_Placeholder placeholder = (TPlsqlCreateType_Placeholder) stmt; 4060 if (placeholder.getObjectStatement() != null && pipelinedAnalyzer != null) { 4061 this.pipelinedAnalyzer.indexObjectType(placeholder.getObjectStatement()); 4062 } 4063 if (placeholder.getNestedTableStatement() != null) { 4064 if (pipelinedAnalyzer != null) { 4065 this.pipelinedAnalyzer.indexCollectionType(placeholder.getNestedTableStatement()); 4066 } 4067 this.analyzePlsqlTableTypeDefStmt(placeholder.getNestedTableStatement()); 4068 } 4069 this.stmtStack.pop(); 4070 } else if (stmt instanceof TPlsqlCreateType) { 4071 this.stmtStack.push(stmt); 4072 if (pipelinedAnalyzer != null) { 4073 this.pipelinedAnalyzer.indexObjectType((TPlsqlCreateType) stmt); 4074 } 4075 this.stmtStack.pop(); 4076 } else if (stmt instanceof TPlsqlTableTypeDefStmt) { 4077 this.stmtStack.push(stmt); 4078 this.analyzePlsqlTableTypeDefStmt((TPlsqlTableTypeDefStmt) stmt); 4079 if (pipelinedAnalyzer != null) { 4080 this.pipelinedAnalyzer.indexCollectionType((TPlsqlTableTypeDefStmt) stmt); 4081 } 4082 this.stmtStack.pop(); 4083 } else if (stmt instanceof TStoredProcedureSqlStatement) { 4084 this.stmtStack.push(stmt); 4085 this.analyzeStoredProcedureStmt((TStoredProcedureSqlStatement) stmt); 4086 this.stmtStack.pop(); 4087 } else if (stmt instanceof TCreateTableSqlStatement) { 4088 stmtStack.push(stmt); 4089 analyzeCreateTableStmt((TCreateTableSqlStatement) stmt); 4090 stmtStack.pop(); 4091 } else if (stmt instanceof TCreateStageStmt) { 4092 stmtStack.push(stmt); 4093 analyzeCreateStageStmt((TCreateStageStmt) stmt); 4094 stmtStack.pop(); 4095 } else if (stmt instanceof TCreateExternalDataSourceStmt) { 4096 stmtStack.push(stmt); 4097 analyzeCreateExternalDataSourceStmt((TCreateExternalDataSourceStmt) stmt); 4098 stmtStack.pop(); 4099 } else if (stmt instanceof TCreateStreamStmt) { 4100 stmtStack.push(stmt); 4101 analyzeCreateStreamStmt((TCreateStreamStmt) stmt); 4102 stmtStack.pop(); 4103 } else if (stmt instanceof TSelectSqlStatement) { 4104 analyzeSelectStmt((TSelectSqlStatement) stmt); 4105 } else if (stmt instanceof TDropTableSqlStatement) { 4106 stmtStack.push(stmt); 4107 analyzeDropTableStmt((TDropTableSqlStatement) stmt); 4108 stmtStack.pop(); 4109 } else if (stmt instanceof TTruncateStatement) { 4110 stmtStack.push(stmt); 4111 analyzeTruncateTableStmt((TTruncateStatement) stmt); 4112 stmtStack.pop(); 4113 } else if (stmt instanceof TCreateMaterializedSqlStatement) { 4114 stmtStack.push(stmt); 4115 TCreateMaterializedSqlStatement view = (TCreateMaterializedSqlStatement) stmt; 4116 analyzeCreateViewStmt(view, view.getSubquery(), view.getViewAliasClause(), view.getViewName()); 4117 stmtStack.pop(); 4118 } else if (stmt instanceof TCreateViewSqlStatement) { 4119 stmtStack.push(stmt); 4120 TCreateViewSqlStatement view = (TCreateViewSqlStatement) stmt; 4121 analyzeCreateViewStmt(view, view.getSubquery(), view.getViewAliasClause(), view.getViewName()); 4122 stmtStack.pop(); 4123 } else if(stmt instanceof TDb2SqlVariableDeclaration){ 4124 stmtStack.push(stmt); 4125 analyzeDb2Declare((TDb2SqlVariableDeclaration)stmt); 4126 stmtStack.pop(); 4127 } else if (stmt instanceof TMssqlCreateType) { 4128 stmtStack.push(stmt); 4129 TMssqlCreateType createType = (TMssqlCreateType) stmt; 4130 analyzeMssqlCreateType(createType); 4131 stmtStack.pop(); 4132 } else if (stmt instanceof TMssqlDeclare) { 4133 stmtStack.push(stmt); 4134 TMssqlDeclare declare = (TMssqlDeclare) stmt; 4135 analyzeMssqlDeclare(declare); 4136 stmtStack.pop(); 4137 } else if (stmt instanceof TInsertSqlStatement) { 4138 stmtStack.push(stmt); 4139 TInsertSqlStatement insert = (TInsertSqlStatement)stmt; 4140 analyzeInsertStmt(insert); 4141 if(insert.getMultiInsertStatements()!=null) { 4142 for(int i=0;i<insert.getMultiInsertStatements().size();i++) { 4143 analyzeInsertStmt(insert.getMultiInsertStatements().get(i)); 4144 } 4145 } 4146 stmtStack.pop(); 4147 } else if (stmt instanceof TRedshiftCopy) { 4148 stmtStack.push(stmt); 4149 analyzeRedshiftCopyStmt((TRedshiftCopy) stmt); 4150 stmtStack.pop(); 4151 } else if (stmt instanceof TSnowflakeCopyIntoStmt) { 4152 stmtStack.push(stmt); 4153 analyzeCopyIntoStmt((TSnowflakeCopyIntoStmt) stmt); 4154 stmtStack.pop(); 4155 } else if (stmt instanceof TUnloadStmt) { 4156 stmtStack.push(stmt); 4157 analyzeUnloadStmt((TUnloadStmt) stmt); 4158 stmtStack.pop(); 4159 } else if (stmt instanceof TUpdateSqlStatement) { 4160 stmtStack.push(stmt); 4161 analyzeUpdateStmt((TUpdateSqlStatement) stmt); 4162 stmtStack.pop(); 4163 } else if (stmt instanceof TMergeSqlStatement) { 4164 stmtStack.push(stmt); 4165 analyzeMergeStmt((TMergeSqlStatement) stmt); 4166 stmtStack.pop(); 4167 } else if (stmt instanceof TDeleteSqlStatement) { 4168 stmtStack.push(stmt); 4169 analyzeDeleteStmt((TDeleteSqlStatement) stmt); 4170 stmtStack.pop(); 4171 } else if (stmt instanceof TCursorDeclStmt) { 4172 stmtStack.push(stmt); 4173 analyzeCursorDeclStmt((TCursorDeclStmt) stmt); 4174 stmtStack.pop(); 4175 } else if (stmt instanceof TFetchStmt) { 4176 stmtStack.push(stmt); 4177 analyzeFetchStmt((TFetchStmt) stmt); 4178 stmtStack.pop(); 4179 } else if (stmt instanceof TMssqlFetch) { 4180 stmtStack.push(stmt); 4181 analyzeFetchStmt((TMssqlFetch) stmt); 4182 stmtStack.pop(); 4183 } else if (stmt instanceof TForStmt) { 4184 stmtStack.push(stmt); 4185 analyzeForStmt((TForStmt) stmt); 4186 stmtStack.pop(); 4187 } else if (stmt instanceof TOpenforStmt) { 4188 stmtStack.push(stmt); 4189 analyzeOpenForStmt((TOpenforStmt) stmt); 4190 stmtStack.pop(); 4191 } else if (stmt instanceof TLoopStmt) { 4192 stmtStack.push(stmt); 4193 analyzeLoopStmt((TLoopStmt) stmt); 4194 stmtStack.pop(); 4195 } else if (stmt instanceof TAssignStmt) { 4196 stmtStack.push(stmt); 4197 analyzeAssignStmt((TAssignStmt) stmt); 4198 stmtStack.pop(); 4199 } else if (stmt instanceof TSetStmt) { 4200 stmtStack.push(stmt); 4201 analyzeSetStmt((TSetStmt) stmt); 4202 stmtStack.pop(); 4203 } else if (stmt instanceof TMssqlSet) { 4204 stmtStack.push(stmt); 4205 analyzeMssqlSetStmt((TMssqlSet) stmt); 4206 stmtStack.pop(); 4207 } else if (stmt instanceof TVarDeclStmt) { 4208 stmtStack.push(stmt); 4209 analyzeVarDeclStmt((TVarDeclStmt) stmt); 4210 stmtStack.pop(); 4211 } else if (stmt instanceof TCreateDatabaseSqlStatement) { 4212 stmtStack.push(stmt); 4213 analyzeCloneDatabaseStmt((TCreateDatabaseSqlStatement) stmt); 4214 stmtStack.pop(); 4215 } else if (stmt instanceof TCreateSchemaSqlStatement) { 4216 stmtStack.push(stmt); 4217 analyzeCloneSchemaStmt((TCreateSchemaSqlStatement) stmt); 4218 stmtStack.pop(); 4219 } else if (stmt instanceof TAlterTableStatement) { 4220 stmtStack.push(stmt); 4221 analyzeAlterTableStmt((TAlterTableStatement) stmt); 4222 stmtStack.pop(); 4223 } else if (stmt instanceof TAlterViewStatement) { 4224 stmtStack.push(stmt); 4225 analyzeAlterViewStmt((TAlterViewStatement) stmt); 4226 stmtStack.pop(); 4227 } else if (stmt instanceof TRenameStmt) { 4228 stmtStack.push(stmt); 4229 analyzeRenameStmt((TRenameStmt) stmt); 4230 stmtStack.pop(); 4231 } else if (stmt instanceof TCreateSynonymStmt) { 4232 stmtStack.push(stmt); 4233 analyzeCreateSynonymStmt((TCreateSynonymStmt) stmt); 4234 stmtStack.pop(); 4235 } else if (stmt instanceof TLoadDataStmt) { 4236 stmtStack.push(stmt); 4237 analyzeLoadDataStmt((TLoadDataStmt) stmt); 4238 stmtStack.pop(); 4239 } else if (stmt instanceof THiveLoad) { 4240 stmtStack.push(stmt); 4241 analyzeHiveLoadStmt((THiveLoad) stmt); 4242 stmtStack.pop(); 4243 } else if (stmt instanceof TDb2ReturnStmt) { 4244 stmtStack.push(stmt); 4245 analyzeDb2ReturnStmt((TDb2ReturnStmt) stmt); 4246 stmtStack.pop(); 4247 } else if (stmt instanceof TReturnStmt) { 4248 stmtStack.push(stmt); 4249 analyzeReturnStmt((TReturnStmt) stmt); 4250 stmtStack.pop(); 4251 } else if (stmt instanceof TMssqlReturn) { 4252 stmtStack.push(stmt); 4253 analyzeMssqlReturnStmt((TMssqlReturn) stmt); 4254 stmtStack.pop(); 4255 } else if (stmt instanceof TMssqlUpdateText) { 4256 stmtStack.push(stmt); 4257 analyzeMssqlUpdateTextStmt((TMssqlUpdateText) stmt); 4258 stmtStack.pop(); 4259 } else if (stmt instanceof TMssqlWriteText) { 4260 stmtStack.push(stmt); 4261 analyzeMssqlWriteTextStmt((TMssqlWriteText) stmt); 4262 stmtStack.pop(); 4263 } else if (stmt instanceof TMssqlReadText) { 4264 stmtStack.push(stmt); 4265 analyzeMssqlReadTextStmt((TMssqlReadText) stmt); 4266 stmtStack.pop(); 4267 } else if (stmt instanceof TExecuteSqlStatement) { 4268 String sqlText = ((TExecuteSqlStatement) stmt).getPreparedSqlText(); 4269 if(sqlText == null) { 4270 sqlText = ((TExecuteSqlStatement) stmt).getSqlText(); 4271 } 4272 if (!option.isAnalyzeDynamicSql()) { 4273 if (sqlText != null 4274 || ((TExecuteSqlStatement) stmt).getStmtString() != null) { 4275 modelManager.collectDynamicSqlHash(stmt); 4276 recordDynamicSqlSite(stmt, DynamicSqlSite.Kind.OTHER, 4277 DynamicSqlSite.Status.UNRESOLVED, 4278 "dynamic SQL inner analysis disabled"); 4279 } 4280 } else if (sqlText != null) { 4281 modelManager.collectDynamicSqlHash(stmt); 4282 TGSqlParser sqlparser = createSqlParser(option.getVendor()); 4283 sqlparser.sqltext = SQLUtil.trimColumnStringQuote(sqlText); 4284 int result = sqlparser.parse(); 4285 if (result != 0) { 4286 errors = sqlparser.getSyntaxErrors(); 4287 if (errors != null && !errors.isEmpty()) { 4288 for (int i = 0; i < errors.size(); i++) { 4289 TSyntaxError error = errors.get(i); 4290 ErrorInfo errorInfo = new ErrorInfo(); 4291 errorInfo.setErrorType(ErrorInfo.SYNTAX_ERROR); 4292 errorInfo.setErrorMessage(getErrorMessage(error, ErrorInfo.SYNTAX_ERROR)); 4293 errorInfo.setStartPosition(new Pair3<Long, Long, String>(error.lineNo, error.columnNo, 4294 ModelBindingManager.getGlobalHash())); 4295 String[] segments = error.tokentext.split("\n"); 4296 if (segments.length == 1) { 4297 errorInfo.setEndPosition(new Pair3<Long, Long, String>(error.lineNo, 4298 error.columnNo + error.tokentext.length(), 4299 ModelBindingManager.getGlobalHash())); 4300 } else { 4301 errorInfo.setEndPosition( 4302 new Pair3<Long, Long, String>(error.lineNo + segments.length - 1, 4303 (long) segments[segments.length - 1].length() + 1, 4304 ModelBindingManager.getGlobalHash())); 4305 } 4306 errorInfo.fillInfo(this); 4307 errorInfos.add(errorInfo); 4308 } 4309 } 4310 } else if (sqlparser.sqlstatements != null) { 4311 dynamicFoldDepth++; 4312 try { 4313 for (int i = 0; i < sqlparser.sqlstatements.size(); i++) { 4314 analyzeCustomSqlStmt(sqlparser.sqlstatements.get(i)); 4315 } 4316 } finally { 4317 dynamicFoldDepth--; 4318 } 4319 } 4320 } 4321 else if (((TExecuteSqlStatement) stmt).getStmtString() != null) { 4322 modelManager.collectDynamicSqlHash(stmt); 4323 } 4324 } else if (stmt instanceof TMssqlExecute) { 4325 TMssqlExecute executeStmt = (TMssqlExecute)stmt; 4326 if (!option.isAnalyzeDynamicSql() 4327 && expandsDynamicSql(executeStmt)) { 4328 modelManager.collectDynamicSqlHash(stmt); 4329 recordDynamicSqlSite(stmt, dynamicKindOf(executeStmt), 4330 DynamicSqlSite.Status.UNRESOLVED, 4331 "dynamic SQL inner analysis disabled"); 4332 } else if (executeStmt.getSqlText() != null) { 4333 modelManager.collectDynamicSqlHash(stmt); 4334 boolean evalHandled = false; 4335 DynamicSqlTrustMode trustMode = option.getDynamicSqlTrustMode(); 4336 boolean nonLiteralDynamicArg = !isLiteralDynamicArg(executeStmt); 4337 gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite evalSite = null; 4338 if (nonLiteralDynamicArg) { 4339 evalSite = evaluatedDynamicSqlFor(stmt); 4340 // Abstract evaluation fully materialized one or more strings 4341 // (constant relation rows and/or literal call bindings). Analyze 4342 // every distinct concrete variant in stable evidence order. 4343 evalHandled = analyzeEvaluatedDynamicSqlVariants(stmt, 4344 dynamicKindOf(executeStmt), true); 4345 } 4346 boolean evaluatorProofUnavailable = nonLiteralDynamicArg 4347 && (evalSite == null || evalSite.sqlText == null 4348 || evalSite.provenanceIncomplete); 4349 boolean provenanceIncomplete = trustMode == DynamicSqlTrustMode.SHADOW 4350 && evaluatorProofUnavailable; 4351 // Hole evidence is a property of the ARGUMENT's provenance as the 4352 // evaluator saw it (which variable, assigned from what, where), not 4353 // of the text analyzed below - the evaluator and the legacy fold 4354 // render the same expression differently (@DeleteWhere vs a dropped 4355 // tail), but its holes are the same holes. Hole SPANS inside the 4356 // analyzed text remain text-gated (markDynamicTemplateEndpoints). 4357 // Whenever the evaluator saw the node its verdict stands, including 4358 // "not known" (-1). Only without an evaluator view (no enclosing 4359 // procedure) does the legacy rule apply: SHADOW says unknown, LEGACY 4360 // keeps its historical zero. 4361 DynamicHoleEvidence holeEvidence = dynamicHoleEvidence(evalSite); 4362 int trustHoleCount = evalSite != null ? holeEvidence.count 4363 : (provenanceIncomplete ? -1 : 0); 4364 boolean trustHoleCountExact = evalSite != null ? holeEvidence.exact 4365 : !provenanceIncomplete; 4366 if (!evalHandled) { 4367 TGSqlParser sqlparser = createSqlParser(option.getVendor()); 4368 sqlparser.sqltext = ((TMssqlExecute) stmt).getSqlText(); 4369 Set<Relationship> shadowBefore = provenanceIncomplete 4370 && trustMode == DynamicSqlTrustMode.SHADOW 4371 ? snapshotDynamicRelationships() : null; 4372 int relsBeforeDynamic = modelManager.getRelations().length; 4373 int dynamicSitesBefore = dynamicSqlSites.size(); 4374 // A data-driven argument (variable / expression) is at most PARTIALLY folded: 4375 // GSP keeps the literal pieces but drops the unresolved trailing @var, leaving a 4376 // legitimately truncated fragment (e.g. "... WHERE " with no predicate -> end of 4377 // input). A parse failure on such text says nothing about the real SQL, so we must 4378 // not surface it as a unit SYNTAX_ERROR — that would discard the whole procedure's 4379 // lineage even though the outer parse succeeded. Only a true compile-time literal 4380 // EXEC('...') surfaces inner parse failures (mirrors the EXECUTE IMMEDIATE 4381 // isDynamicSQLPartial() guard below). 4382 boolean literalArg = isLiteralDynamicArg(executeStmt); 4383 // A partial fold leaves the unresolved @var / @param remnant in the folded 4384 // text. Decided BEFORE the inner analysis so the endpoints this site 4385 // introduces can be told apart from the ones that already existed, and so 4386 // a fully-resolved fold pays nothing (snapshot returns null). 4387 boolean maybePartialFold = !literalArg 4388 && ((TMssqlExecute) stmt).getSqlText().matches("(?s).*@\\w+.*"); 4389 // The conservative @\w+ test is not the only way a hole can be present: 4390 // the evaluator may know this fold has value-unknown holes whose legacy 4391 // rendering carries no '@' (UPPER(@X) folds to the bare function name). 4392 // Snapshot whenever EITHER tier could fire, so provenance is never wasted. 4393 Set<Table> templateTablesBefore = snapshotTemplateCandidateTables(maybePartialFold 4394 || hasValueUnknownHole(evalSite)); 4395 boolean foldStatementsAnalyzed = false; 4396 int result = sqlparser.parse(); 4397 if (result != 0) { 4398 errors = literalArg ? sqlparser.getSyntaxErrors() : null; 4399 if (errors != null && !errors.isEmpty()) { 4400 for (int i = 0; i < errors.size(); i++) { 4401 TSyntaxError error = errors.get(i); 4402 ErrorInfo errorInfo = new ErrorInfo(); 4403 errorInfo.setErrorType(ErrorInfo.SYNTAX_ERROR); 4404 errorInfo.setErrorMessage(getErrorMessage(error, ErrorInfo.SYNTAX_ERROR)); 4405 errorInfo.setStartPosition(new Pair3<Long, Long, String>(error.lineNo, error.columnNo, 4406 ModelBindingManager.getGlobalHash())); 4407 String[] segments = error.tokentext.split("\n"); 4408 if (segments.length == 1) { 4409 errorInfo.setEndPosition(new Pair3<Long, Long, String>(error.lineNo, 4410 error.columnNo + error.tokentext.length(), 4411 ModelBindingManager.getGlobalHash())); 4412 } else { 4413 errorInfo.setEndPosition( 4414 new Pair3<Long, Long, String>(error.lineNo + segments.length - 1, 4415 (long) segments[segments.length - 1].length() + 1, 4416 ModelBindingManager.getGlobalHash())); 4417 } 4418 errorInfo.fillInfo(this); 4419 errorInfos.add(errorInfo); 4420 } 4421 } 4422 } else if (sqlparser.sqlstatements != null) { 4423 foldStatementsAnalyzed = true; 4424 dynamicFoldDepth++; 4425 try { 4426 for (int i = 0; i < sqlparser.sqlstatements.size(); i++) { 4427 analyzeCustomSqlStmt(sqlparser.sqlstatements.get(i)); 4428 } 4429 } finally { 4430 dynamicFoldDepth--; 4431 } 4432 } 4433 // getSqlText()!=null means GSP folded the dynamic text. RESOLVED only when the argument 4434 // was a compile-time literal AND it parsed; a variable/expression argument is data-driven 4435 // (often only partially folded, e.g. an unresolved @parameter left in) -> UNRESOLVED. 4436 // NOTE: a variable 4437 // argument whose folded text fails to parse deliberately stays UNRESOLVED, 4438 // not PARSE_ERROR — the fold is flow-insensitive (the variable may hold a 4439 // different value at runtime), so a parse failure on folded text says 4440 // nothing definitive about the real SQL. Only a compile-time literal 4441 // argument earns PARSE_ERROR. 4442 // Land the placeholder verdict on the model NODES, not only on the site 4443 // status: exact hole spans when the evaluator described this very text, 4444 // literal-aligned spans when it described the same fold's other rendering, 4445 // otherwise the conservative @\w+ scan of the legacy fold. 4446 if (foldStatementsAnalyzed) { 4447 markDynamicTemplateEndpoints(templateTablesBefore, evalSite, sqlparser); 4448 } 4449 int[] dynCounts = classifyDynamicSiteLineage(relsBeforeDynamic, maybePartialFold); 4450 int unprovenCount = countNewDynamicRelationships(shadowBefore); 4451 if (!literalArg) { 4452 // Truthful status: a fold that yielded clean lineage into base tables 4453 // — or a pure dynamic SELECT whose only product is its projection — 4454 // is RESOLVED (with counts). Placeholder-contaminated, partial-target, 4455 // or empty outcomes stay UNRESOLVED with an accurate reason. 4456 boolean usable = dynCounts[2] == 0 4457 && (dynCounts[0] > 0 || (dynCounts[1] == 0 && dynCounts[3] > 0)); 4458 if (provenanceIncomplete && trustMode == DynamicSqlTrustMode.SHADOW) { 4459 recordDynamicSqlSite(stmt, dynamicKindOf(executeStmt), 4460 unprovenCount > 0 ? DynamicSqlSite.Status.PARTIAL 4461 : DynamicSqlSite.Status.UNRESOLVED, 4462 dynamicTrustReason("SHADOW retained legacy publication behavior", 4463 unprovenCount, trustHoleCount, trustHoleCountExact), 4464 dynCounts[0], dynCounts[1], unprovenCount, 4465 trustHoleCount, trustHoleCountExact, trustMode, 4466 relsBeforeDynamic, dynamicSitesBefore, null, null, 4467 holeEvidence.fragments, holeEvidence.materializedText); 4468 } else { 4469 recordDynamicSqlSite(stmt, dynamicKindOf(executeStmt), 4470 usable ? DynamicSqlSite.Status.RESOLVED : DynamicSqlSite.Status.UNRESOLVED, 4471 usable ? null 4472 : (dynCounts[2] != 0 4473 ? "dynamic SQL references a runtime-built object name; lineage source is a placeholder" 4474 : dynamicReasonOf(executeStmt)), 4475 dynCounts[0], dynCounts[1], 0, 4476 trustHoleCount, trustHoleCountExact, trustMode, 4477 relsBeforeDynamic, dynamicSitesBefore, null, null, 4478 holeEvidence.fragments, holeEvidence.materializedText); 4479 } 4480 } else { 4481 recordDynamicSqlSite(stmt, dynamicKindOf(executeStmt), 4482 result != 0 ? DynamicSqlSite.Status.PARSE_ERROR : DynamicSqlSite.Status.RESOLVED, 4483 result != 0 ? "inner dynamic SQL failed to parse" : null, 4484 dynCounts[0], dynCounts[1], 0, 0, true, trustMode, 4485 relsBeforeDynamic, dynamicSitesBefore, null, null, 4486 Collections.<UnresolvedFragment>emptyList(), executeStmt.getSqlText()); 4487 } 4488 } 4489 } else if (executeStmt.getModuleName() != null) { 4490 // getSqlText()==null here: a folded literal would have set sqlText. So this is either an 4491 // opaque sp_executesql (data-driven argument), a dynamic proc name (EXEC @var), or an 4492 // ordinary static procedure call. Record the dynamic ones; leave static EXEC proc alone. 4493 boolean evalHandledSite = false; 4494 if (isSpExecutesql(executeStmt.getModuleName())) { 4495 evalHandledSite = analyzeEvaluatedDynamicSqlVariants(stmt, 4496 DynamicSqlSite.Kind.SP_EXECUTESQL, false); 4497 if (!evalHandledSite) { 4498 recordUnresolvedDynamicSqlSite(stmt, executeStmt, 4499 DynamicSqlSite.Kind.SP_EXECUTESQL); 4500 } 4501 } else if (executeStmt.getModuleName().toString().startsWith("@")) { 4502 // EXEC @var is SQL Server's @module_name_var form: the variable 4503 // holds a PROCEDURE NAME, not SQL text (parenthesized EXEC(@var) 4504 // arrives as metExecStringCmd below) - never evaluate it as SQL. 4505 recordDynamicSqlSite(stmt, DynamicSqlSite.Kind.OTHER, 4506 DynamicSqlSite.Status.UNRESOLVED, "EXEC target procedure name is a runtime variable"); 4507 } 4508 if (!evalHandledSite) { 4509 // When evaluation already analyzed the materialized batch, the 4510 // legacy path would re-execute the same argument and duplicate 4511 // every process/edge - skip it. 4512 stmtStack.push(stmt); 4513 analyzeMssqlExecute(executeStmt); 4514 stmtStack.pop(); 4515 } 4516 } else if (executeStmt.getExecType() == TBaseType.metExecStringCmd) { 4517 // EXEC(<expr>) whose string was not folded: try the no-bindings abstract 4518 // evaluation (template/constant-built strings); otherwise honestly UNRESOLVED. 4519 if (!analyzeEvaluatedDynamicSqlVariants(stmt, 4520 DynamicSqlSite.Kind.EXEC_STRING, false)) { 4521 recordUnresolvedDynamicSqlSite(stmt, executeStmt, 4522 DynamicSqlSite.Kind.EXEC_STRING); 4523 } 4524 } 4525 } else if (stmt instanceof TExecImmeStmt) { 4526 TExecImmeStmt execImmeStmt = (TExecImmeStmt) stmt; 4527 modelManager.collectDynamicSqlHash(stmt); 4528 if (!option.isAnalyzeDynamicSql()) { 4529 recordDynamicSqlSite(stmt, 4530 DynamicSqlSite.Kind.EXECUTE_IMMEDIATE, 4531 DynamicSqlSite.Status.UNRESOLVED, 4532 "dynamic SQL inner analysis disabled"); 4533 return; 4534 } 4535 synchronized (DataFlowAnalyzer.class) { 4536 int relsBeforeDynamic = modelManager.getRelations().length; 4537 int dynamicSitesBefore = dynamicSqlSites.size(); 4538 boolean analyzedInner = false; 4539 boolean innerParseFailed = false; 4540 // PL/SQL call-site binding (plan Phase 5): when the token fold left 4541 // this site partial or empty, interpret the file's statements (plus 4542 // literal callers harvested from other files of a multi-file 4543 // analysis) so parameter-dependent templates can fold fully. 4544 // getDynamicStatements() below then reads the evaluated text. 4545 maybeEvaluatePlsqlDynamicSql(execImmeStmt); 4546 DynamicSqlTrustMode trustMode = option.getDynamicSqlTrustMode(); 4547 boolean legacyPartialFold = execImmeStmt.isDynamicSQLPartial(); 4548 boolean shadowIncomplete = legacyPartialFold 4549 && execImmeStmt.getEvaluatedDynamicSQLs().isEmpty(); 4550 Set<Relationship> shadowBefore = shadowIncomplete 4551 && trustMode == DynamicSqlTrustMode.SHADOW 4552 ? snapshotDynamicRelationships() : null; 4553 TStatementList stmts = execImmeStmt.getDynamicStatements(); 4554 if (stmts != null && stmts.size() > 0) { 4555 dynamicFoldDepth++; 4556 try { 4557 for (int i = 0; i < stmts.size(); i++) { 4558 analyzeCustomSqlStmt(stmts.get(i)); 4559 } 4560 } finally { 4561 dynamicFoldDepth--; 4562 } 4563 analyzedInner = true; 4564 } 4565 4566 // Only re-parse the raw dynamic SQL when getDynamicStatements() 4567 // produced nothing. getDynamicStatements() already parses and 4568 // analyzes the fragment with coordinates remapped back to the 4569 // original file position; re-parsing it here with a fresh parser 4570 // would analyze the same SQL a second time and emit duplicate 4571 // hints/lineage carrying fragment-relative (un-remapped) 4572 // coordinates. This fallback still covers the case where 4573 // getDynamicStatements() failed (e.g. dynamic SQL syntax error), 4574 // reporting the parse error below. 4575 String dynamicSql = (stmts == null || stmts.size() == 0) ? execImmeStmt.getDynamicSQL() : null; 4576 if (!SQLUtil.isEmpty(dynamicSql)) { 4577 TGSqlParser sqlparser = createSqlParser(option.getVendor()); 4578 sqlparser.sqltext = dynamicSql; 4579 int result = sqlparser.parse(); 4580 // This fresh parse runs the dynamic SQL un-padded, so its 4581 // coordinates are relative to the fragment (first char at 4582 // line 1). Shift them back to the position the fragment 4583 // occupies in the original file, mirroring the shift 4584 // TExecImmeStmt.getDynamicStatements() applies; otherwise 4585 // fallback parse errors for EXECUTE IMMEDIATE text located 4586 // later in the file would report fragment-relative lines. 4587 if (execImmeStmt.getDynamicStringExpr() != null 4588 && execImmeStmt.getDynamicStringExpr().getPlainTextLineNo() != -1) { 4589 int deltaLine = (int) execImmeStmt.getDynamicStringExpr().getPlainTextLineNo() - 1; 4590 int deltaColumn = (int) execImmeStmt.getDynamicStringExpr().getPlainTextColumnNo(); 4591 if ((deltaLine != 0 || deltaColumn != 0) && sqlparser.getSyntaxErrors() != null) { 4592 for (int ei = 0; ei < sqlparser.getSyntaxErrors().size(); ei++) { 4593 TSyntaxError dynErr = sqlparser.getSyntaxErrors().get(ei); 4594 if (dynErr == null) continue; 4595 boolean onFirstLine = (dynErr.lineNo == 1); 4596 dynErr.lineNo += deltaLine; 4597 if (onFirstLine) dynErr.columnNo += deltaColumn; 4598 } 4599 } 4600 } 4601 if (result != 0) { 4602 innerParseFailed = true; 4603 // a partially resolved value contains placeholder identifiers 4604 // for unknowable parts; a parse failure on such text says 4605 // nothing about the real SQL, so don't report it 4606 errors = execImmeStmt.isDynamicSQLPartial() ? null : sqlparser.getSyntaxErrors(); 4607 if (errors != null && !errors.isEmpty()) { 4608 for (int i = 0; i < errors.size(); i++) { 4609 TSyntaxError error = errors.get(i); 4610 ErrorInfo errorInfo = new ErrorInfo(); 4611 errorInfo.setErrorType(ErrorInfo.SYNTAX_ERROR); 4612 errorInfo.setErrorMessage(getErrorMessage(error, ErrorInfo.SYNTAX_ERROR)); 4613 errorInfo.setStartPosition(new Pair3<Long, Long, String>(error.lineNo, error.columnNo, 4614 ModelBindingManager.getGlobalHash())); 4615 String[] segments = error.tokentext.split("\n"); 4616 if (segments.length == 1) { 4617 errorInfo.setEndPosition(new Pair3<Long, Long, String>(error.lineNo, 4618 error.columnNo + error.tokentext.length(), 4619 ModelBindingManager.getGlobalHash())); 4620 } else { 4621 errorInfo.setEndPosition( 4622 new Pair3<Long, Long, String>(error.lineNo + segments.length - 1, 4623 (long) segments[segments.length - 1].length() + 1, 4624 ModelBindingManager.getGlobalHash())); 4625 } 4626 errorInfo.fillInfo(this); 4627 errorInfos.add(errorInfo); 4628 } 4629 } 4630 } else if (sqlparser.sqlstatements != null) { 4631 dynamicFoldDepth++; 4632 try { 4633 for (int i = 0; i < sqlparser.sqlstatements.size(); i++) { 4634 analyzeCustomSqlStmt(sqlparser.sqlstatements.get(i)); 4635 } 4636 } finally { 4637 dynamicFoldDepth--; 4638 } 4639 analyzedInner = true; 4640 } 4641 } 4642 // Site accounting mirrors the T-SQL EXEC branch: every 4643 // EXECUTE IMMEDIATE is a dynamic site; status reflects what 4644 // the analyzer actually did with it. 4645 boolean partialFold = legacyPartialFold; 4646 int[] dynCounts = classifyDynamicSiteLineage(relsBeforeDynamic, partialFold); 4647 int unprovenCount = countNewDynamicRelationships(shadowBefore); 4648 boolean fullyKnown = analyzedInner && !partialFold; 4649 boolean usableLineage = dynCounts[2] == 0 4650 && (dynCounts[0] > 0 || (dynCounts[1] == 0 && dynCounts[3] > 0)); 4651 if (shadowIncomplete && trustMode == DynamicSqlTrustMode.SHADOW) { 4652 DynamicSqlSite.Status status = analyzedInner && unprovenCount > 0 4653 ? DynamicSqlSite.Status.PARTIAL : DynamicSqlSite.Status.UNRESOLVED; 4654 recordDynamicSqlSite(stmt, DynamicSqlSite.Kind.EXECUTE_IMMEDIATE, status, 4655 dynamicTrustReason("SHADOW retained legacy publication behavior", 4656 unprovenCount, -1, false), 4657 dynCounts[0], dynCounts[1], unprovenCount, 4658 -1, false, trustMode, 4659 relsBeforeDynamic, dynamicSitesBefore); 4660 } else if (analyzedInner && (fullyKnown || usableLineage)) { 4661 recordDynamicSqlSite(stmt, DynamicSqlSite.Kind.EXECUTE_IMMEDIATE, 4662 DynamicSqlSite.Status.RESOLVED, null, dynCounts[0], dynCounts[1], 4663 0, 0, true, trustMode, 4664 relsBeforeDynamic, dynamicSitesBefore); 4665 } else if (innerParseFailed && !partialFold) { 4666 recordDynamicSqlSite(stmt, DynamicSqlSite.Kind.EXECUTE_IMMEDIATE, 4667 DynamicSqlSite.Status.PARSE_ERROR, "inner dynamic SQL failed to parse", 4668 dynCounts[0], dynCounts[1], 0, 0, true, trustMode, 4669 relsBeforeDynamic, dynamicSitesBefore); 4670 } else if (innerParseFailed) { 4671 recordDynamicSqlSite(stmt, DynamicSqlSite.Kind.EXECUTE_IMMEDIATE, 4672 DynamicSqlSite.Status.UNRESOLVED, 4673 "partially folded dynamic SQL failed to parse; unresolved parts remain", 4674 dynCounts[0], dynCounts[1], 0, -1, false, trustMode, 4675 relsBeforeDynamic, dynamicSitesBefore); 4676 } else if (analyzedInner) { 4677 recordDynamicSqlSite(stmt, DynamicSqlSite.Kind.EXECUTE_IMMEDIATE, 4678 DynamicSqlSite.Status.UNRESOLVED, 4679 dynCounts[2] != 0 4680 ? "dynamic SQL references a runtime-built object name; lineage source is a placeholder" 4681 : "dynamic SQL only partially folded; no resolved lineage produced", 4682 dynCounts[0], dynCounts[1], 0, -1, false, trustMode, 4683 relsBeforeDynamic, dynamicSitesBefore); 4684 } else { 4685 boolean runtimeTextUnavailable = !analyzedInner 4686 && execImmeStmt.getDynamicSQL() == null; 4687 boolean unresolvedFragments = partialFold || runtimeTextUnavailable; 4688 recordDynamicSqlSite(stmt, DynamicSqlSite.Kind.EXECUTE_IMMEDIATE, 4689 DynamicSqlSite.Status.UNRESOLVED, 4690 "dynamic SQL string is a runtime value; no statically-known text to analyze", 4691 dynCounts[0], dynCounts[1], 0, 4692 unresolvedFragments ? -1 : 0, !unresolvedFragments, trustMode, 4693 relsBeforeDynamic, dynamicSitesBefore); 4694 } 4695 } 4696 } else if (stmt instanceof TCallStatement) { 4697 TCallStatement callStmt = (TCallStatement)stmt; 4698 stmtStack.push(stmt); 4699 analyzeCallStmt(callStmt); 4700 stmtStack.pop(); 4701 } else if (stmt instanceof TDb2CallStmt) { 4702 TDb2CallStmt db2CallStmt = (TDb2CallStmt) stmt; 4703 stmtStack.push(stmt); 4704 analyzeDb2CallStmt(db2CallStmt); 4705 stmtStack.pop(); 4706 } else if (stmt instanceof TBasicStmt) { 4707 TBasicStmt oracleBasicStmt = (TBasicStmt) stmt; 4708 stmtStack.push(stmt); 4709 analyzeOracleBasicStmt(oracleBasicStmt); 4710 stmtStack.pop(); 4711 } else if (stmt instanceof TIfStmt) { 4712 TIfStmt ifStmt = (TIfStmt) stmt; 4713 stmtStack.push(stmt); 4714 analyzeIfStmt(ifStmt); 4715 stmtStack.pop(); 4716 } else if (stmt instanceof TElsifStmt) { 4717 TElsifStmt elsIfStmt = (TElsifStmt) stmt; 4718 stmtStack.push(stmt); 4719 analyzeElsIfStmt(elsIfStmt); 4720 stmtStack.pop(); 4721 } else if (stmt instanceof TMssqlIfElse) { 4722 TMssqlIfElse ifStmt = (TMssqlIfElse) stmt; 4723 stmtStack.push(stmt); 4724 analyzeMssqlIfElseStmt(ifStmt); 4725 stmtStack.pop(); 4726 } else if (stmt.getStatements() != null && stmt.getStatements().size() > 0) { 4727 for (int i = 0; i < stmt.getStatements().size(); i++) { 4728 analyzeCustomSqlStmt(stmt.getStatements().get(i)); 4729 } 4730 } else if (stmt instanceof TCreateIndexSqlStatement) { 4731 stmtStack.push(stmt); 4732 analyzeCreateIndexStageStmt((TCreateIndexSqlStatement) stmt); 4733 stmtStack.pop(); 4734 } else if (stmt instanceof gudusoft.gsqlparser.stmt.mdx.TMdxSelect) { 4735 stmtStack.push(stmt); 4736 analyzeMdxSelectStmt((gudusoft.gsqlparser.stmt.mdx.TMdxSelect) stmt); 4737 stmtStack.pop(); 4738 } else if (stmt instanceof TPowerQueryDocumentStmt) { 4739 stmtStack.push(stmt); 4740 analyzePowerQueryDocumentStmt((TPowerQueryDocumentStmt) stmt); 4741 stmtStack.pop(); 4742 } 4743 } catch (Exception e) { 4744 StringBuffer errorMessage = new StringBuffer(); 4745 errorMessage.append("analyze sql stmt failed, "); 4746 if (stmt.getStartToken() != null) { 4747 errorMessage.append("line: " + stmt.getStartToken().lineNo + ", column: " + stmt.getStartToken().columnNo).append(", "); 4748 } 4749 if (stmt.getGsqlparser() != null && !SQLUtil.isEmpty(stmt.getGsqlparser().sqlfilename)) { 4750 errorMessage.append("file: "+ stmt.getGsqlparser().sqlfilename).append(", "); 4751 } 4752 if (stmt.toString() != null) { 4753 errorMessage.append("sql:\n" + stmt.toString()); 4754 } 4755 logger.error(errorMessage.toString(), e); 4756 ErrorInfo errorInfo = new ErrorInfo(); 4757 errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR); 4758 if (e.getMessage() == null) { 4759 if (e.getStackTrace() != null && e.getStackTrace().length > 0) { 4760 errorInfo.setErrorMessage(e.getClass().getSimpleName() + ": " + e.getStackTrace()[0].toString()); 4761 } else { 4762 errorInfo.setErrorMessage(e.getClass().getSimpleName()); 4763 } 4764 } else { 4765 errorInfo.setErrorMessage(e.getClass().getSimpleName() + ": " + e.getMessage()); 4766 } 4767 errorInfo.setStartPosition(new Pair3<Long, Long, String>(stmt.getStartToken().lineNo, 4768 stmt.getStartToken().columnNo, ModelBindingManager.getGlobalHash())); 4769 String[] segments = stmt.getEndToken().getAstext().split("\n", -1); 4770 if (segments.length == 1) { 4771 errorInfo.setEndPosition(new Pair3<Long, Long, String>(stmt.getEndToken().lineNo, 4772 stmt.getEndToken().columnNo + stmt.getEndToken().getAstext().length(), 4773 ModelBindingManager.getGlobalHash())); 4774 } else { 4775 errorInfo.setEndPosition(new Pair3<Long, Long, String>(stmt.getEndToken().lineNo + segments.length - 1, 4776 (long) segments[segments.length - 1].length() + 1, ModelBindingManager.getGlobalHash())); 4777 } 4778 errorInfo.fillInfo(this); 4779 errorInfos.add(errorInfo); 4780 } 4781 } 4782 4783 private void analyzePlsqlRecordTypeDefStmt(TPlsqlRecordTypeDefStmt stmt) { 4784 TObjectName typeName = stmt.getTypeName(); 4785 Variable variable = modelFactory.createVariable(typeName); 4786 variable.setSubType(SubType.record_type); 4787 4788 if (stmt.getFieldDeclarations() != null) { 4789 for (int i = 0; i < stmt.getFieldDeclarations().size(); i++) { 4790 TParameterDeclaration param = stmt.getFieldDeclarations().getParameterDeclarationItem(i); 4791 String dataTypeName = param.getDataType().getDataTypeName(); 4792 TObjectName columnName = param.getParameterName(); 4793 TableColumn variableProperty = modelFactory.createTableColumn(variable, columnName, true); 4794 if(dataTypeName.indexOf(".")!=-1) { 4795 String tableName = dataTypeName.substring(0, dataTypeName.lastIndexOf(".")); 4796 Table table = modelFactory.createTableByName(tableName, true); 4797 if(table!=null) { 4798 TableColumn tableColumn = modelFactory.createInsertTableColumn(table, dataTypeName); 4799 if (tableColumn != null) { 4800 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 4801 relation.setEffectType(EffectType.rowtype); 4802 relation.setTarget(new TableColumnRelationshipElement(variableProperty)); 4803 relation.addSource(new TableColumnRelationshipElement(tableColumn)); 4804 } 4805 } 4806 } 4807 } 4808 variable.setDetermined(true); 4809 } 4810 else { 4811 TObjectName starColumn = new TObjectName(); 4812 starColumn.setString("*"); 4813 } 4814 } 4815 4816 private void analyzePlsqlTableTypeDefStmt(TPlsqlTableTypeDefStmt stmt) { 4817 TTypeName typeName = stmt.getElementDataType(); 4818 if (typeName != null && typeName.toString().toUpperCase().indexOf("ROWTYPE") != -1) { 4819 Variable cursorVariable = modelFactory.createVariable(stmt.getTypeName()); 4820 cursorVariable.setSubType(SubType.record_type); 4821 4822 Table variableTable = modelFactory.createTableByName(typeName.getDataTypeName(), false); 4823 if(!variableTable.isCreateTable()) { 4824 TObjectName starColumn1 = new TObjectName(); 4825 starColumn1.setString("*"); 4826 TableColumn variableTableStarColumn = modelFactory.createTableColumn(variableTable, starColumn1, true); 4827 variableTableStarColumn.setShowStar(false); 4828 variableTableStarColumn.setExpandStar(true); 4829 4830 TObjectName starColumn = new TObjectName(); 4831 starColumn.setString("*"); 4832 TableColumn variableProperty = modelFactory.createTableColumn(cursorVariable, starColumn, true); 4833 variableProperty.setShowStar(false); 4834 variableProperty.setExpandStar(true); 4835 4836 DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation(); 4837 dataflowRelation.setEffectType(EffectType.rowtype); 4838 dataflowRelation.addSource(new TableColumnRelationshipElement(variableTableStarColumn)); 4839 dataflowRelation.setTarget(new TableColumnRelationshipElement(variableProperty)); 4840 } else { 4841 for (TableColumn sourceColumn : variableTable.getColumns()) { 4842 String columnName = sourceColumn.getName(); 4843 TObjectName targetColumn = new TObjectName(); 4844 targetColumn.setString(columnName); 4845 TableColumn variableProperty = modelFactory.createTableColumn(cursorVariable, targetColumn, true); 4846 DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation(); 4847 dataflowRelation.setEffectType(EffectType.rowtype); 4848 dataflowRelation.addSource(new TableColumnRelationshipElement(sourceColumn)); 4849 dataflowRelation.setTarget(new TableColumnRelationshipElement(variableProperty)); 4850 } 4851 } 4852 } 4853 } 4854 4855 private void analyzeMssqlCreateType(TMssqlCreateType createType) { 4856 4857 } 4858 4859 /** 4860 * UPDATETEXT table.dest_column dest_text_ptr {NULL|offset} {NULL|del_len} 4861 * [WITH LOG] [inserted_data | table.src_column src_text_ptr] (Mantis #4607). 4862 * The written column is a real data-write target: emit inserted_data (variable 4863 * or literal) -> dest column, and for the column-copy form the proven 4864 * src_column -> dest_column edge. 4865 */ 4866 private void analyzeMssqlUpdateTextStmt(TMssqlUpdateText stmt) { 4867 TableColumn destColumn = createTextPtrTargetColumn(stmt.getTargetTable(), stmt.getDestColumnName()); 4868 if (destColumn == null) { 4869 return; 4870 } 4871 List<TObjectName> sourceNames = new ArrayList<TObjectName>(); 4872 if (stmt.getInsertedVariable() != null) { 4873 sourceNames.add(stmt.getInsertedVariable()); 4874 } 4875 if (stmt.getSourceColumnName() != null && stmt.getCopySourceTable() != null) { 4876 // make sure the copy-source table has a model so the edge resolves 4877 modelFactory.createTable(stmt.getCopySourceTable()); 4878 sourceNames.add(stmt.getSourceColumnName()); 4879 } 4880 if (!sourceNames.isEmpty()) { 4881 analyzeDataFlowRelation(destColumn, sourceNames, EffectType.update, null); 4882 } 4883 if (stmt.getInsertedLiteral() != null) { 4884 List<TParseTreeNode> constants = new ArrayList<TParseTreeNode>(); 4885 constants.add(stmt.getInsertedLiteral()); 4886 analyzeConstantDataFlowRelation(destColumn, constants, EffectType.update, null); 4887 } 4888 } 4889 4890 /** WRITETEXT table.column text_ptr [WITH LOG] data (Mantis #4607): data -> column. */ 4891 private void analyzeMssqlWriteTextStmt(TMssqlWriteText stmt) { 4892 TableColumn destColumn = createTextPtrTargetColumn(stmt.getTargetTable(), stmt.getDestColumnName()); 4893 if (destColumn == null) { 4894 return; 4895 } 4896 if (stmt.getWriteDataVariable() != null) { 4897 List<TObjectName> sourceNames = new ArrayList<TObjectName>(); 4898 sourceNames.add(stmt.getWriteDataVariable()); 4899 analyzeDataFlowRelation(destColumn, sourceNames, EffectType.update, null); 4900 } 4901 if (stmt.getWriteDataLiteral() != null) { 4902 List<TParseTreeNode> constants = new ArrayList<TParseTreeNode>(); 4903 constants.add(stmt.getWriteDataLiteral()); 4904 analyzeConstantDataFlowRelation(destColumn, constants, EffectType.update, null); 4905 } 4906 } 4907 4908 /** 4909 * READTEXT table.column text_ptr offset size [HOLDLOCK] (Mantis #4607): the 4910 * column is read into the client stream; model the output as the statement's 4911 * result set with the read column so the read is visible in the lineage. 4912 */ 4913 private void analyzeMssqlReadTextStmt(TMssqlReadText stmt) { 4914 TObjectName columnRef = stmt.getSourceColumnName(); 4915 if (columnRef == null || columnRef.getSourceTable() == null) { 4916 return; 4917 } 4918 Table tableModel = modelFactory.createTable(columnRef.getSourceTable()); 4919 if (tableModel == null) { 4920 return; 4921 } 4922 TableColumn sourceColumn = modelFactory.createTableColumn(tableModel, columnRef, false); 4923 if (sourceColumn == null) { 4924 return; 4925 } 4926 ResultSet resultSet = modelFactory.createResultSet(stmt, true); 4927 ResultColumn resultColumn = modelFactory.createResultColumn(resultSet, columnRef); 4928 if (resultColumn == null) { 4929 return; 4930 } 4931 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 4932 relation.setEffectType(EffectType.select); 4933 relation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 4934 relation.addSource(new TableColumnRelationshipElement(sourceColumn)); 4935 } 4936 4937 /** 4938 * The write target of an UPDATETEXT/WRITETEXT statement: the model column of 4939 * the fused table.column reference, or null when the statement carries no 4940 * table-qualified column. 4941 */ 4942 private TableColumn createTextPtrTargetColumn(TTable targetTable, TObjectName columnRef) { 4943 if (targetTable == null || columnRef == null) { 4944 return null; 4945 } 4946 Table tableModel = modelFactory.createTable(targetTable); 4947 if (tableModel == null) { 4948 return null; 4949 } 4950 return modelFactory.createTableColumn(tableModel, columnRef, false); 4951 } 4952 4953 private void analyzeMssqlExecute(TMssqlExecute executeStmt) { 4954 if (executeStmt.getModuleName() != null) { 4955 TObjectName module = executeStmt.getModuleName(); 4956 if(module.toString().toLowerCase().endsWith("sp_rename")) { 4957 String oldTableName = SQLUtil.trimColumnStringQuote(executeStmt.getParameters().getExecParameter(0).toString()); 4958 Table oldNameTableModel = modelFactory.createTableByName(oldTableName, true); 4959 List<String> oldTableNames = SQLUtil.parseNames(oldNameTableModel.getName()); 4960 TObjectName oldStarColumn = new TObjectName(); 4961 oldStarColumn.setString("*"); 4962 TableColumn oldTableStarColumn = modelFactory.createTableColumn(oldNameTableModel, oldStarColumn, true); 4963 4964 String newTableName = SQLUtil.trimColumnStringQuote(executeStmt.getParameters().getExecParameter(1).toString()); 4965 List<String> newTableNames = SQLUtil.parseNames(newTableName); 4966 if (oldTableNames.size() > newTableNames.size()) { 4967 for (int i = oldTableNames.size() - newTableNames.size() - 1; i >= 0; i--) { 4968 newTableName = (oldTableNames.get(i) + ".") + newTableName; 4969 } 4970 } 4971 4972 Table newNameTableModel = modelFactory.createTableByName(newTableName, true); 4973 TObjectName newStarColumn = new TObjectName(); 4974 newStarColumn.setString("*"); 4975 TableColumn newTableStarColumn = modelFactory.createTableColumn(newNameTableModel, newStarColumn, true); 4976 4977 Process process = modelFactory.createProcess(executeStmt); 4978 newNameTableModel.addProcess(process); 4979 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 4980 relation.setEffectType(EffectType.rename_table); 4981 relation.setTarget(new TableColumnRelationshipElement(newTableStarColumn)); 4982 relation.addSource(new TableColumnRelationshipElement(oldTableStarColumn)); 4983 relation.setProcess(process); 4984 4985 if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) { 4986 oldTableStarColumn.setShowStar(false); 4987 relation.setShowStarRelation(false); 4988 } 4989 4990 if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) { 4991 newTableStarColumn.setShowStar(false); 4992 relation.setShowStarRelation(false); 4993 } 4994 } 4995 else if(module.toString().toLowerCase().endsWith("sp_executesql")) { 4996 String sql = SQLUtil.trimColumnStringQuote(executeStmt.getParameters().getExecParameter(0).toString()); 4997 if(sql.startsWith("N")) { 4998 sql = SQLUtil.trimColumnStringQuote(sql.substring(1)); 4999 } 5000 executeDynamicSql(sql); 5001 } 5002 else { 5003 // sp_execute_external_script runs an R / Python / Java / ONNX script 5004 // in-database (SQL Server 2016+ Machine Learning Services). Its 5005 // @input_data_1 parameter is a SQL string literal whose SELECT reads 5006 // real tables to build the InputDataSet the script consumes. Recover 5007 // that source-table -> result-set lineage instead of dropping it 5008 // (MantisBT #4606). This runs IN ADDITION to the generic call-edge 5009 // handling below, so the call to sp_execute_external_script is still 5010 // recorded. 5011 if (option.isAnalyzeDynamicSql() 5012 && isSpExecuteExternalScript(module)) { 5013 analyzeExternalScriptInputData(executeStmt); 5014 } 5015 int argumentSize = executeStmt.getParameters() == null ? 0 : executeStmt.getParameters().size(); 5016 String procedureNameWithArgSize = module.toString() + "(" + argumentSize + ")"; 5017 if (argumentSize <= 0 || !DlineageUtil.supportFunctionOverride(option.getVendor())) { 5018 procedureNameWithArgSize = module.toString(); 5019 } 5020 if (procedureDDLMap.containsKey(procedureNameWithArgSize)) { 5021 analyzeCustomSqlStmt(procedureDDLMap.get(procedureNameWithArgSize)); 5022 } 5023 Procedure procedure = modelFactory.createProcedureByName(module, executeStmt.getParameters() == null ? 0 : executeStmt.getParameters().size()); 5024 String procedureParent = getProcedureParentName(executeStmt); 5025 if (procedureParent != null) { 5026 Procedure caller = modelManager 5027 .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent)); 5028 if (caller != null) { 5029 CallRelationship callRelation = modelFactory.createCallRelation(); 5030 callRelation.setCallObject(executeStmt); 5031 callRelation.setTarget(new ProcedureRelationshipElement(caller)); 5032 callRelation.addSource(new ProcedureRelationshipElement(procedure)); 5033 if(isBuiltInFunctionName(module) || isKeyword(module)){ 5034 callRelation.setBuiltIn(true); 5035 } 5036 } 5037 } 5038 5039 if (procedure.getArguments() != null) { 5040 for (int i = 0; i < procedure.getArguments().size(); i++) { 5041 Argument argument = procedure.getArguments().get(i); 5042 Variable variable = modelFactory.createVariable(procedure, argument.getName(), false); 5043 if (variable != null) { 5044 if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) { 5045 Transform transform = new Transform(); 5046 transform.setType(Transform.FUNCTION); 5047 transform.setCode(module); 5048 compositeBindingColumn(variable).setTransform(transform); 5049 } 5050 Process process = modelFactory.createProcess(executeStmt); 5051 variable.addProcess(process); 5052 analyzeFunctionArgumentsDataFlowRelation(compositeBindingColumn(variable), executeStmt, argument.getName(), i, process); 5053 } 5054 } 5055 } 5056 } 5057 } 5058 } 5059 5060 private void analyzeDropTableStmt(TDropTableSqlStatement stmt) { 5061 TTable dropTable = stmt.getTargetTable(); 5062 if(dropTable == null) { 5063 return; 5064 } 5065 Table tableModel = modelManager.getTableByName(DlineageUtil.getTableFullName(dropTable.getTableName().toString())); 5066 if(tableModel!=null) { 5067 modelManager.dropTable(tableModel); 5068 } 5069 5070 if(option.getAnalyzeMode() == AnalyzeMode.crud) { 5071 tableModel = modelFactory.createTable(dropTable); 5072 CrudRelationship crudRelationship = modelFactory.createCrudRelation(); 5073 crudRelationship.setTarget(new TableRelationshipElement(tableModel)); 5074 crudRelationship.setEffectType(EffectType.drop_table); 5075 } 5076 } 5077 5078 private void analyzeTruncateTableStmt(TTruncateStatement stmt) { 5079 if(option.getAnalyzeMode() == AnalyzeMode.crud) { 5080 TObjectName table = stmt.getTableName(); 5081 Table tableModel = modelFactory.createTableByName(table); 5082 CrudRelationship crudRelationship = modelFactory.createCrudRelation(); 5083 crudRelationship.setTarget(new TableRelationshipElement(tableModel)); 5084 crudRelationship.setEffectType(EffectType.truncate_table); 5085 } 5086 } 5087 5088 private void analyzeCallStmt(TCallStatement callStmt) { 5089 if (callStmt.getRoutineExpr() != null && callStmt.getRoutineExpr().getFunctionCall() != null) { 5090 5091 TFunctionCall functionCall = callStmt.getRoutineExpr().getFunctionCall(); 5092 Procedure callee = modelManager.getProcedureByName( 5093 DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall))); 5094 if (callee == null && procedureDDLMap.containsKey(DlineageUtil.getFunctionNameWithArgNum(functionCall))) { 5095 analyzeCustomSqlStmt(procedureDDLMap.get(DlineageUtil.getFunctionNameWithArgNum(functionCall))); 5096 callee = modelManager.getProcedureByName(DlineageUtil 5097 .getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall))); 5098 } 5099 if (callee != null) { 5100 String procedureParent = getProcedureParentName(callStmt); 5101 if (procedureParent != null) { 5102 Procedure caller = modelManager 5103 .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent)); 5104 if (caller != null) { 5105 CallRelationship callRelation = modelFactory.createCallRelation(); 5106 callRelation.setCallObject(callStmt); 5107 callRelation.setTarget(new ProcedureRelationshipElement(caller)); 5108 callRelation.addSource(new ProcedureRelationshipElement(callee)); 5109 if (isBuiltInFunctionName(functionCall.getFunctionName()) 5110 || isKeyword(functionCall.getFunctionName())) { 5111 callRelation.setBuiltIn(true); 5112 } 5113 } 5114 } 5115 if (callee.getArguments() != null) { 5116 for (int i = 0; i < callee.getArguments().size(); i++) { 5117 Argument argument = callee.getArguments().get(i); 5118 Variable variable = resolveFormalVariable(callee, argument); 5119 if (variable != null) { 5120 if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) { 5121 Transform transform = new Transform(); 5122 transform.setType(Transform.FUNCTION); 5123 transform.setCode(functionCall); 5124 compositeBindingColumn(variable).setTransform(transform); 5125 } 5126 Process process = modelFactory.createProcess(callStmt); 5127 variable.addProcess(process); 5128 analyzeFunctionArgumentsDataFlowRelation(compositeBindingColumn(variable), functionCall, i, 5129 process); 5130 } 5131 } 5132 } 5133 } else { 5134 Function function = (Function) createFunction(functionCall); 5135 String procedureParent = getProcedureParentName(callStmt); 5136 if (procedureParent != null) { 5137 Procedure caller = modelManager 5138 .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent)); 5139 if (caller != null) { 5140 CallRelationship callRelation = modelFactory.createCallRelation(); 5141 callRelation.setCallObject(callStmt); 5142 callRelation.setTarget(new ProcedureRelationshipElement(caller)); 5143 callRelation.addSource(new FunctionRelationshipElement(function)); 5144 if (isBuiltInFunctionName(functionCall.getFunctionName()) 5145 || isKeyword(functionCall.getFunctionName())) { 5146 callRelation.setBuiltIn(true); 5147 } 5148 } 5149 } 5150 } 5151 } 5152 else if (callStmt.getRoutineName() != null) { 5153 TObjectName function = callStmt.getRoutineName(); 5154 String functionName = function.toString(); 5155 Procedure callee = modelManager.getProcedureByName( 5156 DlineageUtil.getIdentifierNormalTableName(functionName)); 5157 if (callee == null && procedureDDLMap.containsKey(functionName)) { 5158 analyzeCustomSqlStmt(procedureDDLMap.get(functionName)); 5159 callee = modelManager.getProcedureByName(functionName); 5160 } 5161 if (callee != null) { 5162 String procedureParent = getProcedureParentName(callStmt); 5163 if (procedureParent != null) { 5164 Procedure caller = modelManager 5165 .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent)); 5166 if (caller != null) { 5167 CallRelationship callRelation = modelFactory.createCallRelation(); 5168 callRelation.setCallObject(callStmt); 5169 callRelation.setTarget(new ProcedureRelationshipElement(caller)); 5170 callRelation.addSource(new ProcedureRelationshipElement(callee)); 5171 if (isBuiltInFunctionName(function) 5172 || isKeyword(function)) { 5173 callRelation.setBuiltIn(true); 5174 } 5175 } 5176 } 5177 if (callee.getArguments() != null) { 5178 for (int i = 0; i < callee.getArguments().size(); i++) { 5179 Argument argument = callee.getArguments().get(i); 5180 Variable variable = resolveFormalVariable(callee, argument); 5181 if (variable != null) { 5182 if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) { 5183 Transform transform = new Transform(); 5184 transform.setType(Transform.FUNCTION); 5185 transform.setCode(callStmt); 5186 compositeBindingColumn(variable).setTransform(transform); 5187 } 5188 Process process = modelFactory.createProcess(callStmt); 5189 variable.addProcess(process); 5190 analyzeFunctionArgumentsDataFlowRelation(compositeBindingColumn(variable), callStmt, i, 5191 process); 5192 } 5193 } 5194 } 5195 } 5196 } 5197 } 5198 5199 /** 5200 * DB2's CALL node is {@link TDb2CallStmt} (a sibling of {@link TCallStatement}, 5201 * not a subclass), so the generic call dispatch never reaches it. Resolve its 5202 * caller/callee and emit the call relationship exactly as the 5203 * {@link #analyzeCallStmt} getRoutineName path does, using the DB2 accessors 5204 * getProcedureName()/getParameters(). 5205 */ 5206 private void analyzeDb2CallStmt(TDb2CallStmt callStmt) { 5207 if (callStmt.getProcedureName() == null) { 5208 return; 5209 } 5210 TObjectName function = callStmt.getProcedureName(); 5211 String functionName = function.toString(); 5212 // DB2 supports procedure overloading by arity, so overloadable procedures 5213 // register under a "name(argCount)" key (see getFunctionNameWithArgNum). 5214 // Resolve by name+argCount first to bind the correct overload, then fall 5215 // back to the plain name for the single-definition / non-overloaded case. 5216 String nameWithArgNum = functionName; 5217 if (callStmt.getParameters() != null 5218 && DlineageUtil.supportFunctionOverride(ModelBindingManager.getGlobalVendor())) { 5219 nameWithArgNum = functionName + "(" + callStmt.getParameters().size() + ")"; 5220 } 5221 Procedure callee = modelManager.getProcedureByName( 5222 DlineageUtil.getIdentifierNormalTableName(nameWithArgNum)); 5223 if (callee == null && !nameWithArgNum.equals(functionName)) { 5224 callee = modelManager.getProcedureByName( 5225 DlineageUtil.getIdentifierNormalTableName(functionName)); 5226 } 5227 if (callee == null && procedureDDLMap.containsKey(nameWithArgNum)) { 5228 analyzeCustomSqlStmt(procedureDDLMap.get(nameWithArgNum)); 5229 callee = modelManager.getProcedureByName(nameWithArgNum); 5230 } 5231 if (callee == null && procedureDDLMap.containsKey(functionName)) { 5232 analyzeCustomSqlStmt(procedureDDLMap.get(functionName)); 5233 callee = modelManager.getProcedureByName(functionName); 5234 } 5235 if (callee != null) { 5236 String procedureParent = getProcedureParentName(callStmt); 5237 if (procedureParent != null) { 5238 Procedure caller = modelManager 5239 .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent)); 5240 if (caller != null) { 5241 CallRelationship callRelation = modelFactory.createCallRelation(); 5242 callRelation.setCallObject(callStmt); 5243 callRelation.setTarget(new ProcedureRelationshipElement(caller)); 5244 callRelation.addSource(new ProcedureRelationshipElement(callee)); 5245 if (isBuiltInFunctionName(function) || isKeyword(function)) { 5246 callRelation.setBuiltIn(true); 5247 } 5248 } 5249 } 5250 if (callee.getArguments() != null) { 5251 for (int i = 0; i < callee.getArguments().size(); i++) { 5252 Argument argument = callee.getArguments().get(i); 5253 Variable variable = resolveFormalVariable(callee, argument); 5254 if (variable != null) { 5255 if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) { 5256 Transform transform = new Transform(); 5257 transform.setType(Transform.FUNCTION); 5258 transform.setCode(callStmt); 5259 compositeBindingColumn(variable).setTransform(transform); 5260 } 5261 Process process = modelFactory.createProcess(callStmt); 5262 variable.addProcess(process); 5263 analyzeFunctionArgumentsDataFlowRelation(compositeBindingColumn(variable), callStmt, i, 5264 process); 5265 } 5266 } 5267 } 5268 } 5269 } 5270 5271 private boolean analyzeCustomFunctionCall(TFunctionCall functionCall) { 5272 Procedure callee = modelManager.getProcedureByName( 5273 DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall))); 5274 if(callee == null && procedureDDLMap.containsKey(DlineageUtil.getFunctionNameWithArgNum(functionCall))) { 5275 analyzeCustomSqlStmt(procedureDDLMap.get(DlineageUtil.getFunctionNameWithArgNum(functionCall))); 5276 callee = modelManager.getProcedureByName( 5277 DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall))); 5278 } 5279 if (callee != null) { 5280 String procedureParent = getProcedureParentName(stmtStack.peek()); 5281 if (procedureParent != null) { 5282 Procedure caller = modelManager 5283 .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent)); 5284 if (caller != null) { 5285 CallRelationship callRelation = modelFactory.createCallRelation(); 5286 callRelation.setCallObject(functionCall); 5287 callRelation.setTarget(new ProcedureRelationshipElement(caller)); 5288 callRelation.addSource(new ProcedureRelationshipElement(callee)); 5289 if(isBuiltInFunctionName(functionCall.getFunctionName()) || isKeyword(functionCall.getFunctionName())){ 5290 callRelation.setBuiltIn(true); 5291 } 5292 } 5293 } 5294 if (callee.getArguments() != null) { 5295 for (int i = 0; i < callee.getArguments().size(); i++) { 5296 Argument argument = callee.getArguments().get(i); 5297 Variable variable = resolveFormalVariable(callee, argument); 5298 if(variable!=null) { 5299 if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) { 5300 Transform transform = new Transform(); 5301 transform.setType(Transform.FUNCTION); 5302 transform.setCode(functionCall); 5303 compositeBindingColumn(variable).setTransform(transform); 5304 } 5305 Process process = modelFactory.createProcess(functionCall); 5306 variable.addProcess(process); 5307 analyzeFunctionArgumentsDataFlowRelation(compositeBindingColumn(variable), functionCall, i, process); 5308 } 5309 } 5310 } 5311 return true; 5312 } 5313 return false; 5314 } 5315 5316 private void analyzeOracleBasicStmt(TBasicStmt oracleBasicStmt) { 5317 if (oracleBasicStmt.getExpr() == null || oracleBasicStmt.getExpr().getFunctionCall() == null) { 5318 return; 5319 } 5320 5321 TFunctionCall functionCall = oracleBasicStmt.getExpr().getFunctionCall(); 5322 recordDbmsSqlSiteIfParseCall(oracleBasicStmt, functionCall); 5323 Procedure callee = modelManager.getProcedureByName( 5324 DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall))); 5325 if(callee == null && procedureDDLMap.containsKey(DlineageUtil.getFunctionNameWithArgNum(functionCall))) { 5326 analyzeCustomSqlStmt(procedureDDLMap.get(DlineageUtil.getFunctionNameWithArgNum(functionCall))); 5327 callee = modelManager.getProcedureByName( 5328 DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall))); 5329 } 5330 if (callee != null) { 5331 String procedureParent = getProcedureParentName(oracleBasicStmt); 5332 if (procedureParent != null) { 5333 Procedure caller = modelManager 5334 .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent)); 5335 if (caller != null) { 5336 CallRelationship callRelation = modelFactory.createCallRelation(); 5337 callRelation.setCallObject(functionCall); 5338 callRelation.setTarget(new ProcedureRelationshipElement(caller)); 5339 callRelation.addSource(new ProcedureRelationshipElement(callee)); 5340 if (functionChecker.isOraclePredefinedPackageFunction(functionCall.getFunctionName().toString()) 5341 || (isBuiltInFunctionName(functionCall.getFunctionName()) 5342 || isKeyword(functionCall.getFunctionName()))) { 5343 callRelation.setBuiltIn(true); 5344 } 5345 } 5346 } 5347 if (callee.getArguments() != null) { 5348 for (int i = 0; i < callee.getArguments().size(); i++) { 5349 Argument argument = callee.getArguments().get(i); 5350 Variable variable = resolveFormalVariable(callee, argument); 5351 if(variable!=null) { 5352 if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) { 5353 Transform transform = new Transform(); 5354 transform.setType(Transform.FUNCTION); 5355 transform.setCode(functionCall); 5356 compositeBindingColumn(variable).setTransform(transform); 5357 } 5358 Process process = modelFactory.createProcess(functionCall); 5359 variable.addProcess(process); 5360 analyzeFunctionArgumentsDataFlowRelation(compositeBindingColumn(variable), functionCall, i, process); 5361 } 5362 } 5363 } 5364 } else { 5365 Object functionModel = createFunction(functionCall); 5366 if (!(functionModel instanceof Function)) { 5367 return; 5368 } 5369 Function function = (Function) functionModel; 5370 String procedureParent = getProcedureParentName(oracleBasicStmt); 5371 if (procedureParent != null) { 5372 Procedure caller = modelManager 5373 .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent)); 5374 if (caller != null) { 5375 CallRelationship callRelation = modelFactory.createCallRelation(); 5376 callRelation.setCallObject(functionCall); 5377 callRelation.setTarget(new ProcedureRelationshipElement(caller)); 5378 callRelation.addSource(new FunctionRelationshipElement(function)); 5379 if (functionChecker.isOraclePredefinedPackageFunction(functionCall.getFunctionName().toString()) 5380 || (isBuiltInFunctionName(functionCall.getFunctionName()) || isKeyword(functionCall.getFunctionName()))) { 5381 callRelation.setBuiltIn(true); 5382 } 5383 } 5384 } 5385 } 5386 } 5387 5388 private void analyzeMssqlIfElseStmt(TMssqlIfElse ifStmt) { 5389 if (ifStmt.getCondition() != null) { 5390 columnsInExpr visitor = new columnsInExpr(); 5391 ifStmt.getCondition().inOrderTraverse(visitor); 5392 List<TParseTreeNode> functions = visitor.getFunctions(); 5393 5394 if (functions != null && !functions.isEmpty()) { 5395 for (int i = 0; i < functions.size(); i++) { 5396 if (!(functions.get(i) instanceof TFunctionCall)) 5397 continue; 5398 TFunctionCall functionCall = (TFunctionCall) functions.get(i); 5399 Procedure callee = modelManager.getProcedureByName(DlineageUtil 5400 .getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall))); 5401 if(callee == null && procedureDDLMap.containsKey(DlineageUtil.getFunctionNameWithArgNum(functionCall))) { 5402 analyzeCustomSqlStmt(procedureDDLMap.get(DlineageUtil.getFunctionNameWithArgNum(functionCall))); 5403 callee = modelManager.getProcedureByName( 5404 DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall))); 5405 } 5406 if (callee != null) { 5407 String procedureParent = getProcedureParentName(ifStmt); 5408 if (procedureParent != null) { 5409 Procedure caller = modelManager 5410 .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent)); 5411 if (caller != null) { 5412 CallRelationship callRelation = modelFactory.createCallRelation(); 5413 callRelation.setCallObject(functionCall); 5414 callRelation.setTarget(new ProcedureRelationshipElement(caller)); 5415 callRelation.addSource(new ProcedureRelationshipElement(callee)); 5416 if (isBuiltInFunctionName(functionCall.getFunctionName()) || isKeyword(functionCall.getFunctionName())) { 5417 callRelation.setBuiltIn(true); 5418 } 5419 } 5420 } 5421 if (callee.getArguments() != null) { 5422 for (int j = 0; j < callee.getArguments().size(); j++) { 5423 Argument argument = callee.getArguments().get(j); 5424 Variable variable = resolveFormalVariable(callee, argument); 5425 if(variable!=null) { 5426 if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) { 5427 Transform transform = new Transform(); 5428 transform.setType(Transform.FUNCTION); 5429 transform.setCode(functionCall); 5430 compositeBindingColumn(variable).setTransform(transform); 5431 } 5432 Process process = modelFactory.createProcess(functionCall); 5433 variable.addProcess(process); 5434 analyzeFunctionArgumentsDataFlowRelation(compositeBindingColumn(variable), functionCall, j, process); 5435 } 5436 } 5437 } 5438 } else { 5439 Object functionModel = createFunction(functionCall); 5440 if (!(functionModel instanceof Function)) { 5441 continue; 5442 } 5443 Function function = (Function) functionModel; 5444 String procedureParent = getProcedureParentName(ifStmt); 5445 if (procedureParent != null) { 5446 Procedure caller = modelManager 5447 .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent)); 5448 if (caller != null) { 5449 CallRelationship callRelation = modelFactory.createCallRelation(); 5450 callRelation.setCallObject(functionCall); 5451 callRelation.setTarget(new ProcedureRelationshipElement(caller)); 5452 callRelation.addSource(new FunctionRelationshipElement(function)); 5453 if (isBuiltInFunctionName(functionCall.getFunctionName()) || isKeyword(functionCall.getFunctionName())) { 5454 callRelation.setBuiltIn(true); 5455 } 5456 } 5457 } 5458 } 5459 } 5460 } 5461 } 5462 5463 if(ifStmt.getStmt()!=null) { 5464 analyzeCustomSqlStmt(ifStmt.getStmt()); 5465 } 5466 5467 if (ifStmt.getElseStmt() != null) { 5468 analyzeCustomSqlStmt(ifStmt.getElseStmt()); 5469 } 5470 } 5471 5472 private void analyzeIfStmt(TIfStmt ifStmt) { 5473 if (ifStmt.getCondition() != null) { 5474 columnsInExpr visitor = new columnsInExpr(); 5475 ifStmt.getCondition().inOrderTraverse(visitor); 5476 List<TParseTreeNode> functions = visitor.getFunctions(); 5477 5478 if (functions != null && !functions.isEmpty()) { 5479 for (int i = 0; i < functions.size(); i++) { 5480 if (!(functions.get(i) instanceof TFunctionCall)) 5481 continue; 5482 TFunctionCall functionCall = (TFunctionCall) functions.get(i); 5483 Procedure callee = modelManager.getProcedureByName(DlineageUtil 5484 .getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall))); 5485 if(callee == null && procedureDDLMap.containsKey(DlineageUtil.getFunctionNameWithArgNum(functionCall))) { 5486 analyzeCustomSqlStmt(procedureDDLMap.get(DlineageUtil.getFunctionNameWithArgNum(functionCall))); 5487 callee = modelManager.getProcedureByName( 5488 DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall))); 5489 } 5490 if (callee != null) { 5491 String procedureParent = getProcedureParentName(ifStmt); 5492 if (procedureParent != null) { 5493 Procedure caller = modelManager 5494 .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent)); 5495 if (caller != null) { 5496 CallRelationship callRelation = modelFactory.createCallRelation(); 5497 callRelation.setCallObject(functionCall); 5498 callRelation.setTarget(new ProcedureRelationshipElement(caller)); 5499 callRelation.addSource(new ProcedureRelationshipElement(callee)); 5500 if (isBuiltInFunctionName(functionCall.getFunctionName()) || isKeyword(functionCall.getFunctionName())) { 5501 callRelation.setBuiltIn(true); 5502 } 5503 } 5504 } 5505 if (callee.getArguments() != null) { 5506 for (int j = 0; j < callee.getArguments().size(); j++) { 5507 Argument argument = callee.getArguments().get(j); 5508 Variable variable = resolveFormalVariable(callee, argument); 5509 if(variable!=null) { 5510 if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) { 5511 Transform transform = new Transform(); 5512 transform.setType(Transform.FUNCTION); 5513 transform.setCode(functionCall); 5514 compositeBindingColumn(variable).setTransform(transform); 5515 } 5516 Process process = modelFactory.createProcess(functionCall); 5517 variable.addProcess(process); 5518 analyzeFunctionArgumentsDataFlowRelation(compositeBindingColumn(variable), functionCall, j, process); 5519 } 5520 } 5521 } 5522 } else { 5523 Object functionModel = createFunction(functionCall); 5524 if (!(functionModel instanceof Function)) { 5525 continue; 5526 } 5527 Function function = (Function) functionModel; 5528 String procedureParent = getProcedureParentName(ifStmt); 5529 if (procedureParent != null) { 5530 Procedure caller = modelManager 5531 .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent)); 5532 if (caller != null) { 5533 CallRelationship callRelation = modelFactory.createCallRelation(); 5534 callRelation.setCallObject(functionCall); 5535 callRelation.setTarget(new ProcedureRelationshipElement(caller)); 5536 callRelation.addSource(new FunctionRelationshipElement(function)); 5537 if (isBuiltInFunctionName(functionCall.getFunctionName()) || isKeyword(functionCall.getFunctionName())) { 5538 callRelation.setBuiltIn(true); 5539 } 5540 } 5541 } 5542 } 5543 } 5544 } 5545 } 5546 5547 if (ifStmt.getThenStatements() != null) { 5548 for (int i = 0; i < ifStmt.getThenStatements().size(); ++i) { 5549 analyzeCustomSqlStmt(ifStmt.getThenStatements().get(i)); 5550 ResultSet returnResult = modelFactory.createResultSet(ifStmt.getThenStatements().get(i), false); 5551 if(returnResult!=null){ 5552 for(ResultColumn resultColumn: returnResult.getColumns()){ 5553 analyzeFilterCondition(resultColumn, ifStmt.getCondition(), null, null, EffectType.function); 5554 } 5555 } 5556 } 5557 } 5558 5559 if (ifStmt.getElseifStatements() != null) { 5560 for (int i = 0; i < ifStmt.getElseifStatements().size(); ++i) { 5561 analyzeCustomSqlStmt(ifStmt.getElseifStatements().get(i)); 5562 } 5563 } 5564 5565 if (ifStmt.getElseStatements() != null) { 5566 for (int i = 0; i < ifStmt.getElseStatements().size(); ++i) { 5567 analyzeCustomSqlStmt(ifStmt.getElseStatements().get(i)); 5568 } 5569 } 5570 } 5571 5572 private void analyzeElsIfStmt(TElsifStmt elsIfStmt) { 5573 if (elsIfStmt.getCondition() != null) { 5574 columnsInExpr visitor = new columnsInExpr(); 5575 elsIfStmt.getCondition().inOrderTraverse(visitor); 5576 List<TParseTreeNode> functions = visitor.getFunctions(); 5577 5578 if (functions != null && !functions.isEmpty()) { 5579 for (int i = 0; i < functions.size(); i++) { 5580 if (!(functions.get(i) instanceof TFunctionCall)) 5581 continue; 5582 TFunctionCall functionCall = (TFunctionCall) functions.get(i); 5583 Procedure callee = modelManager.getProcedureByName(DlineageUtil 5584 .getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall))); 5585 if(callee == null && procedureDDLMap.containsKey(DlineageUtil.getFunctionNameWithArgNum(functionCall))) { 5586 analyzeCustomSqlStmt(procedureDDLMap.get(DlineageUtil.getFunctionNameWithArgNum(functionCall))); 5587 callee = modelManager.getProcedureByName( 5588 DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall))); 5589 } 5590 if (callee != null) { 5591 String procedureParent = getProcedureParentName(elsIfStmt); 5592 if (procedureParent != null) { 5593 Procedure caller = modelManager 5594 .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent)); 5595 if (caller != null) { 5596 CallRelationship callRelation = modelFactory.createCallRelation(); 5597 callRelation.setCallObject(functionCall); 5598 callRelation.setTarget(new ProcedureRelationshipElement(caller)); 5599 callRelation.addSource(new ProcedureRelationshipElement(callee)); 5600 if(isBuiltInFunctionName(functionCall.getFunctionName()) || isKeyword(functionCall.getFunctionName())){ 5601 callRelation.setBuiltIn(true); 5602 } 5603 } 5604 } 5605 if (callee.getArguments() != null) { 5606 for (int j = 0; j < callee.getArguments().size(); j++) { 5607 Argument argument = callee.getArguments().get(j); 5608 Variable variable = resolveFormalVariable(callee, argument); 5609 if(variable!=null) { 5610 if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) { 5611 Transform transform = new Transform(); 5612 transform.setType(Transform.FUNCTION); 5613 transform.setCode(functionCall); 5614 compositeBindingColumn(variable).setTransform(transform); 5615 } 5616 Process process = modelFactory.createProcess(functionCall); 5617 variable.addProcess(process); 5618 analyzeFunctionArgumentsDataFlowRelation(compositeBindingColumn(variable), functionCall, j, process); 5619 } 5620 } 5621 } 5622 } else { 5623 Function function = (Function)createFunction(functionCall); 5624 String procedureParent = getProcedureParentName(elsIfStmt); 5625 if (procedureParent != null) { 5626 Procedure caller = modelManager 5627 .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent)); 5628 if (caller != null) { 5629 CallRelationship callRelation = modelFactory.createCallRelation(); 5630 callRelation.setCallObject(functionCall); 5631 callRelation.setTarget(new ProcedureRelationshipElement(caller)); 5632 callRelation.addSource(new FunctionRelationshipElement(function)); 5633 if(isBuiltInFunctionName(functionCall.getFunctionName()) || isKeyword(functionCall.getFunctionName())){ 5634 callRelation.setBuiltIn(true); 5635 } 5636 } 5637 } 5638 } 5639 } 5640 } 5641 } 5642 5643 if (elsIfStmt.getThenStatements() != null) { 5644 for (int i = 0; i < elsIfStmt.getThenStatements().size(); ++i) { 5645 ResultSet returnResult = modelFactory.createResultSet(elsIfStmt.getThenStatements().get(i), false); 5646 if (returnResult != null) { 5647 for (ResultColumn resultColumn : returnResult.getColumns()) { 5648 analyzeFilterCondition(resultColumn, elsIfStmt.getCondition(), null, null, EffectType.function); 5649 } 5650 } 5651 analyzeCustomSqlStmt(elsIfStmt.getThenStatements().get(i)); 5652 } 5653 } 5654 } 5655 5656 private void analyzeCloneTableStmt(TCreateTableSqlStatement stmt) { 5657 if (stmt.getCloneSourceTable() != null) { 5658 Table sourceTable = modelFactory.createTableByName(stmt.getCloneSourceTable()); 5659 Table cloneTable = modelFactory.createTableByName(stmt.getTableName()); 5660 // Clone creates the target. Parse fact, set regardless of source 5661 // resolution (the gated setFromDDL below stays as-is). See 5662 // dlineage-authoritative-endpoint-classification.md. 5663 cloneTable.setCreatedInSql(true); 5664 cloneTable.setEndpointIntroduction(EndpointIntroduction.CLONE_TABLE); 5665 Process process = modelFactory.createProcess(stmt); 5666 cloneTable.addProcess(process); 5667 5668 if (sourceTable.isDetermined()) { 5669 for (int k = 0; k < sourceTable.getColumns().size(); k++) { 5670 TableColumn sourceColumn = sourceTable.getColumns().get(k); 5671 TObjectName objectName = new TObjectName(); 5672 objectName.setString(sourceColumn.getName()); 5673 TableColumn tableColumn = modelFactory.createTableColumn(cloneTable, objectName, true); 5674 DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation(); 5675 dataflowRelation.setEffectType(EffectType.clone_table); 5676 dataflowRelation 5677 .addSource(new TableColumnRelationshipElement(sourceColumn)); 5678 dataflowRelation.setTarget(new TableColumnRelationshipElement(tableColumn)); 5679 dataflowRelation.setProcess(process); 5680 } 5681 cloneTable.setDetermined(true); 5682 cloneTable.setFromDDL(true); 5683 } else { 5684 TObjectName sourceName = new TObjectName(); 5685 sourceName.setString("*"); 5686 TableColumn sourceTableColumn = modelFactory.createTableColumn(sourceTable, sourceName, false); 5687 TObjectName targetName = new TObjectName(); 5688 targetName.setString("*"); 5689 TableColumn targetTableColumn = modelFactory.createTableColumn(cloneTable, targetName, false); 5690 if(sourceTableColumn!=null && targetTableColumn!=null) { 5691 DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation(); 5692 dataflowRelation.setEffectType(EffectType.clone_table); 5693 dataflowRelation 5694 .addSource(new TableColumnRelationshipElement(sourceTableColumn)); 5695 dataflowRelation.setTarget(new TableColumnRelationshipElement(targetTableColumn)); 5696 dataflowRelation.setProcess(process); 5697 } 5698 } 5699 } 5700 } 5701 5702 private void analyzeCloneDatabaseStmt(TCreateDatabaseSqlStatement stmt) { 5703 if (stmt.getCloneSourceDb() != null) { 5704 Database sourceDatabase = modelFactory.createDatabase(stmt.getCloneSourceDb()); 5705 Database cloneDatabase = modelFactory.createDatabase(stmt.getDatabaseName()); 5706 Process process = modelFactory.createProcess(stmt); 5707 cloneDatabase.addProcess(process); 5708 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 5709 relation.setEffectType(EffectType.clone_database); 5710 relation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(cloneDatabase.getRelationRows())); 5711 relation.addSource( 5712 new RelationRowsRelationshipElement<TableRelationRows>(sourceDatabase.getRelationRows())); 5713 relation.setProcess(process); 5714 } 5715 } 5716 5717 private void analyzeCloneSchemaStmt(TCreateSchemaSqlStatement stmt) { 5718 if (stmt.getCloneSourceSchema() != null) { 5719 Schema sourceSchema = modelFactory.createSchema(stmt.getCloneSourceSchema()); 5720 Schema cloneSchema = modelFactory.createSchema(stmt.getSchemaName()); 5721 Process process = modelFactory.createProcess(stmt); 5722 cloneSchema.addProcess(process); 5723 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 5724 relation.setEffectType(EffectType.clone_schema); 5725 relation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(cloneSchema.getRelationRows())); 5726 relation.addSource(new RelationRowsRelationshipElement<TableRelationRows>(sourceSchema.getRelationRows())); 5727 relation.setProcess(process); 5728 } 5729 } 5730 5731 /** 5732 * Record one dynamic-SQL site. Span is the statement extent (1-based, end-exclusive), tagged with 5733 * the per-statement file hash, mirroring the {@link ErrorInfo} position convention. When the 5734 * {@code reportDynamicSqlSitesAsErrors} option is on, non-RESOLVED sites are also mirrored into an 5735 * {@link ErrorInfo} so legacy {@code getErrorMessages()} consumers can see them. 5736 */ 5737 private void recordDynamicSqlSite(TCustomSqlStatement stmt, DynamicSqlSite.Kind kind, 5738 DynamicSqlSite.Status status, String reason) { 5739 recordDynamicSqlSite(stmt, kind, status, reason, 0, 0); 5740 } 5741 5742 private void recordDynamicSqlSite(TCustomSqlStatement stmt, DynamicSqlSite.Kind kind, 5743 DynamicSqlSite.Status status, String reason, int resolvedCount, int partialCount) { 5744 DynamicSqlTrustMode trustMode = option == null ? DynamicSqlTrustMode.LEGACY 5745 : option.getDynamicSqlTrustMode(); 5746 // This compatibility overload has no fragment provenance. Never turn 5747 // that absence into an exact zero; callers with evidence use the full 5748 // overload below. 5749 recordDynamicSqlSite(stmt, kind, status, reason, resolvedCount, partialCount, 5750 0, -1, false, trustMode); 5751 } 5752 5753 private void recordDynamicSqlSite(TCustomSqlStatement stmt, DynamicSqlSite.Kind kind, 5754 DynamicSqlSite.Status status, String reason, int resolvedCount, int partialCount, 5755 int unprovenCount, int unresolvedHoleCount, 5756 boolean holeCountExact, DynamicSqlTrustMode trustMode) { 5757 recordDynamicSqlSite(stmt, kind, status, reason, resolvedCount, partialCount, 5758 unprovenCount, unresolvedHoleCount, holeCountExact, trustMode, 5759 -1, -1, null); 5760 } 5761 5762 private void recordDynamicSqlSite(TCustomSqlStatement stmt, DynamicSqlSite.Kind kind, 5763 DynamicSqlSite.Status status, String reason, int resolvedCount, int partialCount, 5764 int unprovenCount, int unresolvedHoleCount, 5765 boolean holeCountExact, DynamicSqlTrustMode trustMode, 5766 int relationshipsBefore, int nestedSitesBefore) { 5767 recordDynamicSqlSite(stmt, kind, status, reason, resolvedCount, partialCount, 5768 unprovenCount, unresolvedHoleCount, holeCountExact, trustMode, 5769 relationshipsBefore, nestedSitesBefore, null); 5770 } 5771 5772 private void recordDynamicSqlSite(TCustomSqlStatement stmt, DynamicSqlSite.Kind kind, 5773 DynamicSqlSite.Status status, String reason, int resolvedCount, int partialCount, 5774 int unprovenCount, int unresolvedHoleCount, 5775 boolean holeCountExact, DynamicSqlTrustMode trustMode, 5776 int relationshipsBefore, int nestedSitesBefore, 5777 List<Long> explicitlyProvenRelationshipIds) { 5778 recordDynamicSqlSite(stmt, kind, status, reason, resolvedCount, partialCount, 5779 unprovenCount, unresolvedHoleCount, holeCountExact, trustMode, 5780 relationshipsBefore, nestedSitesBefore, 5781 explicitlyProvenRelationshipIds, null); 5782 } 5783 5784 private void recordDynamicSqlSite(TCustomSqlStatement stmt, DynamicSqlSite.Kind kind, 5785 DynamicSqlSite.Status status, String reason, int resolvedCount, int partialCount, 5786 int unprovenCount, int unresolvedHoleCount, 5787 boolean holeCountExact, DynamicSqlTrustMode trustMode, 5788 int relationshipsBefore, int nestedSitesBefore, 5789 List<Long> explicitlyProvenRelationshipIds, String diagnostic) { 5790 recordDynamicSqlSite(stmt, kind, status, reason, resolvedCount, partialCount, 5791 unprovenCount, unresolvedHoleCount, holeCountExact, trustMode, 5792 relationshipsBefore, nestedSitesBefore, 5793 explicitlyProvenRelationshipIds, diagnostic, 5794 Collections.<UnresolvedFragment>emptyList(), null); 5795 } 5796 5797 /** 5798 * An opaque site the evaluator could not turn into analyzable text: UNRESOLVED with 5799 * the argument-shape reason, carrying whatever hole evidence the evaluator gathered 5800 * on the way (the variable that stopped the fold, its assignment, its span). 5801 */ 5802 private void recordUnresolvedDynamicSqlSite(TCustomSqlStatement stmt, TMssqlExecute executeStmt, 5803 DynamicSqlSite.Kind kind) { 5804 DynamicHoleEvidence holeEvidence = dynamicHoleEvidence(evaluatedDynamicSqlFor(stmt)); 5805 recordDynamicSqlSite(stmt, kind, DynamicSqlSite.Status.UNRESOLVED, dynamicReasonOf(executeStmt), 5806 0, 0, 0, holeEvidence.count, holeEvidence.exact, option.getDynamicSqlTrustMode(), 5807 -1, -1, null, null, holeEvidence.fragments, holeEvidence.materializedText); 5808 } 5809 5810 private void recordDynamicSqlSite(TCustomSqlStatement stmt, DynamicSqlSite.Kind kind, 5811 DynamicSqlSite.Status status, String reason, int resolvedCount, int partialCount, 5812 int unprovenCount, int unresolvedHoleCount, 5813 boolean holeCountExact, DynamicSqlTrustMode trustMode, 5814 int relationshipsBefore, int nestedSitesBefore, 5815 List<Long> explicitlyProvenRelationshipIds, String diagnostic, 5816 List<UnresolvedFragment> unresolvedFragments, String materializedText) { 5817 if (stmt == null || stmt.getStartToken() == null || stmt.getEndToken() == null) { 5818 return; 5819 } 5820 List<Long> observedRelationshipIds = dynamicRelationshipIdsSince( 5821 relationshipsBefore, nestedSitesBefore); 5822 List<Long> provenRelationshipIds = Collections.emptyList(); 5823 if (explicitlyProvenRelationshipIds != null) { 5824 LinkedHashSet<Long> observed = new LinkedHashSet<Long>(observedRelationshipIds); 5825 provenRelationshipIds = new ArrayList<Long>(); 5826 for (Long id : explicitlyProvenRelationshipIds) { 5827 if (id != null && observed.contains(id) 5828 && !provenRelationshipIds.contains(id)) { 5829 provenRelationshipIds.add(id); 5830 } 5831 } 5832 } else if (status == DynamicSqlSite.Status.RESOLVED 5833 && unprovenCount == 0 5834 && holeCountExact && unresolvedHoleCount == 0) { 5835 provenRelationshipIds = provenDynamicRelationshipIdsSince( 5836 relationshipsBefore, nestedSitesBefore); 5837 } 5838 Pair3<Long, Long, String> start = new Pair3<Long, Long, String>(stmt.getStartToken().lineNo, 5839 stmt.getStartToken().columnNo, ModelBindingManager.getGlobalHash()); 5840 Pair3<Long, Long, String> end; 5841 String[] segments = stmt.getEndToken().getAstext().split("\n", -1); 5842 if (segments.length == 1) { 5843 end = new Pair3<Long, Long, String>(stmt.getEndToken().lineNo, 5844 stmt.getEndToken().columnNo + stmt.getEndToken().getAstext().length(), 5845 ModelBindingManager.getGlobalHash()); 5846 } else { 5847 end = new Pair3<Long, Long, String>(stmt.getEndToken().lineNo + segments.length - 1, 5848 (long) segments[segments.length - 1].length() + 1, ModelBindingManager.getGlobalHash()); 5849 } 5850 dynamicSqlSites.add(new DynamicSqlSite(kind, status, reason, start, end, 5851 resolvedCount, partialCount, unprovenCount, 5852 unresolvedHoleCount, holeCountExact, trustMode, 5853 observedRelationshipIds, provenRelationshipIds, diagnostic, 5854 unresolvedFragments, materializedText)); 5855 5856 if (status != DynamicSqlSite.Status.RESOLVED && option != null 5857 && option.isReportDynamicSqlSitesAsErrors()) { 5858 ErrorInfo errorInfo = new ErrorInfo(); 5859 errorInfo.setErrorType(ErrorInfo.DYNAMIC_SQL_UNRESOLVED); 5860 errorInfo.setErrorMessage(reason != null ? reason : "dynamic SQL not resolved"); 5861 errorInfo.setStartPosition(start); 5862 errorInfo.setEndPosition(end); 5863 errorInfo.fillInfo(this); 5864 errorInfos.add(errorInfo); 5865 } 5866 } 5867 5868 /** 5869 * Relationship IDs created by one dynamic site, excluding relationship 5870 * deltas already attributed to nested dynamic sites. The IDs match the 5871 * non-simple XML model emitted by this analyzer instance. 5872 */ 5873 private List<Long> dynamicRelationshipIdsSince(int beforeCount, int nestedSitesBefore) { 5874 if (beforeCount < 0) { 5875 return Collections.emptyList(); 5876 } 5877 Set<Long> nested = new HashSet<Long>(); 5878 if (nestedSitesBefore >= 0) { 5879 for (int i = Math.min(nestedSitesBefore, dynamicSqlSites.size()); 5880 i < dynamicSqlSites.size(); i++) { 5881 nested.addAll(dynamicSqlSites.get(i).getObservedRelationshipIds()); 5882 } 5883 } 5884 Relationship[] relationships = modelManager.getRelations(); 5885 List<Long> ids = new ArrayList<Long>(); 5886 for (int i = Math.min(beforeCount, relationships.length); 5887 i < relationships.length; i++) { 5888 Relationship relationship = relationships[i]; 5889 if (relationship instanceof AbstractRelationship) { 5890 Long id = Long.valueOf(((AbstractRelationship) relationship).getId()); 5891 if (!nested.contains(id)) { 5892 ids.add(id); 5893 } 5894 } 5895 } 5896 return ids; 5897 } 5898 5899 /** 5900 * Per-edge proof filter for a complete materialization. Placeholder-tainted 5901 * relations and every downstream relation that consumes their target stay 5902 * observation-only; independent relations from the same site remain proven. 5903 */ 5904 private List<Long> provenDynamicRelationshipIdsSince( 5905 int beforeCount, int nestedSitesBefore) { 5906 List<Long> observedIds = dynamicRelationshipIdsSince(beforeCount, nestedSitesBefore); 5907 if (observedIds.isEmpty()) { 5908 return observedIds; 5909 } 5910 Set<Long> observed = new HashSet<Long>(observedIds); 5911 Set<Long> nestedUnproven = new HashSet<Long>(); 5912 if (nestedSitesBefore >= 0) { 5913 for (int i = Math.min(nestedSitesBefore, dynamicSqlSites.size()); 5914 i < dynamicSqlSites.size(); i++) { 5915 DynamicSqlSite nestedSite = dynamicSqlSites.get(i); 5916 nestedUnproven.addAll(nestedSite.getObservedRelationshipIds()); 5917 nestedUnproven.removeAll(nestedSite.getProvenRelationshipIds()); 5918 } 5919 } 5920 5921 Set<Object> taintedElements = Collections.newSetFromMap( 5922 new IdentityHashMap<Object, Boolean>()); 5923 Set<Relationship> taintedRelationships = Collections.newSetFromMap( 5924 new IdentityHashMap<Relationship, Boolean>()); 5925 Relationship[] relationships = modelManager.getRelations(); 5926 Map<Long, Relationship> relationshipsById = new HashMap<Long, Relationship>(); 5927 for (Relationship relationship : relationships) { 5928 if (!(relationship instanceof AbstractRelationship)) continue; 5929 Long id = Long.valueOf(((AbstractRelationship) relationship).getId()); 5930 relationshipsById.put(id, relationship); 5931 if (nestedUnproven.contains(id)) { 5932 addDynamicTargetElement(taintedElements, relationship); 5933 } 5934 } 5935 5936 boolean changed; 5937 do { 5938 changed = false; 5939 for (Relationship relationship : relationships) { 5940 if (!(relationship instanceof AbstractRelationship) 5941 || !observed.contains(Long.valueOf( 5942 ((AbstractRelationship) relationship).getId())) 5943 || taintedRelationships.contains(relationship)) { 5944 continue; 5945 } 5946 if (hasUnresolvedDynamicEndpoint(relationship) 5947 || hasDynamicSourceElement(taintedElements, relationship)) { 5948 taintedRelationships.add(relationship); 5949 changed |= addDynamicTargetElement(taintedElements, relationship); 5950 } 5951 } 5952 } while (changed); 5953 5954 List<Long> proven = new ArrayList<Long>(); 5955 for (Long id : observedIds) { 5956 Relationship relationship = relationshipsById.get(id); 5957 if (relationship != null && !taintedRelationships.contains(relationship)) { 5958 proven.add(id); 5959 } 5960 } 5961 return proven; 5962 } 5963 5964 private boolean hasDynamicSourceElement(Set<Object> elements, Relationship relationship) { 5965 if (relationship.getSources() == null) return false; 5966 for (Object value : relationship.getSources()) { 5967 if (value instanceof RelationshipElement 5968 && elements.contains(((RelationshipElement<?>) value).getElement())) { 5969 return true; 5970 } 5971 } 5972 return false; 5973 } 5974 5975 private boolean addDynamicTargetElement(Set<Object> elements, Relationship relationship) { 5976 if (relationship.getTarget() == null 5977 || relationship.getTarget().getElement() == null) { 5978 return false; 5979 } 5980 return elements.add(relationship.getTarget().getElement()); 5981 } 5982 5983 private boolean hasUnresolvedDynamicEndpoint(Relationship relationship) { 5984 if (hasPlaceholderSourceTable(relationship, false)) { 5985 return true; 5986 } 5987 if (relationship.getSources() != null) { 5988 for (Object value : relationship.getSources()) { 5989 if (value instanceof RelationRowsRelationshipElement) { 5990 Object rows = ((RelationRowsRelationshipElement<?>) value).getElement(); 5991 if (rows instanceof TableRelationRows 5992 && isRuntimeObjectPlaceholder( 5993 ((TableRelationRows) rows).getHolder())) { 5994 return true; 5995 } 5996 } 5997 } 5998 } 5999 Object target = relationship.getTarget() == null 6000 ? null : relationship.getTarget().getElement(); 6001 if (target instanceof TableColumn) { 6002 return isRuntimeObjectPlaceholder(((TableColumn) target).getTable()); 6003 } 6004 return target instanceof TableRelationRows 6005 && isRuntimeObjectPlaceholder(((TableRelationRows) target).getHolder()); 6006 } 6007 6008 private boolean isRuntimeObjectPlaceholder(Table table) { 6009 if (table == null || table.isPseudo()) return true; 6010 if (table.isVariable()) return false; 6011 String name = table.getName(); 6012 return name != null && name.trim().startsWith("@"); 6013 } 6014 6015 /** Identity snapshot used only by SHADOW observability; it never drives removal. */ 6016 private Set<Relationship> snapshotDynamicRelationships() { 6017 Set<Relationship> snapshot = Collections.newSetFromMap( 6018 new IdentityHashMap<Relationship, Boolean>()); 6019 Collections.addAll(snapshot, modelManager.getRelations()); 6020 return snapshot; 6021 } 6022 6023 /** Exact number of currently published relationships not present in {@code before}. */ 6024 private int countNewDynamicRelationships(Set<Relationship> before) { 6025 if (before == null) { 6026 return 0; 6027 } 6028 int count = 0; 6029 for (Relationship relation : modelManager.getRelations()) { 6030 if (!before.contains(relation)) { 6031 count++; 6032 } 6033 } 6034 return count; 6035 } 6036 6037 /** 6038 * One unresolved hole of a dynamic fold, located by CHARACTER SPAN in the analyzed text 6039 * rather than by its rendered spelling. Locating by text would over-mark: a hole rendering 6040 * as {@code SRC} also occurs inside the unrelated literal {@code SRC_COPY} in the same batch. 6041 * 6042 * <p>A hole here is always a fragment whose VALUE is unknown. An {@code INEXACT_TRANSFORM} 6043 * fragment is deliberately NOT one: there the evaluator did compute the value from literals 6044 * ({@code REPLACE(tmpl, '{SRC}', 'dbo.SalesOrders')}) and only its byte-exactness against 6045 * vendor runtime semantics is unproven. Treating it as a hole would brand every fully 6046 * template-resolved name a runtime template — the opposite of this feature's purpose. 6047 */ 6048 private static final class FoldHole { 6049 /** Character span [start, end) in the analyzed text; start == end for a splice point. */ 6050 final int start; 6051 final int end; 6052 FoldHole(int start, int end) { 6053 this.start = start; 6054 this.end = end; 6055 } 6056 6057 /** True when this hole can affect any character of [from, to) — splice points included. */ 6058 boolean touches(int from, int to) { 6059 if (start == end) { 6060 return from <= start && start <= to; 6061 } 6062 return start < to && end > from; 6063 } 6064 } 6065 6066 /** Conservative fold-hole spelling: the unresolved {@code @variable} remnant a partial fold leaves behind. */ 6067 private static final Pattern DYNAMIC_FOLD_PLACEHOLDER = Pattern.compile("@\\w+"); 6068 6069 /** 6070 * Depth of dynamic-fold inner analysis currently on the stack. Statements analyzed at 6071 * depth 0 are STATIC statements of the unit; their table references veto dynamic-template 6072 * marks on the same name (see {@link #applyStaticTemplateVetoes}), which is what makes the 6073 * shared-node contract independent of statement ORDER — a static reference after the fold 6074 * withdraws the mark exactly as one before the fold prevents it. 6075 */ 6076 private int dynamicFoldDepth = 0; 6077 6078 /** True once any endpoint of this analysis carries a dynamic-template mark. */ 6079 private boolean anyDynamicTemplateMarked = false; 6080 6081 /** 6082 * Name-registry keys of every table referenced by a STATIC statement after the first 6083 * template mark. A fold never marks (and a later collection unmarks) a name in this set. 6084 */ 6085 private final Set<String> staticTemplateVetoKeys = new HashSet<String>(); 6086 6087 /** Statements already veto-walked (identity), so repeated re-analysis stays O(1). */ 6088 private final Set<TCustomSqlStatement> staticVetoWalkedStmts = 6089 Collections.newSetFromMap(new IdentityHashMap<TCustomSqlStatement, Boolean>()); 6090 6091 /** 6092 * Collects {@code stmt}'s static table references into the veto set and withdraws any 6093 * template mark already sitting on a node they share. Runs only at fold depth 0 and only 6094 * once a mark exists, so the common no-dynamic-SQL unit pays nothing. 6095 */ 6096 private void applyStaticTemplateVetoes(TCustomSqlStatement stmt) { 6097 // Walk each statement object once: the veto keys accumulate, so a repeat walk of a 6098 // re-analyzed procedure body adds nothing but O(AST) cost per call. 6099 if (!staticVetoWalkedStmts.add(stmt)) { 6100 return; 6101 } 6102 TemplateTableCollector collector = new TemplateTableCollector(); 6103 try { 6104 if (stmt instanceof TSelectSqlStatement) { 6105 collector.preVisit((TSelectSqlStatement) stmt); 6106 } 6107 stmt.acceptChildren(collector); 6108 } catch (RuntimeException ex) { 6109 // This is an enrichment-only walk over an error-recovered AST. Keep every 6110 // static veto collected before the malformed slot, and never let a missing 6111 // annotation discard the statement's real lineage. 6112 logger.warn("Static template-veto walk was incomplete; keeping recovered lineage", ex); 6113 } 6114 for (TTable table : collector.tables) { 6115 if (table.getSubquery() != null || table.getCTE() != null 6116 || table.getTableName() == null) { 6117 continue; 6118 } 6119 vetoStaticTemplateName(table.getTableName().toString()); 6120 } 6121 for (TObjectName intoTarget : collector.intoTargets) { 6122 vetoStaticTemplateName(intoTarget.toString()); 6123 } 6124 } 6125 6126 /** 6127 * Records one static reference under the SAME name-registry keys the model-reuse lookups 6128 * use ({@code ModelFactory.createTableByName}), so the veto matches exactly the reuse that 6129 * creates a shared node. 6130 */ 6131 private void vetoStaticTemplateName(String rawName) { 6132 if (SQLUtil.isEmpty(rawName)) { 6133 return; 6134 } 6135 String[] keys = { DlineageUtil.getTableFullName(rawName), 6136 DlineageUtil.getTableFullNameWithDefaultSchema(rawName) }; 6137 for (String key : keys) { 6138 if (SQLUtil.isEmpty(key)) { 6139 continue; 6140 } 6141 staticTemplateVetoKeys.add(key); 6142 Table bound = modelManager.getTableByName(key); 6143 if (bound != null && bound.isDynamicTemplate()) { 6144 bound.clearDynamicTemplate(); 6145 } 6146 } 6147 } 6148 6149 /** 6150 * A delimited identifier whose ENTIRE content is an {@code @variable} remnant — 6151 * {@code [@S]} / {@code "@S"}, the shape a {@code '... [' + @S + '] ...'} splice folds to. 6152 * Deliberately anchored to the whole content: {@code [weird@name]} does NOT match, because 6153 * a delimited name that merely CONTAINS an {@code @} is an ordinary legal identifier. 6154 */ 6155 private static final Pattern DYNAMIC_FOLD_DELIMITED_PLACEHOLDER = 6156 Pattern.compile("\\[(@\\w+)\\]|\"(@\\w+)\""); 6157 6158 /** 6159 * Identity snapshot of the endpoints that already existed before a dynamic fold was 6160 * analyzed, so {@link #markDynamicTemplateEndpoints} can restrict itself to the nodes 6161 * THIS fold introduced. Returns {@code null} when the fold cannot carry a hole, which is 6162 * the signal to skip marking entirely (and pay nothing on the fully-resolved path). 6163 */ 6164 private Set<Table> snapshotTemplateCandidateTables(boolean foldMayCarryHoles) { 6165 if (!foldMayCarryHoles) { 6166 return null; 6167 } 6168 Set<Table> snapshot = Collections.newSetFromMap(new IdentityHashMap<Table, Boolean>()); 6169 snapshot.addAll(modelManager.getTablesByName()); 6170 return snapshot; 6171 } 6172 6173 /** 6174 * Exact hole spans for {@code analyzedText}, from the evaluator's fragment provenance. 6175 * Empty unless the site describes THIS text — spans taken from a different materialization 6176 * would point at arbitrary offsets of the string actually analyzed. 6177 */ 6178 /** True when the site carries fragments that provably render its own text. */ 6179 private static boolean hasCompleteFragmentProvenance( 6180 gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite site) { 6181 if (site == null || site.sqlText == null || site.fragments.isEmpty()) { 6182 return false; 6183 } 6184 StringBuilder rendered = new StringBuilder(site.sqlText.length()); 6185 for (gudusoft.gsqlparser.dlineage.dynamicsql.SqlFragment f : site.fragments) { 6186 rendered.append(f.text); 6187 } 6188 return rendered.toString().equals(site.sqlText); 6189 } 6190 6191 /** True when any fragment of the site stands in for a value nobody knows. */ 6192 private static boolean hasValueUnknownHole( 6193 gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite site) { 6194 if (site == null) { 6195 return false; 6196 } 6197 for (gudusoft.gsqlparser.dlineage.dynamicsql.SqlFragment f : site.fragments) { 6198 if (!f.isLiteral() && f.valueUnknown) { 6199 return true; 6200 } 6201 } 6202 return false; 6203 } 6204 6205 /** 6206 * Value-unknown hole spans of a site whose text IS the analyzed text. A hole whose value 6207 * the evaluator computed from literals ({@code SqlFragment.valueUnknown} false) is an 6208 * unproven RENDERING of a known name, not a runtime template, and yields no span. 6209 */ 6210 private List<FoldHole> exactFoldHoles( 6211 gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite site, 6212 String analyzedText) { 6213 List<FoldHole> holes = new ArrayList<FoldHole>(); 6214 if (site == null || site.sqlText == null || analyzedText == null 6215 || !site.sqlText.equals(analyzedText)) { 6216 return holes; 6217 } 6218 List<int[]> spans = gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlEdgeProof 6219 .holeSpansOf(site); 6220 List<gudusoft.gsqlparser.dlineage.dynamicsql.SqlFragment> fragments = 6221 gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlEdgeProof.holesOf(site); 6222 for (int i = 0; i < spans.size() && i < fragments.size(); i++) { 6223 if (!fragments.get(i).valueUnknown) { 6224 continue; 6225 } 6226 int[] span = spans.get(i); 6227 holes.add(new FoldHole(span[0], span[1])); 6228 } 6229 return holes; 6230 } 6231 6232 /** 6233 * Maps the site's hole positions into a DIFFERENT rendering of the same fold — the legacy 6234 * token fold renders a hole as the variable/function name ({@code [pre_UPPER]}) where the 6235 * evaluator renders {@code [pre_@X]} — by aligning the LITERAL fragments, which both 6236 * renderings share, against the analyzed text. The literals are matched twice, greedily 6237 * from the left and greedily from the right; only when both passes agree is the alignment 6238 * forced, and the gaps between matched literals are then provably the holes. Returns 6239 * {@code null} when alignment is ambiguous or impossible — the caller falls back to the 6240 * conservative tier rather than guess. 6241 */ 6242 private List<FoldHole> alignedFoldHoles( 6243 gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite site, 6244 String analyzedText) { 6245 if (site == null || analyzedText == null) { 6246 return null; 6247 } 6248 List<gudusoft.gsqlparser.dlineage.dynamicsql.SqlFragment> fragments = site.fragments; 6249 // A value-KNOWN hole's text IS its value (that is what valueUnknown=false means), so 6250 // it anchors the alignment exactly like a literal. Without this, a known hole 6251 // adjacent to an unknown one would smear "templated" across every segment the 6252 // combined gap touches — [' + UPPER(@D) + '].[dbo].[x] must report db, not 6253 // db|schema|name. If the legacy rendering of the known hole differs, the anchor 6254 // match fails and the alignment is refused rather than guessed. 6255 List<String> literals = new ArrayList<String>(); 6256 for (gudusoft.gsqlparser.dlineage.dynamicsql.SqlFragment f : fragments) { 6257 if (isAlignmentAnchor(f) && !f.text.isEmpty()) { 6258 literals.add(f.text); 6259 } 6260 } 6261 int n = literals.size(); 6262 int[] posForward = new int[n]; 6263 int cursor = 0; 6264 for (int i = 0; i < n; i++) { 6265 int at = analyzedText.indexOf(literals.get(i), cursor); 6266 if (at < 0) { 6267 return null; 6268 } 6269 posForward[i] = at; 6270 cursor = at + literals.get(i).length(); 6271 } 6272 int[] posBackward = new int[n]; 6273 cursor = analyzedText.length(); 6274 for (int i = n - 1; i >= 0; i--) { 6275 int at = analyzedText.lastIndexOf(literals.get(i), cursor - literals.get(i).length()); 6276 if (at < 0) { 6277 return null; 6278 } 6279 posBackward[i] = at; 6280 cursor = at; 6281 } 6282 for (int i = 0; i < n; i++) { 6283 if (posForward[i] != posBackward[i]) { 6284 return null; 6285 } 6286 } 6287 // Walk the fragments, attributing each inter-literal gap to the hole fragments 6288 // between the anchors. A non-empty gap with no hole, or leftover text at either 6289 // end, means the analyzed text is not this fold — refuse. 6290 List<FoldHole> holes = new ArrayList<FoldHole>(); 6291 int pos = 0; 6292 int literalIndex = 0; 6293 boolean pendingHole = false; 6294 for (gudusoft.gsqlparser.dlineage.dynamicsql.SqlFragment f : fragments) { 6295 if (isAlignmentAnchor(f)) { 6296 if (f.text.isEmpty()) { 6297 continue; 6298 } 6299 int at = posForward[literalIndex++]; 6300 if (at < pos) { 6301 return null; 6302 } 6303 if (at > pos && !pendingHole) { 6304 return null; 6305 } 6306 if (pendingHole) { 6307 holes.add(new FoldHole(pos, at)); 6308 } 6309 pendingHole = false; 6310 pos = at + f.text.length(); 6311 } else { 6312 pendingHole = true; 6313 } 6314 } 6315 if (pos < analyzedText.length() && !pendingHole) { 6316 return null; 6317 } 6318 if (pendingHole) { 6319 holes.add(new FoldHole(pos, analyzedText.length())); 6320 } 6321 return holes; 6322 } 6323 6324 /** Literal fragments AND value-known holes anchor the alignment; only value-unknown holes gap. */ 6325 private static boolean isAlignmentAnchor(gudusoft.gsqlparser.dlineage.dynamicsql.SqlFragment f) { 6326 return f.isLiteral() || !f.valueUnknown; 6327 } 6328 6329 /** 6330 * Fallback for a legacy token fold that carries no fragment provenance: the unresolved 6331 * {@code @variable} remnants left in the text. Restricted to what is PROVABLE — a T-SQL 6332 * family dialect (where an undelimited leading {@code @} cannot begin a legal object name, 6333 * the same reasoning {@link #isPlaceholderNamedTable} relies on), and only occurrences 6334 * outside identifier delimiters and string literals. A bracketed {@code [weird@name]} is a 6335 * legal literal identifier even inside a folded batch, so it is deliberately not a hole 6336 * here; such a name is only marked when exact provenance says so. 6337 */ 6338 private List<FoldHole> conservativeFoldHoles(String analyzedText) { 6339 List<FoldHole> holes = new ArrayList<FoldHole>(); 6340 if (analyzedText == null || !Table.isTsqlFamilyVendor(option.getVendor())) { 6341 return holes; 6342 } 6343 Matcher matcher = DYNAMIC_FOLD_PLACEHOLDER.matcher(analyzedText); 6344 while (matcher.find()) { 6345 if (!isInsideDelimiterOrLiteral(analyzedText, matcher.start())) { 6346 holes.add(new FoldHole(matcher.start(), matcher.end())); 6347 } 6348 } 6349 Matcher delimited = DYNAMIC_FOLD_DELIMITED_PLACEHOLDER.matcher(analyzedText); 6350 while (delimited.find()) { 6351 // The delimiters themselves are literal fold text; only the content is the hole, 6352 // so a '#'/'@' that survives outside them still classifies the endpoint. 6353 if (!isInsideDelimiterOrLiteral(analyzedText, delimited.start())) { 6354 holes.add(new FoldHole(delimited.start() + 1, delimited.end() - 1)); 6355 } 6356 } 6357 return holes; 6358 } 6359 6360 /** True when {@code index} sits inside a delimited identifier or a string literal. */ 6361 private static boolean isInsideDelimiterOrLiteral(String text, int index) { 6362 boolean inBracket = false, inDoubleQuote = false, inSingleQuote = false; 6363 for (int i = 0; i < index && i < text.length(); i++) { 6364 char c = text.charAt(i); 6365 if (inBracket) { 6366 if (c == ']') { 6367 inBracket = false; 6368 } 6369 } else if (inDoubleQuote) { 6370 if (c == '"') { 6371 inDoubleQuote = false; 6372 } 6373 } else if (inSingleQuote) { 6374 if (c == '\'') { 6375 inSingleQuote = false; 6376 } 6377 } else if (c == '[') { 6378 inBracket = true; 6379 } else if (c == '"') { 6380 inDoubleQuote = true; 6381 } else if (c == '\'') { 6382 inSingleQuote = true; 6383 } 6384 } 6385 return inBracket || inDoubleQuote || inSingleQuote; 6386 } 6387 6388 /** 6389 * Land the placeholder verdict on the MODEL NODES a partially-folded dynamic site just 6390 * produced (docs/tmp/dynamic-template-endpoint-provenance.md): every endpoint whose 6391 * identifier OVERLAPS an unresolved hole is marked {@link EndpointKind#DYNAMIC_TEMPLATE} 6392 * and told WHICH name segments are templated, so a consumer never has to sniff {@code @} 6393 * out of a name — which it cannot do soundly, because the fold keeps the original 6394 * delimiters and {@code [weird@name]} is a legal literal identifier. 6395 * 6396 * <p>Overlap is decided on CHARACTER SPANS of this parse's own tokens against the hole 6397 * spans (exact tier) or the provable {@code @\w+} remnants (conservative tier) — never on 6398 * rendered-text coincidence, which marks unrelated literals that happen to contain a 6399 * hole's spelling. 6400 * 6401 * <p>Never marks outside a fold context ({@code before == null} short-circuits), and never 6402 * marks a plain endpoint that already existed before this fold — a static statement's 6403 * {@code dbo.[weird@name]} stays exactly what it was. <b>Known limitation:</b> when a 6404 * static statement and a folded statement name the SAME object, the model carries one 6405 * shared node; it is left unmarked, because marking it would make the static reference 6406 * wrong. Per-reference template provenance would be needed to state both facts. 6407 * 6408 * <p>Oracle {@code EXECUTE IMMEDIATE} / DBMS_SQL legacy folds splice a BARE identifier 6409 * ({@code '... FROM ' || p_tab}), indistinguishable from a real table name and carrying no 6410 * fragment provenance, so they are deliberately NOT marked rather than marked on a guess. 6411 */ 6412 private void markDynamicTemplateEndpoints(Set<Table> before, 6413 gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite site, 6414 TGSqlParser analyzedParser) { 6415 if (before == null || analyzedParser == null || analyzedParser.sqlstatements == null) { 6416 return; 6417 } 6418 String analyzedText = analyzedParser.sqltext; 6419 List<FoldHole> holes; 6420 if (hasCompleteFragmentProvenance(site)) { 6421 // Exact provenance is AUTHORITATIVE in both directions: zero value-unknown 6422 // holes means every part of the string is a known value, so nothing here is 6423 // runtime-templated and the conservative heuristic must NOT run — it would 6424 // mark a legitimate literal-derived [@S]. 6425 if (!hasValueUnknownHole(site)) { 6426 return; 6427 } 6428 if (site.sqlText.equals(analyzedText)) { 6429 holes = exactFoldHoles(site, analyzedText); 6430 } else { 6431 // The analyzed text is the legacy token fold's rendering of the same 6432 // fold; map the holes across via literal alignment, falling back to the 6433 // conservative remnant scan when the alignment cannot be proven. 6434 holes = alignedFoldHoles(site, analyzedText); 6435 if (holes == null) { 6436 holes = conservativeFoldHoles(analyzedText); 6437 } 6438 } 6439 } else { 6440 holes = conservativeFoldHoles(analyzedText); 6441 } 6442 if (holes.isEmpty()) { 6443 return; 6444 } 6445 TemplateTableCollector collector = new TemplateTableCollector(); 6446 for (int i = 0; i < analyzedParser.sqlstatements.size(); i++) { 6447 TCustomSqlStatement stmt = analyzedParser.sqlstatements.get(i); 6448 if (stmt == null) { 6449 continue; 6450 } 6451 try { 6452 // acceptChildren does not preVisit the statement itself, so a top-level 6453 // SELECT ... INTO <target> would lose its write endpoint. 6454 if (stmt instanceof TSelectSqlStatement) { 6455 collector.preVisit((TSelectSqlStatement) stmt); 6456 } 6457 stmt.acceptChildren(collector); 6458 } catch (RuntimeException ex) { 6459 // A clean inner parse can still contain recovery shells. The collector is 6460 // stateless, so keep endpoints reached before the bad slot and continue with 6461 // later statements. A warning is intentional: marks for this site may be 6462 // incomplete, but enrichment must never erase otherwise valid lineage. 6463 logger.warn("Dynamic template endpoint walk was incomplete; keeping recovered lineage", ex); 6464 } 6465 } 6466 for (TTable table : collector.tables) { 6467 if (table.getSubquery() != null || table.getCTE() != null) { 6468 continue; 6469 } 6470 markDynamicTemplateEndpoint(before, table.getTableName(), 6471 templateTableModelOf(table), analyzedText, holes); 6472 } 6473 for (TObjectName intoTarget : collector.intoTargets) { 6474 // SELECT ... INTO targets are bound by NAME (ModelFactory.createTableByName), 6475 // not against the parsed node, so they resolve through the name registry. 6476 Table model = modelManager.getTableByName( 6477 DlineageUtil.getTableFullName(intoTarget.toString())); 6478 markDynamicTemplateEndpoint(before, intoTarget, model, analyzedText, holes); 6479 } 6480 } 6481 6482 /** Collects the endpoint-bearing name nodes of an analyzed dynamic-fold parse. */ 6483 private static final class TemplateTableCollector extends TParseTreeVisitor { 6484 final List<TTable> tables = new ArrayList<TTable>(); 6485 final List<TObjectName> intoTargets = new ArrayList<TObjectName>(); 6486 6487 @Override 6488 public void preVisit(TTable table) { 6489 if (table != null) { 6490 tables.add(table); 6491 } 6492 } 6493 6494 @Override 6495 public void preVisit(TSelectSqlStatement select) { 6496 if (select == null) { 6497 return; 6498 } 6499 if (select.getIntoTableClause() != null 6500 && select.getIntoTableClause().getTableName() != null) { 6501 intoTargets.add(select.getIntoTableClause().getTableName()); 6502 } 6503 // T-SQL SELECT ... INTO <target> arrives as an INTO clause carrying an 6504 // expression list, the same shape analyzeSelectIntoClause() reads. 6505 TIntoClause intoClause = select.getIntoClause(); 6506 if (intoClause == null || intoClause.getExprList() == null) { 6507 return; 6508 } 6509 for (int i = 0; i < intoClause.getExprList().size(); i++) { 6510 TExpression expr = intoClause.getExprList().getExpression(i); 6511 if (expr != null && expr.getObjectOperand() != null) { 6512 intoTargets.add(expr.getObjectOperand()); 6513 } 6514 } 6515 } 6516 } 6517 6518 /** Per-endpoint half of {@link #markDynamicTemplateEndpoints}: span overlap + part labelling. */ 6519 private void markDynamicTemplateEndpoint(Set<Table> before, TObjectName tableName, 6520 Table model, String analyzedText, List<FoldHole> holes) { 6521 if (tableName == null || model == null || tableName.getStartToken() == null 6522 || tableName.getEndToken() == null) { 6523 return; 6524 } 6525 int from = (int) tableName.getStartToken().offset; 6526 int to = (int) (tableName.getEndToken().offset 6527 + tableName.getEndToken().toString().length()); 6528 if (from < 0 || to <= from || to > analyzedText.length()) { 6529 return; 6530 } 6531 List<int[]> segments = nameSegmentSpans(analyzedText, from, to); 6532 if (segments.isEmpty()) { 6533 return; 6534 } 6535 int last = segments.size() - 1; 6536 StringBuilder parts = new StringBuilder(); 6537 boolean identityTemplated = false; 6538 for (int i = 0; i < segments.size(); i++) { 6539 int[] segment = segments.get(i); 6540 boolean touched = false; 6541 for (FoldHole hole : holes) { 6542 if (!hole.touches(segment[0], segment[1])) { 6543 continue; 6544 } 6545 touched = true; 6546 // The hole reaching the FIRST character of the object-name segment means the 6547 // '#'/'@' prefix that would classify this endpoint as temp / table-variable is 6548 // itself unknown runtime text ('INSERT INTO ' + @tab folds to a table named 6549 // @tab, which really names a CATALOG object). 6550 if (i == last 6551 && hole.touches(firstNameChar(analyzedText, segment), 6552 firstNameChar(analyzedText, segment) + 1)) { 6553 identityTemplated = true; 6554 } 6555 } 6556 if (!touched) { 6557 continue; 6558 } 6559 String part = templatedPartLabel(i, last); 6560 if (part == null) { 6561 continue; 6562 } 6563 if (parts.length() > 0) { 6564 parts.append('|'); 6565 } 6566 parts.append(part); 6567 } 6568 if (parts.length() == 0) { 6569 return; 6570 } 6571 if (model.hasSubquery() || model.isPseudo() || model.isVariable()) { 6572 return; 6573 } 6574 // A plain endpoint that predates this fold is shared with a static reference; marking 6575 // it would make that reference wrong. An endpoint already known to be a template may 6576 // accumulate more segments from a second fold. 6577 if (before.contains(model) && !model.isDynamicTemplate()) { 6578 return; 6579 } 6580 // The same contract, order-independent: a name a STATIC statement referenced at any 6581 // earlier point of the unit is never marked (see staticTemplateVetoKeys). 6582 if (staticTemplateVetoKeys.contains(DlineageUtil.getTableFullName(tableName.toString())) 6583 || staticTemplateVetoKeys.contains( 6584 DlineageUtil.getTableFullNameWithDefaultSchema(tableName.toString()))) { 6585 return; 6586 } 6587 model.markDynamicTemplate(parts.toString(), identityTemplated); 6588 anyDynamicTemplateMarked = true; 6589 } 6590 6591 /** Offset of a segment's first NAME character, skipping an opening identifier delimiter. */ 6592 private static int firstNameChar(String text, int[] segment) { 6593 int i = segment[0]; 6594 if (i < text.length()) { 6595 char c = text.charAt(i); 6596 if (c == '[' || c == '"' || c == '`') { 6597 return i + 1; 6598 } 6599 } 6600 return i; 6601 } 6602 6603 /** 6604 * Splits the source span of a qualified name into one span per identifier segment, on the 6605 * dots that sit OUTSIDE identifier delimiters (so {@code [a.b].c} is two segments). 6606 */ 6607 private static List<int[]> nameSegmentSpans(String text, int from, int to) { 6608 List<int[]> segments = new ArrayList<int[]>(); 6609 int segmentStart = from; 6610 boolean inBracket = false, inDoubleQuote = false, inBacktick = false; 6611 for (int i = from; i < to; i++) { 6612 char c = text.charAt(i); 6613 if (inBracket) { 6614 if (c == ']') { 6615 inBracket = false; 6616 } 6617 } else if (inDoubleQuote) { 6618 if (c == '"') { 6619 inDoubleQuote = false; 6620 } 6621 } else if (inBacktick) { 6622 if (c == '`') { 6623 inBacktick = false; 6624 } 6625 } else if (c == '[') { 6626 inBracket = true; 6627 } else if (c == '"') { 6628 inDoubleQuote = true; 6629 } else if (c == '`') { 6630 inBacktick = true; 6631 } else if (c == '.') { 6632 segments.add(new int[] { segmentStart, i }); 6633 segmentStart = i + 1; 6634 } 6635 } 6636 segments.add(new int[] { segmentStart, to }); 6637 return segments; 6638 } 6639 6640 /** 6641 * Outermost-first segment labels for a qualified name, counted from the RIGHT so a 6642 * 2-, 3- or 4-part name labels consistently. Anything deeper than server is unlabelled. 6643 */ 6644 private String templatedPartLabel(int index, int lastIndex) { 6645 switch (lastIndex - index) { 6646 case 0: 6647 return "name"; 6648 case 1: 6649 return "schema"; 6650 case 2: 6651 return "db"; 6652 case 3: 6653 return "server"; 6654 default: 6655 return null; 6656 } 6657 } 6658 6659 /** The dlineage endpoint model bound to a parsed table, across the binding maps. */ 6660 private Table templateTableModelOf(TTable table) { 6661 Object model = modelManager.getModel(table); 6662 if (!(model instanceof Table)) { 6663 model = modelManager.getCreateModel(table); 6664 } 6665 if (!(model instanceof Table)) { 6666 model = modelManager.getInsertModel(table); 6667 } 6668 if (!(model instanceof Table)) { 6669 model = modelManager.getUpdateModel(table); 6670 } 6671 if (!(model instanceof Table)) { 6672 model = modelManager.getMergeModel(table); 6673 } 6674 return model instanceof Table ? (Table) model : null; 6675 } 6676 6677 /** 6678 * What a site can say about its holes. {@code exact} is true only when the evaluator 6679 * saw the site's argument: then {@code count == fragments.size()}, and 6680 * {@code materializedText} is the argument as far as it folded (holes rendered as 6681 * their placeholder text). Otherwise {@code count} is {@code -1} and the list is 6682 * empty - "not known", never an invented zero. 6683 */ 6684 private static final class DynamicHoleEvidence { 6685 final int count; 6686 final boolean exact; 6687 final List<UnresolvedFragment> fragments; 6688 final String materializedText; 6689 6690 DynamicHoleEvidence(int count, boolean exact, List<UnresolvedFragment> fragments, 6691 String materializedText) { 6692 this.count = count; 6693 this.exact = exact; 6694 this.fragments = fragments; 6695 this.materializedText = materializedText; 6696 } 6697 } 6698 6699 private static final DynamicHoleEvidence NO_HOLE_EVIDENCE = new DynamicHoleEvidence(-1, false, 6700 Collections.<UnresolvedFragment>emptyList(), null); 6701 6702 private DynamicHoleEvidence dynamicHoleEvidence( 6703 gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite site) { 6704 if (site == null) { 6705 return NO_HOLE_EVIDENCE; 6706 } 6707 if (!site.provenanceIncomplete) { 6708 return new DynamicHoleEvidence(0, true, Collections.<UnresolvedFragment>emptyList(), 6709 site.sqlText); 6710 } 6711 if (site.fragments.isEmpty()) { 6712 // The value did not reduce and the producer left no provenance (a channel 6713 // without fragments, or a value that never touched a variable). 6714 return new DynamicHoleEvidence(-1, false, Collections.<UnresolvedFragment>emptyList(), 6715 site.sqlText); 6716 } 6717 List<UnresolvedFragment> holes = new ArrayList<UnresolvedFragment>(); 6718 for (gudusoft.gsqlparser.dlineage.dynamicsql.SqlFragment fragment : site.fragments) { 6719 UnresolvedFragment hole = UnresolvedFragment.of(fragment); 6720 if (hole != null) { 6721 holes.add(hole); 6722 } 6723 } 6724 // Only a reduced argument has a materialization; rendering the provenance of an 6725 // unreduced one would fabricate text (the arguments of an unfoldable call glued 6726 // together). Its holes are still reported. 6727 String text = site.sqlText; 6728 if (site.sqlText == null && holes.isEmpty()) { 6729 // The value did not reduce, yet nothing in its provenance is a hole: the 6730 // producer that stopped the fold left no fragment (an undecidable CASE, a 6731 // resource bound, a malformed call). "Not known" - never an exact zero. 6732 return new DynamicHoleEvidence(-1, false, Collections.<UnresolvedFragment>emptyList(), text); 6733 } 6734 return new DynamicHoleEvidence(holes.size(), true, holes, text); 6735 } 6736 6737 /** 6738 * True when the materialized text is nothing but value-unknown holes - a bare 6739 * {@code @sql} whose value the evaluator could not fold. Parsing that as SQL proves 6740 * nothing about the runtime statement, so such a site is reported as its 6741 * argument shape (with the hole evidence) instead of as a parse failure of text 6742 * that was never SQL. 6743 */ 6744 private static boolean isWholeTextHole( 6745 gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite site) { 6746 if (site == null || site.sqlText == null || site.fragments.isEmpty()) { 6747 return false; 6748 } 6749 for (gudusoft.gsqlparser.dlineage.dynamicsql.SqlFragment fragment : site.fragments) { 6750 if (fragment.isLiteral() || !fragment.valueUnknown) { 6751 return false; 6752 } 6753 } 6754 return true; 6755 } 6756 6757 private String dynamicTrustReason(String prefix, int unprovenCount, 6758 int unresolvedHoleCount, boolean holeCountExact) { 6759 String holes = holeCountExact ? unresolvedHoleCount + " unresolved fragment(s)" 6760 : "unresolved fragment count unavailable from the producer"; 6761 return prefix + "; " + holes 6762 + (unprovenCount >= 0 ? "; " + unprovenCount + " unproven candidate relationship(s)" : ""); 6763 } 6764 6765 /** 6766 * Classify the lineage edges a folded dynamic site produced. {@code relsBefore} is the relation 6767 * snapshot taken immediately before the folded SQL was analyzed; the tail of the current relation set 6768 * is this site's delta (relationHolder is insertion-ordered). counts[0]=resolved (edge into a real 6769 * target column), counts[1]=partial (edge into a T-SQL variable / placeholder target). Edges whose 6770 * target is an intermediate result-set are ignored — only final-target edges are counted. 6771 */ 6772 /** 6773 * String-building functions whose names show up as fabricated source "tables" when a partial 6774 * fold leaves a runtime-built object name in the re-parsed text (e.g. 6775 * {@code EXEC('... FROM ' + QUOTENAME(@tab))}). 6776 */ 6777 private static final java.util.Set<String> DYNAMIC_NAME_BUILDER_FUNCTIONS = new HashSet<String>( 6778 Arrays.asList("QUOTENAME", "REPLACE", "CONCAT", "FORMAT", "UPPER", "LOWER", "TRIM", "LTRIM", "RTRIM")); 6779 6780 /** 6781 * Returns {@code {resolvedCount, partialCount, placeholderSeen, resultSetEdgeCount}} for the 6782 * lineage edges added after {@code beforeCount}. 6783 * 6784 * <ul> 6785 * <li>{@code resolvedCount} — edges into a real base-table target column with no placeholder 6786 * contamination anywhere in the new edge set.</li> 6787 * <li>{@code partialCount} — edges into a variable / placeholder target, plus (when the edge 6788 * set is contaminated) every edge that would otherwise have counted as resolved, so the 6789 * public counts never claim usable lineage a placeholder actually feeds.</li> 6790 * <li>{@code placeholderSeen} — 1 when any new edge draws from a fabricated placeholder table: 6791 * a raw name starting with '@' (delimited {@code [@x]} identifiers are legal object names 6792 * and are NOT placeholders), or — during a partial fold only — a variable-backed source 6793 * table (Oracle {@code ' ... FROM ' || p_tab}) or a table named after a string-building 6794 * function ({@code QUOTENAME} etc.).</li> 6795 * <li>{@code resultSetEdgeCount} — clean edges into a result-set column (a folded dynamic 6796 * {@code SELECT}'s projection). These contribute to the site-resolution decision but stay 6797 * out of {@code resolvedCount}, which is documented as base-table targets only.</li> 6798 * </ul> 6799 */ 6800 private int[] classifyDynamicSiteLineage(int beforeCount, boolean partialFold) { 6801 int resolved = 0; 6802 int partial = 0; 6803 int resultSetEdges = 0; 6804 boolean placeholderSeen = false; 6805 Relationship[] after = modelManager.getRelations(); 6806 for (int i = beforeCount; i < after.length; i++) { 6807 Relationship r = after[i]; 6808 if (r == null) { 6809 continue; 6810 } 6811 if (hasPlaceholderSourceTable(r, partialFold)) { 6812 placeholderSeen = true; 6813 } 6814 if (r.getTarget() instanceof ResultColumnRelationshipElement) { 6815 resultSetEdges++; 6816 } else if (r.getTarget() instanceof TableColumnRelationshipElement) { 6817 TableColumn targetColumn = ((TableColumnRelationshipElement) r.getTarget()).getElement(); 6818 Table targetTable = targetColumn == null ? null : targetColumn.getTable(); 6819 if (isPlaceholderNamedTable(targetTable, partialFold)) { 6820 // Runtime-built INSERT destination (e.g. 'INSERT INTO '+QUOTENAME(@tab)+...) 6821 // — the target object is unknown, so the whole site is contaminated. 6822 placeholderSeen = true; 6823 partial++; 6824 } else if (isUnresolvedTargetTable(targetTable)) { 6825 partial++; 6826 } else { 6827 resolved++; 6828 } 6829 } 6830 } 6831 if (placeholderSeen) { 6832 // Contaminated edges must not be surfaced as usable lineage 6833 // (producedLineage() reads resolvedCount). 6834 partial += resolved; 6835 resolved = 0; 6836 resultSetEdges = 0; 6837 } 6838 return new int[] { resolved, partial, placeholderSeen ? 1 : 0, resultSetEdges }; 6839 } 6840 6841 /** 6842 * Oracle DBMS_SQL resolution (plan Phase 5 slice 4; supersedes the Phase 1 6843 * visibility-only recording): a {@code DBMS_SQL.PARSE(c, <sql>, ...)} call is a 6844 * dynamic-SQL execution site. The SQL-text argument is folded through the shared 6845 * literal-bindable whitelist — inline literals, literal-composed locals, and 6846 * parameters bound at vetted literal call sites (same-file or cross-file via the 6847 * routine catalog) — and each folded text is analyzed through the ordinary inner- 6848 * SQL path. Anything that does not fold keeps an honest UNRESOLVED diagnostic; 6849 * PARSE_ERROR is reserved for compile-time literal text, mirroring the EXECUTE 6850 * IMMEDIATE status contract. Recording keys off PARSE, the call that carries the 6851 * SQL text, so a PARSE/EXECUTE pair maps to one site. 6852 */ 6853 private void recordDbmsSqlSiteIfParseCall(TBasicStmt stmt, TFunctionCall functionCall) { 6854 if (functionCall == null || functionCall.getFunctionName() == null 6855 || option.getVendor() != EDbVendor.dbvoracle) { 6856 return; 6857 } 6858 String calledName = functionCall.getFunctionName().toString(); 6859 if (!SQLUtil.compareIdentifier(EDbVendor.dbvoracle, ESQLDataObjectType.dotProcedure, 6860 calledName, "DBMS_SQL.PARSE") 6861 && !SQLUtil.compareIdentifier(EDbVendor.dbvoracle, ESQLDataObjectType.dotProcedure, 6862 calledName, "SYS.DBMS_SQL.PARSE")) { 6863 return; 6864 } 6865 if (!option.isAnalyzeDynamicSql()) { 6866 recordDynamicSqlSite(stmt, DynamicSqlSite.Kind.DBMS_SQL, 6867 DynamicSqlSite.Status.UNRESOLVED, 6868 "dynamic SQL inner analysis disabled"); 6869 return; 6870 } 6871 boolean literalSql = false; 6872 DynamicSqlTrustMode trustMode = option.getDynamicSqlTrustMode(); 6873 TExpression sqlArg = null; 6874 if (functionCall.getArgs() != null && functionCall.getArgs().size() >= 2) { 6875 // Positional form: PARSE(c, <sql>, <lang>). Named association may reorder the 6876 // arguments (statement => '...'), so resolve the named form first. 6877 boolean namedForm = false; 6878 for (int i = 0; i < functionCall.getArgs().size(); i++) { 6879 TExpression arg = functionCall.getArgs().getExpression(i); 6880 if (arg != null && arg.getExpressionType() == EExpressionType.ref_arrow_t) { 6881 namedForm = true; 6882 if (arg.getLeftOperand() != null 6883 && SQLUtil.sameName(EDbVendor.dbvoracle, ESQLDataObjectType.dotColumn, 6884 arg.getLeftOperand().toString(), "statement")) { 6885 sqlArg = arg.getRightOperand(); 6886 break; 6887 } 6888 } 6889 } 6890 if (sqlArg == null && !namedForm) { 6891 sqlArg = functionCall.getArgs().getExpression(1); 6892 } 6893 literalSql = sqlArg != null && sqlArg.getExpressionType() == EExpressionType.simple_constant_t; 6894 } 6895 if (sqlArg != null) { 6896 TCustomSqlStatement routine = null; 6897 for (int i = stmtStack.size() - 1; i >= 0; i--) { 6898 if (stmtStack.get(i) instanceof TStoredProcedureSqlStatement) { 6899 routine = stmtStack.get(i); 6900 break; 6901 } 6902 } 6903 List<String> foldedTexts; 6904 try { 6905 foldedTexts = gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver 6906 .foldDbmsSqlTexts(routine, sqlArg, routineCatalog, 4); 6907 } catch (RuntimeException ex) { 6908 foldedTexts = new ArrayList<String>(); 6909 } 6910 if (!foldedTexts.isEmpty()) { 6911 Set<Relationship> shadowBefore = trustMode == DynamicSqlTrustMode.SHADOW 6912 ? snapshotDynamicRelationships() : null; 6913 int dynamicSitesBefore = dynamicSqlSites.size(); 6914 int relsBeforeDynamic = modelManager.getRelations().length; 6915 int errorsBeforeDynamic = errorInfos.size(); 6916 boolean analyzedAny = false; 6917 boolean variantFailed = false; 6918 List<Long> provenVariantRelationshipIds = new ArrayList<Long>(); 6919 for (String foldedText : foldedTexts) { 6920 int relationshipsBeforeVariant = modelManager.getRelations().length; 6921 int dynamicSitesBeforeVariant = dynamicSqlSites.size(); 6922 int errorsBeforeVariant = errorInfos.size(); 6923 TGSqlParser sqlparser = createSqlParser(option.getVendor()); 6924 sqlparser.sqltext = foldedText; 6925 int result; 6926 try { 6927 result = sqlparser.parse(); 6928 } catch (RuntimeException ex) { 6929 variantFailed = true; 6930 continue; 6931 } 6932 if (result != 0 || sqlparser.sqlstatements == null) { 6933 variantFailed = true; 6934 continue; 6935 } 6936 try { 6937 dynamicFoldDepth++; 6938 try { 6939 for (int i = 0; i < sqlparser.sqlstatements.size(); i++) { 6940 analyzeCustomSqlStmt(sqlparser.sqlstatements.get(i)); 6941 } 6942 } finally { 6943 dynamicFoldDepth--; 6944 } 6945 analyzedAny = true; 6946 if (errorInfos.size() == errorsBeforeVariant 6947 && !hasIncompleteDynamicSqlSitesSince(dynamicSitesBeforeVariant)) { 6948 provenVariantRelationshipIds.addAll(provenDynamicRelationshipIdsSince( 6949 relationshipsBeforeVariant, dynamicSitesBeforeVariant)); 6950 } 6951 } catch (RuntimeException ex) { 6952 // An analyzer crash on folded text must neither surface as an 6953 // AnalyzeError blessed into output nor masquerade as resolved. 6954 variantFailed = true; 6955 } 6956 } 6957 // Inner analysis handlers may swallow their own crashes into 6958 // errorInfos (never rethrowing to the catch above). Errors raised 6959 // while analyzing SYNTHESIZED text carry misleading fragment- 6960 // relative coordinates - roll them back and treat the variant as 6961 // failed so the site status stays honest. 6962 if (errorInfos.size() > errorsBeforeDynamic) { 6963 while (errorInfos.size() > errorsBeforeDynamic) { 6964 errorInfos.remove(errorInfos.size() - 1); 6965 } 6966 variantFailed = true; 6967 } 6968 int[] dynCounts = classifyDynamicSiteLineage(relsBeforeDynamic, false); 6969 boolean nestedIncomplete = trustMode != DynamicSqlTrustMode.LEGACY 6970 && hasIncompleteDynamicSqlSitesSince(dynamicSitesBefore); 6971 boolean aggregateIncomplete = variantFailed || nestedIncomplete; 6972 int shadowUnproven = trustMode == DynamicSqlTrustMode.SHADOW && aggregateIncomplete 6973 ? Math.max(0, countNewDynamicRelationships(shadowBefore) 6974 - provenVariantRelationshipIds.size()) : 0; 6975 if (analyzedAny && !aggregateIncomplete) { 6976 recordDynamicSqlSite(stmt, DynamicSqlSite.Kind.DBMS_SQL, 6977 DynamicSqlSite.Status.RESOLVED, null, dynCounts[0], dynCounts[1], 6978 0, 0, true, trustMode, 6979 relsBeforeDynamic, dynamicSitesBefore, 6980 provenVariantRelationshipIds); 6981 return; 6982 } 6983 if (analyzedAny) { 6984 // Some accepted variants could not be analyzed - the aggregate 6985 // picture is incomplete. SHADOW retains the historical candidates 6986 // and labels the aggregate as lacking per-edge proof. 6987 DynamicSqlSite.Status status = trustMode == DynamicSqlTrustMode.SHADOW 6988 && (shadowUnproven > 0 || !provenVariantRelationshipIds.isEmpty()) 6989 ? DynamicSqlSite.Status.PARTIAL 6990 : DynamicSqlSite.Status.UNRESOLVED; 6991 String reason = nestedIncomplete 6992 ? "nested dynamic SQL in a folded DBMS_SQL variant is incomplete" 6993 : "some folded DBMS_SQL variants failed to parse or analyze; lineage may be incomplete"; 6994 recordDynamicSqlSite(stmt, DynamicSqlSite.Kind.DBMS_SQL, status, 6995 trustMode == DynamicSqlTrustMode.SHADOW 6996 ? dynamicTrustReason(reason + "; SHADOW retained historical candidates", 6997 shadowUnproven, nestedIncomplete ? -1 : 0, 6998 !nestedIncomplete) 6999 : reason, 7000 dynCounts[0], dynCounts[1], shadowUnproven, 7001 nestedIncomplete ? -1 : 0, !nestedIncomplete, trustMode, 7002 relsBeforeDynamic, dynamicSitesBefore, 7003 provenVariantRelationshipIds); 7004 return; 7005 } 7006 if (variantFailed && literalSql) { 7007 recordDynamicSqlSite(stmt, DynamicSqlSite.Kind.DBMS_SQL, 7008 DynamicSqlSite.Status.PARSE_ERROR, 7009 "DBMS_SQL.PARSE literal SQL failed to parse", 0, 0, 7010 0, 0, true, trustMode, 7011 relsBeforeDynamic, dynamicSitesBefore); 7012 return; 7013 } 7014 if (variantFailed) { 7015 recordDynamicSqlSite(stmt, DynamicSqlSite.Kind.DBMS_SQL, 7016 DynamicSqlSite.Status.UNRESOLVED, 7017 "folded DBMS_SQL text failed to parse", 0, 0, 7018 0, 0, true, trustMode, 7019 relsBeforeDynamic, dynamicSitesBefore); 7020 return; 7021 } 7022 } 7023 } 7024 String unresolvedReason = literalSql ? "DBMS_SQL.PARSE with literal SQL; inner SQL not analyzed" 7025 : "DBMS_SQL.PARSE with runtime SQL text; inner SQL not analyzed"; 7026 if (trustMode == DynamicSqlTrustMode.LEGACY) { 7027 recordDynamicSqlSite(stmt, DynamicSqlSite.Kind.DBMS_SQL, 7028 DynamicSqlSite.Status.UNRESOLVED, unresolvedReason); 7029 } else { 7030 boolean runtimeText = !literalSql; 7031 if (runtimeText && trustMode == DynamicSqlTrustMode.SHADOW) { 7032 unresolvedReason = dynamicTrustReason("SHADOW observed no materialized DBMS_SQL candidate", 7033 0, -1, false); 7034 } 7035 recordDynamicSqlSite(stmt, DynamicSqlSite.Kind.DBMS_SQL, 7036 DynamicSqlSite.Status.UNRESOLVED, unresolvedReason, 0, 0, 7037 0, 7038 runtimeText ? -1 : 0, !runtimeText, trustMode); 7039 } 7040 } 7041 7042 private boolean hasIncompleteDynamicSqlSitesSince(int start) { 7043 return hasIncompleteDynamicSqlSitesSince(dynamicSqlSites, start); 7044 } 7045 7046 private static boolean hasIncompleteDynamicSqlSitesSince(List<DynamicSqlSite> sites, int start) { 7047 for (int i = Math.max(0, start); i < sites.size(); i++) { 7048 DynamicSqlSite site = sites.get(i); 7049 if (site.getStatus() != DynamicSqlSite.Status.RESOLVED) { 7050 return true; 7051 } 7052 } 7053 return false; 7054 } 7055 7056 /** True when any source of this edge is a fabricated placeholder table (see caller javadoc). */ 7057 private boolean hasPlaceholderSourceTable(Relationship r, boolean partialFold) { 7058 if (r.getSources() == null) { 7059 return false; 7060 } 7061 for (Object source : r.getSources()) { 7062 if (!(source instanceof TableColumnRelationshipElement)) { 7063 continue; 7064 } 7065 TableColumn sourceColumn = ((TableColumnRelationshipElement) source).getElement(); 7066 Table sourceTable = sourceColumn == null ? null : sourceColumn.getTable(); 7067 if (sourceTable == null || sourceTable.isPseudo()) { 7068 continue; 7069 } 7070 if (sourceTable.isVariable()) { 7071 // A variable-backed source inside a PARTIAL fold is the folded remnant of a 7072 // runtime-built OBJECT NAME only when it stands in table position — its 7073 // "columns" then carry real column names distinct from the variable's own 7074 // name ('... FROM ' || p_tab reparsed as table p_tab with column s1). A 7075 // scalar variable used as a value/predicate operand surfaces as a 7076 // single-column variable whose column IS the variable name — legitimate, 7077 // not contamination. 7078 if (partialFold && sourceColumn != null && sourceColumn.getName() != null 7079 && sourceTable.getName() != null 7080 && !DlineageUtil.sameName(ESQLDataObjectType.dotColumn, 7081 sourceColumn.getName(), sourceTable.getName())) { 7082 return true; 7083 } 7084 continue; 7085 } 7086 if (isPlaceholderNamedTable(sourceTable, partialFold)) { 7087 return true; 7088 } 7089 } 7090 return false; 7091 } 7092 7093 /** 7094 * A table whose NAME betrays a runtime-built object: a raw name starting with '@' (fold 7095 * artifact; delimited {@code [@x]} / {@code "@x"} are legal object names and do not match), or — 7096 * during a partial fold — a name equal to a string-building function left behind by the folder 7097 * ({@code QUOTENAME(...)} etc.). Applied to both edge sources and INSERT targets. 7098 */ 7099 private boolean isPlaceholderNamedTable(Table table, boolean partialFold) { 7100 if (table == null) { 7101 return false; 7102 } 7103 String name = table.getName() == null ? "" : table.getName().trim(); 7104 if (name.startsWith("@")) { 7105 return true; 7106 } 7107 // Locale-stable fold: default-locale toUpperCase turns "ltrim" into "LTRİM" under tr_TR. 7108 return partialFold && DYNAMIC_NAME_BUILDER_FUNCTIONS.contains(name.toUpperCase(Locale.ENGLISH)); 7109 } 7110 7111 /** 7112 * A folded dynamic site's target is "unresolved" when it is a T-SQL variable / placeholder rather 7113 * than a real object. When GSP only partially folds (e.g. {@code EXEC sp_executesql @sql} → 7114 * {@code INSERT INTO @p SELECT ...}), the re-parsed target is a table literally named {@code @p}; 7115 * its {@code isVariable()} flag is not set, so the reliable signal is a name starting with '@'. 7116 */ 7117 private boolean isUnresolvedTargetTable(Table table) { 7118 if (table == null) { 7119 return true; 7120 } 7121 if (table.isVariable() || table.isPseudo()) { 7122 return true; 7123 } 7124 String name = table.getName(); 7125 if (name != null) { 7126 // Raw-name check only: delimited [@x] / "@x" are legal permanent-object names, 7127 // while the fold artifact keeps a bare leading '@'. 7128 if (name.trim().startsWith("@")) { 7129 return true; 7130 } 7131 } 7132 return false; 7133 } 7134 7135 /** True when a module name resolves to sp_executesql, ignoring schema qualifier and brackets/quotes. */ 7136 private boolean isSpExecutesql(TObjectName moduleName) { 7137 if (moduleName == null) { 7138 return false; 7139 } 7140 String s = moduleName.toString(); 7141 int dot = s.lastIndexOf('.'); 7142 if (dot >= 0) { 7143 s = s.substring(dot + 1); 7144 } 7145 s = s.replace("[", "").replace("]", "").replace("\"", "").replace("`", "").trim(); 7146 return "sp_executesql".equalsIgnoreCase(s); 7147 } 7148 7149 /** 7150 * True when a module name is a <em>local</em> invocation of the system 7151 * procedure sp_execute_external_script whose input query therefore executes in 7152 * the caller's current database context. 7153 * 7154 * <p>Uses the parser's structured object/schema segmentation plus the 7155 * identifier-equality facade ({@link SQLUtil#sameName}) so vendor delimiter 7156 * rules are honored: a naive {@code lastIndexOf('.')} or delimiter strip would 7157 * mistake a delimited identifier that merely contains a dot 7158 * ({@code [dbo.sp_execute_external_script]}) or one with escaped delimiters 7159 * ({@code [sp_execute_external_]]script]}, {@code "sp_execute_external_""script"}) 7160 * for the system procedure. 7161 * 7162 * <p>Only the unqualified form or a bare {@code sys} schema is accepted, and any 7163 * server or database qualifier is rejected. A {@code dbo.} (or other user 7164 * schema) qualifier denotes a user procedure, and a server/database qualifier 7165 * (e.g. a four-part {@code Srv.Db.sys.sp_execute_external_script} linked-server 7166 * call) means the input query resolves in a remote context this pass does not 7167 * model. In both cases the argument is left unanalyzed rather than analyzed 7168 * against the wrong objects. 7169 */ 7170 private boolean isSpExecuteExternalScript(TObjectName moduleName) { 7171 if (moduleName == null || moduleName.getObjectToken() == null) { 7172 return false; 7173 } 7174 EDbVendor vendor = option.getVendor(); 7175 if (!SQLUtil.sameName(vendor, gudusoft.gsqlparser.sqlenv.ESQLDataObjectType.dotProcedure, 7176 moduleName.getObjectToken().toString(), "sp_execute_external_script")) { 7177 return false; 7178 } 7179 // A server or database qualifier changes (or makes remote) the context in 7180 // which the input query resolves; only handle the local, unqualified call. 7181 if (moduleName.getServerToken() != null || moduleName.getDatabaseToken() != null) { 7182 return false; 7183 } 7184 TSourceToken schemaTok = moduleName.getSchemaToken(); 7185 if (schemaTok == null) { 7186 return true; 7187 } 7188 return SQLUtil.sameName(vendor, gudusoft.gsqlparser.sqlenv.ESQLDataObjectType.dotSchema, 7189 schemaTok.toString(), "sys"); 7190 } 7191 7192 /** 7193 * Recover lineage for the query passed to sp_execute_external_script via its 7194 * input dataset (MantisBT #4606). The input value is a SQL string literal whose 7195 * SELECT reads real tables; analyze it as a normal statement so its 7196 * source-table -> result-set lineage is emitted. 7197 * 7198 * <p>The input dataset is the {@code @input_data_1} parameter (named form) or, 7199 * for a positional call, the third argument — the documented signature is 7200 * {@code (@language, @script, @input_data_1, ...)}. Only that one parameter 7201 * carries a query; its siblings {@code @input_data_1_name}, 7202 * {@code @input_data_1_order_by_columns} and 7203 * {@code @input_data_1_partition_by_columns} are identifiers / column lists, 7204 * NOT queries — analyzing them would attach lineage to the wrong objects, so 7205 * named matching is exact and positional matching is restricted to the third 7206 * slot. 7207 * 7208 * <p>Only a plain string literal is materialized. A variable-valued argument is 7209 * left unanalyzed: the parse-time symbol table is not available in this pass, 7210 * so its value cannot be statically resolved here (the same unresolved-value 7211 * limitation as the general dynamic-SQL case). 7212 * 7213 * <p>The script body is opaque (arbitrary Python/R can reorder, drop, or 7214 * synthesize columns), so by default the input query's result set is NOT 7215 * connected to the {@code WITH RESULT SETS} output columns — that mapping is 7216 * unprovable. Only when {@link Option#isAssumeExternalScriptPassthrough()} is 7217 * explicitly enabled does {@link #buildExternalScriptPassthrough} add that 7218 * connection, tagged {@link EffectType#external_script_passthrough} to keep it 7219 * distinguishable from proven lineage. 7220 */ 7221 private void analyzeExternalScriptInputData(TMssqlExecute executeStmt) { 7222 TExecParameterList params = executeStmt.getParameters(); 7223 if (params == null) { 7224 return; 7225 } 7226 // T-SQL allows a positional prefix followed by named arguments; once an 7227 // argument is named, every later one must be named too. So a positional 7228 // argument is only valid while no NAMED argument has appeared before it. 7229 // The documented signature is (@language, @script, @input_data_1, ...), so 7230 // the third positional argument (index 2) is @input_data_1. 7231 boolean namedBefore = false; 7232 for (int i = 0; i < params.size(); i++) { 7233 TExecParameter param = params.getExecParameter(i); 7234 TObjectName pname = param.getParameterName(); 7235 boolean isInputData1; 7236 if (pname != null) { 7237 String nm = pname.toString(); 7238 if (nm.startsWith("@")) { 7239 nm = nm.substring(1); 7240 } 7241 isInputData1 = "input_data_1".equalsIgnoreCase(nm.trim()); // non-identifier-compare: T-SQL system-proc parameter name 7242 namedBefore = true; 7243 } else { 7244 isInputData1 = !namedBefore && i == 2; 7245 } 7246 if (!isInputData1 || param.getParameterValue() == null) { 7247 continue; 7248 } 7249 String sql = materializeTSqlStringLiteral(param.getParameterValue().toString()); 7250 if (sql != null && sql.trim().length() > 0) { 7251 ResultSet inputResultSet = analyzeExternalScriptInputQuery(sql); 7252 // Opt-in only: bridge the opaque script by assuming the input dataset 7253 // is passed through to the WITH RESULT SETS output columns positionally 7254 // (EffectType.external_script_passthrough). Off by default because the 7255 // script may reorder/drop/add columns, so these edges are assumed, not 7256 // proven. 7257 if (option.isAssumeExternalScriptPassthrough() && inputResultSet != null) { 7258 buildExternalScriptPassthrough(executeStmt, inputResultSet); 7259 } 7260 } 7261 // Only @input_data_1 carries a query; nothing else to look at. 7262 return; 7263 } 7264 } 7265 7266 /** 7267 * Parse and analyze an sp_execute_external_script input query, returning the 7268 * ResultSet model of its (last) top-level SELECT, or null. Behaves like 7269 * {@link #executeDynamicSql(String)} but hands back the result set so the 7270 * optional passthrough model can wire it to the WITH RESULT SETS output. 7271 */ 7272 private ResultSet analyzeExternalScriptInputQuery(String sql) { 7273 TGSqlParser sqlparser = createSqlParser(option.getVendor()); 7274 sqlparser.sqltext = sql; 7275 if (sqlparser.parse() != 0) { 7276 return null; 7277 } 7278 ResultSet resultSet = null; 7279 // Dynamic inner analysis: its references are NOT static statements of the unit 7280 // and must not veto dynamic-template marks (see dynamicFoldDepth). 7281 dynamicFoldDepth++; 7282 try { 7283 for (int i = 0; i < sqlparser.sqlstatements.size(); i++) { 7284 TCustomSqlStatement stmt = sqlparser.sqlstatements.get(i); 7285 analyzeCustomSqlStmt(stmt); 7286 if (stmt instanceof TSelectSqlStatement) { 7287 Object model = modelManager.getModel(stmt); 7288 if (model instanceof ResultSet) { 7289 resultSet = (ResultSet) model; 7290 } 7291 } 7292 } 7293 } finally { 7294 dynamicFoldDepth--; 7295 } 7296 return resultSet; 7297 } 7298 7299 /** 7300 * Under the opt-in passthrough assumption, wire the external-script input 7301 * dataset to its declared {@code WITH RESULT SETS} output columns positionally, 7302 * emitting {@link EffectType#external_script_passthrough} edges and binding the 7303 * synthesized output ResultSet to the EXEC statement so an 7304 * {@code INSERT ... EXEC} consumer can inherit it (see the INSERT..EXEC path in 7305 * {@link #analyzeInsertStmt}). 7306 */ 7307 private void buildExternalScriptPassthrough(TMssqlExecute executeStmt, ResultSet inputResultSet) { 7308 TColumnDefinitionList outputColumns = firstInlineResultSetColumns(executeStmt); 7309 if (outputColumns == null || outputColumns.size() == 0) { 7310 // No declared output columns (e.g. WITH RESULT SETS NONE / UNDEFINED): 7311 // there is nothing to pass the input through to. 7312 return; 7313 } 7314 List<ResultColumn> inputColumns = inputResultSet.getColumns(); 7315 // A star projection ("SELECT *") has unknown width, so positional 7316 // correspondence to the declared output cannot be established -- decline 7317 // rather than mis-map a single star column onto the first output column. 7318 for (ResultColumn inputColumn : inputColumns) { 7319 if (inputColumn != null && "*".equals(inputColumn.getName())) { 7320 return; 7321 } 7322 } 7323 ResultSet outputResultSet = modelFactory.createResultSet(executeStmt, false); 7324 Process process = modelFactory.createProcess(executeStmt); 7325 // Materialize the FULL declared output schema (so every WITH RESULT SETS 7326 // column is available to a downstream INSERT..EXEC), preserving cardinality 7327 // by position via the raw ResultColumn constructor (the name-keyed factory 7328 // overload would collapse duplicate declared names). Passthrough edges are 7329 // added only for positions that have a matching input column. 7330 for (int i = 0; i < outputColumns.size(); i++) { 7331 TObjectName outputName = outputColumns.getColumn(i).getColumnName(); 7332 if (outputName == null) { 7333 continue; 7334 } 7335 ResultColumn outputColumn = new ResultColumn(outputResultSet, outputName); 7336 if (i < inputColumns.size() && inputColumns.get(i) != null) { 7337 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 7338 relation.setEffectType(EffectType.external_script_passthrough); 7339 relation.setTarget(new ResultColumnRelationshipElement(outputColumn)); 7340 relation.addSource(new ResultColumnRelationshipElement(inputColumns.get(i))); 7341 relation.setProcess(process); 7342 } 7343 } 7344 } 7345 7346 /** The column list of the first inline {@code WITH RESULT SETS ((...))} definition of an EXEC, or null. */ 7347 private gudusoft.gsqlparser.nodes.TColumnDefinitionList firstInlineResultSetColumns(TMssqlExecute executeStmt) { 7348 gudusoft.gsqlparser.nodes.mssql.TExecuteOption execOption = executeStmt.getExecuteOption(); 7349 if (!(execOption instanceof gudusoft.gsqlparser.nodes.mssql.TResultSetsExecuteOption)) { 7350 return null; 7351 } 7352 for (gudusoft.gsqlparser.nodes.mssql.TResultSetDefinition definition 7353 : ((gudusoft.gsqlparser.nodes.mssql.TResultSetsExecuteOption) execOption).getDefinitions()) { 7354 if (definition instanceof gudusoft.gsqlparser.nodes.mssql.TInlineResultSetDefinition) { 7355 return ((gudusoft.gsqlparser.nodes.mssql.TInlineResultSetDefinition) definition) 7356 .getColumnDefinitionList(); 7357 } 7358 } 7359 return null; 7360 } 7361 7362 /** 7363 * Result sets that an {@code INSERT ... EXEC <proc>} can consume: the result 7364 * sets of the procedure's trailing SELECT(s), or — when the procedure has no 7365 * such SELECT — the {@code WITH RESULT SETS} output(s) synthesized for its 7366 * internal sp_execute_external_script call. The latter exist only when the 7367 * passthrough assumption is enabled ({@link Option#isAssumeExternalScriptPassthrough()}), 7368 * so with the option off this reduces to the previous SELECT-only behavior. 7369 */ 7370 private List<ResultSet> collectCalleeResultSets(TStoredProcedureSqlStatement procedureStmt) { 7371 List<ResultSet> resultSets = new ArrayList<ResultSet>(); 7372 List<TSelectSqlStatement> selects = getLastSelectStmt(procedureStmt); 7373 if (selects != null) { 7374 for (TSelectSqlStatement select : selects) { 7375 Object model = modelManager.getModel(select); 7376 if (model instanceof ResultSet) { 7377 resultSets.add((ResultSet) model); 7378 } 7379 } 7380 } 7381 // A procedure may return an ordinary SELECT result set AND an external-script 7382 // result set; INSERT..EXEC consumes every returned (compatible) result set, so 7383 // include the passthrough output(s) as well, not only when no SELECT exists. 7384 // externalScriptOutputResultSets is empty unless the passthrough option is on, 7385 // so with the option off this reduces to the previous SELECT-only behavior. 7386 resultSets.addAll(externalScriptOutputResultSets(procedureStmt)); 7387 return resultSets; 7388 } 7389 7390 /** 7391 * Result sets an {@code INSERT ... EXEC <target>} consumes. When the target is a 7392 * user stored procedure, this is {@link #collectCalleeResultSets}. When it is a 7393 * DIRECT call to sp_execute_external_script (no wrapping procedure), the 7394 * synthesized WITH RESULT SETS output is bound to the EXEC statement itself, so 7395 * it is fetched directly. Empty unless the passthrough assumption is enabled. 7396 */ 7397 private List<ResultSet> calleeResultSetsForInsertExec(TMssqlExecute executeStmt, Procedure procedure) { 7398 if (procedure != null && procedure.getProcedureObject() instanceof TStoredProcedureSqlStatement) { 7399 return collectCalleeResultSets((TStoredProcedureSqlStatement) procedure.getProcedureObject()); 7400 } 7401 if (isSpExecuteExternalScript(executeStmt.getModuleName())) { 7402 Object model = modelManager.getModel(executeStmt); 7403 if (model instanceof ResultSet) { 7404 return java.util.Collections.singletonList((ResultSet) model); 7405 } 7406 } 7407 return new ArrayList<ResultSet>(); 7408 } 7409 7410 /** 7411 * WITH RESULT SETS output result sets bound to sp_execute_external_script calls 7412 * inside a procedure body (the passthrough model binds one to each such EXEC). 7413 * Empty unless the passthrough assumption is enabled. 7414 */ 7415 private List<ResultSet> externalScriptOutputResultSets(TStoredProcedureSqlStatement procedureStmt) { 7416 final List<ResultSet> resultSets = new ArrayList<ResultSet>(); 7417 procedureStmt.acceptChildren(new TParseTreeVisitor() { 7418 @Override 7419 public void preVisit(TMssqlExecute execute) { 7420 if (isSpExecuteExternalScript(execute.getModuleName())) { 7421 Object model = modelManager.getModel(execute); 7422 if (model instanceof ResultSet) { 7423 resultSets.add((ResultSet) model); 7424 } 7425 } 7426 } 7427 }); 7428 return resultSets; 7429 } 7430 7431 /** 7432 * Materialize a T-SQL string literal: strip the {@code N'...'} / {@code '...'} 7433 * wrapper and unescape doubled single quotes ({@code ''} -> {@code '}). 7434 * Unlike {@link TBaseType#getStringInsideLiteral(String)} this does NOT treat 7435 * {@code \'} as an escape — backslash is an ordinary character in T-SQL, so a 7436 * source such as an {@code OPENROWSET} path must survive unchanged. Returns 7437 * null when the argument is not a plain string literal (e.g. a variable). 7438 */ 7439 private static String materializeTSqlStringLiteral(String raw) { 7440 if (raw == null) { 7441 return null; 7442 } 7443 String s = raw.trim(); 7444 int start = -1; 7445 if (s.length() >= 2 && s.charAt(0) == '\'') { 7446 start = 1; 7447 } else if (s.length() >= 3 && (s.charAt(0) == 'N' || s.charAt(0) == 'n') && s.charAt(1) == '\'') { 7448 start = 2; 7449 } 7450 if (start < 0 || s.charAt(s.length() - 1) != '\'') { 7451 return null; 7452 } 7453 return s.substring(start, s.length() - 1).replace("''", "'"); 7454 } 7455 7456 /** True when this EXEC form would expand SQL text rather than invoke an 7457 * ordinary statically named stored procedure. */ 7458 private boolean expandsDynamicSql(TMssqlExecute e) { 7459 return e.getSqlText() != null 7460 || e.getExecType() == TBaseType.metExecStringCmd 7461 || isSpExecutesql(e.getModuleName()); 7462 } 7463 7464 /** Kind of a {@link TMssqlExecute} dynamic site: sp_executesql vs the EXEC(string) family. */ 7465 private DynamicSqlSite.Kind dynamicKindOf(TMssqlExecute e) { 7466 if (e.getExecType() == TBaseType.metExecStringCmd) { 7467 return DynamicSqlSite.Kind.EXEC_STRING; 7468 } 7469 if (isSpExecutesql(e.getModuleName())) { 7470 return DynamicSqlSite.Kind.SP_EXECUTESQL; 7471 } 7472 return DynamicSqlSite.Kind.OTHER; 7473 } 7474 7475 /** The dynamic-SQL argument expression of a {@link TMssqlExecute}, or null. */ 7476 private TExpression dynamicArgOf(TMssqlExecute e) { 7477 if (e.getExecType() == TBaseType.metExecStringCmd) { 7478 if (e.getStringValues() != null && e.getStringValues().size() > 0) { 7479 return e.getStringValues().getExpression(0); 7480 } 7481 return null; 7482 } 7483 if (e.getParameters() != null && e.getParameters().size() > 0) { 7484 TExecParameter p = e.getParameters().getExecParameter(0); 7485 if (p != null) { 7486 return p.getParameterValue(); 7487 } 7488 } 7489 return null; 7490 } 7491 7492 /** 7493 * True only when the dynamic-SQL argument is a compile-time string literal. A variable or 7494 * expression argument is data-driven: even when GSP partially folds it (e.g. {@code @sql} → 7495 * {@code INSERT INTO @t SELECT ...} with an unresolved parameter left in), the site is NOT 7496 * fully resolved, so it must not be reported RESOLVED. 7497 */ 7498 private boolean isLiteralDynamicArg(TMssqlExecute e) { 7499 TExpression arg = dynamicArgOf(e); 7500 return arg != null && arg.getExpressionType() == EExpressionType.simple_constant_t; 7501 } 7502 7503 /** Refine the UNRESOLVED reason by inspecting the dynamic argument expression. */ 7504 private String dynamicReasonOf(TMssqlExecute e) { 7505 TExpression arg = dynamicArgOf(e); 7506 if (arg == null) { 7507 return "dynamic SQL target could not be resolved"; 7508 } 7509 if (arg.getExpressionType() == EExpressionType.simple_object_name_t && arg.getObjectOperand() != null 7510 && arg.getObjectOperand().getDbObjectType() == EDbObjectType.variable) { 7511 return "argument is a runtime variable"; 7512 } 7513 if (arg.getExpressionType() == EExpressionType.simple_constant_t) { 7514 return "inline literal not folded; no lineage produced"; 7515 } 7516 return "argument is a runtime expression or partially-folded variable"; 7517 } 7518 7519 private void executeDynamicSql(String sql) { 7520 TGSqlParser sqlparser = createSqlParser(option.getVendor()); 7521 sqlparser.sqltext = sql; 7522 int result = sqlparser.parse(); 7523 if (result == 0) { 7524 dynamicFoldDepth++; 7525 try { 7526 for (int i = 0; i < sqlparser.sqlstatements.size(); i++) { 7527 analyzeCustomSqlStmt(sqlparser.sqlstatements.get(i)); 7528 } 7529 } finally { 7530 dynamicFoldDepth--; 7531 } 7532 } 7533 } 7534 7535 /** 7536 * Callback invoked from snowflake.js (the script shim run by 7537 * {@link #extractSnowflakeSQLFromProcedure}) for every SQL text a 7538 * JavaScript procedure body passes to snowflake.execute() / 7539 * snowflake.createStatement(). The script resolves this method by name 7540 * at runtime, so it must stay public and keep this exact signature — 7541 * renaming it to executeDynamicSql broke JavaScript-procedure lineage 7542 * with "analyzer.extractSnowflakeSQL is not a function" (Mantis 4590). 7543 */ 7544 public void extractSnowflakeSQL(String sql) { 7545 executeDynamicSql(sql); 7546 } 7547 7548 private void extractSnowflakeSQLFromProcedure(TCreateProcedureStmt procedure) { 7549 Map<String, String> argMap = new LinkedHashMap<String, String>(); 7550 if (procedure.getParameterDeclarations() != null) { 7551 for (int i = 0; i < procedure.getParameterDeclarations().size(); i++) { 7552 TParameterDeclaration def = procedure.getParameterDeclarations().getParameterDeclarationItem(i); 7553 argMap.put(def.getParameterName().toString(), def.getDataType().getDataTypeName()); 7554 } 7555 } 7556 StringBuilder buffer = new StringBuilder(); 7557 buffer.append("(function("); 7558 String[] args = argMap.keySet().toArray(new String[0]); 7559 for (int i = 0; i < args.length; i++) { 7560 buffer.append(args[i].toUpperCase()); 7561 if (i < args.length - 1) { 7562 buffer.append(","); 7563 } 7564 } 7565 buffer.append("){\n"); 7566 7567 int start = -1; 7568 int end = -1; 7569 boolean dollar = false; 7570 boolean quote = false; 7571 if (procedure.getRoutineBody().indexOf("$$") != -1) { 7572 start = procedure.getRoutineBody().indexOf("$$") + 2; 7573 end = procedure.getRoutineBody().lastIndexOf("$$") - 1; 7574 dollar = true; 7575 } else if (procedure.getRoutineBody().indexOf("'") != -1) { 7576 start = procedure.getRoutineBody().indexOf("'") + 1; 7577 end = procedure.getRoutineBody().lastIndexOf("'"); 7578 quote = true; 7579 } 7580 String body = procedure.getRoutineBody().substring(start, end); 7581 if (dollar && body.indexOf("`") != -1) { 7582 Pattern pattern = Pattern.compile("`.+?`", Pattern.CASE_INSENSITIVE | Pattern.DOTALL); 7583 Matcher matcher = pattern.matcher(body); 7584 StringBuffer replaceBuffer = new StringBuffer(); 7585 while (matcher.find()) { 7586 String condition = matcher.group().replace("\r\n", "\n").replace("'", "\\\\'") 7587 .replace("\n", "\\\\n'\n+'").replace("`", "'").replace("$", "RDS_CHAR_DOLLAR"); 7588 matcher.appendReplacement(replaceBuffer, condition); 7589 } 7590 matcher.appendTail(replaceBuffer); 7591 body = replaceBuffer.toString().replace("RDS_CHAR_DOLLAR", "$"); 7592 } 7593 if (quote && body.indexOf("'") != -1) { 7594 body = body.replace("''", "'"); 7595 } 7596 buffer.append(body); 7597 buffer.append("})("); 7598 for (int i = 0; i < args.length; i++) { 7599 String type = argMap.get(args[i]); 7600 if (type.equalsIgnoreCase("VARCHAR")) { 7601 buffer.append("'pseudo'"); 7602 } else if (type.equalsIgnoreCase("STRING")) { 7603 buffer.append("'pseudo'"); 7604 } else if (type.equalsIgnoreCase("CHAR")) { 7605 buffer.append("'pseudo'"); 7606 } else if (type.equalsIgnoreCase("CHARACTER")) { 7607 buffer.append("'pseudo'"); 7608 } else if (type.equalsIgnoreCase("TEXT")) { 7609 buffer.append("'pseudo'"); 7610 } else if (type.equalsIgnoreCase("BINARY")) { 7611 buffer.append("'pseudo'"); 7612 } else if (type.equalsIgnoreCase("VARBINARY")) { 7613 buffer.append("'pseudo'"); 7614 } else if (type.equalsIgnoreCase("BOOLEAN")) { 7615 buffer.append(true); 7616 } else if (type.equalsIgnoreCase("FLOAT")) { 7617 buffer.append("1.0"); 7618 } else if (type.equalsIgnoreCase("FLOAT4")) { 7619 buffer.append("1.0"); 7620 } else if (type.equalsIgnoreCase("FLOAT8")) { 7621 buffer.append("1.0"); 7622 } else if (type.equalsIgnoreCase("DOUBLE")) { 7623 buffer.append("1.0"); 7624 } else if (type.equalsIgnoreCase("DOUBLE PRECISION")) { 7625 buffer.append("1.0"); 7626 } else if (type.equalsIgnoreCase("REAL")) { 7627 buffer.append("1.0"); 7628 } else if (type.equalsIgnoreCase("NUMBER")) { 7629 buffer.append("1.0"); 7630 } else if (type.equalsIgnoreCase("DECIMAL")) { 7631 buffer.append("1.0"); 7632 } else if (type.equalsIgnoreCase("NUMERIC")) { 7633 buffer.append("1.0"); 7634 } else if (type.equalsIgnoreCase("INT")) { 7635 buffer.append("1"); 7636 } else if (type.equalsIgnoreCase("INTEGER")) { 7637 buffer.append("1"); 7638 } else if (type.equalsIgnoreCase("BIGINT")) { 7639 buffer.append("1"); 7640 } else if (type.equalsIgnoreCase("SMALLINT")) { 7641 buffer.append("1"); 7642 } else if (type.equalsIgnoreCase("DATE")) { 7643 buffer.append("new Date()"); 7644 } else if (type.equalsIgnoreCase("DATETIME")) { 7645 buffer.append("new Date()"); 7646 } else if (type.equalsIgnoreCase("TIME")) { 7647 buffer.append("new Date()"); 7648 } else if (type.equalsIgnoreCase("TIMESTAMP")) { 7649 buffer.append("new Date()"); 7650 } else if (type.equalsIgnoreCase("TIMESTAMP_LTZ")) { 7651 buffer.append("new Date()"); 7652 } else if (type.equalsIgnoreCase("TIMESTAMP_NTZ")) { 7653 buffer.append("new Date()"); 7654 } else if (type.equalsIgnoreCase("TIMESTAMP_TZ")) { 7655 buffer.append("new Date()"); 7656 } else if (type.equalsIgnoreCase("VARIANT")) { 7657 buffer.append("{}"); 7658 } else if (type.equalsIgnoreCase("OBJECT")) { 7659 buffer.append("{}"); 7660 } else if (type.equalsIgnoreCase("ARRAY")) { 7661 buffer.append("[]"); 7662 } else if (type.equalsIgnoreCase("GEOGRAPHY")) { 7663 buffer.append("{}"); 7664 } 7665 if (i < args.length - 1) { 7666 buffer.append(","); 7667 } 7668 } 7669 buffer.append(");"); 7670 7671 try { 7672 ScriptEngineManager scriptEngineManager = new ScriptEngineManager(); 7673 ScriptEngine nashorn = scriptEngineManager.getEngineByName("nashorn"); 7674 if (nashorn == null) { 7675 // The Nashorn engine was removed from the JDK in Java 15, so the 7676 // JavaScript body cannot be executed to extract the SQL it runs. 7677 // Try the body as plain SQL; otherwise keep the procedure model 7678 // (with its declared language) and skip the body (Mantis 4590). 7679 TGSqlParser sqlparser = createSqlParser(option.getVendor()); 7680 sqlparser.sqltext = body; 7681 if (sqlparser.parse() == 0) { 7682 dynamicFoldDepth++; 7683 try { 7684 for (int i = 0; i < sqlparser.sqlstatements.size(); i++) { 7685 analyzeCustomSqlStmt(sqlparser.sqlstatements.get(i)); 7686 } 7687 } finally { 7688 dynamicFoldDepth--; 7689 } 7690 } 7691 return; 7692 } 7693 nashorn.put("analyzer", this); 7694 nashorn.eval(new InputStreamReader( 7695 getClass().getResourceAsStream("/gudusoft/gsqlparser/parser/snowflake/snowflake.js"))); 7696 nashorn.eval(new StringReader(buffer.toString())); 7697 } catch (ScriptException e) { 7698 TGSqlParser sqlparser = createSqlParser(option.getVendor()); 7699 sqlparser.sqltext = body; 7700 int result = sqlparser.parse(); 7701 if (result == 0) { 7702 dynamicFoldDepth++; 7703 try { 7704 for (int i = 0; i < sqlparser.sqlstatements.size(); i++) { 7705 analyzeCustomSqlStmt(sqlparser.sqlstatements.get(i)); 7706 } 7707 } finally { 7708 dynamicFoldDepth--; 7709 } 7710 return; 7711 } 7712 ErrorInfo errorInfo = new ErrorInfo(); 7713 errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR); 7714 errorInfo.setErrorMessage("Invoke script error: " + e.getMessage()); 7715 errorInfo.setStartPosition(new Pair3<Long, Long, String>(procedure.getStartToken().lineNo, 7716 procedure.getStartToken().columnNo, ModelBindingManager.getGlobalHash())); 7717 errorInfo.setEndPosition(new Pair3<Long, Long, String>(procedure.getEndToken().lineNo, 7718 procedure.getEndToken().columnNo + procedure.getEndToken().getAstext().length(), 7719 ModelBindingManager.getGlobalHash())); 7720 errorInfos.add(errorInfo); 7721 } 7722 } 7723 7724 private void analyzeHiveLoadStmt(THiveLoad stmt) { 7725 if (stmt.getPath() != null && stmt.getTable() != null) { 7726 Table uriFile = modelFactory.createTableByName(stmt.getPath(), true); 7727 uriFile.setPath(true); 7728 uriFile.setCreateTable(true); 7729 TObjectName fileUri = new TObjectName(); 7730 fileUri.setString("uri=" + stmt.getPath()); 7731 TableColumn fileUriColumn = modelFactory.createFileUri(uriFile, fileUri); 7732 7733 Table tableModel = modelFactory.createTable(stmt.getTable()); 7734 Process process = modelFactory.createProcess(stmt); 7735 tableModel.addProcess(process); 7736 7737 TPartitionExtensionClause p = stmt.getTable().getPartitionExtensionClause(); 7738 if (p.getKeyValues() != null && p.getKeyValues().size() > 0) { 7739 for (int i = 0; i < p.getKeyValues().size(); i++) { 7740 TExpression expression = p.getKeyValues().getExpression(i); 7741 if (expression.getLeftOperand().getExpressionType() == EExpressionType.simple_object_name_t) { 7742 modelFactory.createTableColumn(tableModel, expression.getLeftOperand().getObjectOperand(), 7743 true); 7744 } 7745 } 7746 } 7747 7748 for (int j = 0; j < tableModel.getColumns().size(); j++) { 7749 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 7750 relation.addSource(new TableColumnRelationshipElement(fileUriColumn)); 7751 relation.setTarget(new TableColumnRelationshipElement(tableModel.getColumns().get(j))); 7752 relation.setProcess(process); 7753 } 7754 } 7755 } 7756 7757 private void analyzeLoadDataStmt(TLoadDataStmt stmt) { 7758 7759 } 7760 7761 private void analyzeUnloadStmt(TUnloadStmt unloadStmt) { 7762 if (unloadStmt.getSelectSqlStatement() != null && unloadStmt.getS3() != null) { 7763 7764 Table uriFile = modelFactory.createTableByName(unloadStmt.getS3(), true); 7765 uriFile.setPath(true); 7766 uriFile.setCreateTable(true); 7767 TObjectName fileUri = new TObjectName(); 7768 fileUri.setString("uri=" + unloadStmt.getS3()); 7769 TableColumn fileUriColumn = modelFactory.createFileUri(uriFile, fileUri); 7770 7771 Process process = modelFactory.createProcess(unloadStmt); 7772 uriFile.addProcess(process); 7773 7774 TCustomSqlStatement stmt = unloadStmt.getSelectSqlStatement(); 7775 analyzeCustomSqlStmt(stmt); 7776 if (stmt instanceof TSelectSqlStatement) { 7777 TSelectSqlStatement select = (TSelectSqlStatement) stmt; 7778 ResultSet resultSetModel = (ResultSet) modelManager.getModel(select); 7779 if (resultSetModel != null) { 7780 for (int j = 0; j < resultSetModel.getColumns().size(); j++) { 7781 ResultColumn resultColumn = resultSetModel.getColumns().get(j); 7782 if (resultColumn.hasStarLinkColumn() 7783 && resultColumn.getStarLinkColumnNames().size() > 0) { 7784 for (int k = 0; k < resultColumn.getStarLinkColumnNames().size(); k++) { 7785 ResultColumn expandStarColumn = modelFactory.createResultColumn(resultSetModel, 7786 resultColumn.getStarLinkColumnName(k), false); 7787 DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation(); 7788 dataflowRelation.setEffectType(EffectType.unload); 7789 dataflowRelation 7790 .addSource(new ResultColumnRelationshipElement(expandStarColumn)); 7791 dataflowRelation.setTarget(new TableColumnRelationshipElement(fileUriColumn)); 7792 dataflowRelation.setProcess(process); 7793 } 7794 } 7795 if (!resultColumn.hasStarLinkColumn() || resultColumn.isShowStar()) { 7796 DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation(); 7797 dataflowRelation.setEffectType(EffectType.unload); 7798 dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn)); 7799 dataflowRelation.setTarget(new TableColumnRelationshipElement(fileUriColumn)); 7800 dataflowRelation.setProcess(process); 7801 } 7802 } 7803 } 7804 } 7805 } 7806 7807 } 7808 7809 private void analyzeCopyIntoStmt(TSnowflakeCopyIntoStmt stmt) { 7810 if (stmt.getTableName() != null) { 7811 if (stmt.getStageLocation() != null) { 7812 Table intoTable = modelManager 7813 .getTableByName(DlineageUtil.getTableFullName(stmt.getTableName().toString())); 7814 if (intoTable == null) { 7815 intoTable = modelFactory.createTableByName(stmt.getTableName(), false); 7816 TObjectName starColumn = new TObjectName(); 7817 starColumn.setString("*"); 7818 TableColumn column = modelFactory.createTableColumn(intoTable, starColumn, false); 7819 if (column != null) { 7820 column.setExpandStar(false); 7821 column.setPseduo(true); 7822 } 7823 } 7824 Process process = modelFactory.createProcess(stmt); 7825 intoTable.addProcess(process); 7826 7827 TObjectName stageName = stmt.getStageLocation().getStageName(); 7828 if (stageName == null || stmt.getStageLocation().getTableName() != null) { 7829 stageName = stmt.getStageLocation().getTableName(); 7830 } 7831 if (stageName != null) { 7832 String stageFullName = DlineageUtil.getTableFullName(stageName.toString()); 7833 Table stage = modelManager.getTableByName(stageFullName); 7834 if (stage == null) { 7835 stage = modelFactory.createStage(stageName); 7836 stage.setCreateTable(true); 7837 stage.setStage(true); 7838 String stagePath = stmt.getStageLocation().getPath() == null ? null 7839 : stmt.getStageLocation().getPath().toString(); 7840 if (stagePath != null) { 7841 stage.setLocation(stagePath); 7842 TObjectName location = new TObjectName(); 7843 location.setString(stagePath); 7844 modelFactory.createStageLocation(stage, location); 7845 } else { 7846 stage.setLocation("unknownPath"); 7847 TObjectName location = new TObjectName(); 7848 location.setString("unknownPath"); 7849 modelFactory.createStageLocation(stage, location); 7850 } 7851 } 7852 7853 if (stage != null && intoTable != null) { 7854 if (intoTable != null && !intoTable.getColumns().isEmpty() && !stage.getColumns().isEmpty()) { 7855 for (int i = 0; i < intoTable.getColumns().size(); i++) { 7856 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 7857 relation.addSource(new TableColumnRelationshipElement(stage.getColumns().get(0))); 7858 relation.setTarget(new TableColumnRelationshipElement(intoTable.getColumns().get(i))); 7859 relation.setProcess(process); 7860 } 7861 } 7862 } 7863 } else if (stmt.getStageLocation().getExternalLocation() != null) { 7864 Table pathModel = modelFactory.createTableByName(stmt.getStageLocation().getExternalLocation(), 7865 true); 7866 pathModel.setPath(true); 7867 pathModel.setCreateTable(true); 7868 TableColumn fileUriColumn = modelFactory.createFileUri(pathModel, 7869 stmt.getStageLocation().getExternalLocation()); 7870 if (intoTable != null) { 7871 if (intoTable != null && !intoTable.getColumns().isEmpty()) { 7872 for (int i = 0; i < intoTable.getColumns().size(); i++) { 7873 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 7874 relation.addSource(new TableColumnRelationshipElement(fileUriColumn)); 7875 relation.setTarget(new TableColumnRelationshipElement(intoTable.getColumns().get(i))); 7876 relation.setProcess(process); 7877 } 7878 } 7879 } 7880 } 7881 } 7882 else if (stmt.getSubQuery() != null) { 7883 analyzeSelectStmt(stmt.getSubQuery()); 7884 7885 Table intoTable = modelManager 7886 .getTableByName(DlineageUtil.getTableFullName(stmt.getTableName().toString())); 7887 if (intoTable == null) { 7888 intoTable = modelFactory.createTableByName(stmt.getTableName(), false); 7889 TObjectName starColumn = new TObjectName(); 7890 starColumn.setString("*"); 7891 TableColumn column = modelFactory.createTableColumn(intoTable, starColumn, false); 7892 if (column != null) { 7893 column.setExpandStar(true); 7894 column.setPseduo(true); 7895 } 7896 7897 Process process = modelFactory.createProcess(stmt); 7898 intoTable.addProcess(process); 7899 7900 ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt.getSubQuery()); 7901 if (resultSetModel != null && column != null) { 7902 for (int i = 0; i < resultSetModel.getColumns().size(); i++) { 7903 ResultColumn resultColumn = resultSetModel.getColumns().get(i); 7904 DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation(); 7905 dataflowRelation.setEffectType(EffectType.copy); 7906 dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn)); 7907 dataflowRelation.setTarget(new TableColumnRelationshipElement(column)); 7908 dataflowRelation.setProcess(process); 7909 } 7910 } 7911 } 7912 } 7913 } 7914 } 7915 7916 private void analyzeRedshiftCopyStmt(TRedshiftCopy stmt) { 7917 if (stmt.getTableName() != null && stmt.getFromSource() != null) { 7918 7919 Table intoTable = modelManager 7920 .getTableByName(DlineageUtil.getTableFullName(stmt.getTableName().toString())); 7921 if (intoTable == null) { 7922 intoTable = modelFactory.createTableByName(stmt.getTableName(), false); 7923 if (stmt.getColumnList() == null || stmt.getColumnList().size() == 0) { 7924 TObjectName starColumn = new TObjectName(); 7925 starColumn.setString("*"); 7926 TableColumn column = modelFactory.createTableColumn(intoTable, starColumn, false); 7927 if (column != null) { 7928 column.setExpandStar(false); 7929 column.setPseduo(true); 7930 } 7931 } else { 7932 for (TObjectName columnName : stmt.getColumnList()) { 7933 modelFactory.createTableColumn(intoTable, columnName, true); 7934 } 7935 } 7936 } 7937 Process process = modelFactory.createProcess(stmt); 7938 intoTable.addProcess(process); 7939 7940 if (stmt.getFromSource() != null) { 7941 Table pathModel = modelFactory.createTableByName(stmt.getFromSource(), true); 7942 pathModel.setPath(true); 7943 pathModel.setCreateTable(true); 7944 TObjectName fileUri = new TObjectName(); 7945 fileUri.setString(stmt.getFromSource()); 7946 TableColumn fileUriColumn = modelFactory.createFileUri(pathModel, fileUri); 7947 if (intoTable != null) { 7948 if (intoTable != null && !intoTable.getColumns().isEmpty()) { 7949 for (int i = 0; i < intoTable.getColumns().size(); i++) { 7950 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 7951 relation.addSource(new TableColumnRelationshipElement(fileUriColumn)); 7952 relation.setTarget(new TableColumnRelationshipElement(intoTable.getColumns().get(i))); 7953 relation.setProcess(process); 7954 } 7955 } 7956 } 7957 } 7958 } 7959 } 7960 7961 private void analyzeCreateIndexExpressionOperand(TExpression expr, Table tableModel){ 7962 if(expr == null) return; 7963 Deque<TExpression> stack = new ArrayDeque<>(); 7964 stack.push(expr); 7965 while (!stack.isEmpty()) { 7966 TExpression current = stack.pop(); 7967 if (current == null) continue; 7968 TObjectName columnObj = current.getObjectOperand(); 7969 if (columnObj != null) { 7970 TableColumn tableConstraint = modelFactory.createTableColumn(tableModel, columnObj, true); 7971 tableConstraint.setIndexKey(true); 7972 } else { 7973 if (current.getRightOperand() != null) { 7974 stack.push(current.getRightOperand()); 7975 } 7976 if (current.getLeftOperand() != null) { 7977 stack.push(current.getLeftOperand()); 7978 } 7979 } 7980 } 7981 } 7982 private void analyzeCreateIndexStageStmt(TCreateIndexSqlStatement stmt) { 7983 if(stmt.getTableName() == null){ 7984 return; 7985 } 7986 Table tableModel = modelFactory.createTableByName(stmt.getTableName()); 7987 TOrderByItemList columns = stmt.getColumnNameList(); 7988 if(columns!=null) { 7989 for (int i = 0; i < columns.size(); i++) { 7990 TExpression expr = columns.getOrderByItem(i).getSortKey(); 7991 analyzeCreateIndexExpressionOperand(expr, tableModel); 7992 } 7993 } 7994 } 7995 7996 private void analyzeCreateSynonymStmt(TCreateSynonymStmt stmt) { 7997 TObjectName sourceTableName = stmt.getForName(); 7998 if(sourceTableName == null) { 7999 return; 8000 } 8001 TCustomSqlStatement createView = viewDDLMap 8002 .get(DlineageUtil.getTableFullName(sourceTableName.toString())); 8003 if (createView != null) { 8004 analyzeCustomSqlStmt(createView); 8005 } 8006 Table sourceTableModel = modelFactory.createTableByName(sourceTableName); 8007 Process process = modelFactory.createProcess(stmt); 8008 sourceTableModel.addProcess(process); 8009 if(stmt.getSynonymName()!=null) { 8010 Table synonymTableModel = modelFactory.createTableByName(stmt.getSynonymName()); 8011 synonymTableModel.setSubType(SubType.synonym); 8012 if(sourceTableModel.isCreateTable()) { 8013 synonymTableModel.setCreateTable(true); 8014 for(TableColumn sourceTableColumn: new ArrayList<>(sourceTableModel.getColumns())) { 8015 TObjectName columnName = new TObjectName(); 8016 columnName.setString(sourceTableColumn.getName()); 8017 TableColumn synonymTableColumn = new TableColumn(synonymTableModel, columnName); 8018 synonymTableModel.addColumn(synonymTableColumn); 8019 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 8020 relation.setEffectType(EffectType.create_synonym); 8021 relation.setTarget(new TableColumnRelationshipElement(synonymTableColumn)); 8022 relation.addSource(new TableColumnRelationshipElement(sourceTableColumn)); 8023 relation.setProcess(process); 8024 } 8025 } 8026 else { 8027 TObjectName synonymStarColumn = new TObjectName(); 8028 synonymStarColumn.setString("*"); 8029 TableColumn synonymTableStarColumn = modelFactory.createTableColumn(synonymTableModel, 8030 synonymStarColumn, true); 8031 TObjectName sourceStarColumn = new TObjectName(); 8032 sourceStarColumn.setString("*"); 8033 TableColumn sourceTableStarColumn = modelFactory.createTableColumn(sourceTableModel, sourceStarColumn, true); 8034 8035 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 8036 relation.setEffectType(EffectType.create_synonym); 8037 relation.setTarget(new TableColumnRelationshipElement(synonymTableStarColumn)); 8038 relation.addSource(new TableColumnRelationshipElement(sourceTableStarColumn)); 8039 relation.setProcess(process); 8040 } 8041 } 8042 } 8043 8044 /** 8045 * For every Table model that maps to a {@link TSQLSynonyms} in the current 8046 * {@link #sqlenv}, materialize the synonym's base table as a separate Table 8047 * model and emit per-column {@code fdd}/{@code synonym} relations from the 8048 * synonym's columns to the base table's matching columns. Lets lineage flow 8049 * through synonyms defined in the metadata SQLEnv (e.g. loaded via 8050 * {@code TDDLSQLEnv} or sqlflow metadata) the same way it does for 8051 * {@code CREATE SYNONYM} DDL inside the analyzed script. 8052 */ 8053 private void materializeSqlEnvSynonyms() { 8054 if (sqlenv == null) { 8055 return; 8056 } 8057 List<Table> snapshot = new ArrayList<Table>(modelManager.getTablesByName()); 8058 8059 Map<String, TSQLSynonyms> synonymCache = new HashMap<>(); 8060 List<TSQLCatalog> catalogs = sqlenv.getCatalogList(); 8061 if (catalogs != null) { 8062 for (TSQLCatalog catalog : catalogs) { 8063 if (catalog == null) { 8064 continue; 8065 } 8066 List<TSQLSchema> schemas = catalog.getSchemaList(); 8067 if (schemas == null) { 8068 continue; 8069 } 8070 for (TSQLSchema schema : schemas) { 8071 if (schema == null) { 8072 continue; 8073 } 8074 List<TSQLSchemaObject> schemaObjects = schema.getSchemaObjectList(); 8075 if (schemaObjects == null || schemaObjects.isEmpty()) { 8076 continue; 8077 } 8078 for (TSQLSchemaObject schemaObject : schemaObjects) { 8079 if (schemaObject instanceof TSQLSynonyms) { 8080 TSQLSynonyms synonym = (TSQLSynonyms) schemaObject; 8081 String qualifiedName = synonym.getQualifiedName(); 8082 String normalizedName = DlineageUtil.getIdentifierNormalTableName(qualifiedName); 8083 synonymCache.put(normalizedName, synonym); 8084 } 8085 } 8086 } 8087 } 8088 } 8089 8090 for (Table synonymTableModel : snapshot) { 8091 if (synonymTableModel == null) { 8092 continue; 8093 } 8094 if (SubType.synonym.equals(synonymTableModel.getSubType())) { 8095 continue; 8096 } 8097 String qualifiedName = ModelFactory.getQualifiedTableName(synonymTableModel); 8098 String normalizedName = DlineageUtil.getIdentifierNormalTableName(qualifiedName); 8099 TSQLSynonyms synonym = synonymCache.get(normalizedName); 8100 if (synonym == null) { 8101 continue; 8102 } 8103 8104 String baseSqlTableQualifiedName = synonym.getBaseTableQualifiedName(); 8105 if (baseSqlTableQualifiedName == null) { 8106 continue; 8107 } 8108 8109 synonymTableModel.setSubType(SubType.synonym); 8110 8111 TObjectName baseObjectName = new TObjectName(); 8112 baseObjectName.setString(baseSqlTableQualifiedName); 8113 Table baseTableModel = modelFactory.createTableByName(baseObjectName); 8114 if (baseTableModel == null || baseTableModel == synonymTableModel) { 8115 continue; 8116 } 8117 8118 for (TableColumn synonymColumn : new ArrayList<TableColumn>(synonymTableModel.getColumns())) { 8119 String colName = synonymColumn.getName(); 8120 if (SQLUtil.isEmpty(colName) || "*".equals(colName)) { 8121 continue; 8122 } 8123 TableColumn baseColumn = null; 8124 for (TableColumn bc : baseTableModel.getColumns()) { 8125 if (bc.getName() != null && SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotColumn, bc.getName(), colName)) { 8126 baseColumn = bc; 8127 break; 8128 } 8129 } 8130 if (baseColumn == null) { 8131 TObjectName columnName = new TObjectName(); 8132 columnName.setString(colName); 8133 baseColumn = new TableColumn(baseTableModel, columnName); 8134 baseTableModel.addColumn(baseColumn); 8135 } 8136 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 8137 relation.setEffectType(EffectType.synonym); 8138 relation.setTarget(new TableColumnRelationshipElement(synonymColumn)); 8139 relation.addSource(new TableColumnRelationshipElement(baseColumn)); 8140 } 8141 } 8142 } 8143 8144 private void analyzeRenameStmt(TRenameStmt stmt) { 8145 TObjectName oldTableName = stmt.getOldName(); 8146 TObjectName newTableName = stmt.getNewName(); 8147 8148 Table oldNameTableModel = modelFactory.createTableByName(oldTableName); 8149 TObjectName oldStarColumn = new TObjectName(); 8150 oldStarColumn.setString("*"); 8151 TableColumn oldTableStarColumn = modelFactory.createTableColumn(oldNameTableModel, oldStarColumn, true); 8152 8153 Table newNameTableModel = modelFactory.createTableByName(newTableName); 8154 TObjectName newStarColumn = new TObjectName(); 8155 newStarColumn.setString("*"); 8156 TableColumn newTableStarColumn = modelFactory.createTableColumn(newNameTableModel, newStarColumn, true); 8157 8158 Process process = modelFactory.createProcess(stmt); 8159 newNameTableModel.addProcess(process); 8160 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 8161 relation.setEffectType( EffectType.rename_table); 8162 relation.setTarget(new TableColumnRelationshipElement(newTableStarColumn)); 8163 relation.addSource(new TableColumnRelationshipElement(oldTableStarColumn)); 8164 relation.setProcess(process); 8165 8166 if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) { 8167 oldTableStarColumn.setShowStar(false); 8168 relation.setShowStarRelation(false); 8169 } 8170 8171 if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) { 8172 newTableStarColumn.setShowStar(false); 8173 relation.setShowStarRelation(false); 8174 } 8175 } 8176 8177 private void analyzeAlterTableStmt(TAlterTableStatement stmt) { 8178 TTable oldNameTable = stmt.getTargetTable(); 8179 if (oldNameTable == null) { 8180 return; 8181 } 8182 8183 Table oldNameTableModel = modelFactory.createTable(oldNameTable); 8184 TObjectName oldStarColumn = new TObjectName(); 8185 oldStarColumn.setString("*"); 8186 TableColumn oldTableStarColumn = modelFactory.createTableColumn(oldNameTableModel, oldStarColumn, true); 8187 8188 for (int i = 0; stmt.getAlterTableOptionList() != null && i < stmt.getAlterTableOptionList().size(); i++) { 8189 TAlterTableOption option = stmt.getAlterTableOptionList().getAlterTableOption(i); 8190 if (option.getOptionType() == EAlterTableOptionType.RenameTable 8191 || option.getOptionType() == EAlterTableOptionType.swapWith) { 8192 TObjectName newTableName = option.getNewTableName(); 8193 Stack<TParseTreeNode> list = newTableName.getStartToken().getNodesStartFromThisToken(); 8194 boolean containsTable = false; 8195 for (int j = 0; j < list.size(); j++) { 8196 if (list.get(j) instanceof TTable) { 8197 TTable newTableTable = (TTable) list.get(j); 8198 Table newNameTableModel = modelFactory.createTable(newTableTable); 8199 newNameTableModel.setStarStmt("rename_table"); 8200 8201 TObjectName newStarColumn = new TObjectName(); 8202 newStarColumn.setString("*"); 8203 TableColumn newTableStarColumn = modelFactory.createTableColumn(newNameTableModel, 8204 newStarColumn, true); 8205 8206 Process process = modelFactory.createProcess(stmt); 8207 newNameTableModel.addProcess(process); 8208 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 8209 relation.setEffectType( 8210 option.getOptionType() == EAlterTableOptionType.RenameTable ? EffectType.rename_table 8211 : EffectType.swap_table); 8212 if (option.getOptionType() == EAlterTableOptionType.RenameTable) { 8213 relation.setTarget(new TableColumnRelationshipElement(newTableStarColumn)); 8214 relation.addSource(new TableColumnRelationshipElement(oldTableStarColumn)); 8215 } else if (option.getOptionType() == EAlterTableOptionType.swapWith) { 8216 relation.setTarget(new TableColumnRelationshipElement(oldTableStarColumn)); 8217 relation.addSource(new TableColumnRelationshipElement(newTableStarColumn)); 8218 } 8219 relation.setProcess(process); 8220 containsTable = true; 8221 8222 if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) { 8223 oldTableStarColumn.setShowStar(false); 8224 relation.setShowStarRelation(false); 8225 } 8226 8227 if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) { 8228 newTableStarColumn.setShowStar(false); 8229 relation.setShowStarRelation(false); 8230 } 8231 } 8232 } 8233 if (!containsTable) { 8234 Table newNameTableModel = modelFactory.createTableByName(newTableName); 8235 newNameTableModel.setStarStmt("rename_table"); 8236 TObjectName newStarColumn = new TObjectName(); 8237 newStarColumn.setString("*"); 8238 TableColumn newTableStarColumn = modelFactory.createTableColumn(newNameTableModel, newStarColumn, 8239 true); 8240 8241 Process process = modelFactory.createProcess(stmt); 8242 newNameTableModel.addProcess(process); 8243 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 8244 relation.setEffectType( 8245 option.getOptionType() == EAlterTableOptionType.RenameTable ? EffectType.rename_table 8246 : EffectType.swap_table); 8247 if (option.getOptionType() == EAlterTableOptionType.RenameTable) { 8248 relation.setTarget(new TableColumnRelationshipElement(newTableStarColumn)); 8249 relation.addSource(new TableColumnRelationshipElement(oldTableStarColumn)); 8250 } else if (option.getOptionType() == EAlterTableOptionType.swapWith) { 8251 relation.setTarget(new TableColumnRelationshipElement(oldTableStarColumn)); 8252 relation.addSource(new TableColumnRelationshipElement(newTableStarColumn)); 8253 } 8254 relation.setProcess(process); 8255 if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) { 8256 oldTableStarColumn.setShowStar(false); 8257 relation.setShowStarRelation(false); 8258 } 8259 8260 if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) { 8261 newTableStarColumn.setShowStar(false); 8262 relation.setShowStarRelation(false); 8263 } 8264 8265 } 8266 } 8267 else if (option.getOptionType() == EAlterTableOptionType.appendFrom) { 8268 TObjectName newTableName = option.getSourceTableName(); 8269 Stack<TParseTreeNode> list = newTableName.getStartToken().getNodesStartFromThisToken(); 8270 boolean containsTable = false; 8271 for (int j = 0; j < list.size(); j++) { 8272 if (list.get(j) instanceof TTable) { 8273 TTable newTableTable = (TTable) list.get(j); 8274 Table newNameTableModel = modelFactory.createTable(newTableTable); 8275 newNameTableModel.setStarStmt("append_from"); 8276 8277 TObjectName newStarColumn = new TObjectName(); 8278 newStarColumn.setString("*"); 8279 TableColumn newTableStarColumn = modelFactory.createTableColumn(newNameTableModel, 8280 newStarColumn, true); 8281 8282 Process process = modelFactory.createProcess(stmt); 8283 newNameTableModel.addProcess(process); 8284 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 8285 relation.setEffectType(EffectType.append_from); 8286 relation.setTarget(new TableColumnRelationshipElement(oldTableStarColumn)); 8287 relation.addSource(new TableColumnRelationshipElement(newTableStarColumn)); 8288 relation.setProcess(process); 8289 containsTable = true; 8290 8291 if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) { 8292 oldTableStarColumn.setShowStar(false); 8293 relation.setShowStarRelation(false); 8294 } 8295 8296 if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) { 8297 newTableStarColumn.setShowStar(false); 8298 relation.setShowStarRelation(false); 8299 } 8300 } 8301 } 8302 if (!containsTable) { 8303 Table newNameTableModel = modelFactory.createTableByName(newTableName); 8304 newNameTableModel.setStarStmt("append_from"); 8305 TObjectName newStarColumn = new TObjectName(); 8306 newStarColumn.setString("*"); 8307 TableColumn newTableStarColumn = modelFactory.createTableColumn(newNameTableModel, newStarColumn, 8308 true); 8309 8310 Process process = modelFactory.createProcess(stmt); 8311 newNameTableModel.addProcess(process); 8312 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 8313 relation.setEffectType(EffectType.append_from); 8314 relation.setTarget(new TableColumnRelationshipElement(oldTableStarColumn)); 8315 relation.addSource(new TableColumnRelationshipElement(newTableStarColumn)); 8316 relation.setProcess(process); 8317 if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) { 8318 oldTableStarColumn.setShowStar(false); 8319 relation.setShowStarRelation(false); 8320 } 8321 8322 if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) { 8323 newTableStarColumn.setShowStar(false); 8324 relation.setShowStarRelation(false); 8325 } 8326 8327 } 8328 } 8329 else if (option.getOptionType() == EAlterTableOptionType.exchangePartition) { 8330 TObjectName newTableName = option.getNewTableName(); 8331 Stack<TParseTreeNode> list = newTableName.getStartToken().getNodesStartFromThisToken(); 8332 boolean containsTable = false; 8333 for (int j = 0; j < list.size(); j++) { 8334 if (list.get(j) instanceof TTable) { 8335 TTable newTableTable = (TTable) list.get(j); 8336 Table newNameTableModel = modelFactory.createTable(newTableTable); 8337 newNameTableModel.setStarStmt("exchange_partition"); 8338 8339 TObjectName newStarColumn = new TObjectName(); 8340 newStarColumn.setString("*"); 8341 TableColumn newTableStarColumn = modelFactory.createTableColumn(newNameTableModel, 8342 newStarColumn, true); 8343 8344 Process process = modelFactory.createProcess(stmt); 8345 newNameTableModel.addProcess(process); 8346 { 8347 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 8348 relation.setEffectType(EffectType.exchange_partition); 8349 relation.setTarget(new TableColumnRelationshipElement(oldTableStarColumn)); 8350 relation.addSource(new TableColumnRelationshipElement(newTableStarColumn)); 8351 relation.setProcess(process); 8352 if (option.getPartitionName() != null) { 8353 relation.setPartition(option.getPartitionName().toString()); 8354 } 8355 if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) { 8356 oldTableStarColumn.setShowStar(false); 8357 relation.setShowStarRelation(false); 8358 } 8359 8360 if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) { 8361 newTableStarColumn.setShowStar(false); 8362 relation.setShowStarRelation(false); 8363 } 8364 } 8365 { 8366 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 8367 relation.setEffectType(EffectType.exchange_partition); 8368 relation.setTarget(new TableColumnRelationshipElement(newTableStarColumn)); 8369 relation.addSource(new TableColumnRelationshipElement(oldTableStarColumn)); 8370 relation.setProcess(process); 8371 if (option.getPartitionName() != null) { 8372 relation.setPartition(option.getPartitionName().toString()); 8373 } 8374 if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) { 8375 oldTableStarColumn.setShowStar(false); 8376 relation.setShowStarRelation(false); 8377 } 8378 8379 if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) { 8380 newTableStarColumn.setShowStar(false); 8381 relation.setShowStarRelation(false); 8382 } 8383 } 8384 containsTable = true; 8385 } 8386 } 8387 if (!containsTable) { 8388 Table newNameTableModel = modelFactory.createTableByName(newTableName); 8389 newNameTableModel.setStarStmt("exchange_partition"); 8390 TObjectName newStarColumn = new TObjectName(); 8391 newStarColumn.setString("*"); 8392 TableColumn newTableStarColumn = modelFactory.createTableColumn(newNameTableModel, newStarColumn, 8393 true); 8394 8395 Process process = modelFactory.createProcess(stmt); 8396 newNameTableModel.addProcess(process); 8397 { 8398 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 8399 relation.setEffectType(EffectType.exchange_partition); 8400 relation.setTarget(new TableColumnRelationshipElement(oldTableStarColumn)); 8401 relation.addSource(new TableColumnRelationshipElement(newTableStarColumn)); 8402 if (option.getPartitionName() != null) { 8403 relation.setPartition(option.getPartitionName().toString()); 8404 } 8405 relation.setProcess(process); 8406 if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) { 8407 oldTableStarColumn.setShowStar(false); 8408 relation.setShowStarRelation(false); 8409 } 8410 8411 if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) { 8412 newTableStarColumn.setShowStar(false); 8413 relation.setShowStarRelation(false); 8414 } 8415 } 8416 { 8417 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 8418 relation.setEffectType(EffectType.exchange_partition); 8419 relation.setTarget(new TableColumnRelationshipElement(newTableStarColumn)); 8420 relation.addSource(new TableColumnRelationshipElement(oldTableStarColumn)); 8421 if (option.getPartitionName() != null) { 8422 relation.setPartition(option.getPartitionName().toString()); 8423 } 8424 relation.setProcess(process); 8425 if ((oldNameTableModel.isCreateTable() || oldNameTableModel.hasSQLEnv())) { 8426 oldTableStarColumn.setShowStar(false); 8427 relation.setShowStarRelation(false); 8428 } 8429 8430 if ((newNameTableModel.isCreateTable() || newNameTableModel.hasSQLEnv())) { 8431 newTableStarColumn.setShowStar(false); 8432 relation.setShowStarRelation(false); 8433 } 8434 } 8435 } 8436 } 8437 else if(option.getOptionType() == EAlterTableOptionType.setLocation) { 8438 TObjectName location = option.getTableLocation(); 8439 Process process = modelFactory.createProcess(stmt); 8440 process.setType("Set Table Location"); 8441 oldNameTableModel.addProcess(process); 8442 Table uriFile = modelFactory.createTableByName(location, true); 8443 uriFile.setPath(true); 8444 for (int j = 0; j < oldNameTableModel.getColumns().size(); j++) { 8445 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 8446 TObjectName fileUri = new TObjectName(); 8447 fileUri.setString("*"); 8448 TableColumn fileUriColumn = modelFactory.createFileUri(uriFile, fileUri); 8449 relation.addSource(new TableColumnRelationshipElement(fileUriColumn)); 8450 relation.setTarget(new TableColumnRelationshipElement(oldNameTableModel.getColumns().get(j))); 8451 relation.setProcess(process); 8452 } 8453 } 8454 else if (option.getOptionType() == EAlterTableOptionType.AddColumn 8455 || option.getOptionType() == EAlterTableOptionType.addColumnIfNotExists) { 8456 if (option.getColumnDefinitionList() != null) { 8457 for (TColumnDefinition column : option.getColumnDefinitionList()) { 8458 if (column != null && column.getColumnName() != null) { 8459 TableColumn tableColumn = modelFactory.createTableColumn(oldNameTableModel, column.getColumnName(), true); 8460 if(this.option.getAnalyzeMode() == AnalyzeMode.crud) { 8461 CrudRelationship crudRelationship = modelFactory.createCrudRelation(); 8462 crudRelationship.setTarget(new TableColumnRelationshipElement(tableColumn)); 8463 crudRelationship.setEffectType(EffectType.add_table_column); 8464 } 8465 } 8466 } 8467 } 8468 } 8469 else if (option.getOptionType() == EAlterTableOptionType.DropColumn && this.option.getAnalyzeMode() == AnalyzeMode.crud) { 8470 if (option.getColumnNameList() != null) { 8471 for (TObjectName column : option.getColumnNameList()) { 8472 TableColumn tableColumn = modelFactory.createTableColumn(oldNameTableModel, column, true); 8473 CrudRelationship crudRelationship = modelFactory.createCrudRelation(); 8474 crudRelationship.setTarget(new TableColumnRelationshipElement(tableColumn)); 8475 crudRelationship.setEffectType(EffectType.drop_table_column); 8476 } 8477 } 8478 } 8479 else if(option.getOptionType() == EAlterTableOptionType.AddConstraint || option.getOptionType() == EAlterTableOptionType.AddConstraintFK 8480 || option.getOptionType() == EAlterTableOptionType.AddConstraintPK || option.getOptionType() == EAlterTableOptionType.AddConstraintUnique 8481 || option.getOptionType() == EAlterTableOptionType.AddConstraintIndex){ 8482 if (option.getTableConstraint() != null) { 8483 TConstraint alertTableConstraint = option.getTableConstraint(); 8484 TPTNodeList<TColumnWithSortOrder> keyNames = alertTableConstraint.getColumnList(); 8485 if (keyNames == null) { 8486 continue; 8487 } 8488 for (int k = 0; k < keyNames.size(); k++) { 8489 TObjectName keyName = keyNames.getElement(k).getColumnName(); 8490 TObjectName referencedTableName = alertTableConstraint.getReferencedObject(); 8491 Table tableModel = modelFactory.createTableByName(stmt.getTableName()); 8492 TableColumn tableConstraint = modelFactory.createTableColumn(tableModel, keyName, true); 8493 if(alertTableConstraint.getConstraint_type() == EConstraintType.primary_key){ 8494 tableConstraint.setPrimaryKey(true); 8495 } 8496 else if(alertTableConstraint.getConstraint_type() == EConstraintType.table_index){ 8497 tableConstraint.setIndexKey(true); 8498 } 8499 else if(alertTableConstraint.getConstraint_type() == EConstraintType.unique){ 8500 tableConstraint.setUnqiueKey(true); 8501 } 8502 else if (alertTableConstraint.getConstraint_type() == EConstraintType.foreign_key) { 8503 tableConstraint.setForeignKey(true); 8504 Table referencedTable = modelManager.getTableByName(DlineageUtil.getTableFullName(referencedTableName.toString())); 8505 if (referencedTable == null) { 8506 referencedTable = modelFactory.createTableByName(referencedTableName); 8507 } 8508 TObjectNameList referencedTableColumns = alertTableConstraint.getReferencedColumnList(); 8509 if (k == keyNames.size() - 1) { 8510 captureCompositeForeignKey(tableModel, keyNames, referencedTable, 8511 referencedTableColumns); 8512 } 8513 if (referencedTableColumns != null) { 8514 for (int j = 0; j < referencedTableColumns.size(); j++) { 8515 TableColumn tableColumn = modelFactory.createTableColumn(referencedTable, 8516 referencedTableColumns.getObjectName(j), false); 8517 if (tableColumn != null) { 8518 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 8519 relation.addSource(new TableColumnRelationshipElement(tableColumn)); 8520 relation.setTarget(new TableColumnRelationshipElement(tableConstraint)); 8521 relation.setEffectType(EffectType.foreign_key); 8522 Process process = modelFactory.createProcess(stmt); 8523 relation.setProcess(process); 8524 if(this.option.isShowERDiagram()){ 8525 ERRelationship erRelation = modelFactory.createERRelation(); 8526 erRelation.addSource(new TableColumnRelationshipElement(tableColumn)); 8527 erRelation.setTarget(new TableColumnRelationshipElement(tableConstraint)); 8528 } 8529 } 8530 } 8531 } 8532 else{ 8533 TableColumn tableColumn = modelFactory.createTableColumn(referencedTable, keyName, false); 8534 if (tableColumn != null) { 8535 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 8536 relation.addSource(new TableColumnRelationshipElement(tableColumn)); 8537 relation.setTarget(new TableColumnRelationshipElement(tableConstraint)); 8538 relation.setEffectType(EffectType.foreign_key); 8539 if(this.option.isShowERDiagram()){ 8540 ERRelationship erRelation = modelFactory.createERRelation(); 8541 erRelation.addSource(new TableColumnRelationshipElement(tableColumn)); 8542 erRelation.setTarget(new TableColumnRelationshipElement(tableConstraint)); 8543 } 8544 } 8545 } 8546 } 8547 } 8548 } 8549 else if (option.getConstraintList() != null) { 8550 for(int iCons=0; iCons<option.getConstraintList().size(); iCons++){ 8551 TConstraint alertTableConstraint = option.getConstraintList().getConstraint(iCons); 8552 TPTNodeList<TColumnWithSortOrder> keyNames = alertTableConstraint.getColumnList(); 8553 if(keyNames != null){ 8554 for (int k = 0; k < keyNames.size(); k++) { 8555 TObjectName keyName = keyNames.getElement(k).getColumnName(); 8556 TObjectName referencedTableName = alertTableConstraint.getReferencedObject(); 8557 Table tableModel = modelFactory.createTableByName(stmt.getTableName()); 8558 TableColumn tableConstraint = modelFactory.createTableColumn(tableModel, keyName, true); 8559 if(alertTableConstraint.getConstraint_type() == EConstraintType.primary_key){ 8560 tableConstraint.setPrimaryKey(true); 8561 } 8562 else if(alertTableConstraint.getConstraint_type() == EConstraintType.table_index){ 8563 tableConstraint.setIndexKey(true); 8564 } 8565 else if(alertTableConstraint.getConstraint_type() == EConstraintType.unique){ 8566 tableConstraint.setUnqiueKey(true); 8567 } 8568 else if (alertTableConstraint.getConstraint_type() == EConstraintType.foreign_key) { 8569 tableConstraint.setForeignKey(true); 8570 Table referencedTable = modelManager.getTableByName(DlineageUtil.getTableFullName(referencedTableName.toString())); 8571 if (referencedTable == null) { 8572 referencedTable = modelFactory.createTableByName(referencedTableName); 8573 } 8574 TObjectNameList referencedTableColumns = alertTableConstraint.getReferencedColumnList(); 8575 if (k == keyNames.size() - 1) { 8576 captureCompositeForeignKey(tableModel, keyNames, referencedTable, 8577 referencedTableColumns); 8578 } 8579 if (referencedTableColumns != null) { 8580 for (int j = 0; j < referencedTableColumns.size(); j++) { 8581 TableColumn tableColumn = modelFactory.createTableColumn(referencedTable, 8582 referencedTableColumns.getObjectName(j), false); 8583 if (tableColumn != null) { 8584 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 8585 relation.addSource(new TableColumnRelationshipElement(tableColumn)); 8586 relation.setTarget(new TableColumnRelationshipElement(tableConstraint)); 8587 relation.setEffectType(EffectType.foreign_key); 8588 Process process = modelFactory.createProcess(stmt); 8589 relation.setProcess(process); 8590 if(this.option.isShowERDiagram()){ 8591 ERRelationship erRelation = modelFactory.createERRelation(); 8592 erRelation.addSource(new TableColumnRelationshipElement(tableColumn)); 8593 erRelation.setTarget(new TableColumnRelationshipElement(tableConstraint)); 8594 } 8595 } 8596 } 8597 } 8598 else{ 8599 TableColumn tableColumn = modelFactory.createTableColumn(referencedTable, keyName, false); 8600 if (tableColumn != null) { 8601 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 8602 relation.addSource(new TableColumnRelationshipElement(tableColumn)); 8603 relation.setTarget(new TableColumnRelationshipElement(tableConstraint)); 8604 relation.setEffectType(EffectType.foreign_key); 8605 Process process = modelFactory.createProcess(stmt); 8606 relation.setProcess(process); 8607 if(this.option.isShowERDiagram()){ 8608 ERRelationship erRelation = modelFactory.createERRelation(); 8609 erRelation.addSource(new TableColumnRelationshipElement(tableColumn)); 8610 erRelation.setTarget(new TableColumnRelationshipElement(tableConstraint)); 8611 } 8612 } 8613 } 8614 } 8615 } 8616 } 8617 } 8618 } 8619 else if (option.getIndexCols() != null){ 8620 TPTNodeList<TColumnWithSortOrder> keyNames = option.getIndexCols(); 8621 for (int k = 0; k < keyNames.size(); k++) { 8622 TObjectName keyName = keyNames.getElement(k).getColumnName(); 8623 Table tableModel = modelFactory.createTableByName(stmt.getTableName()); 8624 TableColumn tableConstraint = modelFactory.createTableColumn(tableModel, keyName, true); 8625 if(option.getOptionType() == EAlterTableOptionType.AddConstraintPK){ 8626 tableConstraint.setPrimaryKey(true); 8627 } 8628 else if(option.getOptionType() == EAlterTableOptionType.AddConstraintIndex){ 8629 tableConstraint.setIndexKey(true); 8630 } 8631 else if(option.getOptionType() == EAlterTableOptionType.AddConstraintUnique){ 8632 tableConstraint.setUnqiueKey(true); 8633 } 8634 else if (option.getOptionType() == EAlterTableOptionType.AddConstraintFK) { 8635 TObjectName referencedTableName = option.getReferencedObjectName(); 8636 tableConstraint.setForeignKey(true); 8637 Table referencedTable = modelManager.getTableByName(DlineageUtil.getTableFullName(referencedTableName.toString())); 8638 if (referencedTable == null) { 8639 referencedTable = modelFactory.createTableByName(referencedTableName); 8640 } 8641 TObjectNameList referencedTableColumns = option.getReferencedColumnList(); 8642 if (k == keyNames.size() - 1) { 8643 captureCompositeForeignKey(tableModel, keyNames, referencedTable, 8644 referencedTableColumns); 8645 } 8646 if (referencedTableColumns != null) { 8647 for (int j = 0; j < referencedTableColumns.size(); j++) { 8648 TableColumn tableColumn = modelFactory.createTableColumn(referencedTable, 8649 referencedTableColumns.getObjectName(j), false); 8650 if (tableColumn != null) { 8651 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 8652 relation.addSource(new TableColumnRelationshipElement(tableColumn)); 8653 relation.setTarget(new TableColumnRelationshipElement(tableConstraint)); 8654 relation.setEffectType(EffectType.foreign_key); 8655 Process process = modelFactory.createProcess(stmt); 8656 relation.setProcess(process); 8657 if(this.option.isShowERDiagram()){ 8658 ERRelationship erRelation = modelFactory.createERRelation(); 8659 erRelation.addSource(new TableColumnRelationshipElement(tableColumn)); 8660 erRelation.setTarget(new TableColumnRelationshipElement(tableConstraint)); 8661 } 8662 } 8663 } 8664 } 8665 else{ 8666 TableColumn tableColumn = modelFactory.createTableColumn(referencedTable, keyName, false); 8667 if (tableColumn != null) { 8668 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 8669 relation.addSource(new TableColumnRelationshipElement(tableColumn)); 8670 relation.setTarget(new TableColumnRelationshipElement(tableConstraint)); 8671 relation.setEffectType(EffectType.foreign_key); 8672 Process process = modelFactory.createProcess(stmt); 8673 relation.setProcess(process); 8674 if(this.option.isShowERDiagram()){ 8675 ERRelationship erRelation = modelFactory.createERRelation(); 8676 erRelation.addSource(new TableColumnRelationshipElement(tableColumn)); 8677 erRelation.setTarget(new TableColumnRelationshipElement(tableConstraint)); 8678 } 8679 } 8680 } 8681 } 8682 } 8683 } 8684 } 8685 } 8686 } 8687 8688 private void analyzeAlterViewStmt(TAlterViewStatement stmt) { 8689 if (stmt.getAlterViewOption() == EAlterViewOption.asSelect) { 8690 analyzeCreateViewStmt(stmt, stmt.getSelectSqlStatement(), null, stmt.getViewName()); 8691 } else { 8692 throw new UnsupportedOperationException("Can't handle this alter view statement case, alter option = "+ stmt.getAlterViewOption().name()); 8693 } 8694 } 8695 8696 private void analyzeDeleteStmt(TDeleteSqlStatement stmt) { 8697 TTable targetTable = stmt.getTargetTable(); 8698 if (targetTable == null) 8699 return; 8700 8701 TTable table = targetTable; 8702 Table tableModel = null; 8703 TSelectSqlStatement projectedSubquery = null; 8704 TObjectNameList exposedColumns = null; 8705 if (targetTable.getCTE() != null) { 8706 projectedSubquery = targetTable.getCTE().getSubquery(); 8707 exposedColumns = targetTable.getCTE().getColumnList(); 8708 } else if (targetTable.getLinkTable() != null 8709 && targetTable.getLinkTable().getSubquery() != null) { 8710 projectedSubquery = targetTable.getLinkTable().getSubquery(); 8711 exposedColumns = getProjectedTargetColumns(targetTable.getLinkTable()); 8712 } else if (targetTable.getSubquery() != null) { 8713 projectedSubquery = targetTable.getSubquery(); 8714 exposedColumns = getProjectedTargetColumns(targetTable); 8715 } 8716 8717 if (projectedSubquery != null) { 8718 CteWriteTargetResolution resolution = resolveCteDeleteTarget(projectedSubquery); 8719 if (resolution.isWritable()) { 8720 analyzeProjectedWriteTarget(targetTable, projectedSubquery, exposedColumns, null); 8721 table = resolution.baseTable; 8722 tableModel = modelFactory.createTable(table); 8723 createProjectedWriteImpact(projectedSubquery, tableModel, EffectType.delete, null); 8724 } else { 8725 analyzeProjectedWriteTarget(targetTable, projectedSubquery, exposedColumns, null); 8726 addCteWriteTargetHint(targetTable, resolution.failureReason); 8727 } 8728 } else { 8729 tableModel = modelFactory.createTable(table); 8730 } 8731 if (tableModel != null && getTableLinkedColumns(table) != null 8732 && getTableLinkedColumns(table).size() > 0) { 8733 for (int j = 0; j < getTableLinkedColumns(table).size(); j++) { 8734 TObjectName object = getTableLinkedColumns(table).getObjectName(j); 8735 8736 if (object.getDbObjectType() == EDbObjectType.variable) { 8737 continue; 8738 } 8739 8740 if (object.getColumnNameOnly().startsWith("@") 8741 && (option.getVendor() == EDbVendor.dbvmssql || option.getVendor() == EDbVendor.dbvazuresql)) { 8742 continue; 8743 } 8744 8745 if (object.getColumnNameOnly().startsWith(":") 8746 && (option.getVendor() == EDbVendor.dbvhana || option.getVendor() == EDbVendor.dbvteradata)) { 8747 continue; 8748 } 8749 8750 if (!isBuiltInFunctionName(object)) { 8751 if (object.getSourceTable() == null || object.getSourceTable() == table) { 8752 modelFactory.createTableColumn(tableModel, object, false); 8753 } 8754 } 8755 } 8756 } 8757 8758 if(option.getAnalyzeMode() == AnalyzeMode.crud && tableModel != null) { 8759 CrudRelationship crudRelationship = modelFactory.createCrudRelation(); 8760 crudRelationship.setTarget(new TableRelationshipElement(tableModel)); 8761 crudRelationship.setEffectType(EffectType.delete); 8762 } 8763 8764 if (stmt.getWhereClause() != null && stmt.getWhereClause().getCondition() != null) { 8765 analyzeFilterCondition(tableModel, stmt.getWhereClause().getCondition(), null, JoinClauseType.where, 8766 EffectType.delete); 8767 } 8768 } 8769 8770 private TObjectName getProcedureName(TStoredProcedureSqlStatement stmt) { 8771 if (stmt instanceof TTeradataCreateProcedure) { 8772 return ((TTeradataCreateProcedure) stmt).getProcedureName(); 8773 } 8774 return DlineageUtil.getProcedureOrFunctionName(stmt); 8775 } 8776 8777 private void analyzePlsqlCreatePackage(TPlsqlCreatePackage stmt) { 8778 TObjectName procedureName = getProcedureName(stmt); 8779 OraclePackage oraclePackage; 8780 if (procedureName != null) { 8781 if (this.modelManager.getOraclePackageByName( 8782 DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getProcedureNameWithArgs(stmt))) == null) { 8783 oraclePackage = this.modelFactory.createOraclePackage(stmt); 8784 8785 if (stmt.getParameterDeclarations() != null) { 8786 TParameterDeclarationList parameters = stmt.getParameterDeclarations(); 8787 8788 for (int i = 0; i < parameters.size(); ++i) { 8789 TParameterDeclaration parameter = parameters.getParameterDeclarationItem(i); 8790 if (parameter.getParameterName() != null) { 8791 this.modelFactory.createProcedureArgument(oraclePackage, parameter, i + 1); 8792 } else if (parameter.getDataType() != null) { 8793 this.modelFactory.createProcedureArgument(oraclePackage, parameter, i + 1); 8794 } 8795 } 8796 } 8797 8798 ModelBindingManager.setGlobalOraclePackage(oraclePackage); 8799 } else { 8800 ModelBindingManager.setGlobalOraclePackage(this.modelManager.getOraclePackageByName( 8801 DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getProcedureNameWithArgs(stmt)))); 8802 } 8803 8804 try { 8805 if (stmt.getDeclareStatements() != null) { 8806 for (int i = 0; i < stmt.getDeclareStatements().size(); ++i) { 8807 analyzeCustomSqlStmt(stmt.getDeclareStatements().get(i)); 8808 } 8809 } 8810 } finally { 8811 ModelBindingManager.removeGlobalOraclePackage(); 8812 } 8813 } 8814 } 8815 8816 /** 8817 * B1 (routine-summary-scc-design §2.1a): canonical signature descriptor of 8818 * a routine DECLARATION, built once at declaration time. Null when the 8819 * statement carries no resolvable name. 8820 */ 8821 private gudusoft.gsqlparser.dlineage.dynamicsql.RoutineIdentity declarationIdentityOf( 8822 TStoredProcedureSqlStatement stmt) { 8823 TObjectName name = getProcedureName(stmt); 8824 if (name == null || name.getObjectString() == null 8825 || name.getObjectString().length() == 0) { 8826 return null; 8827 } 8828 boolean isFunction = stmt instanceof gudusoft.gsqlparser.stmt.TCreateFunctionStmt 8829 || stmt instanceof gudusoft.gsqlparser.stmt.oracle.TPlsqlCreateFunction; 8830 boolean isTrigger = stmt instanceof TCreateTriggerStmt 8831 || stmt instanceof TPlsqlCreateTrigger; 8832 String packageName = null; 8833 gudusoft.gsqlparser.dlineage.dynamicsql.RoutineKind kind; 8834 if (isTrigger) { 8835 kind = gudusoft.gsqlparser.dlineage.dynamicsql.RoutineKind.TRIGGER; 8836 } else if (ModelBindingManager.getGlobalOraclePackage() != null) { 8837 packageName = ModelBindingManager.getGlobalOraclePackage().getName(); 8838 kind = isFunction 8839 ? gudusoft.gsqlparser.dlineage.dynamicsql.RoutineKind.PACKAGE_MEMBER_FUNCTION 8840 : gudusoft.gsqlparser.dlineage.dynamicsql.RoutineKind.PACKAGE_MEMBER_PROCEDURE; 8841 } else { 8842 kind = isFunction 8843 ? gudusoft.gsqlparser.dlineage.dynamicsql.RoutineKind.FUNCTION 8844 : gudusoft.gsqlparser.dlineage.dynamicsql.RoutineKind.PROCEDURE; 8845 } 8846 // mssql numbered procedures: the parser drops ";N" from the AST name; 8847 // recover it from the token stream so grp;1 / grp;2 keep DISTINCT 8848 // identity scope keys (per-overload pools — codex-B3-r2 finding 6). 8849 String overloadDiscriminator = option.getVendor() == EDbVendor.dbvmssql 8850 ? gudusoft.gsqlparser.dlineage.dynamicsql.RoutineCatalog 8851 .numberedGroupAfter(name) 8852 : null; 8853 return gudusoft.gsqlparser.dlineage.dynamicsql.RoutineIdentity.of( 8854 option.getVendor(), kind, name.getDatabaseString(), 8855 name.getSchemaString(), packageName, name.getObjectString(), 8856 stmt.getParameterDeclarations(), overloadDiscriminator); 8857 } 8858 8859 private void analyzeStoredProcedureStmt(TStoredProcedureSqlStatement stmt) { 8860 8861 if (stmt instanceof TPlsqlCreatePackage) { 8862 analyzePlsqlCreatePackage((TPlsqlCreatePackage) stmt); 8863 return; 8864 } 8865 8866 ModelBindingManager.setGlobalProcedure(stmt); 8867 8868 boolean pushedIdentityScope = false; 8869 try { 8870 Procedure procedure = null; 8871 8872 TObjectName procedureName = getProcedureName(stmt); 8873 8874 // Honor procedure-exclusion patterns/names for procedures defined in actual SQL 8875 // (CREATE PROCEDURE). Previously exclusion was only applied to procedures loaded 8876 // from metadata JSON; a CREATE PROCEDURE parsed from SQL still produced its 8877 // procedure node plus all of its inner-statement processes and relationships 8878 // regardless of the exclusion configuration (MantisBT 4533). 8879 // 8880 // analyzeStoredProcedureStmt also handles functions and triggers (every dialect's 8881 // CREATE FUNCTION / CREATE TRIGGER class extends TStoredProcedureSqlStatement), which 8882 // are NOT procedures and must never be dropped by procedure exclusion. Gate on the 8883 // statement type rather than enumerating classes: across all dialects the procedure 8884 // statement types are exactly those whose ESqlStatementType name contains "procedure" 8885 // (e.g. sstcreateprocedure, sstoraclecreateprocedure, sstmssqlcreateprocedure), 8886 // while function/trigger types contain "function"/"trigger". 8887 boolean isProcedureStmt = stmt.sqlstatementtype != null 8888 && stmt.sqlstatementtype.name().toLowerCase().contains("procedure"); 8889 if (procedureName != null && isProcedureStmt 8890 && DlineageUtil.isProcedureExcluded(procedureName.toString())) { 8891 return; 8892 } 8893 8894 if (procedureName != null) { 8895 procedure = this.modelFactory.createProcedure(stmt); 8896 if (procedure != null) { 8897 modelManager.bindModel(stmt, procedure); 8898 } 8899 if (ModelBindingManager.getGlobalOraclePackage() != null) { 8900 ModelBindingManager.getGlobalOraclePackage().addProcedure(procedure); 8901 procedure.setParentPackage(ModelBindingManager.getGlobalOraclePackage()); 8902 // Canonical scope key of this member's variables: the same 8903 // value createVariable(String) keys them under while the 8904 // body is analyzed (fullName/name are display forms and 8905 // must not be touched). 8906 procedure.setVariableScopeKey(DlineageUtil.getProcedureParentName(stmt)); 8907 } else if (procedure != null) { 8908 // Non-package routines: body references register their 8909 // variables under the getTableFullName-composed scope (the 8910 // parent-statement branch of getProcedureParentName), which 8911 // under a USE database context differs BOTH from the raw 8912 // declared name and from the display name — leaving nested 8913 // EXEC argument binding without a matching key. Capture the 8914 // body-side composition here, under the declaration's own 8915 // ambient context. 8916 procedure.setVariableScopeKey(DlineageUtil.getTableFullName( 8917 DlineageUtil.getProcedureParentName(stmt))); 8918 } 8919 if (procedure != null) { 8920 // B1 (routine-summary-scc-design §3): additive identity 8921 // metadata. The identityScopeKey discriminates same-named 8922 // overloads; LEGACY paths never consult it (B3's 8923 // identity-first lookup is the consumer). 8924 gudusoft.gsqlparser.dlineage.dynamicsql.RoutineIdentity routineIdentity = 8925 declarationIdentityOf(stmt); 8926 procedure.setRoutineIdentity(routineIdentity); 8927 if (procedure.getVariableScopeKey() != null && routineIdentity != null) { 8928 procedure.setIdentityScopeKey(DlineageUtil.identityScopeKey( 8929 procedure.getVariableScopeKey(), routineIdentity)); 8930 } 8931 // B3 (routine-summary-scc-design §3): object-level 8932 // per-overload variable pools. Mode-gated — LEGACY never 8933 // pushes, so the identity pool path stays cold there. 8934 if (option.isIdentityFirstVariablePools() 8935 && procedure.getIdentityScopeKey() != null) { 8936 modelFactory.pushIdentityScope(procedure.getIdentityScopeKey()); 8937 pushedIdentityScope = true; 8938 } 8939 } 8940 if (stmt.getParameterDeclarations() != null) { 8941 TParameterDeclarationList parameters = stmt.getParameterDeclarations(); 8942 8943 for (int i = 0; i < parameters.size(); ++i) { 8944 TParameterDeclaration parameter = parameters.getParameterDeclarationItem(i); 8945 Argument argument = null; 8946 TObjectName argumentName = null; 8947 if (parameter.getParameterName() != null) { 8948 argument = this.modelFactory.createProcedureArgument(procedure, parameter, i + 1); 8949 argumentName = parameter.getParameterName(); 8950 } else if (parameter.getDataType() != null) { 8951 argument = this.modelFactory.createProcedureArgument(procedure, parameter, i + 1); 8952 } 8953 8954 if (argument != null) { 8955 if (argumentName == null) { 8956 argumentName = new TObjectName(); 8957 argumentName.setString(argument.getName()); 8958 } 8959 Variable variable = modelFactory.createVariable(argument.getName()); 8960 // B1: index the formal under the overload- 8961 // discriminating identity scope too (lookup-map 8962 // only; consumed by B3's identity-first path). 8963 modelFactory.bindVariableUnderIdentityScope( 8964 procedure, argument.getName(), variable); 8965 if (argument.getMode() == EParameterMode.in || argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) { 8966 variable.setSubType(SubType.of(argument.getMode().name())); 8967 } else { 8968 variable.setSubType(SubType.argument); 8969 } 8970 TableColumn parameterField; 8971 if(isSimpleDataType(argument.getDataType())){ 8972 parameterField = modelFactory.createTableColumn(variable, argumentName, true); 8973 } 8974 else { 8975 TObjectName variableProperties = new TObjectName(); 8976 variableProperties.setString("*"); 8977 parameterField = modelFactory.createTableColumn(variable, variableProperties, true); 8978 } 8979 if (shouldCollectAuthoritativeLineageEvidence()) { 8980 authoritativeEvidenceCollector.bindRoutineParameter(argument, parameterField); 8981 } 8982 } 8983 } 8984 } 8985 } 8986 8987 if (stmt instanceof TCreateTriggerStmt) { 8988 TCreateTriggerStmt trigger = (TCreateTriggerStmt) stmt; 8989 8990 if (trigger.getFunctionCall() != null) { 8991 modelFactory.createProcedureFromFunctionCall(trigger.getFunctionCall()); 8992 } 8993 8994 if (trigger.getTables() != null) { 8995 for (int i = 0; i < trigger.getTables().size(); i++) { 8996 Table tableModel = this.modelFactory.createTriggerOnTable(trigger.getTables().getTable(i)); 8997 } 8998 } 8999 } 9000 9001 if (stmt instanceof TPlsqlCreateTrigger && ((TPlsqlCreateTrigger) stmt).getTriggeringClause()!=null 9002 && ((TPlsqlCreateTrigger) stmt).getTriggeringClause().getEventClause() instanceof TDmlEventClause) { 9003 TPlsqlCreateTrigger trigger = (TPlsqlCreateTrigger) stmt; 9004 TDmlEventClause clause = (TDmlEventClause) ((TPlsqlCreateTrigger) stmt).getTriggeringClause() 9005 .getEventClause(); 9006 Table sourceTable = modelFactory.createTableByName(clause.getTableName()); 9007 9008 bindTriggerCorrelationNames(trigger, sourceTable); 9009 9010 for (TTriggerEventItem item : clause.getEventItems()) { 9011 if (item instanceof TDmlEventItem) { 9012 if (((TDmlEventItem) item).getColumnList() != null) { 9013 for (TObjectName column : ((TDmlEventItem) item).getColumnList()) { 9014 modelFactory.createTableColumn(sourceTable, column, true); 9015 } 9016 } 9017 } 9018 } 9019 9020 for (TCustomSqlStatement subStmt : ((TPlsqlCreateTrigger) stmt).getStatements()) { 9021 if (!(subStmt instanceof TCommonBlock)) 9022 continue; 9023 for (TCustomSqlStatement blockSubStmt : ((TCommonBlock) subStmt).getStatements()) { 9024 if (!(blockSubStmt instanceof TBasicStmt)) 9025 continue; 9026 TBasicStmt basicStmt = (TBasicStmt) blockSubStmt; 9027 TExpression expression = basicStmt.getExpr(); 9028 if (expression != null && expression.getExpressionType() == EExpressionType.function_t) { 9029 Procedure targetProcedure = modelManager.getProcedureByName(DlineageUtil 9030 .getTableFullName(expression.getFunctionCall().getFunctionName().toString())); 9031 if (targetProcedure == null) { 9032 targetProcedure = modelManager 9033 .getProcedureByName(DlineageUtil.getTableFullName(procedure.getSchema() + "." 9034 + expression.getFunctionCall().getFunctionName().toString())); 9035 } 9036 if (targetProcedure != null && expression.getFunctionCall().getArgs() != null && expression 9037 .getFunctionCall().getArgs().size() == targetProcedure.getArguments().size()) { 9038 for (int j = 0; j < expression.getFunctionCall().getArgs().size(); j++) { 9039 TExpression columnExpr = expression.getFunctionCall().getArgs().getExpression(j); 9040 if (columnExpr.getExpressionType() == EExpressionType.simple_object_name_t) { 9041 TObjectName columnObject = columnExpr.getObjectOperand(); 9042 if (columnObject.toString().indexOf(":") != -1) { 9043 TableColumn tableColumn = modelFactory.createTableColumn(sourceTable, 9044 columnObject, true); 9045 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 9046 relation.setEffectType(EffectType.trigger); 9047 relation.addSource(new TableColumnRelationshipElement(tableColumn)); 9048 relation.setTarget( 9049 new ArgumentRelationshipElement(targetProcedure.getArguments().get(j))); 9050 Process process = modelFactory.createProcess(stmt); 9051 relation.setProcess(process); 9052 } 9053 } 9054 } 9055 } 9056 } 9057 } 9058 } 9059 } 9060 9061 if (stmt instanceof TMssqlCreateFunction) { 9062 TMssqlCreateFunction createFunction = (TMssqlCreateFunction) stmt; 9063 if (createFunction.getReturnTableVaraible() != null && createFunction.getReturnTableDefinitions() != null) { 9064 Variable tableModel = this.modelFactory.createVariable(createFunction.getReturnTableVaraible()); 9065 tableModel.setVariable(true); 9066 tableModel.setCreateTable(true); 9067 if (procedure != null) { 9068 tableModel.setProcedureId(String.valueOf(procedure.getId())); 9069 tableModel.setTarget(true); 9070 } 9071 String procedureParent = createFunction.getFunctionName().toString(); 9072 if (procedureParent != null) { 9073 tableModel.setParent(procedureParent); 9074 } 9075 modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent), tableModel); 9076 9077 if (createFunction.getReturnTableDefinitions() != null) { 9078 for (int j = 0; j < createFunction.getReturnTableDefinitions().size(); j++) { 9079 TTableElement tableElement = createFunction.getReturnTableDefinitions().getTableElement(j); 9080 TColumnDefinition column = tableElement.getColumnDefinition(); 9081 if (column != null && column.getColumnName() != null) { 9082 modelFactory.createTableColumn(tableModel, column.getColumnName(), true); 9083 } 9084 } 9085 } 9086 } 9087 9088 if (createFunction.getReturnStmt() != null && createFunction.getReturnStmt().getSubquery() != null) { 9089 String procedureParent = createFunction.getFunctionName().toString(); 9090 analyzeSelectStmt(createFunction.getReturnStmt().getSubquery()); 9091 ResultSet resultSetModel = (ResultSet) modelManager 9092 .getModel(createFunction.getReturnStmt().getSubquery()); 9093 if (procedure != null) { 9094 resultSetModel.setProcedureId(String.valueOf(procedure.getId())); 9095 resultSetModel.setTarget(true); 9096 } 9097 modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent), 9098 resultSetModel); 9099 } 9100 } else if (stmt instanceof TCreateFunctionStmt) { 9101 TCreateFunctionStmt createFunction = (TCreateFunctionStmt) stmt; 9102 if (createFunction.getReturnDataType() != null 9103 && createFunction.getReturnDataType().getColumnDefList() != null) { 9104 Table tableModel = this.modelFactory.createTableByName(createFunction.getFunctionName(), true); 9105 tableModel.setCreateTable(true); 9106 String procedureParent = createFunction.getFunctionName().toString(); 9107 if (procedureParent != null) { 9108 tableModel.setParent(procedureParent); 9109 } 9110 9111 modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent), tableModel); 9112 for (int j = 0; j < createFunction.getReturnDataType().getColumnDefList().size(); j++) { 9113 TColumnDefinition column = createFunction.getReturnDataType().getColumnDefList().getColumn(j); 9114 if (column != null && column.getColumnName() != null) { 9115 modelFactory.createTableColumn(tableModel, column.getColumnName(), true); 9116 } 9117 } 9118 9119 if (createFunction.getSqlQuery() != null) { 9120 analyzeSelectStmt(createFunction.getSqlQuery()); 9121 ResultSet resultSetModel = (ResultSet) modelManager.getModel(createFunction.getSqlQuery()); 9122 if (resultSetModel != null) { 9123 for (int i = 0; i < resultSetModel.getColumns().size(); i++) { 9124 ResultColumn resultColumn = resultSetModel.getColumns().get(i); 9125 for (int j = 0; j < tableModel.getColumns().size(); j++) { 9126 TableColumn tableColumn = tableModel.getColumns().get(j); 9127 if (DlineageUtil.compareColumnIdentifier(getColumnName(resultColumn.getName()), 9128 DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) { 9129 DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation(); 9130 dataflowRelation.setEffectType(EffectType.select); 9131 dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn)); 9132 dataflowRelation.setTarget(new TableColumnRelationshipElement(tableColumn)); 9133 } 9134 } 9135 } 9136 } 9137 } 9138 } 9139 9140 if (createFunction.getReturnStmt() != null && createFunction.getReturnStmt().getSubquery() != null) { 9141 String procedureParent = createFunction.getFunctionName().toString(); 9142 analyzeSelectStmt(createFunction.getReturnStmt().getSubquery()); 9143 ResultSet resultSetModel = (ResultSet) modelManager 9144 .getModel(createFunction.getReturnStmt().getSubquery()); 9145 if (procedure != null) { 9146 resultSetModel.setProcedureId(String.valueOf(procedure.getId())); 9147 resultSetModel.setTarget(true); 9148 } 9149 modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent), 9150 resultSetModel); 9151 } 9152 9153 if (createFunction.getReturnStmt() != null && createFunction.getReturnStmt().getReturnExpr() != null) { 9154 TExpression returnExpression = createFunction.getReturnStmt().getReturnExpr(); 9155 ResultSet returnResult = modelFactory.createResultSet(createFunction.getReturnStmt(), false); 9156 ResultColumn resultColumn = modelFactory.createResultColumn(returnResult, returnExpression); 9157 if (shouldCollectAuthoritativeLineageEvidence()) { 9158 authoritativeEvidenceCollector.bindRoutineReturn(procedure, resultColumn); 9159 } 9160 9161 TExpression expression = createFunction.getReturnStmt().getReturnExpr(); 9162 analyzeResultColumnExpressionRelation(resultColumn, expression); 9163 9164 if (procedure != null) { 9165 returnResult.setProcedureId(String.valueOf(procedure.getId())); 9166 returnResult.setTarget(true); 9167 } 9168 String procedureParent = createFunction.getFunctionName().toString(); 9169 modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent), 9170 returnResult); 9171 } 9172 } 9173 9174 if (stmt instanceof TCreateProcedureStmt) { 9175 TCreateProcedureStmt createProcedure = (TCreateProcedureStmt) stmt; 9176 if (EDbVendor.dbvsnowflake == option.getVendor() && createProcedure.getRoutineBodyInConstant() != null) { 9177 String language = createProcedure.getProcedureLanguage() == null ? null 9178 : createProcedure.getProcedureLanguage().toString(); 9179 // Only a JAVASCRIPT body can be executed by the script-based 9180 // SQL extractor. Other handler languages (PYTHON, JAVA, SCALA) 9181 // are opaque to SQL analysis: keep the procedure model with its 9182 // declared language and skip the body (Mantis 4590). 9183 if (language == null || language.equalsIgnoreCase("JAVASCRIPT")) { // non-identifier-compare: LANGUAGE keyword, not a db object name 9184 extractSnowflakeSQLFromProcedure(createProcedure); 9185 } 9186 } 9187 } 9188 9189 if (stmt.getStatements().size() > 0) { 9190 for (int i = 0; i < stmt.getStatements().size(); ++i) { 9191 this.analyzeCustomSqlStmt(stmt.getStatements().get(i)); 9192 } 9193 } 9194 9195 if (stmt.getBodyStatements().size() > 0) { 9196 for (int i = 0; i < stmt.getBodyStatements().size(); ++i) { 9197 this.analyzeCustomSqlStmt(stmt.getBodyStatements().get(i)); 9198 } 9199 } 9200 9201 // Detect pipelined functions and build signatures 9202 if (stmt instanceof TPlsqlCreateFunction && pipelinedAnalyzer != null) { 9203 try { 9204 pipelinedAnalyzer.analyzePipelinedFunction((TPlsqlCreateFunction) stmt); 9205 } catch (Exception e) { 9206 // Don't let pipelined analysis failure break main flow 9207 } 9208 } 9209 9210 if (procedure != null && !getLastSelectStmt(stmt).isEmpty()) { 9211 for (TSelectSqlStatement select : getLastSelectStmt(stmt)) { 9212 ResultSet resultSet = (ResultSet) modelManager.getModel(select); 9213 resultSet.setProcedureId(String.valueOf(procedure.getId())); 9214 resultSet.setTarget(true); 9215 modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedure.getName()), 9216 resultSet); 9217 } 9218// List<Argument> outArgs = new ArrayList<Argument>(); 9219// for (int i = 0; i < procedure.getArguments().size(); i++) { 9220// Argument argument = procedure.getArguments().get(i); 9221// if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) { 9222// outArgs.add(argument); 9223// } 9224// } 9225// 9226// if (resultSet != null && resultSet.getColumns().size() == outArgs.size()) { 9227// for (int i = 0; i < outArgs.size(); i++) { 9228// Argument argument = outArgs.get(i); 9229// Variable variable = modelFactory.createVariable(argument.getName(), false); 9230// if (variable != null) { 9231// DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation(); 9232// dataflowRelation.setEffectType(EffectType.output); 9233// dataflowRelation 9234// .addSource(new ResultColumnRelationshipElement(resultSet.getColumns().get(i))); 9235// dataflowRelation 9236// .setTarget(new TableColumnRelationshipElement(variable.getColumns().get(0))); 9237// } 9238// } 9239// 9240// } 9241 } 9242 9243 if (stmt instanceof TCreateFunctionStmt && ((TCreateFunctionStmt)stmt).getSqlExpression()!=null) { 9244 TCreateFunctionStmt createFunction = (TCreateFunctionStmt) stmt; 9245 TExpression returnExpression = createFunction.getSqlExpression(); 9246 9247 ResultSet returnResult = modelFactory.createResultSet(stmt, false); 9248 ResultColumn resultColumn = modelFactory.createResultColumn(returnResult, returnExpression); 9249 if (shouldCollectAuthoritativeLineageEvidence()) { 9250 authoritativeEvidenceCollector.bindRoutineReturn(procedure, resultColumn); 9251 } 9252 9253 columnsInExpr visitor = new columnsInExpr(); 9254 returnExpression.inOrderTraverse(visitor); 9255 9256 List<TObjectName> objectNames = visitor.getObjectNames(); 9257 List<TParseTreeNode> functions = visitor.getFunctions(); 9258 List<TParseTreeNode> constants = visitor.getConstants(); 9259 List<TSelectSqlStatement> subquerys = visitor.getSubquerys(); 9260 9261 if (functions != null && !functions.isEmpty()) { 9262 analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function); 9263 } 9264 if (subquerys != null && !subquerys.isEmpty()) { 9265 analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.select); 9266 } 9267 if (objectNames != null && !objectNames.isEmpty()) { 9268 analyzeDataFlowRelation(resultColumn, objectNames, EffectType.select, functions); 9269 } 9270 if (constants != null && !constants.isEmpty()) { 9271 analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.select, functions); 9272 } 9273 9274 String procedureParent = SQLUtil.trimColumnStringQuote(getProcedureParentName(stmt)); 9275 modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent), 9276 returnResult); 9277 } 9278 } finally { 9279 if (pushedIdentityScope) { 9280 modelFactory.popIdentityScope(); 9281 } 9282 ModelBindingManager.removeGlobalProcedure(); 9283 } 9284 9285 } 9286 9287 private boolean isSimpleDataType(TTypeName dataType) { 9288 if (dataType.getDataType() == EDataType.variant_t) { 9289 return false; 9290 } 9291 if (dataType.getDataType() == EDataType.cursor_t) { 9292 return false; 9293 } 9294 if (dataType.getDataType() == EDataType.generic_t) { 9295 return false; 9296 } 9297 if (dataType.getDataType() == EDataType.unknown_t) { 9298 return false; 9299 } 9300 if (dataType.getDataType() == EDataType.sql_variant_t) { 9301 return false; 9302 } 9303 if (dataType.getDataType() == EDataType.table_t) { 9304 return false; 9305 } 9306 if (dataType.getDataType() == EDataType.raw_t) { 9307 return false; 9308 } 9309 if (dataType.getDataType() == EDataType.resultset_t) { 9310 return false; 9311 } 9312 if (dataType.getDataType() == EDataType.row_t) { 9313 return false; 9314 } 9315 if (dataType.getDataType() == EDataType.map_t) { 9316 return false; 9317 } 9318 if (dataType.getDataType() == EDataType.anyType_t) { 9319 return false; 9320 } 9321 if (dataType.getDataType() == EDataType.struct_t) { 9322 return false; 9323 } 9324 if (dataType.getDataType() == EDataType.structType_t) { 9325 return false; 9326 } 9327 if (dataType.getDataType() == EDataType.mapType_t) { 9328 return false; 9329 } 9330 return true; 9331 } 9332 9333 /** 9334 * ClickHouse remote()/remoteSecure() expose a real db.table on another 9335 * host as a table; modelling them as plain function nodes dropped the 9336 * upstream node and every edge into it. Returns true when the call was 9337 * modelled — the model is bound to the TTable, so the columns resolver2 9338 * already linked flow through the normal machinery. The model registers 9339 * through the ordinary table registry: a remote db.table shares identity 9340 * semantics with every other ClickHouse table reference. 9341 * 9342 * Only PROVABLE sources are modelled: plain object names or quoted 9343 * string constants, taken verbatim from the SQL text. Runtime 9344 * expressions and non-string constants (NULL, numbers) fall back to the 9345 * generic function node (never publish an unprovable edge). 9346 * 9347 * The PATH functions (file/s3/url/hdfs, their *Cluster variants and 9348 * azureBlobStorage) are deliberately NOT modelled: resource paths are 9349 * case-sensitive identities that do not fit the folded table-name 9350 * registry (bare filenames collide with same-spelling tables, case 9351 * variants coalesce, and Azure's first argument is a credential-bearing 9352 * connection string). Their missing upstream remains an HONEST, VISIBLE 9353 * gap — a generic function node — until a dedicated resource-identity 9354 * layer exists; see external_source_lineage_gap.md. 9355 * 9356 * Generator table functions (numbers(), generateRandom(), zeros(), ...) 9357 * correctly have NO upstream, which the generic node already expresses. 9358 */ 9359 private boolean modelClickhouseExternalTableFunction(TTable table, TFunctionCall functionCall) { 9360 if (option.getVendor() != EDbVendor.dbvclickhouse) { 9361 return false; 9362 } 9363 if (functionCall.getFunctionName() == null || functionCall.getArgs() == null 9364 || functionCall.getArgs().size() < 2) { 9365 return false; 9366 } 9367 String functionName = functionCall.getFunctionName().toString(); 9368 if (!isClickhouseTableFunction(functionName, "remote") 9369 && !isClickhouseTableFunction(functionName, "remoteSecure")) { 9370 return false; 9371 } 9372 // remote(addresses, db, table[, user, password]) 9373 // remote(addresses, db.table[, user, password]) 9374 TExpressionList args = functionCall.getArgs(); 9375 String first = literalTableFunctionArg(args.getExpression(1)); 9376 if (first == null) { 9377 return false; 9378 } 9379 String remoteTableName; 9380 if (first.indexOf('.') > 0) { 9381 remoteTableName = first; 9382 } else { 9383 String second = args.size() >= 3 ? literalTableFunctionArg(args.getExpression(2)) : null; 9384 if (second == null) { 9385 return false; // db without a provable table argument 9386 } 9387 remoteTableName = first + "." + second; 9388 } 9389 Table model = clickhouseExternalSourceModels.get(remoteTableName); 9390 if (model == null) { 9391 model = modelFactory.createTableFromCreateDDL(table, false, remoteTableName); 9392 clickhouseExternalSourceModels.put(remoteTableName, model); 9393 } 9394 modelManager.bindCreateModel(table, model); 9395 modelManager.bindModel(table, model); 9396 return true; 9397 } 9398 9399 /** Same-literal remote() calls merge here; cleared in init(). */ 9400 private final Map<String, Table> clickhouseExternalSourceModels = new LinkedHashMap<String, Table>(); 9401 9402 /** 9403 * ClickHouse function names are CASE-SENSITIVE (FiLe(...) is 9404 * UNKNOWN_FUNCTION natively) — matched through the canonical 9405 * per-object-type comparison, never a case fold. 9406 */ 9407 private static boolean isClickhouseTableFunction(String rawName, String candidate) { 9408 return DlineageUtil.sameName(ESQLDataObjectType.dotFunction, rawName, candidate); 9409 } 9410 9411 /** A provably literal identifier-ish argument: a plain object name or a 9412 * QUOTED STRING constant (quotes stripped); null for anything computed — 9413 * including non-string constants (NULL, numbers), which are not resource 9414 * identities. */ 9415 private static String literalTableFunctionArg(TExpression arg) { 9416 if (arg == null) { 9417 return null; 9418 } 9419 if (arg.getObjectOperand() != null) { 9420 return arg.getObjectOperand().toString(); 9421 } 9422 return literalStringConstant(arg); 9423 } 9424 9425 /** The DECODED value of a quoted string constant, or null. Delegates to 9426 * the shared ClickHouse literal decoder (IdentifierCodec) — the raw 9427 * spelling is not the value ('db\x2etable' IS db.table), the escape 9428 * table must match ClickHouse's parseComplexEscapeSequence exactly, and 9429 * a value that cannot be represented safely returns null so the caller 9430 * falls back to the generic function node. */ 9431 private static String literalStringConstant(TExpression arg) { 9432 if (arg == null || arg.getConstantOperand() == null) { 9433 return null; 9434 } 9435 return gudusoft.gsqlparser.sqlenv.IdentifierCodec 9436 .decodeClickHouseStringLiteral(arg.getConstantOperand().toString()); 9437 } 9438 9439 protected void analyzeResultColumnExpressionRelation(Object resultColumn, TExpression expression) { 9440 columnsInExpr visitor = new columnsInExpr(); 9441 expression.inOrderTraverse(visitor); 9442 List<TObjectName> objectNames = visitor.getObjectNames(); 9443 List<TParseTreeNode> functions = visitor.getFunctions(); 9444 List<TParseTreeNode> constants = visitor.getConstants(); 9445 List<TSelectSqlStatement> subquerys = visitor.getSubquerys(); 9446 9447 if (functions != null && !functions.isEmpty()) { 9448 analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function); 9449 } 9450 if (subquerys != null && !subquerys.isEmpty()) { 9451 analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.select); 9452 } 9453 if (objectNames != null && !objectNames.isEmpty()) { 9454 analyzeDataFlowRelation(resultColumn, objectNames, EffectType.select, functions); 9455 } 9456 if (constants != null && !constants.isEmpty()) { 9457 analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.select, functions); 9458 } 9459 } 9460 9461 private void analyzeDb2ReturnStmt(TDb2ReturnStmt stmt) { 9462 if (stmt.getReturnExpr() != null) { 9463 TExpression returnExpression = stmt.getReturnExpr(); 9464 ResultSet returnResult = modelFactory.createResultSet(stmt, true); 9465 ResultColumn resultColumn = modelFactory.createResultColumn(returnResult, returnExpression); 9466 bindCurrentRoutineReturn(resultColumn); 9467 9468 columnsInExpr visitor = new columnsInExpr(); 9469 stmt.getReturnExpr().inOrderTraverse(visitor); 9470 9471 List<TObjectName> objectNames = visitor.getObjectNames(); 9472 List<TParseTreeNode> functions = visitor.getFunctions(); 9473 List<TParseTreeNode> constants = visitor.getConstants(); 9474 List<TSelectSqlStatement> subquerys = visitor.getSubquerys(); 9475 9476 if (functions != null && !functions.isEmpty()) { 9477 analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function); 9478 } 9479 if (subquerys != null && !subquerys.isEmpty()) { 9480 analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.select); 9481 } 9482 if (objectNames != null && !objectNames.isEmpty()) { 9483 analyzeDataFlowRelation(resultColumn, objectNames, EffectType.select, functions); 9484 } 9485 if (constants != null && !constants.isEmpty()) { 9486 analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.select, functions); 9487 } 9488 9489 String procedureParent = SQLUtil.trimColumnStringQuote(getProcedureParentName(stmt)); 9490 if (procedureParent != null) { 9491 modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent), 9492 returnResult); 9493 } 9494 } 9495 } 9496 9497 private void analyzeReturnStmt(TReturnStmt stmt) { 9498 if (stmt.getResultColumnList() != null) { 9499 ResultSet returnResult = modelFactory.createResultSet(stmt, true); 9500 for (TResultColumn column : stmt.getResultColumnList()) { 9501 ResultColumn resultColumn = modelFactory.createResultColumn(returnResult, column); 9502 if (stmt.getResultColumnList().size() == 1) bindCurrentRoutineReturn(resultColumn); 9503 9504 columnsInExpr visitor = new columnsInExpr(); 9505 column.getExpr().inOrderTraverse(visitor); 9506 9507 List<TObjectName> objectNames = visitor.getObjectNames(); 9508 List<TParseTreeNode> functions = visitor.getFunctions(); 9509 List<TParseTreeNode> constants = visitor.getConstants(); 9510 List<TSelectSqlStatement> subquerys = visitor.getSubquerys(); 9511 9512 if (functions != null && !functions.isEmpty()) { 9513 analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function); 9514 } 9515 if (subquerys != null && !subquerys.isEmpty()) { 9516 analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.select); 9517 } 9518 if (objectNames != null && !objectNames.isEmpty()) { 9519 analyzeDataFlowRelation(resultColumn, objectNames, EffectType.select, functions); 9520 } 9521 if (constants != null && !constants.isEmpty()) { 9522 analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.select, functions); 9523 } 9524 9525 } 9526 9527 String procedureParent = SQLUtil.trimColumnStringQuote(getProcedureParentName(stmt)); 9528 modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent), 9529 returnResult); 9530 } 9531 else if(stmt.getExpression()!=null){ 9532 TExpression returnExpression = stmt.getExpression(); 9533 ResultSet returnResult = modelFactory.createResultSet(stmt, true); 9534 9535 columnsInExpr visitor = null; 9536 List<TSelectSqlStatement> subquerys = null; 9537 9538 if (returnExpression.getFunctionCall() != null 9539 && returnExpression.getFunctionCall().getFunctionName().toString().equalsIgnoreCase("table") 9540 && returnExpression.getFunctionCall().getArgs() != null 9541 && returnExpression.getFunctionCall().getArgs().size()>0) { 9542 visitor = new columnsInExpr(); 9543 returnExpression.getFunctionCall().getArgs().getExpression(0).inOrderTraverse(visitor); 9544 subquerys = visitor.getSubquerys(); 9545 if (subquerys != null && !subquerys.isEmpty()) { 9546 analyzeSelectStmt(subquerys.get(0)); 9547 ResultSet resultSet = (ResultSet) modelManager.getModel(subquerys.get(0)); 9548 if (resultSet != null && resultSet.getColumns() != null) { 9549 for (ResultColumn column : resultSet.getColumns()) { 9550 TObjectName columnName = new TObjectName(); 9551 columnName.setString(column.getName()); 9552 ResultColumn resultColumn = modelFactory.createResultColumn(returnResult, columnName); 9553 DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation(); 9554 dataflowRelation.setEffectType(EffectType.select); 9555 dataflowRelation.addSource(new ResultColumnRelationshipElement(column)); 9556 dataflowRelation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 9557 } 9558 } 9559 returnResult.setDetermined(resultSet.isDetermined()); 9560 String procedureParent = SQLUtil.trimColumnStringQuote(getProcedureParentName(stmt)); 9561 modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent), 9562 returnResult); 9563 return; 9564 } 9565 } 9566 9567 ResultColumn resultColumn = null; 9568 if (returnExpression.getExpressionType() == EExpressionType.simple_object_name_t) { 9569 TObjectName columnName = new TObjectName(); 9570 columnName.setString("*"); 9571 resultColumn = modelFactory.createResultColumn(returnResult, columnName); 9572 } 9573 else { 9574 resultColumn = modelFactory.createResultColumn(returnResult, returnExpression); 9575 } 9576 9577 visitor = new columnsInExpr(); 9578 stmt.getExpression().inOrderTraverse(visitor); 9579 9580 List<TObjectName> objectNames = visitor.getObjectNames(); 9581 List<TParseTreeNode> functions = visitor.getFunctions(); 9582 List<TParseTreeNode> constants = visitor.getConstants(); 9583 subquerys = visitor.getSubquerys(); 9584 9585 if (functions != null && !functions.isEmpty()) { 9586 analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function); 9587 } 9588 if (subquerys != null && !subquerys.isEmpty()) { 9589 analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.select); 9590 } 9591 if (objectNames != null && !objectNames.isEmpty()) { 9592 DataFlowRelationship relation = analyzeDataFlowRelation(resultColumn, objectNames, EffectType.select, functions); 9593 //如果variable对应的不是一个复杂结构,则resultColumn不要设置为* 9594 if (relation != null && relation.getTarget().getElement() == resultColumn && relation.getSources().size() == 1) { 9595 Object column = relation.getSources().iterator().next().getElement(); 9596 boolean star = true; 9597 if (column instanceof TableColumn && ((TableColumn) column).getName().indexOf("*") == -1) { 9598 star = false; 9599 } 9600 if (column instanceof ResultColumn && ((ResultColumn) column).getName().indexOf("*") == -1) { 9601 star = false; 9602 } 9603 if (!star && returnExpression.getExpressionType() == EExpressionType.simple_object_name_t) { 9604 resultColumn = modelFactory.createResultColumn(returnResult, returnExpression.getObjectOperand()); 9605 returnResult.getColumns().clear(); 9606 returnResult.addColumn(resultColumn); 9607 relation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 9608 } 9609 } 9610 } 9611 if (constants != null && !constants.isEmpty()) { 9612 analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.select, functions); 9613 } 9614 bindCurrentRoutineReturn(resultColumn); 9615 9616 String procedureParent = SQLUtil.trimColumnStringQuote(getProcedureParentName(stmt)); 9617 if (procedureParent != null) { 9618 modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent), 9619 returnResult); 9620 } 9621 } 9622 } 9623 9624 private void bindCurrentRoutineReturn(ResultColumn resultColumn) { 9625 if (!shouldCollectAuthoritativeLineageEvidence()) return; 9626 TStoredProcedureSqlStatement declaration = ModelBindingManager.getGlobalProcedure(); 9627 Object routine = declaration == null ? null : modelManager.getModel(declaration); 9628 if (routine instanceof Procedure) { 9629 authoritativeEvidenceCollector.bindRoutineReturn((Procedure) routine, resultColumn); 9630 } 9631 } 9632 9633 private void analyzeMssqlReturnStmt(TMssqlReturn stmt) { 9634 if (stmt.getResultColumnList() != null) { 9635 ResultSet returnResult = modelFactory.createResultSet(stmt, true); 9636 for (TResultColumn column : stmt.getResultColumnList()) { 9637 ResultColumn resultColumn = modelFactory.createResultColumn(returnResult, column); 9638 if (stmt.getResultColumnList().size() == 1) bindCurrentRoutineReturn(resultColumn); 9639 9640 columnsInExpr visitor = new columnsInExpr(); 9641 column.getExpr().inOrderTraverse(visitor); 9642 9643 List<TObjectName> objectNames = visitor.getObjectNames(); 9644 List<TParseTreeNode> functions = visitor.getFunctions(); 9645 List<TParseTreeNode> constants = visitor.getConstants(); 9646 List<TSelectSqlStatement> subquerys = visitor.getSubquerys(); 9647 9648 if (functions != null && !functions.isEmpty()) { 9649 analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function); 9650 } 9651 if (subquerys != null && !subquerys.isEmpty()) { 9652 analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.select); 9653 } 9654 if (objectNames != null && !objectNames.isEmpty()) { 9655 analyzeDataFlowRelation(resultColumn, objectNames, EffectType.select, functions); 9656 } 9657 if (constants != null && !constants.isEmpty()) { 9658 analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.select, functions); 9659 } 9660 9661 } 9662 9663 String procedureParent = SQLUtil.trimColumnStringQuote(getProcedureParentName(stmt)); 9664 modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent), 9665 returnResult); 9666 } 9667 else if(stmt.getReturnExpr()!=null){ 9668 TExpression returnExpression = stmt.getReturnExpr(); 9669 ResultSet returnResult = modelFactory.createResultSet(stmt, true); 9670 9671 if (returnExpression.getSubQuery() == null) { 9672 ResultColumn resultColumn = modelFactory.createResultColumn(returnResult, returnExpression); 9673 columnsInExpr visitor = new columnsInExpr(); 9674 stmt.getReturnExpr().inOrderTraverse(visitor); 9675 9676 List<TObjectName> objectNames = visitor.getObjectNames(); 9677 List<TParseTreeNode> functions = visitor.getFunctions(); 9678 List<TParseTreeNode> constants = visitor.getConstants(); 9679 List<TSelectSqlStatement> subquerys = visitor.getSubquerys(); 9680 9681 if (functions != null && !functions.isEmpty()) { 9682 analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function); 9683 } 9684 if (subquerys != null && !subquerys.isEmpty()) { 9685 analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.select); 9686 } 9687 if (objectNames != null && !objectNames.isEmpty()) { 9688 analyzeDataFlowRelation(resultColumn, objectNames, EffectType.select, functions); 9689 } 9690 if (constants != null && !constants.isEmpty()) { 9691 analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.select, functions); 9692 } 9693 bindCurrentRoutineReturn(resultColumn); 9694 } 9695 else { 9696 analyzeSelectStmt(returnExpression.getSubQuery()); 9697 ResultSet subResultSet = (ResultSet)modelManager.getModel(returnExpression.getSubQuery()); 9698 if (subResultSet != null) { 9699 for (ResultColumn subResultColumn : subResultSet.getColumns()) { 9700 TObjectName objectName = new TObjectName(); 9701 objectName.setString(subResultColumn.getName()); 9702 ResultColumn resultColumn = modelFactory.createResultColumn(returnResult, objectName); 9703 DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation(); 9704 dataflowRelation.setEffectType(EffectType.select); 9705 dataflowRelation.addSource(new ResultColumnRelationshipElement(subResultColumn)); 9706 dataflowRelation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 9707 9708 } 9709 9710 if(subResultSet.getRelationRows().hasRelation()) { 9711 ImpactRelationship impactRelation = modelFactory.createImpactRelation(); 9712 impactRelation.setEffectType(EffectType.select); 9713 impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>( 9714 subResultSet.getRelationRows())); 9715 impactRelation.setTarget( 9716 new RelationRowsRelationshipElement<ResultSetRelationRows>(returnResult.getRelationRows())); 9717 } 9718 } 9719 } 9720 9721 String procedureParent = SQLUtil.trimColumnStringQuote(getProcedureParentName(stmt)); 9722 if (procedureParent != null) { 9723 modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent), 9724 returnResult); 9725 } 9726 } 9727 else if (stmt.getSubquery() != null) { 9728 analyzeSelectStmt(stmt.getSubquery()); 9729 ResultSet subResultSet = (ResultSet)modelManager.getModel(stmt.getSubquery()); 9730 String procedureParent = SQLUtil.trimColumnStringQuote(getProcedureParentName(stmt)); 9731 if (procedureParent != null && subResultSet!=null) { 9732 modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent), 9733 subResultSet); 9734 } 9735 } 9736 } 9737 9738 /** 9739 * The column that stands for a composite variable as a WHOLE (#695). 9740 * 9741 * <p>A PL/SQL record variable used to be modelled as one opaque column, so 9742 * "the" column of a variable was always {@code getColumns().get(0)}. Once 9743 * records expand to per-field columns, {@code get(0)} silently means "an 9744 * arbitrary field" — the edge that FILLS the record then lands on one 9745 * field (or, worse, every producer field is fanned onto it). The whole- 9746 * record binding must instead land on the record's own {@code *} column, 9747 * which stays distinct per variable instance and can be expanded once the 9748 * pairing is known. 9749 */ 9750 private TableColumn compositeBindingColumn(Variable variable) { 9751 if (variable.getColumns() == null || variable.getColumns().isEmpty()) { 9752 return tableStarColumn(variable); 9753 } 9754 if (variable.getColumns().size() == 1) { 9755 return variable.getColumns().get(0); 9756 } 9757 if (variable.isTypeOrderedFields()) { 9758 // Fields from the TYPE declaration: the star expands onto exactly 9759 // those fields at serialization — every one is genuinely filled. 9760 return tableStarColumn(variable); 9761 } 9762 // Consumption-shaped fields (pseudo-columns, expression names) carry no 9763 // per-field meaning; expanding a star over them fans every producer onto 9764 // every artifact. Keep the historical whole-record binding. 9765 return variable.getColumns().get(0); 9766 } 9767 9768 /** 9769 * The {@code *} column of {@code table}, created on first use with the 9770 * expand-star convention (as the %ROWTYPE declaration path does): the 9771 * serializer expands a relation targeting it onto the sibling fields it 9772 * star-links, so the edge always cites DECLARED columns — a relationship 9773 * must never cite a column the model does not declare. 9774 */ 9775 private TableColumn tableStarColumn(Table table) { 9776 if (table.getColumns() != null) { 9777 for (TableColumn existing : table.getColumns()) { 9778 if ("*".equals(existing.getName())) { 9779 return existing; 9780 } 9781 } 9782 } 9783 TableColumn starColumn = modelFactory.createTableColumn(table, starObjectName(), true); 9784 starColumn.setShowStar(false); 9785 starColumn.setExpandStar(true); 9786 return starColumn; 9787 } 9788 9789 private TObjectName starObjectName() { 9790 TObjectName starColumn = new TObjectName(); 9791 starColumn.setString("*"); 9792 return starColumn; 9793 } 9794 9795 private void analyzeFetchStmt(TFetchStmt stmt) { 9796 if (stmt.getVariableNames() != null) { 9797 for (int i = 0; i < stmt.getVariableNames().size(); i++) { 9798 TExpression variableExpression = stmt.getVariableNames().getExpression(i); 9799 if (variableExpression.getExpressionType() == EExpressionType.simple_object_name_t) { 9800 TObjectName columnObject = variableExpression.getObjectOperand(); 9801 if (columnObject.getDbObjectType() == EDbObjectType.variable) { 9802 continue; 9803 } 9804 9805 if (columnObject.getColumnNameOnly().startsWith("@") && (option.getVendor() == EDbVendor.dbvmssql 9806 || option.getVendor() == EDbVendor.dbvazuresql)) { 9807 continue; 9808 } 9809 9810 if (columnObject.getColumnNameOnly().startsWith(":") && (option.getVendor() == EDbVendor.dbvhana 9811 || option.getVendor() == EDbVendor.dbvteradata)) { 9812 continue; 9813 } 9814 9815 Variable cursorVariable = modelFactory.createVariable(columnObject); 9816 cursorVariable.setSubType(SubType.record); 9817 if (cursorVariable.isDetermined()) { 9818 if (stmt.getCursorName() != null) { 9819 String procedureName = DlineageUtil.getProcedureParentName(stmt); 9820 String variableString = stmt.getCursorName().toString(); 9821 if (variableString.startsWith(":")) { 9822 variableString = variableString.substring(variableString.indexOf(":") + 1); 9823 } 9824 if (!SQLUtil.isEmpty(procedureName)) { 9825 variableString = procedureName + "." 9826 + SQLUtil.getIdentifierNormalTableName(variableString); 9827 } 9828 Table cursor = modelManager.getTableByName(DlineageUtil.getTableFullName(variableString)); 9829 if (cursor != null) { 9830 // FETCH ... INTO binds POSITIONALLY (#695): the k-th 9831 // cursor column fills the k-th record field, whatever 9832 // either is called. Matching by name paired 9833 // like-named columns that are NOT each other's source 9834 // and, when nothing matched, fanned every cursor 9835 // column onto every field — edges the SQL never 9836 // asserts. A field the cursor's known width cannot 9837 // account for keeps the cursor's own * as its source: 9838 // filled, but by an unresolved component. 9839 List<TableColumn> recordFields = cursorVariable.getColumns(); 9840 List<TableColumn> cursorColumns = cursor.getColumns(); 9841 if (columnObject.toString().indexOf('.') != -1) { 9842 // FETCH ... INTO rec.field names its target 9843 // explicitly (#695): bind THAT field to the 9844 // cursor column at this INTO position — 9845 // iterating every field here fanned each 9846 // cursor column onto the whole record. 9847 TableColumn namedField = modelFactory.createTableColumn(cursorVariable, 9848 columnObject, true); 9849 if (namedField != null) { 9850 DataFlowRelationship dottedRelation = modelFactory.createDataFlowRelation(); 9851 dottedRelation.setEffectType(EffectType.cursor); 9852 if (cursorColumns != null && i < cursorColumns.size()) { 9853 dottedRelation.addSource( 9854 new TableColumnRelationshipElement(cursorColumns.get(i))); 9855 } else { 9856 dottedRelation.addSource( 9857 new TableColumnRelationshipElement(tableStarColumn(cursor))); 9858 } 9859 dottedRelation.setTarget(new TableColumnRelationshipElement(namedField)); 9860 } 9861 continue; 9862 } 9863 boolean ordinalPairing = stmt.getVariableNames().size() == 1 9864 && cursorColumns != null && !cursorColumns.isEmpty() 9865 && cursorVariable.isTypeOrderedFields(); 9866 // A SCALAR at INTO position i pairs with cursor 9867 // column i when the widths agree — positional by 9868 // the INTO list itself, no type order involved. 9869 boolean intoPositional = recordFields.size() == 1 9870 && cursorColumns != null 9871 && cursorColumns.size() == stmt.getVariableNames().size(); 9872 boolean hasRealField = false; 9873 for (TableColumn field : recordFields) { 9874 if (field.getName() != null && !"*".equals(field.getName())) { 9875 hasRealField = true; 9876 break; 9877 } 9878 } 9879 int fieldOrdinal = 0; 9880 for (int k = 0; k < recordFields.size(); k++) { 9881 TableColumn variableProperty = recordFields.get(k); 9882 if ("*".equals(variableProperty.getName()) && hasRealField) { 9883 // With real fields present the wildcard is a 9884 // binding artifact (codex-696 F2). When it is 9885 // the ONLY column it IS the whole-record 9886 // binding — skipping it dropped every fill 9887 // whose consumption comes after the FETCH. 9888 continue; 9889 } 9890 DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation(); 9891 dataflowRelation.setEffectType(EffectType.cursor); 9892 if (ordinalPairing && fieldOrdinal < cursorColumns.size()) { 9893 dataflowRelation.addSource( 9894 new TableColumnRelationshipElement(cursorColumns.get(fieldOrdinal))); 9895 } else if (intoPositional) { 9896 dataflowRelation.addSource( 9897 new TableColumnRelationshipElement(cursorColumns.get(i))); 9898 } else if (stmt.getVariableNames().size() > 1) { 9899 dataflowRelation.addSource( 9900 new TableColumnRelationshipElement(tableStarColumn(cursor), i)); 9901 } else { 9902 dataflowRelation.addSource( 9903 new TableColumnRelationshipElement(tableStarColumn(cursor))); 9904 } 9905 dataflowRelation.setTarget(new TableColumnRelationshipElement(variableProperty)); 9906 fieldOrdinal++; 9907 } 9908 } 9909 } 9910 9911 } else { 9912 TableColumn variableProperty = null; 9913 if (stmt.getVariableNames().size() == 1) { 9914 if (cursorVariable.getColumns() == null || cursorVariable.getColumns().isEmpty()) { 9915 TObjectName starColumn = new TObjectName(); 9916 starColumn.setString("*"); 9917 variableProperty = modelFactory.createTableColumn(cursorVariable, starColumn, true); 9918 } else { 9919 variableProperty = cursorVariable.getColumns().get(0); 9920 } 9921 } else { 9922 variableProperty = modelFactory.createTableColumn(cursorVariable, columnObject, true); 9923 } 9924 9925 if (stmt.getCursorName() != null) { 9926 String procedureName = DlineageUtil.getProcedureParentName(stmt); 9927 String variableString = stmt.getCursorName().toString(); 9928 if (variableString.startsWith(":")) { 9929 variableString = variableString.substring(variableString.indexOf(":") + 1); 9930 } 9931 if (!SQLUtil.isEmpty(procedureName)) { 9932 variableString = procedureName + "." 9933 + SQLUtil.getIdentifierNormalTableName(variableString); 9934 } 9935 Table cursor = modelManager.getTableByName(DlineageUtil.getTableFullName(variableString)); 9936 if (cursor != null) { 9937 for (int j = 0; j < cursor.getColumns().size(); j++) { 9938 DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation(); 9939 dataflowRelation.setEffectType(EffectType.cursor); 9940 if (stmt.getVariableNames().size() == 1) { 9941 dataflowRelation.addSource( 9942 new TableColumnRelationshipElement(cursor.getColumns().get(j))); 9943 } else { 9944 dataflowRelation.addSource( 9945 new TableColumnRelationshipElement(cursor.getColumns().get(j), i)); 9946 } 9947 dataflowRelation.setTarget(new TableColumnRelationshipElement(variableProperty)); 9948 } 9949 } 9950 } 9951 } 9952 } 9953 } 9954 } 9955 } 9956 9957 private void analyzeFetchStmt(TMssqlFetch stmt) { 9958 if (stmt.getVariableNames() != null) { 9959 for (int i = 0; i < stmt.getVariableNames().size(); i++) { 9960 TObjectName columnObject = stmt.getVariableNames().getObjectName(i); 9961 Variable cursorVariable = modelFactory.createVariable(columnObject); 9962 cursorVariable.setCreateTable(true); 9963 cursorVariable.setSubType(SubType.record); 9964 TableColumn variableProperty = null; 9965 if (stmt.getVariableNames().size() == 1) { 9966 if (cursorVariable.getColumns() == null || cursorVariable.getColumns().isEmpty()) { 9967 TObjectName starColumn = new TObjectName(); 9968 starColumn.setString("*"); 9969 variableProperty = modelFactory.createTableColumn(cursorVariable, starColumn, true); 9970 } else { 9971 variableProperty = cursorVariable.getColumns().get(0); 9972 } 9973 } else { 9974 variableProperty = modelFactory.createTableColumn(cursorVariable, columnObject, true); 9975 } 9976 9977 if (stmt.getCursorName() != null) { 9978 String procedureName = DlineageUtil.getProcedureParentName(stmt); 9979 String variableString = stmt.getCursorName().toString(); 9980 if (variableString.startsWith(":")) { 9981 variableString = variableString.substring(variableString.indexOf(":") + 1); 9982 } 9983 if (!SQLUtil.isEmpty(procedureName)) { 9984 variableString = procedureName + "." + SQLUtil.getIdentifierNormalTableName(variableString); 9985 } 9986 Table cursor = modelManager 9987 .getTableByName(DlineageUtil.getTableFullName(variableString)); 9988 if (cursor != null) { 9989 for (int j = 0; j < cursor.getColumns().size(); j++) { 9990 DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation(); 9991 dataflowRelation.setEffectType(EffectType.cursor); 9992 if (stmt.getVariableNames().size() == 1) { 9993 dataflowRelation 9994 .addSource(new TableColumnRelationshipElement(cursor.getColumns().get(j))); 9995 } else { 9996 dataflowRelation 9997 .addSource(new TableColumnRelationshipElement(cursor.getColumns().get(j), i)); 9998 } 9999 dataflowRelation.setTarget(new TableColumnRelationshipElement(variableProperty)); 10000 } 10001 } 10002 } 10003 } 10004 10005 } 10006 } 10007 10008 private void analyzeLoopStmt(TLoopStmt stmt) { 10009 10010 if (stmt.getCursorName() != null && stmt.getIndexName() != null) { 10011 modelManager.bindCursorIndex(stmt.getIndexName(), stmt.getCursorName()); 10012 } 10013 10014 if (stmt.getRecordName() != null && stmt.getSubquery() != null) { 10015 Variable cursorTempTable = modelFactory.createCursor(stmt); 10016 cursorTempTable.setVariable(true); 10017 cursorTempTable.setSubType(SubType.cursor); 10018 modelManager.bindCursorModel(stmt, cursorTempTable); 10019 analyzeSelectStmt(stmt.getSubquery()); 10020 10021 TableColumn cursorColumn = null; 10022 if (cursorTempTable.getColumns() == null || cursorTempTable.getColumns().isEmpty()) { 10023 TObjectName starColumn = new TObjectName(); 10024 starColumn.setString("*"); 10025 cursorColumn = modelFactory.createTableColumn(cursorTempTable, starColumn, true); 10026 } else { 10027 cursorColumn = cursorTempTable.getColumns().get(0); 10028 } 10029 10030 10031 ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt.getSubquery()); 10032 if (resultSetModel != null) { 10033 for (int i = 0; i < resultSetModel.getColumns().size(); i++) { 10034 ResultColumn resultColumn = resultSetModel.getColumns().get(i); 10035 DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation(); 10036 dataflowRelation.setEffectType(EffectType.cursor); 10037 dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn)); 10038 dataflowRelation.setTarget(new TableColumnRelationshipElement(cursorColumn)); 10039 } 10040 } 10041 } 10042 10043 for (int i = 0; i < stmt.getStatements().size(); i++) { 10044 analyzeCustomSqlStmt(stmt.getStatements().get(i)); 10045 } 10046 } 10047 10048 private void analyzeForStmt(TForStmt stmt) { 10049 if (stmt.getSubquery() == null) { 10050 return; 10051 } 10052 10053 Variable cursorTempTable = modelFactory.createCursor(stmt); 10054 cursorTempTable.setVariable(true); 10055 cursorTempTable.setSubType(SubType.cursor); 10056 cursorTempTable.setCreateTable(true); 10057 modelManager.bindCursorModel(stmt, cursorTempTable); 10058 analyzeSelectStmt(stmt.getSubquery()); 10059 10060 TableColumn cursorColumn = null; 10061 if (cursorTempTable.getColumns() == null || cursorTempTable.getColumns().isEmpty()) { 10062 TObjectName starColumn = new TObjectName(); 10063 starColumn.setString("*"); 10064 cursorColumn = modelFactory.createTableColumn(cursorTempTable, starColumn, true); 10065 } else { 10066 cursorColumn = cursorTempTable.getColumns().get(0); 10067 } 10068 10069 10070 ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt.getSubquery()); 10071 if (resultSetModel != null) { 10072 for (int i = 0; i < resultSetModel.getColumns().size(); i++) { 10073 ResultColumn resultColumn = resultSetModel.getColumns().get(i); 10074 DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation(); 10075 dataflowRelation.setEffectType(EffectType.cursor); 10076 dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn)); 10077 dataflowRelation.setTarget(new TableColumnRelationshipElement(cursorColumn)); 10078 } 10079 } 10080 10081 if (stmt.getStatements() != null && stmt.getStatements().size() > 0) { 10082 for (int i = 0; i < stmt.getStatements().size(); i++) { 10083 analyzeCustomSqlStmt(stmt.getStatements().get(i)); 10084 } 10085 } 10086 } 10087 10088 /** 10089 * Resolve a declared name visible from the CURRENT scope (#695): a TYPE 10090 * declared at package level is visible from every nested procedure, but 10091 * variable models are keyed under their declaring scope only, so a 10092 * same-scope lookup misses it and the declaring variable silently falls 10093 * back to one opaque variant column — the record-fill edges then have no 10094 * per-field target and are lost. Walk the scope chain outward 10095 * (procedure → package → global); never create. 10096 */ 10097 private Variable resolveVariableInScope(String name) { 10098 if (SQLUtil.isEmpty(name)) { 10099 return null; 10100 } 10101 Variable direct = modelFactory.createVariable(name, false); 10102 if (direct != null) { 10103 return direct; 10104 } 10105 String normalized = SQLUtil.getIdentifierNormalTableName(name); 10106 // Lexical chain first (codex-696 F1): a declaration in an ENCLOSING 10107 // procedure shadows the package's. getProcedureParentName() of the 10108 // current statement omits intermediate nesting levels, so probe each 10109 // enclosing stored-procedure statement on the stack, innermost first. 10110 for (int depth = stmtStack.size() - 1; depth >= 0; depth--) { 10111 String enclosingScope = DlineageUtil.getProcedureParentName(stmtStack.get(depth)); 10112 if (SQLUtil.isEmpty(enclosingScope)) { 10113 continue; 10114 } 10115 Object model = modelManager.getTableByName(DlineageUtil.getTableFullName( 10116 DlineageUtil.getTableFullName(enclosingScope) + "." + normalized)); 10117 if (model instanceof Variable) { 10118 return (Variable) model; 10119 } 10120 } 10121 String scope = stmtStack.isEmpty() ? null : DlineageUtil.getProcedureParentName(stmtStack.peek()); 10122 while (!SQLUtil.isEmpty(scope)) { 10123 int cut = scope.lastIndexOf('.'); 10124 scope = cut == -1 ? null : scope.substring(0, cut); 10125 List<String> keys = new ArrayList<String>(); 10126 if (SQLUtil.isEmpty(scope)) { 10127 keys.add(normalized); 10128 } else { 10129 keys.add(DlineageUtil.getTableFullName(scope) + "." + normalized); 10130 // A declaration at the top level of a package BODY is scoped 10131 // "<pkg>.<pkg>" (the body's own block), so a sibling procedure 10132 // walking up to "<pkg>" must probe that block scope too. 10133 String tail = scope.indexOf('.') == -1 ? scope : scope.substring(scope.lastIndexOf('.') + 1); 10134 keys.add(DlineageUtil.getTableFullName(scope) + "." + tail + "." + normalized); 10135 } 10136 for (String key : keys) { 10137 Object model = modelManager.getTableByName(DlineageUtil.getTableFullName(key)); 10138 if (model instanceof Variable) { 10139 return (Variable) model; 10140 } 10141 } 10142 if (cut == -1) { 10143 break; 10144 } 10145 } 10146 return null; 10147 } 10148 10149 /** 10150 * Materialize the fields of a record-typed FORMAL argument from its 10151 * declared type before binding an actual to it (#695). Without this the 10152 * formal is one opaque column and the argument-fill edge either lands on 10153 * an arbitrary field or is lost. Field order follows the TYPE declaration 10154 * unless consumption already created some fields (name-deduped), so 10155 * ordinal use of formal fields must not rely on this method alone. 10156 */ 10157 private void materializeArgumentFields(Variable variable, Argument argument) { 10158 if (variable == null || argument == null || argument.getDataType() == null) { 10159 return; 10160 } 10161 if (variable.isDetermined() && variable.getColumns() != null && variable.getColumns().size() > 1) { 10162 return; 10163 } 10164 Variable typeModel = resolveVariableInScope(argument.getDataType().getDataTypeName()); 10165 if (typeModel == null || typeModel == variable || !typeModel.isDetermined() 10166 || typeModel.getColumns() == null || typeModel.getColumns().isEmpty()) { 10167 return; 10168 } 10169 // Fields created by earlier consumption sit BEFORE the type fields we 10170 // append here, so the column list is only in type order when nothing 10171 // preceded it (codex-696 F2). Ordinal pairing must not trust a mixed 10172 // list. 10173 boolean hadRealFields = false; 10174 if (variable.getColumns() != null) { 10175 for (TableColumn existing : variable.getColumns()) { 10176 if (existing.getName() != null && !"*".equals(existing.getName())) { 10177 hadRealFields = true; 10178 break; 10179 } 10180 } 10181 } 10182 for (TableColumn typeField : typeModel.getColumns()) { 10183 if (typeField.getName() == null || "*".equals(typeField.getName())) { 10184 continue; 10185 } 10186 TObjectName fieldName = new TObjectName(); 10187 fieldName.setString(typeField.getName()); 10188 modelFactory.createTableColumn(variable, fieldName, true); 10189 } 10190 // A formal declared before this had only its wildcard column; now that 10191 // the real fields exist, a relation binding the record as a whole must 10192 // expand onto them at serialization instead of citing the bare star. 10193 if (variable.getColumns() != null) { 10194 for (TableColumn existing : variable.getColumns()) { 10195 if ("*".equals(existing.getName())) { 10196 existing.setShowStar(false); 10197 existing.setExpandStar(true); 10198 } 10199 } 10200 } 10201 variable.setDetermined(true); 10202 if (!hadRealFields) { 10203 variable.setTypeOrderedFields(true); 10204 } 10205 } 10206 10207 /** 10208 * The variable model of a callee's FORMAL argument, shaped for binding 10209 * (#695). {@code createVariable(Procedure, name, false)} deliberately 10210 * refuses a record-typed formal it recovers through the declaration-time 10211 * scope key (one wildcard column = nothing exact to bind to). Recover the 10212 * variable anyway, materialize its fields from the declared type, and only 10213 * bind when a real field emerged — otherwise keep the historical refusal. 10214 */ 10215 private Variable resolveFormalVariable(Procedure callee, Argument argument) { 10216 Variable variable = modelFactory.createVariable(callee, argument.getName(), false); 10217 if (variable != null) { 10218 materializeArgumentFields(variable, argument); 10219 return variable; 10220 } 10221 Variable recovered = modelFactory.findFormalVariable(callee, argument.getName()); 10222 if (recovered == null) { 10223 return null; 10224 } 10225 materializeArgumentFields(recovered, argument); 10226 if (recovered.getColumns() != null) { 10227 for (TableColumn column : recovered.getColumns()) { 10228 if (!"*".equals(column.getName())) { 10229 return recovered; 10230 } 10231 } 10232 } 10233 return null; 10234 } 10235 10236 private void analyzeVarDeclStmt(TVarDeclStmt stmt) { 10237 TTypeName typeName = stmt.getDataType(); 10238 if (typeName != null && typeName.toString().toUpperCase().indexOf("ROWTYPE") != -1) { 10239 Variable cursorVariable = modelFactory.createVariable(stmt.getElementName()); 10240 cursorVariable.setSubType(SubType.record_type); 10241 10242 Table variableTable = modelFactory.createTableByName(typeName.getDataTypeName(), false); 10243 if(!variableTable.isCreateTable()) { 10244 TObjectName starColumn1 = new TObjectName(); 10245 starColumn1.setString("*"); 10246 TableColumn variableTableStarColumn = modelFactory.createTableColumn(variableTable, starColumn1, true); 10247 variableTableStarColumn.setShowStar(false); 10248 variableTableStarColumn.setExpandStar(true); 10249 10250 TObjectName starColumn = new TObjectName(); 10251 starColumn.setString("*"); 10252 TableColumn variableProperty = modelFactory.createTableColumn(cursorVariable, starColumn, true); 10253 variableProperty.setShowStar(false); 10254 variableProperty.setExpandStar(true); 10255 10256 DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation(); 10257 dataflowRelation.setEffectType(EffectType.rowtype); 10258 dataflowRelation.addSource(new TableColumnRelationshipElement(variableTableStarColumn)); 10259 dataflowRelation.setTarget(new TableColumnRelationshipElement(variableProperty)); 10260 } else { 10261 for (TableColumn sourceColumn : variableTable.getColumns()) { 10262 String columnName = sourceColumn.getName(); 10263 TObjectName targetColumn = new TObjectName(); 10264 targetColumn.setString(columnName); 10265 TableColumn variableProperty = modelFactory.createTableColumn(cursorVariable, targetColumn, true); 10266 DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation(); 10267 dataflowRelation.setEffectType(EffectType.rowtype); 10268 dataflowRelation.addSource(new TableColumnRelationshipElement(sourceColumn)); 10269 dataflowRelation.setTarget(new TableColumnRelationshipElement(variableProperty)); 10270 } 10271 } 10272 } else if (stmt.getElementName() != null) { 10273 Variable variable = modelFactory.createVariable(stmt.getElementName()); 10274 variable.setCreateTable(true); 10275 variable.setSubType(SubType.record); 10276 TableColumn tableColumn = null; 10277 if (stmt.getDataType() != null && (resolveVariableInScope(stmt.getDataType().getDataTypeName())!=null || isCursorType(stmt.getDataType().getDataTypeName()))) { 10278 Variable cursorVariable = resolveVariableInScope(stmt.getDataType().getDataTypeName()); 10279 if (cursorVariable != null) { 10280 if (cursorVariable.getSubType() == SubType.record_type) { 10281 variable.setSubType(SubType.record_type); 10282 } 10283 if(cursorVariable.isDetermined()) { 10284 for (int k = 0; k < cursorVariable.getColumns().size(); k++) { 10285 TableColumn sourceColumn = cursorVariable.getColumns().get(k); 10286 TObjectName objectName = new TObjectName(); 10287 objectName.setString(sourceColumn.getName()); 10288 tableColumn = modelFactory.createTableColumn(variable, objectName, true); 10289 DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation(); 10290 dataflowRelation.setEffectType(EffectType.cursor); 10291 dataflowRelation 10292 .addSource(new TableColumnRelationshipElement(sourceColumn)); 10293 dataflowRelation.setTarget(new TableColumnRelationshipElement(tableColumn)); 10294 } 10295 variable.setDetermined(true); 10296 variable.setTypeOrderedFields(true); 10297 return; 10298 } 10299 else { 10300 TObjectName objectName = new TObjectName(); 10301 objectName.setString("*"); 10302 tableColumn = modelFactory.createTableColumn(variable, objectName, true); 10303 DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation(); 10304 dataflowRelation.setEffectType(EffectType.cursor); 10305 dataflowRelation 10306 .addSource(new TableColumnRelationshipElement(cursorVariable.getColumns().get(0))); 10307 dataflowRelation.setTarget(new TableColumnRelationshipElement(tableColumn)); 10308 } 10309 } 10310 else { 10311 TObjectName objectName = new TObjectName(); 10312 objectName.setString("*"); 10313 tableColumn = modelFactory.createTableColumn(variable, objectName, true); 10314 } 10315 } else { 10316 tableColumn = modelFactory.createTableColumn(variable, stmt.getElementName(), true); 10317 tableColumn.setVariant(true); 10318 } 10319 10320 if (stmt.getDefaultValue() != null) { 10321 columnsInExpr visitor = new columnsInExpr(); 10322 stmt.getDefaultValue().inOrderTraverse(visitor); 10323 List<TObjectName> objectNames = visitor.getObjectNames(); 10324 List<TParseTreeNode> functions = visitor.getFunctions(); 10325 List<TParseTreeNode> constants = visitor.getConstants(); 10326 List<TSelectSqlStatement> subquerys = visitor.getSubquerys(); 10327 10328 if (functions != null && !functions.isEmpty()) { 10329 analyzeFunctionDataFlowRelation(tableColumn, functions, EffectType.function); 10330 } 10331 if (subquerys != null && !subquerys.isEmpty()) { 10332 analyzeSubqueryDataFlowRelation(tableColumn, subquerys, EffectType.select); 10333 } 10334 if (objectNames != null && !objectNames.isEmpty()) { 10335 analyzeDataFlowRelation(tableColumn, objectNames, EffectType.select, functions); 10336 } 10337 if (constants != null && !constants.isEmpty()) { 10338 analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select, functions); 10339 } 10340 } 10341 10342 10343 } 10344 } 10345 10346 private boolean isCursorType(String dataTypeName) { 10347 if (dataTypeName != null && dataTypeName.toLowerCase().contains("cursor")) { 10348 return true; 10349 } 10350 return false; 10351 } 10352 10353 private void analyzeSetStmt(TSetStmt stmt) { 10354 TExpression right = stmt.getVariableValue(); 10355 TObjectName columnObject = stmt.getVariableName(); 10356 if (columnObject != null) { 10357 TableColumn tableColumn = null; 10358 Variable tableModel; 10359 if (columnObject.toString().indexOf(".") != -1) { 10360 List<String> splits = SQLUtil.parseNames(columnObject.toString()); 10361 tableModel = modelFactory.createVariable(splits.get(splits.size() - 2)); 10362 } else { 10363 tableModel = modelFactory.createVariable(columnObject); 10364 } 10365 tableModel.setCreateTable(true); 10366 tableModel.setSubType(SubType.record); 10367 10368 if (tableModel.getColumns() == null || tableModel.getColumns().isEmpty()) { 10369 tableColumn = modelFactory.createTableColumn(tableModel, columnObject, true); 10370 } else { 10371 tableColumn = tableModel.getColumns().get(0); 10372 } 10373 10374 if (tableColumn != null && right!=null) { 10375 columnsInExpr visitor = new columnsInExpr(); 10376 right.inOrderTraverse(visitor); 10377 List<TObjectName> objectNames = visitor.getObjectNames(); 10378 List<TParseTreeNode> functions = visitor.getFunctions(); 10379 List<TParseTreeNode> constants = visitor.getConstants(); 10380 List<TSelectSqlStatement> subquerys = visitor.getSubquerys(); 10381 10382 if (functions != null && !functions.isEmpty()) { 10383 analyzeFunctionDataFlowRelation(tableColumn, functions, EffectType.function); 10384 } 10385 if (subquerys != null && !subquerys.isEmpty()) { 10386 analyzeSubqueryDataFlowRelation(tableColumn, subquerys, EffectType.select); 10387 } 10388 if (objectNames != null && !objectNames.isEmpty()) { 10389 analyzeDataFlowRelation(tableColumn, objectNames, EffectType.select, functions); 10390 } 10391 if (constants != null && !constants.isEmpty()) { 10392 analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select, functions); 10393 } 10394 10395 if(columnObject.toString().equalsIgnoreCase("search_path") && !constants.isEmpty()) { 10396 ModelBindingManager.setGlobalSchema(constants.get(0).toString()); 10397 } 10398 } 10399 } 10400 10401 if(stmt.getAssignments()!=null){ 10402 for (int i = 0; i < stmt.getAssignments().size(); i++) { 10403 TSetAssignment assignStmt = stmt.getAssignments().getElement(i); 10404 analyzeSetAssignmentStmt(assignStmt); 10405 } 10406 } 10407 } 10408 10409 private void analyzeSetAssignmentStmt(TSetAssignment stmt) { 10410 TExpression right = stmt.getParameterValue(); 10411 TObjectName columnObject = stmt.getParameterName(); 10412 if (columnObject != null) { 10413 TableColumn tableColumn = null; 10414 Variable tableModel; 10415 if (columnObject.toString().indexOf(".") != -1) { 10416 List<String> splits = SQLUtil.parseNames(columnObject.toString()); 10417 tableModel = modelFactory.createVariable(splits.get(splits.size() - 2)); 10418 } else { 10419 tableModel = modelFactory.createVariable(columnObject); 10420 } 10421 tableModel.setCreateTable(true); 10422 tableModel.setSubType(SubType.record); 10423 if (tableModel.getColumns() == null || tableModel.getColumns().isEmpty()) { 10424 tableColumn = modelFactory.createTableColumn(tableModel, columnObject, true); 10425 } else { 10426 tableColumn = tableModel.getColumns().get(0); 10427 } 10428 10429 if (tableColumn != null && right!=null) { 10430 columnsInExpr visitor = new columnsInExpr(); 10431 right.inOrderTraverse(visitor); 10432 List<TObjectName> objectNames = visitor.getObjectNames(); 10433 List<TParseTreeNode> functions = visitor.getFunctions(); 10434 List<TParseTreeNode> constants = visitor.getConstants(); 10435 List<TSelectSqlStatement> subquerys = visitor.getSubquerys(); 10436 10437 if (functions != null && !functions.isEmpty()) { 10438 analyzeFunctionDataFlowRelation(tableColumn, functions, EffectType.function); 10439 } 10440 if (subquerys != null && !subquerys.isEmpty()) { 10441 analyzeSubqueryDataFlowRelation(tableColumn, subquerys, EffectType.select); 10442 } 10443 if (objectNames != null && !objectNames.isEmpty()) { 10444 analyzeDataFlowRelation(tableColumn, objectNames, EffectType.select, functions); 10445 } 10446 if (constants != null && !constants.isEmpty()) { 10447 analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select, functions); 10448 } 10449 10450 if(columnObject.toString().equalsIgnoreCase("search_path") && !constants.isEmpty()) { 10451 ModelBindingManager.setGlobalSchema(constants.get(0).toString()); 10452 } 10453 } 10454 } 10455 } 10456 10457 private void analyzeMssqlSetStmt(TMssqlSet stmt) { 10458 TExpression right = stmt.getVarExpr(); 10459 TObjectName columnObject = stmt.getVarName(); 10460 if (columnObject != null) { 10461 TableColumn tableColumn = null; 10462 Variable tableModel; 10463 if (columnObject.toString().indexOf(".") != -1) { 10464 List<String> splits = SQLUtil.parseNames(columnObject.toString()); 10465 tableModel = modelFactory.createVariable(splits.get(splits.size() - 2)); 10466 } else { 10467 tableModel = modelFactory.createVariable(columnObject); 10468 } 10469 tableModel.setCreateTable(true); 10470 tableModel.setSubType(SubType.record); 10471 if (tableModel.getColumns() == null || tableModel.getColumns().isEmpty()) { 10472 tableColumn = modelFactory.createTableColumn(tableModel, columnObject, true); 10473 } 10474 else { 10475 tableColumn = tableModel.getColumns().get(0); 10476 } 10477 10478 if (tableColumn != null && right!=null) { 10479 columnsInExpr visitor = new columnsInExpr(); 10480 right.inOrderTraverse(visitor); 10481 List<TObjectName> objectNames = visitor.getObjectNames(); 10482 List<TParseTreeNode> functions = visitor.getFunctions(); 10483 List<TParseTreeNode> constants = visitor.getConstants(); 10484 List<TSelectSqlStatement> subquerys = visitor.getSubquerys(); 10485 10486 if (functions != null && !functions.isEmpty()) { 10487 analyzeFunctionDataFlowRelation(tableColumn, functions, EffectType.function); 10488 } 10489 if (subquerys != null && !subquerys.isEmpty()) { 10490 analyzeSubqueryDataFlowRelation(tableColumn, subquerys, EffectType.select); 10491 } 10492 if (objectNames != null && !objectNames.isEmpty()) { 10493 analyzeDataFlowRelation(tableColumn, objectNames, EffectType.select, functions); 10494 } 10495 if (constants != null && !constants.isEmpty()) { 10496 analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select, functions); 10497 } 10498 10499 if(columnObject.toString().equalsIgnoreCase("search_path") && !constants.isEmpty()) { 10500 ModelBindingManager.setGlobalSchema(constants.get(0).toString()); 10501 } 10502 } 10503 else if(tableColumn!=null && stmt.getSubquery()!=null){ 10504 analyzeCustomSqlStmt(stmt.getSubquery()); 10505 analyzeSubqueryDataFlowRelation(tableColumn, Arrays.asList(stmt.getSubquery()), EffectType.select); 10506 } 10507 } 10508 } 10509 10510 /** 10511 * Oracle analog of the T-SQL default-path evaluation (plan Phase 5): when the 10512 * parser's token fold left an EXECUTE IMMEDIATE partial (parameter placeholders) 10513 * or empty, run the AST interpreter over the file's statements — same-file 10514 * literal calls bind parameters naturally — plus literal caller statements 10515 * harvested from OTHER files of a multi-file analysis. The interpreter stores 10516 * evaluated text on each TExecImmeStmt (getEvaluatedDynamicSQLs), which 10517 * getDynamicSQL()/getDynamicStatements() then prefer over the token fold; the 10518 * existing placeholder-contamination classifier keeps half-bound results honest. 10519 * Best-effort: any interpreter failure leaves the token-fold behavior intact. 10520 */ 10521 private void maybeEvaluatePlsqlDynamicSql(TExecImmeStmt execImmeStmt) { 10522 if (option.getVendor() != EDbVendor.dbvoracle) { 10523 return; 10524 } 10525 if (!execImmeStmt.getEvaluatedDynamicSQLs().isEmpty()) { 10526 return; 10527 } 10528 if (execImmeStmt.getDynamicSQL() != null && !execImmeStmt.isDynamicSQLPartial()) { 10529 // Fully folded already — do not disturb resolved sites. 10530 return; 10531 } 10532 TGSqlParser owner = execImmeStmt.getGsqlparser(); 10533 if (owner == null || owner.getSqlstatements() == null) { 10534 return; 10535 } 10536 String ownText = owner.sqltext; 10537 if (ownText == null || !plsqlEvaluatedTexts.add(ownText)) { 10538 return; // one interpreter run per analysis unit 10539 } 10540 try { 10541 // Definitions plus fully-literal-vetted caller statements only (same-file 10542 // first, then cross-file): a caller passing a variable would make the 10543 // interpreter fold the unevaluated operand to an EMPTY string, silently 10544 // corrupting object names - such callers never enter the list. Empty 10545 // list = no vetted caller anywhere = nothing to bind, skip evaluation. 10546 List<TCustomSqlStatement> toInterpret = gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver 10547 .plsqlInterpretationStatements(owner.getSqlstatements(), ownText, 10548 routineCatalog, 4); 10549 if (toInterpret.isEmpty()) { 10550 return; 10551 } 10552 TStatementList combined = new TStatementList(); 10553 for (TCustomSqlStatement stmt : toInterpret) { 10554 combined.add(stmt); 10555 } 10556 TSQLEnv evalEnv = new TSQLEnv(EDbVendor.dbvoracle) { 10557 @Override 10558 public void initSQLEnv() { 10559 } 10560 }; 10561 new gudusoft.gsqlparser.compiler.TASTEvaluator(combined, 10562 new gudusoft.gsqlparser.compiler.TGlobalScope(evalEnv)).eval(); 10563 } catch (RuntimeException ex) { 10564 // interpretation is best-effort; token-fold behavior remains 10565 } catch (java.lang.Error err) { 10566 if (err instanceof ThreadDeath) { 10567 throw (ThreadDeath) err; 10568 } 10569 // e.g. StackOverflow on pathological nesting — keep the analysis alive 10570 } 10571 } 10572 10573 /** 10574 * Default-path dynamic-SQL evaluation (plan Phase 4a): when the token-level fold 10575 * left an EXEC / sp_executesql / EXEC(@var) site opaque, abstractly evaluate the 10576 * enclosing procedure's string building with no call-site bindings and try to 10577 * materialize this site's SQL. Returns null when no usable text was obtained. 10578 */ 10579 private gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite evaluatedDynamicSqlFor( 10580 TCustomSqlStatement execStmt) { 10581 if (option.getVendor() != EDbVendor.dbvmssql) { 10582 return null; 10583 } 10584 TCustomSqlStatement proc = null; 10585 for (int i = stmtStack.size() - 1; i >= 0; i--) { 10586 if (stmtStack.get(i) instanceof TStoredProcedureSqlStatement) { 10587 proc = stmtStack.get(i); 10588 break; 10589 } 10590 } 10591 if (proc == null) { 10592 return null; 10593 } 10594 List<Map<TCustomSqlStatement, gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite>> variants = dynamicEvalCache 10595 .get(proc); 10596 if (variants == null) { 10597 variants = new ArrayList<Map<TCustomSqlStatement, gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite>>(); 10598 try { 10599 variants.addAll(gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver 10600 .materializeSiteVariants(proc, option.getVendor(), null, null)); 10601 // Call-site binding discovery (plan Phase 2/4a): literal EXEC calls to 10602 // this proc in the same file supply parameter values that can turn a 10603 // placeholder-bearing template into fully-known SQL. 10604 List<Map<String, gudusoft.gsqlparser.dlineage.dynamicsql.SqlValue>> sameFileBindings = gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver 10605 .discoverCallSiteBindings(proc, 4); 10606 for (Map<String, gudusoft.gsqlparser.dlineage.dynamicsql.SqlValue> bindings : sameFileBindings) { 10607 variants.addAll(gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver 10608 .materializeSiteVariants(proc, option.getVendor(), bindings, null)); 10609 } 10610 // Cross-file discovery (plan Phase 5): literal EXEC calls in OTHER 10611 // files of this multi-file analysis. Same-file bindings keep priority; 10612 // the overall binding-set cap stays at 4. 10613 if (routineCatalog != null && sameFileBindings.size() < 4) { 10614 for (Map<String, gudusoft.gsqlparser.dlineage.dynamicsql.SqlValue> bindings : gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver 10615 .discoverCrossFileCallSiteBindings(proc, routineCatalog, 10616 4 - sameFileBindings.size())) { 10617 variants.addAll(gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver 10618 .materializeSiteVariants(proc, option.getVendor(), bindings, null)); 10619 } 10620 } 10621 } catch (RuntimeException ex) { 10622 // evaluation is best-effort; fall through with whatever variants exist 10623 } 10624 dynamicEvalCache.put(proc, variants); 10625 } 10626 // Prefer the first fully-concrete materialization; otherwise the first 10627 // usable partial text; otherwise the no-bindings entry (carries the reason). 10628 gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite best = null; 10629 for (Map<TCustomSqlStatement, gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite> variant : variants) { 10630 gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite candidate = variant.get(execStmt); 10631 if (candidate == null) { 10632 continue; 10633 } 10634 if (candidate.sqlText != null && !candidate.partial) { 10635 return candidate; 10636 } 10637 if (best == null || (best.sqlText == null && candidate.sqlText != null)) { 10638 best = candidate; 10639 } 10640 } 10641 return best; 10642 } 10643 10644 /** All distinct usable materializations for one EXEC, in stable variant order. */ 10645 private List<gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite> evaluatedDynamicSqlVariantsFor( 10646 TCustomSqlStatement execStmt, boolean concreteOnly) { 10647 // Builds and caches the variant list on first use. 10648 evaluatedDynamicSqlFor(execStmt); 10649 TCustomSqlStatement proc = null; 10650 for (int i = stmtStack.size() - 1; i >= 0; i--) { 10651 if (stmtStack.get(i) instanceof TStoredProcedureSqlStatement) { 10652 proc = stmtStack.get(i); 10653 break; 10654 } 10655 } 10656 List<gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite> result = 10657 new ArrayList<gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite>(); 10658 if (proc == null) { 10659 return result; 10660 } 10661 List<Map<TCustomSqlStatement, gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite>> variants = 10662 dynamicEvalCache.get(proc); 10663 Set<String> seen = new LinkedHashSet<String>(); 10664 for (Map<TCustomSqlStatement, gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite> variant : variants) { 10665 gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite candidate = 10666 variant.get(execStmt); 10667 if (candidate == null || candidate.sqlText == null 10668 || (concreteOnly && candidate.partial) 10669 || isWholeTextHole(candidate)) { 10670 continue; 10671 } 10672 String key = candidate.partial + "\u0000" + candidate.sqlText; 10673 if (seen.add(key)) { 10674 result.add(candidate); 10675 } 10676 } 10677 return result; 10678 } 10679 10680 private boolean analyzeEvaluatedDynamicSqlVariants(TCustomSqlStatement stmt, 10681 DynamicSqlSite.Kind kind, boolean concreteOnly) { 10682 boolean handled = false; 10683 for (gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite site : 10684 evaluatedDynamicSqlVariantsFor(stmt, concreteOnly)) { 10685 handled |= analyzeEvaluatedDynamicSql(stmt, kind, site); 10686 } 10687 return handled; 10688 } 10689 10690 /** 10691 * Analyze the materialized text of a dynamic site through the ordinary inner-SQL 10692 * path and record a truthful site status. Returns true when the site was handled 10693 * (recorded) here; false when the text was unusable and the caller should record 10694 * its own UNRESOLVED diagnostic. 10695 */ 10696 private boolean analyzeEvaluatedDynamicSql(TCustomSqlStatement stmt, DynamicSqlSite.Kind kind, 10697 gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.MaterializedSite site) { 10698 if (site == null || site.sqlText == null) { 10699 return false; 10700 } 10701 DynamicSqlTrustMode trustMode = option.getDynamicSqlTrustMode(); 10702 boolean provenanceIncomplete = site.provenanceIncomplete; 10703 DynamicHoleEvidence holeEvidence = dynamicHoleEvidence(site); 10704 int holeCount = holeEvidence.count; 10705 boolean holeCountExact = holeEvidence.exact; 10706 Set<Relationship> shadowBefore = provenanceIncomplete && trustMode == DynamicSqlTrustMode.SHADOW 10707 ? snapshotDynamicRelationships() : null; 10708 int relsBeforeDynamic = modelManager.getRelations().length; 10709 int dynamicSitesBefore = dynamicSqlSites.size(); 10710 // This is the ONE place partially-materialized text reaches the ordinary inner-SQL 10711 // path, and the site carries exact provenance — so the endpoints it introduces can be 10712 // marked from real HOLE fragments rather than from a name heuristic. 10713 Set<Table> templateTablesBefore = snapshotTemplateCandidateTables( 10714 site.partial || provenanceIncomplete); 10715 TGSqlParser sqlparser = createSqlParser(option.getVendor()); 10716 sqlparser.sqltext = site.sqlText; 10717 boolean foldStatementsAnalyzed = false; 10718 int result = sqlparser.parse(); 10719 if (result == 0 && sqlparser.sqlstatements != null) { 10720 foldStatementsAnalyzed = true; 10721 dynamicFoldDepth++; 10722 try { 10723 for (int i = 0; i < sqlparser.sqlstatements.size(); i++) { 10724 analyzeCustomSqlStmt(sqlparser.sqlstatements.get(i)); 10725 } 10726 } finally { 10727 dynamicFoldDepth--; 10728 } 10729 } 10730 if (foldStatementsAnalyzed) { 10731 markDynamicTemplateEndpoints(templateTablesBefore, site, sqlparser); 10732 } 10733 int[] dynCounts = classifyDynamicSiteLineage(relsBeforeDynamic, site.partial); 10734 int unprovenCount = countNewDynamicRelationships(shadowBefore); 10735 if (result != 0) { 10736 // Evaluated text always derives from variables, and the evaluation is 10737 // flow-insensitive — the variable may hold a different value at runtime. 10738 // A parse failure on it is therefore data-driven UNRESOLVED, never 10739 // PARSE_ERROR (which stays reserved for compile-time literal arguments, 10740 // matching the fold path's contract). 10741 String reason = site.partial ? "partially evaluated dynamic SQL failed to parse; unresolved parts remain" 10742 : "evaluated dynamic SQL failed to parse; value is flow-insensitive"; 10743 if (provenanceIncomplete && trustMode == DynamicSqlTrustMode.SHADOW) { 10744 reason = dynamicTrustReason(reason + "; SHADOW retained legacy publication behavior", 10745 unprovenCount, holeCount, holeCountExact); 10746 } 10747 recordDynamicSqlSite(stmt, kind, DynamicSqlSite.Status.UNRESOLVED, reason, 10748 dynCounts[0], dynCounts[1], unprovenCount, 10749 holeCount, holeCountExact, trustMode, 10750 relsBeforeDynamic, dynamicSitesBefore, null, site.diagnostic, 10751 holeEvidence.fragments, holeEvidence.materializedText); 10752 return true; 10753 } 10754 boolean usable = dynCounts[2] == 0 10755 && (dynCounts[0] > 0 || (dynCounts[1] == 0 && dynCounts[3] > 0)); 10756 DynamicSqlSite.Status status = usable ? DynamicSqlSite.Status.RESOLVED 10757 : DynamicSqlSite.Status.UNRESOLVED; 10758 String reason = usable ? null 10759 : (dynCounts[2] != 0 10760 ? "dynamic SQL references a runtime-built object name; lineage source is a placeholder" 10761 : "evaluated dynamic SQL produced no resolved lineage"); 10762 if (provenanceIncomplete && trustMode == DynamicSqlTrustMode.SHADOW) { 10763 status = unprovenCount > 0 ? DynamicSqlSite.Status.PARTIAL 10764 : DynamicSqlSite.Status.UNRESOLVED; 10765 reason = dynamicTrustReason("SHADOW retained legacy publication behavior", 10766 unprovenCount, holeCount, holeCountExact); 10767 } 10768 recordDynamicSqlSite(stmt, kind, 10769 status, reason, dynCounts[0], dynCounts[1], unprovenCount, 10770 holeCount, holeCountExact, trustMode, 10771 relsBeforeDynamic, dynamicSitesBefore, null, site.diagnostic, 10772 holeEvidence.fragments, holeEvidence.materializedText); 10773 return true; 10774 } 10775 10776 /** 10777 * Trigger correlation names (:NEW / :OLD and REFERENCING aliases) denote rows of the 10778 * trigger's ON table. Bind them so name-based lookups from body statements (e.g. the 10779 * assignment target in ":alias.col := expr") resolve to the base table. Only 10780 * ':'-prefixed and trigger-scoped keys are registered - bare alias names could collide 10781 * with real tables elsewhere in the file. 10782 */ 10783 private void bindTriggerCorrelationNames(TPlsqlCreateTrigger trigger, Table sourceTable) { 10784 List<String> correlationNames = new ArrayList<String>(); 10785 correlationNames.add("NEW"); 10786 correlationNames.add("OLD"); 10787 if (trigger.getTriggeringClause() != null 10788 && trigger.getTriggeringClause().getReferencingClause() != null 10789 && trigger.getTriggeringClause().getReferencingClause().getReferencingItems() != null) { 10790 for (TTriggerReferencingItem item : trigger.getTriggeringClause().getReferencingClause() 10791 .getReferencingItems()) { 10792 if (item.getCorrelationName() != null) { 10793 correlationNames.add(item.getCorrelationName().toString()); 10794 } 10795 } 10796 } 10797 String scope = null; 10798 if (trigger.getStatements() != null && trigger.getStatements().size() > 0) { 10799 scope = DlineageUtil.getProcedureParentName(trigger.getStatements().get(0)); 10800 } 10801 for (String correlationName : correlationNames) { 10802 modelManager.bindTableByName(DlineageUtil.getTableFullName(":" + correlationName), sourceTable); 10803 if (!SQLUtil.isEmpty(scope)) { 10804 modelManager.bindTableByName(DlineageUtil.getTableFullName( 10805 scope + "." + SQLUtil.getIdentifierNormalTableName(correlationName)), sourceTable); 10806 } 10807 } 10808 } 10809 10810 private void analyzeAssignStmt(TAssignStmt stmt) { 10811 TExpression left = stmt.getLeft(); 10812 TExpression right = stmt.getExpression(); 10813 TObjectName columnObject = null; 10814 if (left == null) { 10815 columnObject = stmt.getVariableName(); 10816 } else if (left.getExpressionType() == EExpressionType.simple_object_name_t) { 10817 columnObject = left.getObjectOperand(); 10818 } 10819 if (columnObject != null) { 10820 TableColumn tableColumn = null; 10821 List<TableColumn> recordFillTargets = null; 10822 if (columnObject.getDbObjectType() == EDbObjectType.variable || stmt.getVariableName() != null) { 10823 Variable tableModel; 10824 if (columnObject.toString().indexOf(".") != -1) { 10825 List<String> splits = SQLUtil.parseNames(columnObject.toString()); 10826 tableModel = modelFactory.createVariable(splits.get(splits.size() - 2)); 10827 } else { 10828 tableModel = modelFactory.createVariable(columnObject); 10829 } 10830 tableModel.setCreateTable(true); 10831 tableModel.setSubType(SubType.record); 10832 if (tableModel.getColumns() == null || tableModel.getColumns().isEmpty()) { 10833 tableColumn = modelFactory.createTableColumn(tableModel, columnObject, true); 10834 } else if (tableModel.getColumns().size() == 1) { 10835 tableColumn = tableModel.getColumns().get(0); 10836 } else { 10837 // Whole-record assignment to a field-expanded record (#695): 10838 // every field is filled by the right-hand side, so the fill 10839 // edge is emitted per field. Binding get(0) instead attached 10840 // the whole assignment to one arbitrary field and left the 10841 // rest with no upstream at all. 10842 tableColumn = tableModel.getColumns().get(0); 10843 recordFillTargets = tableModel.getColumns(); 10844 } 10845 } else { 10846 List<String> splits = SQLUtil.parseNames(columnObject.toString()); 10847 if (splits.size() > 1) { 10848 Table tableModel = modelManager 10849 .getTableByName(DlineageUtil.getTableFullName(splits.get(splits.size() - 2))); 10850 if (tableModel == null) { 10851 String procedureName = DlineageUtil.getProcedureParentName(stmt); 10852 String variableString = splits.get(splits.size() - 2).toString(); 10853 if (variableString.startsWith(":")) { 10854 variableString = variableString.substring(variableString.indexOf(":") + 1); 10855 } 10856 if (!SQLUtil.isEmpty(procedureName)) { 10857 variableString = procedureName + "." + SQLUtil.getIdentifierNormalTableName(variableString); 10858 } 10859 tableModel = modelManager.getTableByName(DlineageUtil.getTableFullName(variableString)); 10860 } 10861 if (tableModel != null) { 10862 tableColumn = modelFactory.createTableColumn(tableModel, columnObject, true); 10863 } 10864 } 10865 } 10866 10867 if (tableColumn != null && right != null) { 10868 columnsInExpr visitor = new columnsInExpr(); 10869 right.inOrderTraverse(visitor); 10870 List<TObjectName> objectNames = visitor.getObjectNames(); 10871 List<TParseTreeNode> functions = visitor.getFunctions(); 10872 List<TParseTreeNode> constants = visitor.getConstants(); 10873 List<TSelectSqlStatement> subquerys = visitor.getSubquerys(); 10874 10875 List<TableColumn> bindTargets = recordFillTargets != null ? recordFillTargets 10876 : Arrays.asList(tableColumn); 10877 for (TableColumn bindTarget : bindTargets) { 10878 Transform transform = new Transform(); 10879 transform.setType(Transform.EXPRESSION); 10880 transform.setCode(right); 10881 bindTarget.setTransform(transform); 10882 if (functions != null && !functions.isEmpty()) { 10883 analyzeFunctionDataFlowRelation(bindTarget, functions, EffectType.function); 10884 } 10885 if (subquerys != null && !subquerys.isEmpty()) { 10886 analyzeSubqueryDataFlowRelation(bindTarget, subquerys, EffectType.select); 10887 } 10888 if (objectNames != null && !objectNames.isEmpty()) { 10889 analyzeDataFlowRelation(bindTarget, objectNames, EffectType.select, functions); 10890 } 10891 if (constants != null && !constants.isEmpty()) { 10892 analyzeConstantDataFlowRelation(bindTarget, constants, EffectType.select, functions); 10893 } 10894 } 10895 } 10896 } 10897 } 10898 10899 private void analyzeOpenForStmt(TOpenforStmt stmt) { 10900 if (stmt.getSubquery() == null) { 10901 return; 10902 } 10903 10904 Variable cursorTempTable = modelFactory.createCursor(stmt); 10905 cursorTempTable.setVariable(true); 10906 cursorTempTable.setSubType(SubType.cursor); 10907 modelManager.bindCursorModel(stmt, cursorTempTable); 10908 analyzeSelectStmt(stmt.getSubquery()); 10909 10910 TableColumn cursorColumn = null; 10911 if (cursorTempTable.getColumns() == null || cursorTempTable.getColumns().isEmpty()) { 10912 TObjectName starColumn = new TObjectName(); 10913 starColumn.setString("*"); 10914 cursorColumn = modelFactory.createTableColumn(cursorTempTable, starColumn, true); 10915 } else { 10916 cursorColumn = cursorTempTable.getColumns().get(0); 10917 } 10918 10919 ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt.getSubquery()); 10920 if (resultSetModel != null) { 10921 for (int i = 0; i < resultSetModel.getColumns().size(); i++) { 10922 ResultColumn resultColumn = resultSetModel.getColumns().get(i); 10923 DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation(); 10924 dataflowRelation.setEffectType(EffectType.cursor); 10925 dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn)); 10926 dataflowRelation.setTarget(new TableColumnRelationshipElement(cursorColumn)); 10927 } 10928 } 10929 } 10930 10931 private void analyzeCursorDeclStmt(TCursorDeclStmt stmt) { 10932 if (stmt.getSubquery() == null) { 10933 return; 10934 } 10935 10936 Variable cursorTempTable = modelFactory.createCursor(stmt); 10937 cursorTempTable.setVariable(true); 10938 cursorTempTable.setSubType(SubType.cursor); 10939 modelManager.bindCursorModel(stmt, cursorTempTable); 10940 analyzeSelectStmt(stmt.getSubquery()); 10941 10942 ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt.getSubquery()); 10943 if(resultSetModel!=null && resultSetModel.isDetermined()) { 10944 for (int i = 0; i < resultSetModel.getColumns().size(); i++) { 10945 ResultColumn resultColumn = resultSetModel.getColumns().get(i); 10946 TObjectName columnName = new TObjectName(); 10947 columnName.setString(resultColumn.getName()); 10948 TableColumn cursorColumn = modelFactory.createTableColumn(cursorTempTable, columnName, true); 10949 DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation(); 10950 dataflowRelation.setEffectType(EffectType.cursor); 10951 dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn)); 10952 dataflowRelation.setTarget(new TableColumnRelationshipElement(cursorColumn)); 10953 } 10954 } 10955 else { 10956 TableColumn cursorColumn = null; 10957 if (cursorTempTable.getColumns() == null || cursorTempTable.getColumns().isEmpty()) { 10958 TObjectName starColumn = new TObjectName(); 10959 starColumn.setString("*"); 10960 cursorColumn = modelFactory.createTableColumn(cursorTempTable, starColumn, true); 10961 } else { 10962 cursorColumn = cursorTempTable.getColumns().get(0); 10963 } 10964 if (resultSetModel != null) { 10965 for (int i = 0; i < resultSetModel.getColumns().size(); i++) { 10966 ResultColumn resultColumn = resultSetModel.getColumns().get(i); 10967 DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation(); 10968 dataflowRelation.setEffectType(EffectType.cursor); 10969 dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn)); 10970 dataflowRelation.setTarget(new TableColumnRelationshipElement(cursorColumn)); 10971 } 10972 } 10973 } 10974 10975 } 10976 10977 private void analyzeDb2Declare(TDb2SqlVariableDeclaration stmt) { 10978 TDeclareVariableList variables = stmt.getVariables(); 10979 if (variables == null) { 10980 return; 10981 } 10982 for (int i = 0; i < variables.size(); i++) { 10983 TDeclareVariable variable = variables.getDeclareVariable(i); 10984 if (variable.getTableTypeDefinitions() != null && variable.getTableTypeDefinitions().size() > 0) { 10985 10986 10987 TObjectName tableName = variable.getVariableName(); 10988 TTableElementList columns = variable.getTableTypeDefinitions(); 10989 10990 Table tableModel = modelFactory.createTableByName(tableName, true); 10991 tableModel.setCreateTable(true); 10992 String procedureParent = getProcedureParentName(stmt); 10993 if (procedureParent != null) { 10994 tableModel.setParent(procedureParent); 10995 } 10996 modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent), tableModel); 10997 10998 for (int j = 0; j < columns.size(); j++) { 10999 TTableElement tableElement = columns.getTableElement(j); 11000 TColumnDefinition column = tableElement.getColumnDefinition(); 11001 if (column != null && column.getColumnName() != null) { 11002 modelFactory.createTableColumn(tableModel, column.getColumnName(), true); 11003 } 11004 } 11005 } else if (variable.getVariableName() != null) { 11006 Variable cursorVariable = modelFactory.createVariable(variable.getVariableName()); 11007 cursorVariable.setCreateTable(true); 11008 cursorVariable.setSubType(SubType.record); 11009 if(variable.getDatatype()!=null && isSimpleDataType(variable.getDatatype())){ 11010 TableColumn variableProperty = modelFactory.createTableColumn(cursorVariable, 11011 variable.getVariableName(), true); 11012 } 11013 else { 11014 TObjectName variableProperties = new TObjectName(); 11015 variableProperties.setString("*"); 11016 TableColumn variableProperty = modelFactory.createTableColumn(cursorVariable, 11017 variableProperties, true); 11018 } 11019 } 11020 } 11021 } 11022 11023 private void analyzeMssqlDeclare(TMssqlDeclare stmt) { 11024 if (stmt.getDeclareType() == EDeclareType.variable) { 11025 TDeclareVariableList variables = stmt.getVariables(); 11026 if (variables == null) { 11027 return; 11028 } 11029 for (int i = 0; i < variables.size(); i++) { 11030 TDeclareVariable variable = variables.getDeclareVariable(i); 11031 if (variable.getTableTypeDefinitions() != null && variable.getTableTypeDefinitions().size() > 0) { 11032 11033 11034 TObjectName tableName = variable.getVariableName(); 11035 TTableElementList columns = variable.getTableTypeDefinitions(); 11036 11037 Variable tableModel = modelFactory.createVariable(tableName); 11038 tableModel.setVariable(true); 11039 tableModel.setCreateTable(true); 11040 String procedureParent = getProcedureParentName(stmt); 11041 if (procedureParent != null) { 11042 tableModel.setParent(procedureParent); 11043 } 11044 modelManager.bindTableFunction(DlineageUtil.getIdentifierNormalTableName(procedureParent), tableModel); 11045 11046 for (int j = 0; j < columns.size(); j++) { 11047 TTableElement tableElement = columns.getTableElement(j); 11048 TColumnDefinition column = tableElement.getColumnDefinition(); 11049 if (column != null && column.getColumnName() != null) { 11050 modelFactory.createTableColumn(tableModel, column.getColumnName(), true); 11051 } 11052 } 11053 } else if (variable.getVariableName() != null) { 11054 Variable cursorVariable = modelFactory.createVariable(variable.getVariableName()); 11055 cursorVariable.setCreateTable(true); 11056 cursorVariable.setSubType(SubType.record); 11057 TableColumn variableProperty = null; 11058 if (variable.getDatatype() != null && isSimpleDataType(variable.getDatatype())) { 11059 variableProperty = modelFactory.createTableColumn(cursorVariable, variable.getVariableName(), 11060 true); 11061 } else { 11062 TObjectName variableProperties = new TObjectName(); 11063 variableProperties.setString("*"); 11064 variableProperty = modelFactory.createTableColumn(cursorVariable, variableProperties, true); 11065 } 11066 11067 if (variable.getDefaultValue() != null && variable.getDefaultValue().getSubQuery() != null) { 11068 analyzeSelectStmt(variable.getDefaultValue().getSubQuery()); 11069 ResultSet resultSetModel = (ResultSet) modelManager 11070 .getModel(variable.getDefaultValue().getSubQuery()); 11071 if (variableProperty != null && resultSetModel != null) { 11072 for (ResultColumn column : resultSetModel.getColumns()) { 11073 DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation(); 11074 dataflowRelation.setEffectType(EffectType.select); 11075 dataflowRelation.addSource(new ResultColumnRelationshipElement(column)); 11076 dataflowRelation.setTarget(new TableColumnRelationshipElement(variableProperty)); 11077 } 11078 } 11079 } 11080 } 11081 } 11082 } else if (stmt.getDeclareType() == EDeclareType.cursor) { 11083 Variable cursorTempTable = modelFactory.createCursor(stmt); 11084 cursorTempTable.setVariable(true); 11085 cursorTempTable.setSubType(SubType.cursor); 11086 modelManager.bindCursorModel(stmt, cursorTempTable); 11087 analyzeSelectStmt(stmt.getSubquery()); 11088 ResultSet resultSetModel = (ResultSet)modelManager.getModel(stmt.getSubquery()); 11089 if (resultSetModel != null && resultSetModel.isDetermined()) { 11090 for(ResultColumn resultColumn: resultSetModel.getColumns()){ 11091 TObjectName starColumn = new TObjectName(); 11092 starColumn.setString(resultColumn.getName()); 11093 TableColumn cursorColumn = modelFactory.createTableColumn(cursorTempTable, starColumn, true); 11094 DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation(); 11095 dataflowRelation.setEffectType(EffectType.cursor); 11096 dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn)); 11097 dataflowRelation.setTarget(new TableColumnRelationshipElement(cursorColumn)); 11098 } 11099 } 11100 else { 11101 TableColumn cursorColumn = null; 11102 if (cursorTempTable.getColumns() == null || cursorTempTable.getColumns().isEmpty()) { 11103 TObjectName starColumn = new TObjectName(); 11104 starColumn.setString("*"); 11105 cursorColumn = modelFactory.createTableColumn(cursorTempTable, starColumn, true); 11106 cursorColumn.setShowStar(false); 11107 cursorColumn.setExpandStar(true); 11108 } else { 11109 cursorColumn = cursorTempTable.getColumns().get(0); 11110 } 11111 11112 if (resultSetModel != null) { 11113 for (int i = 0; i < resultSetModel.getColumns().size(); i++) { 11114 ResultColumn resultColumn = resultSetModel.getColumns().get(i); 11115 DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation(); 11116 dataflowRelation.setEffectType(EffectType.cursor); 11117 dataflowRelation.addSource(new ResultColumnRelationshipElement(resultColumn)); 11118 dataflowRelation.setTarget(new TableColumnRelationshipElement(cursorColumn)); 11119 } 11120 } 11121 } 11122 } 11123 } 11124 11125 11126 private boolean analyzeMssqlJsonDeclare(TMssqlDeclare stmt, String jsonName, Table jsonTable) { 11127 TDeclareVariableList variables = stmt.getVariables(); 11128 if (variables == null) { 11129 return false; 11130 } 11131 for (int i = 0; i < variables.size(); i++) { 11132 TDeclareVariable variable = variables.getDeclareVariable(i); 11133 TObjectName variableName = variable.getVariableName(); 11134 if (DlineageUtil.getIdentifierNormalTableName(variableName.toString()) 11135 .equals(DlineageUtil.getIdentifierNormalTableName(jsonName))) { 11136 if (variable.getDefaultValue() != null) { 11137 Table variableTable = modelFactory.createJsonVariable(variableName); 11138 variableTable.setVariable(true); 11139 variableTable.setSubType(SubType.scalar); 11140 variableTable.setCreateTable(true); 11141 TableColumn property = modelFactory.createVariableProperty(variableTable, variable); 11142 11143 for (int j = 0; j < jsonTable.getColumns().size(); j++) { 11144 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 11145 relation.setEffectType(EffectType.select); 11146 relation.setTarget(new TableColumnRelationshipElement(jsonTable.getColumns().get(j))); 11147 relation.addSource(new TableColumnRelationshipElement(property)); 11148 } 11149 } 11150 return true; 11151 } 11152 } 11153 return false; 11154 } 11155 11156 private void analyzeCreateTableStmt(TCreateTableSqlStatement stmt) { 11157 if (stmt.getCloneSourceTable() != null) { 11158 analyzeCloneTableStmt(stmt); 11159 return; 11160 } 11161 11162 TTable table = stmt.getTargetTable(); 11163 11164 boolean hasDefinition = false; 11165 11166 if (stmt.getColumnList() != null && stmt.getColumnList().size() > 0) { 11167 hasDefinition = true; 11168 } 11169 11170 if (table != null) { 11171 Table tableModel = modelFactory.createTableFromCreateDDL(table, hasDefinition || (stmt.getSubQuery() == null && hasDefinition) 11172 || (stmt.getSubQuery()!=null && stmt.getSubQuery().getSetOperatorType() == ESetOperatorType.none && stmt.getSubQuery().getResultColumnList().toString().indexOf("*") == -1)); 11173 // Authoritative create-effect classification (parse fact, NOT gated on 11174 // isDetermined() like fromDDL below). CTAS when there is a defining query, 11175 // plain DDL otherwise. See dlineage-authoritative-endpoint-classification.md. 11176 tableModel.setCreatedInSql(true); 11177 tableModel.setEndpointIntroduction(stmt.getSubQuery() != null 11178 ? EndpointIntroduction.CTAS : EndpointIntroduction.CREATE_TABLE_DDL); 11179 if (stmt.isExternal()) { 11180 tableModel.setExternal(true); 11181 } 11182 11183 if (stmt.getSubQuery() != null) { 11184 Process process = modelFactory.createProcess(stmt); 11185 tableModel.addProcess(process); 11186 } 11187 11188 String procedureParent = getProcedureParentName(stmt); 11189 if (procedureParent != null) { 11190 tableModel.setParent(procedureParent); 11191 } 11192 11193 if (stmt.isUsingTemplate()) { 11194 // Snowflake CREATE TABLE ... USING TEMPLATE <query>: the query only 11195 // infers the column definitions (e.g. ARRAY_AGG(OBJECT_CONSTRUCT(*)) 11196 // over INFER_SCHEMA); it does NOT populate the table. Analyze it so its 11197 // own sources (the stage / INFER_SCHEMA) still resolve, but do not 11198 // project its result columns onto the target table or emit CTAS-style 11199 // data-flow edges that would misrepresent it as data population. 11200 if (stmt.getSubQuery() != null) { 11201 analyzeSelectStmt(stmt.getSubQuery()); 11202 } 11203 return; 11204 } 11205 11206 if (hasDefinition) { 11207 if(stmt.getSubQuery()!=null) { 11208 TSelectSqlStatement subquery = stmt.getSubQuery(); 11209 analyzeSelectStmt(subquery); 11210 Process process = modelFactory.createProcess(stmt); 11211 ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt.getSubQuery()); 11212 if(resultSetModel.isDetermined()){ 11213 tableModel.setFromDDL(true); 11214 } 11215 if (resultSetModel != null) { 11216 int resultSetSize = resultSetModel.getColumns().size(); 11217 int stmtColumnSize = stmt.getColumnList().size(); 11218 int j = 0; 11219 int tableColumnSize = stmtColumnSize; 11220 if (resultSetModel.isDetermined() && resultSetSize > tableColumnSize) { 11221 tableColumnSize = resultSetSize; 11222 } 11223 for (int i = 0; i < tableColumnSize && j < resultSetSize; i++) { 11224 ResultColumn resultColumn = resultSetModel.getColumns().get(j); 11225 if (i < stmtColumnSize) { 11226 TObjectName alias = stmt.getColumnList().getColumn(i).getColumnName(); 11227 11228 if (!resultSetModel.getColumns().get(j).getName().contains("*")) { 11229 j++; 11230 } else { 11231 if (resultSetSize - j == stmt.getColumnList().size() - i) { 11232 j++; 11233 } 11234 } 11235 11236 if (alias != null) { 11237 TableColumn viewColumn = modelFactory.createTableColumn(tableModel, alias, true); 11238 if (resultColumn != null) { 11239 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 11240 relation.setEffectType(EffectType.create_table); 11241 relation.setTarget(new ViewColumnRelationshipElement(viewColumn)); 11242 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 11243 relation.setProcess(process); 11244 } 11245 } else if (resultColumn.getColumnObject() instanceof TObjectName) { 11246 TableColumn viewColumn = modelFactory.createTableColumn(tableModel, 11247 (TObjectName) resultColumn.getColumnObject(), true); 11248 if (resultColumn != null) { 11249 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 11250 relation.setEffectType(EffectType.create_table); 11251 relation.setTarget(new ViewColumnRelationshipElement(viewColumn)); 11252 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 11253 relation.setProcess(process); 11254 } 11255 } else if (resultColumn.getColumnObject() instanceof TResultColumn) { 11256 TableColumn viewColumn = modelFactory.createTableColumn(tableModel, 11257 ((TResultColumn) resultColumn.getColumnObject()).getFieldAttr(), true); 11258 ResultColumn column = (ResultColumn) modelManager 11259 .getModel(resultColumn.getColumnObject()); 11260 if (column != null && !column.getStarLinkColumns().isEmpty()) { 11261 viewColumn.bindStarLinkColumns(column.getStarLinkColumns()); 11262 } 11263 if (resultColumn != null) { 11264 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 11265 relation.setEffectType(EffectType.create_table); 11266 relation.setTarget(new ViewColumnRelationshipElement(viewColumn)); 11267 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 11268 relation.setProcess(process); 11269 } 11270 } 11271 } 11272 else if(resultSetModel.isDetermined()){ 11273 TObjectName tableName = new TObjectName(); 11274 tableName.setString(resultColumn.getName()); 11275 TableColumn viewColumn = modelFactory.createTableColumn(tableModel, tableName, true); 11276 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 11277 relation.setEffectType(EffectType.create_table); 11278 relation.setTarget(new ViewColumnRelationshipElement(viewColumn)); 11279 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 11280 relation.setProcess(process); 11281 j++; 11282 } 11283 } 11284 if (resultSetModel != null && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) { 11285 ImpactRelationship impactRelation = modelFactory.createImpactRelation(); 11286 impactRelation.setEffectType(EffectType.create_table); 11287 impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>( 11288 resultSetModel.getRelationRows())); 11289 impactRelation.setTarget( 11290 new RelationRowsRelationshipElement<TableRelationRows>(tableModel.getRelationRows())); 11291 } 11292 } 11293 11294 if (subquery.getResultColumnList() == null && subquery.getValueClause() != null 11295 && subquery.getValueClause().getValueRows().size() == stmt.getColumnList().size()) { 11296 for (int i = 0; i < stmt.getColumnList().size(); i++) { 11297 TObjectName alias = stmt.getColumnList().getColumn(i).getColumnName(); 11298 11299 if (alias != null) { 11300 TableColumn viewColumn = modelFactory.createTableColumn(tableModel, alias, true); 11301 11302 TExpression expression = subquery.getValueClause().getValueRows().getValueRowItem(i).getExpr(); 11303 11304 columnsInExpr visitor = new columnsInExpr(); 11305 expression.inOrderTraverse(visitor); 11306 List<TObjectName> objectNames = visitor.getObjectNames(); 11307 List<TParseTreeNode> functions = visitor.getFunctions(); 11308 11309 if (functions != null && !functions.isEmpty()) { 11310 analyzeFunctionDataFlowRelation(viewColumn, functions, EffectType.select); 11311 11312 } 11313 11314 List<TSelectSqlStatement> subquerys = visitor.getSubquerys(); 11315 if (subquerys != null && !subquerys.isEmpty()) { 11316 analyzeSubqueryDataFlowRelation(viewColumn, subquerys, EffectType.select); 11317 } 11318 11319 analyzeDataFlowRelation(viewColumn, objectNames, EffectType.select, functions); 11320 List<TParseTreeNode> constants = visitor.getConstants(); 11321 analyzeConstantDataFlowRelation(viewColumn, constants, EffectType.select, functions); 11322 } 11323 } 11324 } 11325 11326 return; 11327 } 11328 else { 11329 for (int i = 0; i < stmt.getColumnList().size(); i++) { 11330 TColumnDefinition column = stmt.getColumnList().getColumn(i); 11331 if (column.getDatatype() != null && column.getDatatype().getTypeOfList() != null 11332 && column.getDatatype().getTypeOfList().getColumnDefList() != null) { 11333 for (int j = 0; j < column.getDatatype().getTypeOfList().getColumnDefList().size(); j++) { 11334 TObjectName columnName = new TObjectName(); 11335 if (column.getDatatype().getDataType() == EDataType.array_t) { 11336// columnName.setString(column.getColumnName().getColumnNameOnly() + ".array." 11337// + column.getDatatype().getTypeOfList().getColumnDefList().getColumn(j) 11338// .getColumnName().getColumnNameOnly()); 11339 columnName.setString(column.getColumnName().getColumnNameOnly() + "." 11340 + column.getDatatype().getTypeOfList().getColumnDefList().getColumn(j) 11341 .getColumnName().getColumnNameOnly()); 11342 } else { 11343 columnName.setString(column.getColumnName().getColumnNameOnly() + "." 11344 + column.getDatatype().getTypeOfList().getColumnDefList().getColumn(j) 11345 .getColumnName().getColumnNameOnly()); 11346 } 11347 TableColumn tableColumn = modelFactory.createTableColumn(tableModel, columnName, 11348 hasDefinition); 11349 tableColumn.setStruct(true); 11350 tableColumn.setColumnIndex(i); 11351 appendTableColumnToSQLEnv(tableModel, tableColumn); 11352 11353 if(option.getAnalyzeMode() == AnalyzeMode.crud) { 11354 CrudRelationship crudRelationship = modelFactory.createCrudRelation(); 11355 crudRelationship.setTarget(new TableColumnRelationshipElement(tableColumn)); 11356 crudRelationship.setEffectType(EffectType.create_table); 11357 } 11358 } 11359 continue; 11360 } 11361 if (column.getDatatype() != null && column.getDatatype().getColumnDefList() != null) { 11362 Stack<TColumnDefinition> columnPaths = new Stack<TColumnDefinition>(); 11363 flattenStructColumns(hasDefinition, tableModel, column, columnPaths, i); 11364 continue; 11365 } 11366 TableColumn tableColumn = modelFactory.createTableColumn(tableModel, column.getColumnName(), 11367 hasDefinition); 11368 if (tableColumn == null) { 11369 continue; 11370 } 11371 11372 if(option.getAnalyzeMode() == AnalyzeMode.crud) { 11373 CrudRelationship crudRelationship = modelFactory.createCrudRelation(); 11374 crudRelationship.setTarget(new TableColumnRelationshipElement(tableColumn)); 11375 crudRelationship.setEffectType(EffectType.create_table); 11376 } 11377 11378 if (column.getDatatype() != null && column.getDatatype().getDataType() == EDataType.variant_t) { 11379 if (tableColumn != null) { 11380 tableColumn.setVariant(true); 11381 } 11382 } 11383 if (column.getDatatype() != null) { 11384 String dType = column.getDatatype().getDataTypeName(); 11385 if ((!SQLUtil.isEmpty(dType)) && (dType.indexOf("_") > 0)) { 11386 dType = dType.split("_")[0]; 11387 } 11388 tableColumn.setDataType(dType); 11389 } 11390 11391 // Inline (column-level) PK / FK constraint FLAGS. The 11392 // table-level pass below (stmt.getTableConstraints()) 11393 // never sees inline constraints, so set the flags here. 11394 // Without this, "id INTEGER PRIMARY KEY" or 11395 // "fk INTEGER REFERENCES t (c)" produced no 11396 // isPrimaryKey()/isForeignKey() flag, unlike the 11397 // equivalent table-level CONSTRAINT form. 11398 // EConstraintType.reference is the inline REFERENCES 11399 // form; foreign_key is the (rare) inline FOREIGN KEY. 11400 // Only flags are set here (no model objects allocated) 11401 // so column ids are unchanged; the FK relationship 11402 // edges are emitted in a deferred pass after the column 11403 // loop (see "inline FK relationships" below), matching 11404 // the table-level pass ordering so ids stay stable. 11405 TConstraintList inlineConstraints = column.getConstraints(); 11406 if (inlineConstraints != null) { 11407 for (int c = 0; c < inlineConstraints.size(); c++) { 11408 EConstraintType inlineType = inlineConstraints.getConstraint(c) 11409 .getConstraint_type(); 11410 if (inlineType == EConstraintType.primary_key) { 11411 tableColumn.setPrimaryKey(true); 11412 } else if (inlineType == EConstraintType.foreign_key 11413 || inlineType == EConstraintType.reference) { 11414 tableColumn.setForeignKey(true); 11415 } 11416 } 11417 } 11418 11419 appendTableColumnToSQLEnv(tableModel, tableColumn); 11420 } 11421 } 11422 } 11423 11424 if (stmt.getExternalTableOption("DATA_SOURCE") != null) { 11425 String dataSourceName = stmt.getExternalTableOption("DATA_SOURCE"); 11426 Table dataSource = modelManager.getTableByName(DlineageUtil.getTableFullName(dataSourceName)); 11427 if (dataSource != null) { 11428 TableColumn dataSourceColumn = dataSource.getColumns().get(0); 11429 for (int i = 0; i < tableModel.getColumns().size(); i++) { 11430 TableColumn column = tableModel.getColumns().get(i); 11431 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 11432 relation.setTarget(new TableColumnRelationshipElement(column)); 11433 relation.addSource(new TableColumnRelationshipElement(dataSourceColumn)); 11434 11435 appendTableColumnToSQLEnv(tableModel, column); 11436 } 11437 } 11438 } 11439 11440 if (stmt.getSubQuery() != null) { 11441 11442 analyzeSelectStmt(stmt.getSubQuery()); 11443 11444 ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt.getSubQuery()); 11445 if (resultSetModel != null && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) { 11446 ImpactRelationship impactRelation = modelFactory.createImpactRelation(); 11447 impactRelation.setEffectType(EffectType.create_table); 11448 impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>( 11449 resultSetModel.getRelationRows())); 11450 impactRelation.setTarget( 11451 new RelationRowsRelationshipElement<TableRelationRows>(tableModel.getRelationRows())); 11452 } 11453 } 11454 11455 if (stmt.getSubQuery() != null && !stmt.getSubQuery().isCombinedQuery()) { 11456 SelectResultSet resultSetModel = (SelectResultSet) modelManager 11457 .getModel(stmt.getSubQuery().getResultColumnList()); 11458 if(resultSetModel.isDetermined()){ 11459 tableModel.setDetermined(true); 11460 tableModel.setFromDDL(true); 11461 } 11462 for (int i = 0; i < resultSetModel.getColumns().size(); i++) { 11463 ResultColumn resultColumn = resultSetModel.getColumns().get(i); 11464 if (resultSetModel.isDetermined() && resultColumn.getName().endsWith("*")) { 11465 continue; 11466 } 11467 11468 if (resultColumn.getColumnObject() instanceof TResultColumn) { 11469 TResultColumn columnObject = (TResultColumn) resultColumn.getColumnObject(); 11470 11471 TAliasClause alias = columnObject.getAliasClause(); 11472 if (alias != null && alias.getAliasName() != null) { 11473 TableColumn tableColumn = null; 11474 if (!hasDefinition) { 11475 tableColumn = modelFactory.createTableColumn(tableModel, alias.getAliasName(), 11476 !hasDefinition); 11477 } else { 11478 tableColumn = tableModel.getColumns().get(i); 11479 } 11480 11481 if (!tableColumn.getName().endsWith("*") && tableModel.isDetermined()) { 11482 appendTableColumnToSQLEnv(tableModel, tableColumn); 11483 } 11484 11485 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 11486 relation.setEffectType(EffectType.create_table); 11487 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 11488 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 11489 Process process = modelFactory.createProcess(stmt); 11490 relation.setProcess(process); 11491 } else if (columnObject.getFieldAttr() != null || (columnObject.getExpr()!=null && columnObject.getExpr().getExpressionType() == EExpressionType.typecast_t)) { 11492 TableColumn tableColumn = null; 11493 TObjectName columnObj = columnObject.getFieldAttr(); 11494 if(columnObj == null) { 11495 columnObj = columnObject.getExpr().getLeftOperand().getObjectOperand(); 11496 } 11497 if ((columnObj == null || columnObj.toString().endsWith("*")) && !resultColumn.getName().endsWith("*")) { 11498 columnObj = new TObjectName(); 11499 columnObj.setString(resultColumn.getName()); 11500 } 11501 11502 if(columnObj == null){ 11503 logger.info("Can't handle column " + resultColumn.getName()); 11504 continue; 11505 } 11506 11507 if (!hasDefinition) { 11508 tableColumn = modelFactory.createTableColumn(tableModel, columnObj, 11509 !hasDefinition); 11510 if (tableColumn == null) { 11511 if (tableModel.getColumns().isEmpty()) { 11512 logger.info("Add table " + tableModel.getName() + " column " + columnObj.toString() + " failed"); 11513 } 11514 else if (resultColumn.getName().endsWith("*")) { 11515 for (int j = 0; j < tableModel.getColumns().size(); j++) { 11516 tableColumn = tableModel.getColumns().get(j); 11517 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 11518 relation.setEffectType(EffectType.create_table); 11519 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 11520 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 11521 if (tableColumn.getName().endsWith("*") 11522 && resultColumn.getName().endsWith("*")) { 11523 tableModel.setStarStmt("create_table"); 11524 } 11525 Process process = modelFactory.createProcess(stmt); 11526 relation.setProcess(process); 11527 } 11528 } 11529 continue; 11530 } 11531 if (!tableColumn.getName().endsWith("*") && tableModel.isDetermined()) { 11532 appendTableColumnToSQLEnv(tableModel, tableColumn); 11533 } 11534 11535 Object model = modelManager 11536 .getModel(resultColumn.getColumnObject()); 11537 if (model instanceof ResultColumn) { 11538 ResultColumn column = (ResultColumn) model; 11539 if (tableColumn.getName().endsWith("*") && column != null 11540 && !column.getStarLinkColumns().isEmpty()) { 11541 tableColumn.bindStarLinkColumns(column.getStarLinkColumns()); 11542 } 11543 11544 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 11545 relation.setEffectType(EffectType.create_table); 11546 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 11547 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 11548 if (tableColumn.getName().endsWith("*") && resultColumn.getName().endsWith("*")) { 11549 tableModel.setStarStmt("create_table"); 11550 } 11551 Process process = modelFactory.createProcess(stmt); 11552 relation.setProcess(process); 11553 } 11554 else if(model instanceof LinkedHashMap) { 11555 String columnName = getColumnNameOnly(resultColumn.getName()); 11556 LinkedHashMap<String, ResultColumn> resultColumns = (LinkedHashMap<String, ResultColumn>)model; 11557 if (columnObj.toString().endsWith("*")) { 11558 for (String key : resultColumns.keySet()) { 11559 tableColumn = modelFactory.createInsertTableColumn(tableModel, resultColumns.get(key).getName()); 11560 DataFlowRelationship relation = modelFactory 11561 .createDataFlowRelation(); 11562 relation.setEffectType(EffectType.create_table); 11563 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 11564 relation.addSource( 11565 new ResultColumnRelationshipElement(resultColumns.get(key))); 11566 Process process = modelFactory.createProcess(stmt); 11567 relation.setProcess(process); 11568 } 11569 } else if (resultColumns.containsKey(columnName)) { 11570 ResultColumn column = resultColumns.get(columnName); 11571 tableColumn = modelFactory.createInsertTableColumn(tableModel, column.getName()); 11572 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 11573 relation.setEffectType(EffectType.create_table); 11574 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 11575 relation.addSource(new ResultColumnRelationshipElement(column)); 11576 Process process = modelFactory.createProcess(stmt); 11577 relation.setProcess(process); 11578 } 11579 } 11580 } else { 11581 if (resultColumn.getName().endsWith("*")) { 11582 for (int j = 0; j < tableModel.getColumns().size(); j++) { 11583 tableColumn = tableModel.getColumns().get(j); 11584 ResultColumn column = (ResultColumn) modelManager 11585 .getModel(resultColumn.getColumnObject()); 11586 if (tableColumn.getName().endsWith("*") && column != null 11587 && !column.getStarLinkColumns().isEmpty()) { 11588 tableColumn.bindStarLinkColumns(column.getStarLinkColumns()); 11589 } 11590 11591 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 11592 relation.setEffectType(EffectType.create_table); 11593 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 11594 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 11595 if (tableColumn.getName().endsWith("*") 11596 && resultColumn.getName().endsWith("*")) { 11597 tableModel.setStarStmt("create_table"); 11598 ; 11599 } 11600 Process process = modelFactory.createProcess(stmt); 11601 relation.setProcess(process); 11602 } 11603 } else { 11604 tableColumn = tableModel.getColumns().get(i); 11605 Object model = modelManager 11606 .getModel(resultColumn.getColumnObject()); 11607 String columnName = getColumnNameOnly(resultColumn.getName()); 11608 if(model instanceof LinkedHashMap) { 11609 LinkedHashMap<String, ResultColumn> resultColumns = (LinkedHashMap<String, ResultColumn>)model; 11610 if (resultColumns.size() == tableModel.getColumns().size()) { 11611 int j = 0; 11612 for (String key : resultColumns.keySet()) { 11613 if (j == i) { 11614 ResultColumn column = resultColumns.get(key); 11615 DataFlowRelationship relation = modelFactory 11616 .createDataFlowRelation(); 11617 relation.setEffectType(EffectType.create_table); 11618 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 11619 relation.addSource(new ResultColumnRelationshipElement(column)); 11620 Process process = modelFactory.createProcess(stmt); 11621 relation.setProcess(process); 11622 } 11623 j++; 11624 } 11625 } else if (resultColumns.containsKey(columnName)) { 11626 ResultColumn column = resultColumns.get(columnName); 11627 tableColumn = modelFactory.createInsertTableColumn(tableModel, column.getName()); 11628 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 11629 relation.setEffectType(EffectType.create_table); 11630 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 11631 relation.addSource(new ResultColumnRelationshipElement(column)); 11632 Process process = modelFactory.createProcess(stmt); 11633 relation.setProcess(process); 11634 } else { 11635 throw new UnsupportedOperationException("Can't handle this star case."); 11636 } 11637 } 11638 else if (model instanceof ResultColumn) { 11639 ResultColumn column = (ResultColumn) modelManager 11640 .getModel(resultColumn.getColumnObject()); 11641 if (tableColumn.getName().endsWith("*") && column != null 11642 && !column.getStarLinkColumns().isEmpty()) { 11643 tableColumn.bindStarLinkColumns(column.getStarLinkColumns()); 11644 } 11645 11646 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 11647 relation.setEffectType(EffectType.create_table); 11648 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 11649 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 11650 if (tableColumn.getName().endsWith("*") 11651 && resultColumn.getName().endsWith("*")) { 11652 tableModel.setStarStmt("create_table"); 11653 } 11654 Process process = modelFactory.createProcess(stmt); 11655 relation.setProcess(process); 11656 } 11657 } 11658 } 11659 } else { 11660 TableColumn tableColumn = null; 11661 if (!hasDefinition) { 11662 TObjectName columnName = new TObjectName(); 11663 columnName.setString(resultColumn.getColumnObject().toString()); 11664 tableColumn = modelFactory.createTableColumn(tableModel, columnName, !hasDefinition); 11665 if(tableColumn == null){ 11666 continue; 11667 } 11668 } else { 11669 tableColumn = tableModel.getColumns().get(i); 11670 } 11671 ResultColumn column = (ResultColumn) modelManager.getModel(resultColumn.getColumnObject()); 11672 if (column != null && !column.getStarLinkColumns().isEmpty()) { 11673 tableColumn.bindStarLinkColumns(column.getStarLinkColumns()); 11674 } 11675 11676 if (!tableColumn.getName().endsWith("*") && tableModel.isDetermined()) { 11677 appendTableColumnToSQLEnv(tableModel, tableColumn); 11678 } 11679 11680 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 11681 relation.setEffectType(EffectType.create_table); 11682 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 11683 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 11684 Process process = modelFactory.createProcess(stmt); 11685 relation.setProcess(process); 11686 } 11687 } else if (resultColumn.getColumnObject() instanceof TObjectName) { 11688 TableColumn tableColumn = null; 11689 if (!hasDefinition) { 11690 tableColumn = modelFactory.createTableColumn(tableModel, 11691 (TObjectName) resultColumn.getColumnObject(), !hasDefinition); 11692 } else { 11693 tableColumn = tableModel.getColumns().get(i); 11694 } 11695 11696 if (!tableColumn.getName().endsWith("*") && tableModel.isDetermined()) { 11697 appendTableColumnToSQLEnv(tableModel, tableColumn); 11698 } 11699 11700 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 11701 relation.setEffectType(EffectType.create_table); 11702 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 11703 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 11704 Process process = modelFactory.createProcess(stmt); 11705 relation.setProcess(process); 11706 } 11707 } 11708 } else if (stmt.getSubQuery() != null) { 11709 SelectSetResultSet resultSetModel = (SelectSetResultSet) modelManager.getModel(stmt.getSubQuery()); 11710 if(resultSetModel.isDetermined()){ 11711 tableModel.setDetermined(true); 11712 tableModel.setFromDDL(true); 11713 } 11714 for (int i = 0; i < resultSetModel.getColumns().size(); i++) { 11715 ResultColumn resultColumn = resultSetModel.getColumns().get(i); 11716 if (resultColumn.getColumnObject() instanceof TResultColumn) { 11717 TResultColumn columnObject = (TResultColumn) resultColumn.getColumnObject(); 11718 11719 TAliasClause alias = columnObject.getAliasClause(); 11720 if (alias != null && alias.getAliasName() != null) { 11721 TableColumn tableColumn = null; 11722 if (!hasDefinition) { 11723 tableColumn = modelFactory.createTableColumn(tableModel, alias.getAliasName(), 11724 !hasDefinition); 11725 if (tableColumn == null) { 11726 continue; 11727 } 11728 } else { 11729 tableColumn = tableModel.getColumns().get(i); 11730 } 11731 11732 if (!tableColumn.getName().endsWith("*") && tableModel.isDetermined()) { 11733 appendTableColumnToSQLEnv(tableModel, tableColumn); 11734 } 11735 11736 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 11737 relation.setEffectType(EffectType.create_table); 11738 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 11739 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 11740 Process process = modelFactory.createProcess(stmt); 11741 relation.setProcess(process); 11742 } else if (columnObject.getFieldAttr() != null || (columnObject.getExpr()!=null && columnObject.getExpr().getExpressionType() == EExpressionType.typecast_t)) { 11743 TableColumn tableColumn = null; 11744 TObjectName columnObj = columnObject.getFieldAttr(); 11745 if(columnObj == null) { 11746 columnObj = columnObject.getExpr().getLeftOperand().getObjectOperand(); 11747 } 11748 if (!hasDefinition) { 11749 tableColumn = modelFactory.createTableColumn(tableModel, columnObj, 11750 !hasDefinition); 11751 if (tableColumn == null) { 11752 continue; 11753 } 11754 } else { 11755 tableColumn = tableModel.getColumns().get(i); 11756 } 11757 ResultColumn column = (ResultColumn) modelManager.getModel(resultColumn.getColumnObject()); 11758 if (column != null && !column.getStarLinkColumns().isEmpty()) { 11759 tableColumn.bindStarLinkColumns(column.getStarLinkColumns()); 11760 } 11761 11762 if (!tableColumn.getName().endsWith("*") && tableModel.isDetermined()) { 11763 appendTableColumnToSQLEnv(tableModel, tableColumn); 11764 } 11765 11766 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 11767 relation.setEffectType(EffectType.create_table); 11768 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 11769 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 11770 Process process = modelFactory.createProcess(stmt); 11771 relation.setProcess(process); 11772 } else { 11773 ErrorInfo errorInfo = new ErrorInfo(); 11774 errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR); 11775 errorInfo.setErrorMessage("Can't handle the table column " + columnObject.toString()); 11776 errorInfo.setStartPosition(new Pair3<Long, Long, String>( 11777 columnObject.getStartToken().lineNo, columnObject.getStartToken().columnNo, 11778 ModelBindingManager.getGlobalHash())); 11779 errorInfo.setEndPosition(new Pair3<Long, Long, String>(columnObject.getEndToken().lineNo, 11780 columnObject.getEndToken().columnNo + columnObject.getEndToken().getAstext().length(), 11781 ModelBindingManager.getGlobalHash())); 11782 errorInfos.add(errorInfo); 11783 continue; 11784 } 11785 } else if (resultColumn.getColumnObject() instanceof TObjectName) { 11786 TableColumn tableColumn = null; 11787 if (!hasDefinition) { 11788 tableColumn = modelFactory.createTableColumn(tableModel, 11789 (TObjectName) resultColumn.getColumnObject(), !hasDefinition); 11790 } else { 11791 tableColumn = tableModel.getColumns().get(i); 11792 } 11793 11794 if (!tableColumn.getName().endsWith("*") && tableModel.isDetermined()) { 11795 appendTableColumnToSQLEnv(tableModel, tableColumn); 11796 } 11797 11798 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 11799 relation.setEffectType(EffectType.create_table); 11800 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 11801 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 11802 Process process = modelFactory.createProcess(stmt); 11803 relation.setProcess(process); 11804 } 11805 } 11806 } else if (stmt.getLikeTableName() != null) { 11807 Table likeTableModel = modelFactory.createTableByName(stmt.getLikeTableName()); 11808 11809 if(likeTableModel.isCreateTable()){ 11810 for(TableColumn column: likeTableModel.getColumns()){ 11811 TObjectName tableColumn = new TObjectName(); 11812 tableColumn.setString(column.getName()); 11813 TableColumn createTableColumn = modelFactory.createTableColumn(tableModel, tableColumn, true); 11814 createTableColumn.setPrimaryKey(column.getPrimaryKey()); 11815 createTableColumn.setForeignKey(column.getForeignKey()); 11816 createTableColumn.setIndexKey(column.getIndexKey()); 11817 createTableColumn.setUnqiueKey(column.getUnqiueKey()); 11818 createTableColumn.setDataType(column.getDataType()); 11819 } 11820 tableModel.setCreateTable(true); 11821 } 11822 11823// Process process = modelFactory.createProcess(stmt); 11824// process.setType("Like Table"); 11825// tableModel.addProcess(process); 11826// 11827// 11828// DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 11829// relation.setEffectType(EffectType.like_table); 11830// relation.setTarget( 11831// new RelationRowsRelationshipElement<TableRelationRows>(tableModel.getRelationRows())); 11832// relation.addSource( 11833// new RelationRowsRelationshipElement<TableRelationRows>(likeTableModel.getRelationRows())); 11834// relation.setProcess(process); 11835 } 11836 11837 if (stmt.getStageLocation() != null && stmt.getStageLocation().getStageName() != null) { 11838 tableModel 11839 .setLocation(DlineageUtil.getTableFullName(stmt.getStageLocation().getStageName().toString())); 11840 Process process = modelFactory.createProcess(stmt); 11841 process.setType("Create External Table"); 11842 tableModel.addProcess(process); 11843 Table stage = modelManager.getTableByName(DlineageUtil.getTableFullName(tableModel.getLocation())); 11844 if (stage == null) { 11845 stage = modelManager.getTableByName( 11846 DlineageUtil.getTableFullName(stmt.getStageLocation().toString().replaceFirst("@", ""))); 11847 } 11848 if (stage == null) { 11849 stage = modelFactory.createTableByName(stmt.getStageLocation().toString().replaceFirst("@", ""), 11850 false); 11851 stage.setCreateTable(false); 11852 stage.setStage(true); 11853 if (stmt.getStageLocation().getPath() != null) { 11854 stage.setLocation(stmt.getStageLocation().getPath().toString()); 11855 TObjectName location = new TObjectName(); 11856 location.setString(stmt.getStageLocation().getPath().toString()); 11857 modelFactory.createStageLocation(stage, location); 11858 } else if (stmt.getRegex_pattern() != null) { 11859 stage.setLocation(stmt.getRegex_pattern()); 11860 TObjectName location = new TObjectName(); 11861 location.setString(stmt.getRegex_pattern()); 11862 modelFactory.createStageLocation(stage, location); 11863 } else { 11864 stage.setLocation("unknownPath"); 11865 TObjectName location = new TObjectName(); 11866 location.setString("unknownPath"); 11867 modelFactory.createStageLocation(stage, location); 11868 } 11869 } 11870 if (stage != null && !stage.getColumns().isEmpty()) { 11871 if (!tableModel.getColumns().isEmpty()) { 11872 for (int i = 0; i < tableModel.getColumns().size(); i++) { 11873 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 11874 relation.addSource(new TableColumnRelationshipElement(stage.getColumns().get(0))); 11875 relation.setTarget(new TableColumnRelationshipElement(tableModel.getColumns().get(i))); 11876 relation.setProcess(process); 11877 } 11878 } else { 11879 TObjectName starColumn = new TObjectName(); 11880 starColumn.setString("*"); 11881 TableColumn tableColumn = modelFactory.createTableColumn(tableModel, starColumn, true); 11882 tableColumn.setExpandStar(false); 11883 tableColumn.setShowStar(true); 11884 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 11885 relation.addSource(new TableColumnRelationshipElement(stage.getColumns().get(0))); 11886 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 11887 relation.setProcess(process); 11888 } 11889 } 11890 } else if (stmt.getTableOptions() != null && !stmt.getTableOptions().isEmpty()) { 11891 for (int i = 0; i < stmt.getTableOptions().size(); i++) { 11892 TCreateTableOption createTableOption = stmt.getTableOptions().get(i); 11893 if (createTableOption.getCreateTableOptionType() != ECreateTableOption.etoBigQueryExternal) 11894 continue; 11895 List<String> uris = createTableOption.getUris(); 11896 if (uris == null || uris.isEmpty()) 11897 continue; 11898 Process process = modelFactory.createProcess(stmt); 11899 process.setType("Create External Table"); 11900 tableModel.addProcess(process); 11901 if (tableModel.getColumns().isEmpty()) { 11902 TObjectName starColumn = new TObjectName(); 11903 starColumn.setString("*"); 11904 TableColumn tableColumn = modelFactory.createTableColumn(tableModel, starColumn, true); 11905 tableColumn.setExpandStar(false); 11906 tableColumn.setShowStar(true); 11907 } 11908 for (String uri : uris) { 11909 Table uriFile = modelFactory.createTableByName(uri, true); 11910 uriFile.setPath(true); 11911 uriFile.setFileFormat(createTableOption.getFormat()); 11912 for (int j = 0; j < tableModel.getColumns().size(); j++) { 11913 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 11914 if (stmt.getColumnList() != null) { 11915 TObjectName fileUri = new TObjectName(); 11916 fileUri.setString(tableModel.getColumns().get(j).getColumnObject().toString()); 11917 TableColumn fileUriColumn = modelFactory.createFileUri(uriFile, fileUri); 11918 relation.addSource(new TableColumnRelationshipElement(fileUriColumn)); 11919 } else { 11920 TObjectName fileUri = new TObjectName(); 11921 fileUri.setString("*"); 11922 TableColumn fileUriColumn = modelFactory.createFileUri(uriFile, fileUri); 11923 relation.addSource(new TableColumnRelationshipElement(fileUriColumn)); 11924 } 11925 relation.setTarget(new TableColumnRelationshipElement(tableModel.getColumns().get(j))); 11926 relation.setProcess(process); 11927 } 11928 if (tableModel.getColumns().isEmpty()) { 11929 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 11930 TObjectName fileUri = new TObjectName(); 11931 fileUri.setString("*"); 11932 TableColumn fileUriColumn = modelFactory.createFileUri(uriFile, fileUri); 11933 relation.addSource(new TableColumnRelationshipElement(fileUriColumn)); 11934 relation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>( 11935 tableModel.getRelationRows())); 11936 relation.setProcess(process); 11937 } 11938 } 11939 } 11940 } else if (stmt.getSubQuery() == null && stmt.getTableLocation() != null) { 11941 Process process = modelFactory.createProcess(stmt); 11942 process.setType("Create External Table"); 11943 tableModel.addProcess(process); 11944 Table uriFile = modelFactory.createTableByName(stmt.getTableLocation(), true); 11945 uriFile.setPath(true); 11946 for (int j = 0; j < tableModel.getColumns().size(); j++) { 11947 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 11948 if (stmt.getColumnList() != null) { 11949 TObjectName fileUri = new TObjectName(); 11950 fileUri.setString(tableModel.getColumns().get(j).getColumnObject().toString()); 11951 TableColumn fileUriColumn = modelFactory.createFileUri(uriFile, fileUri); 11952 relation.addSource(new TableColumnRelationshipElement(fileUriColumn)); 11953 } else { 11954 TObjectName fileUri = new TObjectName(); 11955 fileUri.setString("*"); 11956 TableColumn fileUriColumn = modelFactory.createFileUri(uriFile, fileUri); 11957 relation.addSource(new TableColumnRelationshipElement(fileUriColumn)); 11958 } 11959 relation.setTarget(new TableColumnRelationshipElement(tableModel.getColumns().get(j))); 11960 relation.setProcess(process); 11961 } 11962 } 11963 11964 // Inline (column-level) FK relationships. Emitted here, after the 11965 // column loop, rather than inline in it, so the referenced-column / 11966 // relation / process model objects allocate their ids AFTER all 11967 // column ids (matching the table-level pass below) and column ids 11968 // stay stable. Mirrors the table-level FK emission: source = 11969 // referenced column, target = this FK column, effect = foreign_key, 11970 // plus the optional ER relationship. EConstraintType.reference is 11971 // the inline REFERENCES form; foreign_key the inline FOREIGN KEY. 11972 if (stmt.getColumnList() != null) { 11973 for (int i = 0; i < stmt.getColumnList().size(); i++) { 11974 TColumnDefinition fkColumnDef = stmt.getColumnList().getColumn(i); 11975 TConstraintList fkConstraints = fkColumnDef.getConstraints(); 11976 if (fkConstraints == null) { 11977 continue; 11978 } 11979 for (int c = 0; c < fkConstraints.size(); c++) { 11980 TConstraint inlineConstraint = fkConstraints.getConstraint(c); 11981 EConstraintType inlineType = inlineConstraint.getConstraint_type(); 11982 if (inlineType != EConstraintType.foreign_key 11983 && inlineType != EConstraintType.reference) { 11984 continue; 11985 } 11986 TObjectName referencedTableName = inlineConstraint.getReferencedObject(); 11987 TObjectNameList referencedTableColumns = inlineConstraint.getReferencedColumnList(); 11988 if (referencedTableName == null || referencedTableColumns == null) { 11989 continue; 11990 } 11991 TableColumn fkColumn = modelFactory.createTableColumn(tableModel, 11992 fkColumnDef.getColumnName(), true); 11993 if (fkColumn == null) { 11994 continue; 11995 } 11996 Table referencedTable = modelManager.getTableByName( 11997 DlineageUtil.getTableFullName(referencedTableName.toString())); 11998 if (referencedTable == null) { 11999 referencedTable = modelFactory.createTableByName(referencedTableName); 12000 } 12001 for (int j = 0; j < referencedTableColumns.size(); j++) { 12002 TableColumn referencedColumn = modelFactory.createTableColumn(referencedTable, 12003 referencedTableColumns.getObjectName(j), false); 12004 if (referencedColumn != null) { 12005 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 12006 relation.addSource(new TableColumnRelationshipElement(referencedColumn)); 12007 relation.setTarget(new TableColumnRelationshipElement(fkColumn)); 12008 relation.setEffectType(EffectType.foreign_key); 12009 Process process = modelFactory.createProcess(stmt); 12010 relation.setProcess(process); 12011 if (this.option.isShowERDiagram()) { 12012 ERRelationship erRelation = modelFactory.createERRelation(); 12013 erRelation.addSource(new TableColumnRelationshipElement(referencedColumn)); 12014 erRelation.setTarget(new TableColumnRelationshipElement(fkColumn)); 12015 } 12016 } 12017 } 12018 } 12019 } 12020 } 12021 12022 if (stmt.getTableConstraints() != null && stmt.getTableConstraints().size() > 0) { 12023 for (int i = 0; i < stmt.getTableConstraints().size(); i++) { 12024 TConstraint createTableConstraint = stmt.getTableConstraints().getConstraint(i); 12025 TPTNodeList<TColumnWithSortOrder> keyNames = createTableConstraint.getColumnList(); 12026 if (keyNames != null) { 12027 for (int k = 0; k < keyNames.size(); k++) { 12028 TObjectName keyName = keyNames.getElement(k).getColumnName(); 12029 // Skip functional indexes (expression-based indexes) where columnName is null 12030 if (keyName == null) { 12031 continue; 12032 } 12033 TObjectName referencedTableName = createTableConstraint.getReferencedObject(); 12034 TObjectNameList referencedTableColumns = createTableConstraint.getReferencedColumnList(); 12035 12036 TableColumn tableConstraint = modelFactory.createTableColumn(tableModel, keyName, true); 12037 if (createTableConstraint.getConstraint_type() == EConstraintType.primary_key) { 12038 tableConstraint.setPrimaryKey(true); 12039 } else if (createTableConstraint.getConstraint_type() == EConstraintType.table_index) { 12040 tableConstraint.setIndexKey(true); 12041 } else if (createTableConstraint.getConstraint_type() == EConstraintType.unique) { 12042 tableConstraint.setUnqiueKey(true); 12043 } else if (createTableConstraint.getConstraint_type() == EConstraintType.foreign_key) { 12044 tableConstraint.setForeignKey(true); 12045 Table referencedTable = modelManager 12046 .getTableByName(DlineageUtil.getTableFullName(referencedTableName.toString())); 12047 if (referencedTable == null) { 12048 referencedTable = modelFactory.createTableByName(referencedTableName); 12049 } 12050 if (k == keyNames.size() - 1) { 12051 captureCompositeForeignKey(tableModel, keyNames, referencedTable, 12052 referencedTableColumns); 12053 } 12054 12055 if (referencedTableColumns != null) { 12056 for (int j = 0; j < referencedTableColumns.size(); j++) { 12057 TableColumn tableColumn = modelFactory.createTableColumn(referencedTable, 12058 referencedTableColumns.getObjectName(j), false); 12059 if (tableColumn != null) { 12060 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 12061 relation.addSource(new TableColumnRelationshipElement(tableColumn)); 12062 relation.setTarget(new TableColumnRelationshipElement(tableConstraint)); 12063 relation.setEffectType(EffectType.foreign_key); 12064 Process process = modelFactory.createProcess(stmt); 12065 relation.setProcess(process); 12066 if (this.option.isShowERDiagram()) { 12067 ERRelationship erRelation = modelFactory.createERRelation(); 12068 erRelation.addSource(new TableColumnRelationshipElement(tableColumn)); 12069 erRelation 12070 .setTarget(new TableColumnRelationshipElement(tableConstraint)); 12071 } 12072 } 12073 } 12074 } 12075 } 12076 } 12077 } 12078 } 12079 } 12080 12081 if (stmt.getHiveTablePartition() != null && stmt.getHiveTablePartition().getColumnDefList() != null) { 12082 for (int i = 0; i < stmt.getHiveTablePartition().getColumnDefList().size(); i++) { 12083 TColumnDefinition column = stmt.getHiveTablePartition().getColumnDefList().getColumn(i); 12084 modelFactory.createTableColumn(tableModel, column.getColumnName(), true); 12085 appendTableColumnToSQLEnv(tableModel, column.getColumnName()); 12086 } 12087 } 12088 12089 analyzeDynamicTableRefresh(stmt, tableModel); 12090 12091 } else { 12092 ErrorInfo errorInfo = new ErrorInfo(); 12093 errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR); 12094 errorInfo.setErrorMessage("Can't get target table. CreateTableSqlStatement is " + stmt.toString()); 12095 errorInfo.setStartPosition(new Pair3<Long, Long, String>(stmt.getStartToken().lineNo, 12096 stmt.getStartToken().columnNo, ModelBindingManager.getGlobalHash())); 12097 errorInfo.setEndPosition(new Pair3<Long, Long, String>(stmt.getEndToken().lineNo, 12098 stmt.getEndToken().columnNo + stmt.getEndToken().getAstext().length(), 12099 ModelBindingManager.getGlobalHash())); 12100 errorInfos.add(errorInfo); 12101 } 12102 } 12103 12104 private void analyzeDynamicTableRefresh(TCreateTableSqlStatement stmt, 12105 Table dynamicTable) { 12106 TCustomSqlStatement refreshStatement = stmt.getRefreshUsingStatement(); 12107 if (refreshStatement == null || refreshStatement.getTargetTable() == null) { 12108 return; 12109 } 12110 String targetToken = refreshStatement.getTargetTable().getTableName() 12111 .toString(); 12112 if (!"SELF".equalsIgnoreCase(targetToken)) { // non-identifier-compare: Snowflake pseudo-target keyword 12113 return; 12114 } 12115 12116 // Snowflake defines SELF as the dynamic table currently being created. 12117 // Bind the lexical SELF target to that existing model before running the 12118 // normal INSERT/MERGE analyzer, so all emitted target edges use the real 12119 // dynamic-table identity and its declared columns. 12120 modelManager.bindModel(refreshStatement.getTargetTable(), dynamicTable); 12121 analyzeCustomSqlStmt(refreshStatement); 12122 } 12123 12124 private void captureCompositeForeignKey(Table foreignKeyTable, 12125 TPTNodeList<TColumnWithSortOrder> foreignKeyNames, Table referencedTable, 12126 TObjectNameList referencedNames) { 12127 if (!shouldCollectAuthoritativeLineageEvidence() || foreignKeyTable == null 12128 || referencedTable == null || foreignKeyNames == null 12129 || referencedNames == null || foreignKeyNames.size() < 2 12130 || foreignKeyNames.size() != referencedNames.size()) return; 12131 List<TableColumn> foreignKeyFields = new ArrayList<TableColumn>(); 12132 List<TableColumn> referencedFields = new ArrayList<TableColumn>(); 12133 for (int ordinal = 0; ordinal < foreignKeyNames.size(); ordinal++) { 12134 TColumnWithSortOrder foreignKey = foreignKeyNames.getElement(ordinal); 12135 if (foreignKey == null || foreignKey.getColumnName() == null 12136 || referencedNames.getObjectName(ordinal) == null) return; 12137 TableColumn local = findExistingTableColumn(foreignKeyTable, 12138 foreignKey.getColumnName()); 12139 TableColumn referenced = findExistingTableColumn(referencedTable, 12140 referencedNames.getObjectName(ordinal)); 12141 if (local == null || referenced == null) return; 12142 foreignKeyFields.add(local); 12143 referencedFields.add(referenced); 12144 } 12145 authoritativeEvidenceCollector.addCompositeForeignKey(foreignKeyFields, referencedFields); 12146 } 12147 12148 /** Lookup-only: evidence capture must never create or mutate legacy model nodes. */ 12149 private TableColumn findExistingTableColumn(Table table, TObjectName columnName) { 12150 Object exact = modelManager.getModel(new Pair<Table, TObjectName>(table, columnName)); 12151 if (exact instanceof TableColumn && ((TableColumn) exact).getTable() == table) { 12152 return (TableColumn) exact; 12153 } 12154 if (table.getColumns() == null) return null; 12155 for (TableColumn candidate : table.getColumns()) { 12156 if (candidate != null && DlineageUtil.sameColumnName(candidate.getName(), columnName)) { 12157 return candidate; 12158 } 12159 } 12160 return null; 12161 } 12162 12163 protected void flattenStructColumns(boolean hasDefinition, Table tableModel, TColumnDefinition column, 12164 Stack<TColumnDefinition> columnPaths, int index) { 12165 columnPaths.push(column); 12166 for (int j = 0; j < column.getDatatype().getColumnDefList().size(); j++) { 12167 TColumnDefinition columnDefinition = column.getDatatype().getColumnDefList().getColumn(j); 12168 if (columnDefinition.getDatatype().getColumnDefList() != null) { 12169 flattenStructColumns(hasDefinition, tableModel, columnDefinition, columnPaths, index); 12170 } else { 12171 TObjectName columnName = new TObjectName(); 12172 columnName.setString(getColumnName(columnPaths, columnDefinition)); 12173 TableColumn tableColumn = modelFactory.createTableColumn(tableModel, columnName, hasDefinition); 12174 tableColumn.setColumnIndex(index); 12175 tableColumn.setStruct(true); 12176 12177 if(option.getAnalyzeMode() == AnalyzeMode.crud) { 12178 CrudRelationship crudRelationship = modelFactory.createCrudRelation(); 12179 crudRelationship.setTarget(new TableColumnRelationshipElement(tableColumn)); 12180 crudRelationship.setEffectType(EffectType.create_table); 12181 } 12182 12183 appendTableColumnToSQLEnv(tableModel, tableColumn); 12184 } 12185 } 12186 columnPaths.pop(); 12187 } 12188 12189 private String getColumnName(Stack<TColumnDefinition> columnPaths, TColumnDefinition column) { 12190 StringBuilder buffer = new StringBuilder(); 12191 Iterator<TColumnDefinition> iter = columnPaths.iterator(); 12192 while(iter.hasNext()) { 12193 buffer.append(iter.next().getColumnName().getColumnNameOnly()).append("."); 12194 } 12195 buffer.append(column.getColumnName().getColumnNameOnly()); 12196 return buffer.toString(); 12197 } 12198 12199 private void appendTableColumnToSQLEnv(Table tableModel, TableColumn tableColumn) { 12200 //tableModel如果非determined,请不要添加到sqlenv里 12201 if (sqlenv != null && tableColumn!=null) { 12202 TSQLSchema schema = sqlenv.getSQLSchema(DlineageUtil.getTableSchema(tableModel), true); 12203 if (schema != null) { 12204 TSQLTable tempTable = schema.createTable(DlineageUtil.getSimpleTableName(tableModel.getName())); 12205 if (tableColumn.hasStarLinkColumn()) { 12206 for (String column : tableColumn.getStarLinkColumnNames()) { 12207 tempTable.addColumn(DlineageUtil.getColumnNameOnly(column)); 12208 } 12209 } else { 12210 tempTable.addColumn(DlineageUtil.getColumnNameOnly(tableColumn.getName())); 12211 } 12212 } 12213 } 12214 } 12215 12216 private void appendTableColumnToSQLEnv(Table tableModel, TObjectName tableColumn) { 12217 if (sqlenv != null) { 12218 TSQLSchema schema = sqlenv.getSQLSchema(DlineageUtil.getTableSchema(tableModel), true); 12219 if (schema != null) { 12220 TSQLTable tempTable = schema.createTable(DlineageUtil.getSimpleTableName(tableModel.getName())); 12221 tempTable.addColumn(DlineageUtil.getColumnNameOnly(tableColumn.getColumnNameOnly())); 12222 } 12223 } 12224 } 12225 12226 private void analyzeCreateStageStmt(TCreateStageStmt stmt) { 12227 TObjectName stageName = stmt.getStageName(); 12228 12229 if (stageName != null) { 12230 12231 Table tableModel = modelFactory.createStage(stageName); 12232 tableModel.setCreateTable(true); 12233 tableModel.setStage(true); 12234 tableModel.setLocation(stmt.getExternalStageURL()); 12235 12236 Process process = modelFactory.createProcess(stmt); 12237 tableModel.addProcess(process); 12238 12239 TableColumn locationColumn = null; 12240 if (stmt.getExternalStageURL() != null) { 12241 TObjectName location = new TObjectName(); 12242 location.setString(stmt.getExternalStageURL()); 12243 locationColumn = modelFactory.createStageLocation(tableModel, location); 12244 12245 Table pathModel = modelFactory.createTableByName(stmt.getExternalStageURL(), true); 12246 pathModel.setPath(true); 12247 pathModel.setCreateTable(true); 12248 TObjectName fileUri = new TObjectName(); 12249 fileUri.setString(stmt.getExternalStageURL()); 12250 TableColumn fileUriColumn = modelFactory.createFileUri(pathModel, fileUri); 12251 12252 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 12253 relation.addSource(new TableColumnRelationshipElement(fileUriColumn)); 12254 relation.setTarget(new TableColumnRelationshipElement(locationColumn)); 12255 relation.setProcess(process); 12256 12257 } else { 12258 for (TCustomSqlStatement temp : stmt.getGsqlparser().getSqlstatements()) { 12259 if (temp instanceof TPutStmt) { 12260 TPutStmt put = (TPutStmt) temp; 12261 TStageLocation stageLocation = put.getStageLocation(); 12262 if (stageLocation != null && stageLocation.getStageName() != null) { 12263 Table stage = modelManager.getTableByName( 12264 DlineageUtil.getTableFullName(stageLocation.getStageName().toString())); 12265 if (stage == tableModel) { 12266 TObjectName location = new TObjectName(); 12267 if (!SQLUtil.isEmpty(put.getFileName())) { 12268 location.setString(put.getFileName()); 12269 locationColumn = modelFactory.createStageLocation(tableModel, location); 12270 } 12271 break; 12272 } 12273 } 12274 } 12275 } 12276 } 12277 12278 String fileFormat = stmt.getFileFormatName(); 12279 if (fileFormat != null) { 12280 for (TCustomSqlStatement temp : stmt.getGsqlparser().getSqlstatements()) { 12281 if (temp instanceof TCreateFileFormatStmt) { 12282 TCreateFileFormatStmt fileFormatStmt = (TCreateFileFormatStmt) temp; 12283 if (fileFormatStmt.getFileFormatName() != null 12284 && SQLUtil.compareIdentifier(option.getVendor(), ESQLDataObjectType.dotTable, fileFormatStmt.getFileFormatName().toString(), fileFormat)) { 12285 tableModel.setFileType(fileFormatStmt.getTypeName()); 12286 break; 12287 } 12288 } 12289 } 12290 } 12291 12292 String procedureParent = getProcedureParentName(stmt); 12293 if (procedureParent != null) { 12294 tableModel.setParent(procedureParent); 12295 } 12296 12297 if (locationColumn != null) { 12298 List<Table> tables = modelManager.getTablesByName(); 12299 if (tables != null) { 12300 for (Table referTable : tables) { 12301 if (referTable.getLocation() != null && referTable.getLocation() 12302 .equals(DlineageUtil.getIdentifierNormalTableName(tableModel.getName()))) { 12303 for (int i = 0; i < referTable.getColumns().size(); i++) { 12304 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 12305 relation.addSource(new TableColumnRelationshipElement(locationColumn)); 12306 relation.setTarget(new TableColumnRelationshipElement(referTable.getColumns().get(i))); 12307 } 12308 } 12309 } 12310 } 12311 } 12312 } else { 12313 ErrorInfo errorInfo = new ErrorInfo(); 12314 errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR); 12315 errorInfo.setErrorMessage("Can't get target table. CreateStageStmt is " + stmt.toString()); 12316 errorInfo.setStartPosition(new Pair3<Long, Long, String>(stmt.getStartToken().lineNo, 12317 stmt.getStartToken().columnNo, ModelBindingManager.getGlobalHash())); 12318 errorInfo.setEndPosition(new Pair3<Long, Long, String>(stmt.getEndToken().lineNo, 12319 stmt.getEndToken().columnNo + stmt.getEndToken().getAstext().length(), 12320 ModelBindingManager.getGlobalHash())); 12321 errorInfo.fillInfo(this); 12322 errorInfos.add(errorInfo); 12323 } 12324 } 12325 12326 private void analyzeCreateExternalDataSourceStmt(TCreateExternalDataSourceStmt stmt) { 12327 TObjectName dataSourceName = stmt.getDataSourceName(); 12328 String locationUrl = stmt.getOption("LOCATION"); 12329 if (dataSourceName != null && locationUrl != null) { 12330 Table tableModel = modelFactory.createDataSource(dataSourceName); 12331 tableModel.setCreateTable(true); 12332 tableModel.setDataSource(true); 12333 tableModel.setLocation(locationUrl); 12334 12335 Process process = modelFactory.createProcess(stmt); 12336 tableModel.addProcess(process); 12337 12338 TObjectName location = new TObjectName(); 12339 location.setString(locationUrl); 12340 modelFactory.createTableColumn(tableModel, location, true); 12341 12342 String procedureParent = getProcedureParentName(stmt); 12343 if (procedureParent != null) { 12344 tableModel.setParent(procedureParent); 12345 } 12346 } else { 12347 ErrorInfo errorInfo = new ErrorInfo(); 12348 errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR); 12349 errorInfo.setErrorMessage("Can't get target table. CreateExternalDataSourceStmt is " + stmt.toString()); 12350 errorInfo.setStartPosition(new Pair3<Long, Long, String>(stmt.getStartToken().lineNo, 12351 stmt.getStartToken().columnNo, ModelBindingManager.getGlobalHash())); 12352 errorInfo.setEndPosition(new Pair3<Long, Long, String>(stmt.getEndToken().lineNo, 12353 stmt.getEndToken().columnNo + stmt.getEndToken().getAstext().length(), 12354 ModelBindingManager.getGlobalHash())); 12355 errorInfo.fillInfo(this); 12356 errorInfos.add(errorInfo); 12357 } 12358 } 12359 12360 private void analyzeCreateStreamStmt(TCreateStreamStmt stmt) { 12361 TObjectName streamName = stmt.getStreamName(); 12362 12363 if (streamName != null) { 12364 Table tableModel = modelFactory.createTableByName(stmt.getTableName()); 12365 tableModel.setCreateTable(true); 12366 12367 Table streamModel = modelFactory.createStream(streamName); 12368 streamModel.setCreateTable(true); 12369 streamModel.setStream(true); 12370 12371 Process process = modelFactory.createProcess(stmt); 12372 tableModel.addProcess(process); 12373 12374 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 12375 relation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(streamModel.getRelationRows())); 12376 relation.addSource(new RelationRowsRelationshipElement<TableRelationRows>(tableModel.getRelationRows())); 12377 relation.setProcess(process); 12378 } 12379 } 12380 12381 private String getProcedureParentName(TCustomSqlStatement stmt) { 12382 if (stmt instanceof TStoredProcedureSqlStatement) { 12383 if (((TStoredProcedureSqlStatement) stmt).getStoredProcedureName() != null) { 12384 return ((TStoredProcedureSqlStatement) stmt).getStoredProcedureName().toString(); 12385 } 12386 } 12387 12388 stmt = stmt.getParentStmt(); 12389 if (stmt == null) 12390 return null; 12391 12392 if (stmt instanceof TCommonBlock) { 12393 if(((TCommonBlock) stmt).getBlockBody().getParentObjectName() instanceof TStoredProcedureSqlStatement) { 12394 stmt = (TStoredProcedureSqlStatement)((TCommonBlock) stmt).getBlockBody().getParentObjectName(); 12395 } 12396 } 12397 12398 if (stmt instanceof TStoredProcedureSqlStatement) { 12399 if (((TStoredProcedureSqlStatement) stmt).getStoredProcedureName() != null) { 12400 return ((TStoredProcedureSqlStatement) stmt).getStoredProcedureName().toString(); 12401 } 12402 } 12403 if (stmt instanceof TTeradataCreateProcedure) { 12404 if (((TTeradataCreateProcedure) stmt).getProcedureName() != null) { 12405 return ((TTeradataCreateProcedure) stmt).getProcedureName().toString(); 12406 } 12407 } 12408 12409 return getProcedureParentName(stmt); 12410 } 12411 12412 private TStoredProcedureSqlStatement getProcedureParent(TCustomSqlStatement stmt) { 12413 if (stmt instanceof TStoredProcedureSqlStatement) { 12414 return (TStoredProcedureSqlStatement) stmt; 12415 } 12416 12417 stmt = stmt.getParentStmt(); 12418 if (stmt == null) 12419 return null; 12420 12421 if (stmt instanceof TCommonBlock) { 12422 if (((TCommonBlock) stmt).getBlockBody().getParentObjectName() instanceof TStoredProcedureSqlStatement) { 12423 stmt = (TStoredProcedureSqlStatement) ((TCommonBlock) stmt).getBlockBody().getParentObjectName(); 12424 } 12425 } 12426 12427 if (stmt instanceof TStoredProcedureSqlStatement) { 12428 return ((TStoredProcedureSqlStatement) stmt); 12429 } 12430 if (stmt instanceof TTeradataCreateProcedure) { 12431 return ((TTeradataCreateProcedure) stmt); 12432 } 12433 12434 return getProcedureParent(stmt); 12435 } 12436 12437 private static final class CteWriteIntent { 12438 private final List<TObjectName> columns = new ArrayList<TObjectName>(); 12439 private String unsupportedReason; 12440 } 12441 12442 private static final class CteTargetColumnMapping { 12443 private final String exposedName; 12444 private final String baseColumnName; 12445 12446 private CteTargetColumnMapping(String exposedName, String baseColumnName) { 12447 this.exposedName = exposedName; 12448 this.baseColumnName = baseColumnName; 12449 } 12450 } 12451 12452 private static final class CteWriteTargetResolution { 12453 private TTable baseTable; 12454 private final List<CteTargetColumnMapping> columns = new ArrayList<CteTargetColumnMapping>(); 12455 private String failureReason; 12456 12457 private boolean isWritable() { 12458 return baseTable != null; 12459 } 12460 } 12461 12462 private static final class ProjectedColumnResolution { 12463 private TTable baseTable; 12464 private String baseColumnName; 12465 private String failureReason; 12466 } 12467 12468 private static final class CteSelectRelationKey { 12469 private final ResultColumn target; 12470 private final ResultColumn source; 12471 12472 private CteSelectRelationKey(ResultColumn target, ResultColumn source) { 12473 this.target = target; 12474 this.source = source; 12475 } 12476 12477 @Override 12478 public int hashCode() { 12479 return 31 * System.identityHashCode(target) + System.identityHashCode(source); 12480 } 12481 12482 @Override 12483 public boolean equals(Object other) { 12484 if (!(other instanceof CteSelectRelationKey)) { 12485 return false; 12486 } 12487 CteSelectRelationKey key = (CteSelectRelationKey) other; 12488 return target == key.target && source == key.source; 12489 } 12490 } 12491 12492 private CteWriteIntent collectMergeCteWriteIntent(TMergeSqlStatement stmt, TTable targetTable) { 12493 CteWriteIntent intent = new CteWriteIntent(); 12494 if (stmt.getWhenClauses() == null) { 12495 intent.unsupportedReason = "the MERGE has no writable target columns"; 12496 return intent; 12497 } 12498 for (int i = 0; i < stmt.getWhenClauses().size(); i++) { 12499 TMergeWhenClause clause = stmt.getWhenClauses().getElement(i); 12500 if (clause.getUpdateClause() != null) { 12501 TResultColumnList assignments = clause.getUpdateClause().getUpdateColumnList(); 12502 if (assignments != null) { 12503 for (int j = 0; j < assignments.size(); j++) { 12504 TExpression assignment = assignments.getResultColumn(j).getExpr(); 12505 TExpression target = assignment == null ? null : assignment.getLeftOperand(); 12506 if (target == null || target.getExpressionType() != EExpressionType.simple_object_name_t) { 12507 addCteWriteIntentWarning(intent, 12508 "an UPDATE assignment target cannot be mapped to a CTE projection"); 12509 continue; 12510 } 12511 addCteWriteIntentColumn(intent, target.getObjectOperand()); 12512 } 12513 } 12514 } 12515 if (clause.getInsertClause() != null) { 12516 if (clause.getInsertClause().getInsertValue() != null) { 12517 addCteWriteIntentWarning(intent, 12518 "an object-constructor INSERT target cannot be mapped to CTE projections"); 12519 continue; 12520 } 12521 TObjectNameList columns = clause.getInsertClause().getColumnList(); 12522 if (columns == null || columns.size() == 0) { 12523 List<TObjectName> inferredColumns = inferProjectedInsertColumns(targetTable, 12524 clause.getInsertClause().getValuelist()); 12525 if (inferredColumns == null) { 12526 addCteWriteIntentWarning(intent, 12527 "an INSERT without an explicit target column list cannot be mapped safely"); 12528 continue; 12529 } 12530 for (TObjectName inferredColumn : inferredColumns) { 12531 addCteWriteIntentColumn(intent, inferredColumn); 12532 } 12533 continue; 12534 } 12535 for (int j = 0; j < columns.size(); j++) { 12536 addCteWriteIntentColumn(intent, columns.getObjectName(j)); 12537 } 12538 } 12539 } 12540 if (intent.columns.isEmpty() && intent.unsupportedReason == null) { 12541 intent.unsupportedReason = "the MERGE has no modified columns from which to resolve a base table"; 12542 } 12543 return intent; 12544 } 12545 12546 private void addCteWriteIntentWarning(CteWriteIntent intent, String reason) { 12547 if (intent.unsupportedReason == null) { 12548 intent.unsupportedReason = reason; 12549 } else if (!intent.unsupportedReason.contains(reason)) { 12550 intent.unsupportedReason += "; " + reason; 12551 } 12552 } 12553 12554 private List<TObjectName> inferProjectedInsertColumns(TTable targetTable, TResultColumnList values) { 12555 if (targetTable == null || values == null) { 12556 return null; 12557 } 12558 TSelectSqlStatement subquery = null; 12559 TObjectNameList exposedColumns = null; 12560 if (targetTable.getCTE() != null) { 12561 subquery = targetTable.getCTE().getSubquery(); 12562 exposedColumns = targetTable.getCTE().getColumnList(); 12563 } else if (targetTable.getSubquery() != null) { 12564 subquery = targetTable.getSubquery(); 12565 exposedColumns = getProjectedTargetColumns(targetTable); 12566 } else if (targetTable.getLinkTable() != null 12567 && targetTable.getLinkTable().getSubquery() != null) { 12568 subquery = targetTable.getLinkTable().getSubquery(); 12569 exposedColumns = getProjectedTargetColumns(targetTable.getLinkTable()); 12570 } 12571 TResultColumnList projections = subquery == null ? null : subquery.getResultColumnList(); 12572 if (projections == null || projections.size() != values.size()) { 12573 return null; 12574 } 12575 if (exposedColumns != null && exposedColumns.size() > 0 12576 && exposedColumns.size() != projections.size()) { 12577 return null; 12578 } 12579 List<TObjectName> inferredColumns = new ArrayList<TObjectName>(); 12580 for (int i = 0; i < projections.size(); i++) { 12581 TResultColumn projection = projections.getResultColumn(i); 12582 if (isCteStarProjection(projection)) { 12583 return null; 12584 } 12585 String exposedName = getProjectedColumnName(exposedColumns, projection, i); 12586 if (SQLUtil.isEmpty(exposedName)) { 12587 return null; 12588 } 12589 TObjectName column = new TObjectName(); 12590 TSourceToken nameToken = new TSourceToken(exposedName); 12591 TResultColumn value = values.getResultColumn(i); 12592 column.setPartToken(nameToken); 12593 column.setObjectType(TObjectName.ttobjColumn); 12594 column.setSourceTable(targetTable); 12595 column.setStartToken(value.getStartToken() == null 12596 ? nameToken : value.getStartToken()); 12597 column.setEndToken(value.getEndToken() == null 12598 ? nameToken : value.getEndToken()); 12599 inferredColumns.add(column); 12600 } 12601 return inferredColumns; 12602 } 12603 12604 private CteWriteIntent collectUpdateCteWriteIntent(TUpdateSqlStatement stmt) { 12605 CteWriteIntent intent = new CteWriteIntent(); 12606 TResultColumnList assignments = stmt.getResultColumnList(); 12607 if (assignments == null) { 12608 intent.unsupportedReason = "the UPDATE has no writable target columns"; 12609 return intent; 12610 } 12611 for (int i = 0; i < assignments.size(); i++) { 12612 TExpression assignment = assignments.getResultColumn(i).getExpr(); 12613 if (assignment == null || assignment.getExpressionType() == EExpressionType.function_t) { 12614 addCteWriteIntentWarning(intent, 12615 "an UPDATE assignment target cannot be mapped to a CTE projection"); 12616 continue; 12617 } 12618 TExpression target = assignment.getLeftOperand(); 12619 if (target == null) { 12620 addCteWriteIntentWarning(intent, 12621 "an UPDATE assignment target cannot be mapped to a CTE projection"); 12622 continue; 12623 } 12624 if (target.getExpressionType() == EExpressionType.simple_object_name_t) { 12625 addCteWriteIntentColumn(intent, target.getObjectOperand()); 12626 } else if (target.getExpressionType() == EExpressionType.list_t && target.getExprList() != null) { 12627 for (int j = 0; j < target.getExprList().size(); j++) { 12628 TExpression item = target.getExprList().getExpression(j); 12629 if (item.getExpressionType() == EExpressionType.simple_object_name_t) { 12630 addCteWriteIntentColumn(intent, item.getObjectOperand()); 12631 } else { 12632 addCteWriteIntentWarning(intent, 12633 "an UPDATE assignment target cannot be mapped to a CTE projection"); 12634 } 12635 } 12636 } else { 12637 addCteWriteIntentWarning(intent, 12638 "an UPDATE assignment target cannot be mapped to a CTE projection"); 12639 } 12640 } 12641 if (intent.columns.isEmpty() && intent.unsupportedReason == null) { 12642 intent.unsupportedReason = "the UPDATE has no modified columns from which to resolve a base table"; 12643 } 12644 return intent; 12645 } 12646 12647 private void addCteWriteIntentColumn(CteWriteIntent intent, TObjectName column) { 12648 if (column == null || column.getDbObjectType() == EDbObjectType.variable) { 12649 return; 12650 } 12651 String name = column.getColumnNameOnly(); 12652 if (name == null) { 12653 return; 12654 } 12655 if (name.startsWith("@") && (option.getVendor() == EDbVendor.dbvmssql 12656 || option.getVendor() == EDbVendor.dbvazuresql)) { 12657 return; 12658 } 12659 if (name.startsWith(":") && (option.getVendor() == EDbVendor.dbvhana 12660 || option.getVendor() == EDbVendor.dbvteradata)) { 12661 return; 12662 } 12663 intent.columns.add(column); 12664 } 12665 12666 private CteWriteTargetResolution resolveCteWriteTarget(TTable targetTable, CteWriteIntent intent) { 12667 TSelectSqlStatement subquery = targetTable == null || targetTable.getCTE() == null 12668 ? null : targetTable.getCTE().getSubquery(); 12669 TObjectNameList exposedColumns = targetTable == null || targetTable.getCTE() == null 12670 ? null : targetTable.getCTE().getColumnList(); 12671 return resolveProjectedWriteTarget(subquery, exposedColumns, intent); 12672 } 12673 12674 private CteWriteTargetResolution resolveProjectedWriteTarget(TSelectSqlStatement subquery, 12675 TObjectNameList exposedColumns, CteWriteIntent intent) { 12676 CteWriteTargetResolution resolution = new CteWriteTargetResolution(); 12677 if (intent.unsupportedReason != null) { 12678 appendCteResolutionWarning(resolution, intent.unsupportedReason); 12679 } 12680 if (subquery == null) { 12681 appendCteResolutionWarning(resolution, "the target CTE has no SELECT body"); 12682 return resolution; 12683 } 12684 12685 if (subquery.isCombinedQuery() || subquery.getSetOperatorType() != ESetOperatorType.none) { 12686 appendCteResolutionWarning(resolution, 12687 "a CTE with a set operator is not a writable single-table projection"); 12688 return resolution; 12689 } 12690 if (subquery.getSelectDistinct() != null && !subquery.getSelectDistinct().isAll()) { 12691 appendCteResolutionWarning(resolution, 12692 "a DISTINCT CTE is not a writable single-table projection"); 12693 return resolution; 12694 } 12695 if (hasCteAggregate(subquery)) { 12696 appendCteResolutionWarning(resolution, 12697 "a grouped or aggregate CTE is not a writable single-table projection"); 12698 return resolution; 12699 } 12700 TResultColumnList projections = subquery.getResultColumnList(); 12701 if (projections == null || projections.size() == 0) { 12702 appendCteResolutionWarning(resolution, "the target CTE has no selectable columns"); 12703 return resolution; 12704 } 12705 12706 for (TObjectName modifiedColumn : intent.columns) { 12707 int projectionIndex = findCteProjectionIndex(exposedColumns, modifiedColumn, projections); 12708 if (projectionIndex < 0) { 12709 appendCteResolutionWarning(resolution, "modified column '" + modifiedColumn.getColumnNameOnly() 12710 + "' does not map uniquely to a CTE projection"); 12711 continue; 12712 } 12713 TResultColumn projection = projections.getResultColumn(projectionIndex); 12714 boolean starProjection = isCteStarProjection(projection); 12715 if (starProjection && exposedColumns != null && exposedColumns.size() > 0) { 12716 appendCteResolutionWarning(resolution, "modified column '" + modifiedColumn.getColumnNameOnly() 12717 + "' cannot be mapped safely through an explicitly named wildcard projection"); 12718 continue; 12719 } 12720 TObjectName projectedColumn = projection.getColumnFullname(); 12721 if (starProjection && projectedColumn != null 12722 && SQLUtil.isEmpty(projectedColumn.getTableString()) 12723 && subquery.tables != null && subquery.tables.size() > 1) { 12724 appendCteResolutionWarning(resolution, 12725 "an unqualified wildcard over multiple tables is ambiguous"); 12726 continue; 12727 } 12728 if (projectedColumn == null || projectedColumn.getColumnSource() == null) { 12729 appendCteResolutionWarning(resolution, "modified column '" + modifiedColumn.getColumnNameOnly() 12730 + "' is calculated or has no resolvable base column"); 12731 continue; 12732 } 12733 12734 gudusoft.gsqlparser.resolver2.model.ColumnSource source = projectedColumn.getColumnSource(); 12735 if (!starProjection && SQLUtil.isEmpty(projectedColumn.getTableString()) 12736 && isAmbiguousUnqualifiedProjection(subquery, projectedColumn)) { 12737 appendCteResolutionWarning(resolution, 12738 "an unqualified projection over multiple tables is ambiguous"); 12739 continue; 12740 } 12741 if (source.isAmbiguous()) { 12742 appendCteResolutionWarning(resolution, "modified column '" + modifiedColumn.getColumnNameOnly() 12743 + "' has more than one possible base table"); 12744 continue; 12745 } 12746 String projectedColumnName = starProjection 12747 ? modifiedColumn.getColumnNameOnly() : projectedColumn.getColumnNameOnly(); 12748 ProjectedColumnResolution projectedResolution = resolveProjectedColumn( 12749 getResolvedProjectedSourceTable(projectedColumn), projectedColumnName, 12750 new HashSet<TSelectSqlStatement>()); 12751 if (projectedResolution.failureReason != null) { 12752 appendCteResolutionWarning(resolution, projectedResolution.failureReason); 12753 continue; 12754 } 12755 TTable baseTable = projectedResolution.baseTable; 12756 if (!isPhysicalCteWriteTable(baseTable)) { 12757 appendCteResolutionWarning(resolution, "modified column '" + modifiedColumn.getColumnNameOnly() 12758 + "' does not resolve to a physical base table"); 12759 continue; 12760 } 12761 if (resolution.baseTable != null && resolution.baseTable != baseTable) { 12762 appendCteResolutionWarning(resolution, 12763 "modified columns resolve to more than one base table"); 12764 resolution.baseTable = null; 12765 resolution.columns.clear(); 12766 return resolution; 12767 } 12768 12769 String baseColumnName = projectedResolution.baseColumnName; 12770 if (SQLUtil.isEmpty(baseColumnName)) { 12771 baseColumnName = projectedColumn.getColumnNameOnly(); 12772 } 12773 if (SQLUtil.isEmpty(baseColumnName)) { 12774 appendCteResolutionWarning(resolution, "modified column '" + modifiedColumn.getColumnNameOnly() 12775 + "' has no resolvable physical column name"); 12776 continue; 12777 } 12778 12779 resolution.baseTable = baseTable; 12780 resolution.columns.add(new CteTargetColumnMapping(modifiedColumn.getColumnNameOnly(), baseColumnName)); 12781 } 12782 if (resolution.baseTable == null && resolution.failureReason == null) { 12783 resolution.failureReason = "no modified column resolves safely to a physical base table"; 12784 } 12785 return resolution; 12786 } 12787 12788 private boolean isAmbiguousUnqualifiedProjection(TSelectSqlStatement subquery, 12789 TObjectName projectedColumn) { 12790 if (subquery.tables == null || subquery.tables.size() <= 1) { 12791 return false; 12792 } 12793 String columnName = projectedColumn.getColumnNameOnly(); 12794 int matches = 0; 12795 int unknowns = 0; 12796 TTable onlyUnknown = null; 12797 for (int i = 0; i < subquery.tables.size(); i++) { 12798 TTable candidate = subquery.tables.getTable(i); 12799 Boolean hasColumn = hasAuthoritativeColumn(candidate, columnName); 12800 if (hasColumn == null) { 12801 unknowns++; 12802 onlyUnknown = candidate; 12803 continue; 12804 } 12805 if (hasColumn.booleanValue() && ++matches > 1) { 12806 return true; 12807 } 12808 } 12809 if (matches == 1) { 12810 return unknowns != 0; 12811 } 12812 return unknowns != 1 || getResolvedProjectedSourceTable(projectedColumn) != onlyUnknown; 12813 } 12814 12815 private TTable getResolvedProjectedSourceTable(TObjectName column) { 12816 if (column == null) { 12817 return null; 12818 } 12819 TTable sourceTable = column.getSourceTable(); 12820 if (sourceTable != null || column.getColumnSource() == null 12821 || column.getColumnSource().getSourceNamespace() == null) { 12822 return sourceTable; 12823 } 12824 return column.getColumnSource().getSourceNamespace().getSourceTable(); 12825 } 12826 12827 private Boolean hasAuthoritativeColumn(TTable table, String columnName) { 12828 if (table == null || SQLUtil.isEmpty(columnName)) { 12829 return null; 12830 } 12831 if (!isPhysicalCteWriteTable(table)) { 12832 TSelectSqlStatement subquery = null; 12833 TObjectNameList exposedColumns = null; 12834 if (table.getCTE() != null) { 12835 subquery = table.getCTE().getSubquery(); 12836 exposedColumns = table.getCTE().getColumnList(); 12837 } else if (table.getSubquery() != null) { 12838 subquery = table.getSubquery(); 12839 exposedColumns = getProjectedTargetColumns(table); 12840 } else if (table.getLinkTable() != null 12841 && table.getLinkTable().getSubquery() != null) { 12842 subquery = table.getLinkTable().getSubquery(); 12843 exposedColumns = getProjectedTargetColumns(table.getLinkTable()); 12844 } 12845 TResultColumnList projections = subquery == null 12846 ? null : subquery.getResultColumnList(); 12847 if (projections == null || projections.size() == 0 12848 || (exposedColumns != null && exposedColumns.size() > 0 12849 && exposedColumns.size() != projections.size())) { 12850 return null; 12851 } 12852 boolean matched = false; 12853 for (int i = 0; i < projections.size(); i++) { 12854 TResultColumn projection = projections.getResultColumn(i); 12855 if (isCteStarProjection(projection)) { 12856 return null; 12857 } 12858 String exposedName = getProjectedColumnName(exposedColumns, projection, i); 12859 if (SQLUtil.isEmpty(exposedName)) { 12860 return null; 12861 } 12862 if (SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotColumn, 12863 columnName, exposedName)) { 12864 matched = true; 12865 } 12866 } 12867 return Boolean.valueOf(matched); 12868 } 12869 TColumnDefinitionList definitions = table.getColumnDefinitions(); 12870 if (definitions != null && definitions.size() > 0) { 12871 for (int i = 0; i < definitions.size(); i++) { 12872 TObjectName name = definitions.getColumn(i).getColumnName(); 12873 if (name != null && SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotColumn, 12874 columnName, name.getColumnNameOnly())) { 12875 return Boolean.TRUE; 12876 } 12877 } 12878 return Boolean.FALSE; 12879 } 12880 if (sqlenv == null || table.getTableName() == null) { 12881 return null; 12882 } 12883 TSQLTable sqlTable = sqlenv.searchTable(table.getTableName()); 12884 if (sqlTable == null || sqlTable.getColumnList() == null 12885 || sqlTable.getColumnList().isEmpty()) { 12886 return null; 12887 } 12888 for (TSQLColumn column : sqlTable.getColumnList()) { 12889 if (SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotColumn, 12890 columnName, column.getName())) { 12891 return Boolean.TRUE; 12892 } 12893 } 12894 return Boolean.FALSE; 12895 } 12896 12897 private ProjectedColumnResolution resolveProjectedColumn(TTable sourceTable, String columnName, 12898 Set<TSelectSqlStatement> visited) { 12899 ProjectedColumnResolution resolution = new ProjectedColumnResolution(); 12900 if (sourceTable == null) { 12901 resolution.failureReason = "a projected column has no uniquely resolved source table"; 12902 return resolution; 12903 } 12904 if (isPhysicalCteWriteTable(sourceTable)) { 12905 resolution.baseTable = sourceTable; 12906 resolution.baseColumnName = columnName; 12907 return resolution; 12908 } 12909 TSelectSqlStatement subquery = null; 12910 TObjectNameList exposedColumns = null; 12911 if (sourceTable.getCTE() != null) { 12912 subquery = sourceTable.getCTE().getSubquery(); 12913 exposedColumns = sourceTable.getCTE().getColumnList(); 12914 } else if (sourceTable.getSubquery() != null) { 12915 subquery = sourceTable.getSubquery(); 12916 exposedColumns = getProjectedTargetColumns(sourceTable); 12917 } else if (sourceTable.getLinkTable() != null 12918 && sourceTable.getLinkTable().getSubquery() != null) { 12919 subquery = sourceTable.getLinkTable().getSubquery(); 12920 exposedColumns = getProjectedTargetColumns(sourceTable.getLinkTable()); 12921 } 12922 if (subquery == null || !visited.add(subquery)) { 12923 resolution.failureReason = subquery == null 12924 ? "an intermediate projection has no SELECT body" 12925 : "the projected target contains a cyclic intermediate query"; 12926 return resolution; 12927 } 12928 if (subquery.isCombinedQuery() || subquery.getSetOperatorType() != ESetOperatorType.none) { 12929 resolution.failureReason = "an intermediate projection with a set operator is not writable"; 12930 return resolution; 12931 } 12932 if (subquery.getSelectDistinct() != null && !subquery.getSelectDistinct().isAll()) { 12933 resolution.failureReason = "an intermediate DISTINCT projection is not writable"; 12934 return resolution; 12935 } 12936 if (hasCteAggregate(subquery)) { 12937 resolution.failureReason = "an intermediate grouped or aggregate projection is not writable"; 12938 return resolution; 12939 } 12940 TResultColumnList projections = subquery.getResultColumnList(); 12941 if (projections == null || projections.size() == 0 || SQLUtil.isEmpty(columnName)) { 12942 resolution.failureReason = "an intermediate projection has no mappable target column"; 12943 return resolution; 12944 } 12945 TObjectName exposedColumn = new TObjectName(); 12946 TSourceToken columnToken = new TSourceToken(columnName); 12947 exposedColumn.setPartToken(columnToken); 12948 exposedColumn.setObjectType(TObjectName.ttobjColumn); 12949 int projectionIndex = findCteProjectionIndex(exposedColumns, exposedColumn, projections); 12950 if (projectionIndex < 0) { 12951 resolution.failureReason = "modified column '" + columnName 12952 + "' does not map uniquely through an intermediate projection"; 12953 return resolution; 12954 } 12955 TResultColumn projection = projections.getResultColumn(projectionIndex); 12956 boolean starProjection = isCteStarProjection(projection); 12957 if (starProjection && exposedColumns != null && exposedColumns.size() > 0) { 12958 resolution.failureReason = "modified column '" + columnName 12959 + "' cannot be mapped safely through an explicitly named intermediate wildcard projection"; 12960 return resolution; 12961 } 12962 TObjectName innerColumn = projection.getColumnFullname(); 12963 if (innerColumn == null || innerColumn.getColumnSource() == null) { 12964 resolution.failureReason = "modified column '" + columnName 12965 + "' has no resolvable intermediate base column"; 12966 return resolution; 12967 } 12968 if (starProjection && SQLUtil.isEmpty(innerColumn.getTableString()) 12969 && subquery.tables != null && subquery.tables.size() > 1) { 12970 resolution.failureReason = "an unqualified intermediate wildcard over multiple tables is ambiguous"; 12971 return resolution; 12972 } 12973 gudusoft.gsqlparser.resolver2.model.ColumnSource source = innerColumn.getColumnSource(); 12974 if (!starProjection && SQLUtil.isEmpty(innerColumn.getTableString()) 12975 && isAmbiguousUnqualifiedProjection(subquery, innerColumn)) { 12976 resolution.failureReason = "an unqualified intermediate projection over multiple tables is ambiguous"; 12977 return resolution; 12978 } 12979 if (source.isAmbiguous()) { 12980 resolution.failureReason = "modified column '" + columnName 12981 + "' has more than one possible intermediate base table"; 12982 return resolution; 12983 } 12984 String innerName = starProjection ? columnName : innerColumn.getColumnNameOnly(); 12985 return resolveProjectedColumn(getResolvedProjectedSourceTable(innerColumn), innerName, visited); 12986 } 12987 12988 private void appendCteResolutionWarning(CteWriteTargetResolution resolution, String reason) { 12989 if (resolution.failureReason == null) { 12990 resolution.failureReason = reason; 12991 } else if (!resolution.failureReason.contains(reason)) { 12992 resolution.failureReason += "; " + reason; 12993 } 12994 } 12995 12996 private boolean hasCteAggregate(TSelectSqlStatement subquery) { 12997 if (subquery.getGroupByClause() != null) { 12998 return true; 12999 } 13000 TResultColumnList projections = subquery.getResultColumnList(); 13001 if (projections == null) { 13002 return false; 13003 } 13004 for (int i = 0; i < projections.size(); i++) { 13005 TExpression expression = projections.getResultColumn(i).getExpr(); 13006 if (expression == null) { 13007 continue; 13008 } 13009 columnsInExpr visitor = new columnsInExpr(); 13010 expression.inOrderTraverse(visitor); 13011 List<TParseTreeNode> functions = visitor.getFunctions(); 13012 if (functions == null) { 13013 continue; 13014 } 13015 for (TParseTreeNode function : functions) { 13016 if (function instanceof TFunctionCall 13017 && ((TFunctionCall) function).getWindowDef() == null 13018 && isAggregateFunction((TFunctionCall) function)) { 13019 return true; 13020 } 13021 } 13022 } 13023 return false; 13024 } 13025 13026 private int findCteProjectionIndex(TObjectNameList exposedColumns, TObjectName modifiedColumn, 13027 TResultColumnList projections) { 13028 int match = -1; 13029 int starMatch = -1; 13030 for (int i = 0; i < projections.size(); i++) { 13031 TResultColumn projection = projections.getResultColumn(i); 13032 String exposedName; 13033 if (exposedColumns != null && exposedColumns.size() > 0) { 13034 if (i >= exposedColumns.size()) { 13035 continue; 13036 } 13037 exposedName = getProjectedColumnName(exposedColumns, projection, i); 13038 } else { 13039 exposedName = SQLUtil.isEmpty(projection.getColumnAlias()) 13040 ? projection.getDisplayName() : projection.getColumnAlias(); 13041 if (isCteStarProjection(projection)) { 13042 if (starMatch == -1) { 13043 starMatch = i; 13044 } else { 13045 starMatch = -2; 13046 } 13047 } 13048 } 13049 if (SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotColumn, 13050 modifiedColumn.getColumnNameOnly(), exposedName)) { 13051 if (match >= 0) { 13052 return -1; 13053 } 13054 match = i; 13055 } 13056 } 13057 return match >= 0 ? match : starMatch; 13058 } 13059 13060 private String getProjectedColumnName(TObjectNameList exposedColumns, 13061 TResultColumn projection, int index) { 13062 if (exposedColumns != null && exposedColumns.size() > 0) { 13063 if (index >= exposedColumns.size()) { 13064 return null; 13065 } 13066 TObjectName exposedColumn = exposedColumns.getObjectName(index); 13067 String exposedName = exposedColumn.getColumnNameOnly(); 13068 return SQLUtil.isEmpty(exposedName) ? exposedColumn.toString() : exposedName; 13069 } 13070 return SQLUtil.isEmpty(projection.getColumnAlias()) 13071 ? projection.getDisplayName() : projection.getColumnAlias(); 13072 } 13073 13074 private boolean isCteStarProjection(TResultColumn projection) { 13075 if (projection == null || projection.getExpr() == null 13076 || projection.getExpr().getExpressionType() != EExpressionType.simple_object_name_t 13077 || projection.getExpr().getObjectOperand() == null) { 13078 return false; 13079 } 13080 return "*".equals(projection.getExpr().getObjectOperand().getColumnNameOnly()); 13081 } 13082 13083 private boolean isPhysicalCteWriteTable(TTable table) { 13084 return table != null && table.getTableName() != null && table.getCTE() == null 13085 && table.getSubquery() == null 13086 && (table.getLinkTable() == null || table.getLinkTable().getSubquery() == null); 13087 } 13088 13089 private CteWriteTargetResolution resolveCteDeleteTarget(TSelectSqlStatement subquery) { 13090 return resolveCteDeleteTarget(subquery, new HashSet<TSelectSqlStatement>()); 13091 } 13092 13093 private CteWriteTargetResolution resolveCteDeleteTarget(TSelectSqlStatement subquery, 13094 Set<TSelectSqlStatement> visited) { 13095 CteWriteTargetResolution resolution = new CteWriteTargetResolution(); 13096 if (subquery == null || !visited.add(subquery)) { 13097 resolution.failureReason = "the target CTE has no acyclic SELECT body"; 13098 return resolution; 13099 } 13100 if (subquery.isCombinedQuery() || subquery.getSetOperatorType() != ESetOperatorType.none) { 13101 resolution.failureReason = "a CTE with a set operator is not a writable single-table projection"; 13102 return resolution; 13103 } 13104 if (subquery.getSelectDistinct() != null && !subquery.getSelectDistinct().isAll()) { 13105 resolution.failureReason = "a DISTINCT CTE is not a writable single-table projection"; 13106 return resolution; 13107 } 13108 if (hasCteAggregate(subquery)) { 13109 resolution.failureReason = "a grouped or aggregate CTE is not a writable single-table projection"; 13110 return resolution; 13111 } 13112 if (subquery.tables == null || subquery.tables.size() != 1) { 13113 resolution.failureReason = "a DELETE target CTE resolves to more than one base table"; 13114 return resolution; 13115 } 13116 TTable table = subquery.tables.getTable(0); 13117 if (isPhysicalCteWriteTable(table)) { 13118 resolution.baseTable = table; 13119 return resolution; 13120 } 13121 if (table.getCTE() != null) { 13122 return resolveCteDeleteTarget(table.getCTE().getSubquery(), visited); 13123 } 13124 if (table.getSubquery() != null) { 13125 return resolveCteDeleteTarget(table.getSubquery(), visited); 13126 } 13127 if (table.getLinkTable() != null && table.getLinkTable().getSubquery() != null) { 13128 return resolveCteDeleteTarget(table.getLinkTable().getSubquery(), visited); 13129 } 13130 resolution.failureReason = "the DELETE target CTE does not resolve to a physical base table"; 13131 return resolution; 13132 } 13133 13134 private TObjectNameList getProjectedTargetColumns(TTable targetTable) { 13135 return targetTable.getAliasClause() == null 13136 ? null : targetTable.getAliasClause().getColumns(); 13137 } 13138 13139 private TObjectName getCteBaseColumn(CteWriteTargetResolution resolution, TObjectName targetColumn) { 13140 if (resolution == null) { 13141 return targetColumn; 13142 } 13143 if (!resolution.isWritable() || targetColumn == null) { 13144 return null; 13145 } 13146 for (CteTargetColumnMapping mapping : resolution.columns) { 13147 if (SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotColumn, 13148 targetColumn.getColumnNameOnly(), mapping.exposedName)) { 13149 TObjectName baseColumn = new TObjectName(); 13150 TSourceToken nameToken = new TSourceToken(mapping.baseColumnName); 13151 baseColumn.setPartToken(nameToken); 13152 baseColumn.setObjectType(TObjectName.ttobjColumn); 13153 baseColumn.setSourceTable(resolution.baseTable); 13154 baseColumn.setStartToken(targetColumn.getStartToken() == null 13155 ? nameToken : targetColumn.getStartToken()); 13156 baseColumn.setEndToken(targetColumn.getEndToken() == null 13157 ? nameToken : targetColumn.getEndToken()); 13158 return baseColumn; 13159 } 13160 } 13161 return null; 13162 } 13163 13164 private QueryTable analyzeCteWriteTarget(TTable targetTable, Process process) { 13165 return analyzeProjectedWriteTarget(targetTable, targetTable.getCTE().getSubquery(), 13166 targetTable.getCTE().getColumnList(), process); 13167 } 13168 13169 private QueryTable analyzeProjectedWriteTarget(TTable targetTable, TSelectSqlStatement subquery, 13170 TObjectNameList exposedColumns, Process process) { 13171 QueryTable queryTable = modelFactory.createQueryTable(targetTable); 13172 if (exposedColumns != null) { 13173 for (int i = 0; i < exposedColumns.size(); i++) { 13174 modelFactory.createResultColumn(queryTable, exposedColumns.getObjectName(i)); 13175 } 13176 } 13177 13178 if (subquery == null || stmtStack.contains(subquery)) { 13179 return queryTable; 13180 } 13181 analyzeSelectStmt(subquery); 13182 ResultSet resultSetModel = (ResultSet) modelManager.getModel(subquery); 13183 if (resultSetModel == null || resultSetModel == queryTable) { 13184 return queryTable; 13185 } 13186 13187 if (!resultSetModel.getRelationRows().getHoldRelations().isEmpty()) { 13188 ImpactRelationship impactRelation = modelFactory.createImpactRelation(); 13189 impactRelation.setEffectType(EffectType.select); 13190 impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>( 13191 resultSetModel.getRelationRows())); 13192 impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>( 13193 queryTable.getRelationRows())); 13194 } 13195 13196 if (subquery.getSetOperatorType() != ESetOperatorType.none) { 13197 SelectSetResultSet selectSetResultSetModel = (SelectSetResultSet) resultSetModel; 13198 for (int i = 0; i < selectSetResultSetModel.getColumns().size(); i++) { 13199 ResultColumn sourceColumn = selectSetResultSetModel.getColumns().get(i); 13200 ResultColumn targetColumn = exposedColumns != null && i < queryTable.getColumns().size() 13201 ? queryTable.getColumns().get(i) 13202 : modelFactory.createSelectSetResultColumn(queryTable, sourceColumn); 13203 for (Set<TObjectName> starLinkColumns : sourceColumn.getStarLinkColumns().values()) { 13204 for (TObjectName starLinkColumn : starLinkColumns) { 13205 targetColumn.bindStarLinkColumn(starLinkColumn); 13206 } 13207 } 13208 createCteSelectRelation(process, targetColumn, sourceColumn); 13209 } 13210 } else { 13211 for (int i = 0; i < resultSetModel.getColumns().size(); i++) { 13212 ResultColumn sourceColumn = resultSetModel.getColumns().get(i); 13213 ResultColumn targetColumn = exposedColumns != null && i < queryTable.getColumns().size() 13214 ? queryTable.getColumns().get(i) 13215 : modelFactory.createSelectSetResultColumn(queryTable, sourceColumn); 13216 for (TObjectName starLinkColumn : sourceColumn.getStarLinkColumnList()) { 13217 targetColumn.bindStarLinkColumn(starLinkColumn); 13218 } 13219 createCteSelectRelation(process, targetColumn, sourceColumn); 13220 } 13221 } 13222 return queryTable; 13223 } 13224 13225 private void createProjectedWriteImpact(TSelectSqlStatement subquery, Table tableModel, 13226 EffectType effectType, Process process) { 13227 if (subquery == null || tableModel == null) { 13228 return; 13229 } 13230 Object model = modelManager.getModel(subquery); 13231 if (!(model instanceof ResultSet)) { 13232 return; 13233 } 13234 ResultSet resultSet = (ResultSet) model; 13235 if (resultSet.getRelationRows().getHoldRelations().isEmpty()) { 13236 return; 13237 } 13238 ImpactRelationship impactRelation = modelFactory.createImpactRelation(); 13239 impactRelation.setEffectType(effectType); 13240 impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>( 13241 resultSet.getRelationRows())); 13242 impactRelation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>( 13243 tableModel.getRelationRows())); 13244 impactRelation.setProcess(process); 13245 } 13246 13247 private void createCteSelectRelation(Process process, ResultColumn targetColumn, ResultColumn sourceColumn) { 13248 if (!cteSelectRelationKeys.add(new CteSelectRelationKey(targetColumn, sourceColumn))) { 13249 return; 13250 } 13251 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 13252 relation.setEffectType(EffectType.select); 13253 relation.setTarget(new ResultColumnRelationshipElement(targetColumn)); 13254 relation.addSource(new ResultColumnRelationshipElement(sourceColumn)); 13255 relation.setProcess(process); 13256 } 13257 13258 private void addCteWriteTargetHint(TTable targetTable, String reason) { 13259 addCteWriteTargetHint(targetTable, reason, false); 13260 } 13261 13262 private void addCteWriteTargetHint(TTable targetTable, String reason, boolean partiallyResolved) { 13263 ErrorInfo errorInfo = new ErrorInfo(); 13264 errorInfo.setErrorType(ErrorInfo.SYNTAX_HINT); 13265 String targetKind = targetTable.getCTE() == null ? "Projected write target" : "CTE write target"; 13266 String targetName = targetTable.getTableName() == null 13267 ? targetTable.getAliasName() : targetTable.getTableName().toString(); 13268 errorInfo.setErrorMessage(targetKind + " '" + targetName + "' " 13269 + (partiallyResolved ? "was only partially resolved: " : "was kept as a result set: ") 13270 + reason + "."); 13271 if (targetTable.getStartToken() != null && targetTable.getEndToken() != null) { 13272 errorInfo.setStartPosition(new Pair3<Long, Long, String>(targetTable.getStartToken().lineNo, 13273 targetTable.getStartToken().columnNo, ModelBindingManager.getGlobalHash())); 13274 errorInfo.setEndPosition(new Pair3<Long, Long, String>(targetTable.getEndToken().lineNo, 13275 targetTable.getEndToken().columnNo 13276 + targetTable.getEndToken().getAstext().length(), ModelBindingManager.getGlobalHash())); 13277 } 13278 errorInfo.fillInfo(this); 13279 errorInfos.add(errorInfo); 13280 } 13281 13282 private void analyzeMergeStmt(TMergeSqlStatement stmt) { 13283 Object tableModel; 13284 CteWriteTargetResolution cteTargetResolution = null; 13285 if (stmt.getUsingTable() != null) { 13286 TTable table = stmt.getTargetTable(); 13287 Process process = modelFactory.createProcess(stmt); 13288 if (table.getCTE() != null) { 13289 QueryTable cteTargetModel = analyzeCteWriteTarget(table, process); 13290 cteTargetResolution = resolveCteWriteTarget(table, collectMergeCteWriteIntent(stmt, table)); 13291 if (cteTargetResolution.isWritable()) { 13292 tableModel = modelFactory.createTable(cteTargetResolution.baseTable); 13293 ((Table) tableModel).addProcess(process); 13294 createProjectedWriteImpact(table.getCTE().getSubquery(), (Table) tableModel, 13295 EffectType.merge, process); 13296 } else { 13297 tableModel = cteTargetModel; 13298 } 13299 if (cteTargetResolution.failureReason != null) { 13300 addCteWriteTargetHint(table, cteTargetResolution.failureReason, 13301 cteTargetResolution.isWritable()); 13302 } 13303 } else if(table.getSubquery()!=null) { 13304 TObjectNameList exposedColumns = getProjectedTargetColumns(table); 13305 QueryTable projectedTargetModel = analyzeProjectedWriteTarget(table, 13306 table.getSubquery(), exposedColumns, process); 13307 cteTargetResolution = resolveProjectedWriteTarget(table.getSubquery(), exposedColumns, 13308 collectMergeCteWriteIntent(stmt, table)); 13309 if (cteTargetResolution.isWritable()) { 13310 tableModel = modelFactory.createTable(cteTargetResolution.baseTable); 13311 ((Table) tableModel).addProcess(process); 13312 createProjectedWriteImpact(table.getSubquery(), (Table) tableModel, 13313 EffectType.merge, process); 13314 } else { 13315 tableModel = projectedTargetModel; 13316 } 13317 if (cteTargetResolution.failureReason != null) { 13318 addCteWriteTargetHint(table, cteTargetResolution.failureReason, 13319 cteTargetResolution.isWritable()); 13320 } 13321 } 13322 else { 13323 tableModel = modelFactory.createTable(table); 13324 ((Table)tableModel).addProcess(process); 13325 } 13326 13327 for(TTable item: stmt.tables) { 13328 // Skip subqueries and CTE references: a CTE feeding USING is resolved 13329 // through its subquery (see the getCTE() branch below), so materializing 13330 // it here as a physical base table leaves a spurious table named after 13331 // the CTE in the lineage output. (MantisBT #4493) 13332 if(item.getSubquery()!=null || item.getCTE()!=null) { 13333 continue; 13334 } 13335 Table tableItemModel = modelFactory.createTable(item); 13336 if (tableItemModel.getColumns() == null || tableItemModel.getColumns().isEmpty()) { 13337 tableItemModel.addColumnsFromSQLEnv(); 13338 } 13339 } 13340 13341 if (stmt.getUsingTable().getSubquery() != null) { 13342 QueryTable queryTable = modelFactory.createQueryTable(stmt.getUsingTable()); 13343 analyzeSelectStmt(stmt.getUsingTable().getSubquery()); 13344 13345 ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt.getUsingTable().getSubquery()); 13346 13347 if (queryTable != null && resultSetModel != null && queryTable != resultSetModel) { 13348 if (queryTable.getColumns().size() == resultSetModel.getColumns().size()) { 13349 for (int i = 0; i < queryTable.getColumns().size(); i++) { 13350 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 13351 relation.setEffectType(EffectType.select); 13352 relation.setTarget(new ResultColumnRelationshipElement(queryTable.getColumns().get(i))); 13353 relation.addSource(new ResultColumnRelationshipElement(resultSetModel.getColumns().get(i))); 13354 relation.setProcess(process); 13355 } 13356 } 13357 } 13358 13359 if (resultSetModel != null && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) { 13360 ImpactRelationship impactRelation = modelFactory.createImpactRelation(); 13361 impactRelation.setEffectType(EffectType.merge); 13362 impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>( 13363 resultSetModel.getRelationRows())); 13364 if (tableModel instanceof Table) { 13365 impactRelation.setTarget( 13366 new RelationRowsRelationshipElement<TableRelationRows>(((Table)tableModel).getRelationRows())); 13367 } 13368 else { 13369 impactRelation.setTarget( 13370 new RelationRowsRelationshipElement<ResultSetRelationRows>(((ResultSet)tableModel).getRelationRows())); 13371 } 13372 } 13373 } else if (stmt.getUsingTable().getCTE() != null) { 13374 // USING references a CTE defined in the WITH clause attached to the 13375 // MERGE. Mirror the SELECT-side CTE handling so the CTE subquery (and 13376 // the base tables inside it) are analyzed and linked through to the 13377 // merge source; otherwise the CTE is treated as an opaque base table 13378 // and the tables feeding it are dropped from the lineage. (MantisBT #4493) 13379 QueryTable queryTable = modelFactory.createQueryTable(stmt.getUsingTable()); 13380 13381 TObjectNameList cteColumns = stmt.getUsingTable().getCTE().getColumnList(); 13382 if (cteColumns != null) { 13383 for (int j = 0; j < cteColumns.size(); j++) { 13384 modelFactory.createResultColumn(queryTable, cteColumns.getObjectName(j)); 13385 } 13386 } 13387 13388 TSelectSqlStatement subquery = stmt.getUsingTable().getCTE().getSubquery(); 13389 if (subquery != null && !stmtStack.contains(subquery)) { 13390 analyzeSelectStmt(subquery); 13391 13392 ResultSet resultSetModel = (ResultSet) modelManager.getModel(subquery); 13393 13394 if (resultSetModel != null && resultSetModel != queryTable) { 13395 if (subquery.getSetOperatorType() != ESetOperatorType.none) { 13396 SelectSetResultSet selectSetResultSetModel = (SelectSetResultSet) modelManager 13397 .getModel(subquery); 13398 for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) { 13399 ResultColumn sourceColumn = selectSetResultSetModel.getColumns().get(j); 13400 ResultColumn targetColumn = null; 13401 if (cteColumns != null && j < queryTable.getColumns().size()) { 13402 targetColumn = queryTable.getColumns().get(j); 13403 } else { 13404 targetColumn = modelFactory.createSelectSetResultColumn(queryTable, sourceColumn); 13405 } 13406 for (Set<TObjectName> starLinkColumns : sourceColumn.getStarLinkColumns().values()) { 13407 for (TObjectName starLinkColumn : starLinkColumns) { 13408 targetColumn.bindStarLinkColumn(starLinkColumn); 13409 } 13410 } 13411 createCteSelectRelation(process, targetColumn, sourceColumn); 13412 } 13413 } else { 13414 for (int j = 0; j < resultSetModel.getColumns().size(); j++) { 13415 ResultColumn sourceColumn = resultSetModel.getColumns().get(j); 13416 ResultColumn targetColumn = null; 13417 if (cteColumns != null && j < queryTable.getColumns().size()) { 13418 targetColumn = queryTable.getColumns().get(j); 13419 } else { 13420 targetColumn = modelFactory.createSelectSetResultColumn(queryTable, sourceColumn); 13421 } 13422 for (TObjectName starLinkColumn : sourceColumn.getStarLinkColumnList()) { 13423 targetColumn.bindStarLinkColumn(starLinkColumn); 13424 } 13425 createCteSelectRelation(process, targetColumn, sourceColumn); 13426 } 13427 } 13428 } 13429 13430 if (resultSetModel != null && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) { 13431 ImpactRelationship impactRelation = modelFactory.createImpactRelation(); 13432 impactRelation.setEffectType(EffectType.merge); 13433 impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>( 13434 resultSetModel.getRelationRows())); 13435 if (tableModel instanceof Table) { 13436 impactRelation.setTarget( 13437 new RelationRowsRelationshipElement<TableRelationRows>(((Table) tableModel).getRelationRows())); 13438 } else { 13439 impactRelation.setTarget( 13440 new RelationRowsRelationshipElement<ResultSetRelationRows>(((ResultSet) tableModel).getRelationRows())); 13441 } 13442 } 13443 } 13444 } else { 13445 if (stmt.getUsingTable().getAliasClause() != null && stmt.getUsingTable().getAliasClause().getColumns() != null && stmt.getUsingTable().getValueClause().getRows()!=null) { 13446 Table usingTable = modelFactory.createTableFromCreateDDL(stmt.getUsingTable(), false, stmt.getUsingTable().getAliasName() + stmt.getUsingTable().getTableName()); 13447 usingTable.setCreateTable(true); 13448 usingTable.setSubType(SubType.function); 13449 for (int z=0;z<stmt.getUsingTable().getAliasClause().getColumns().size();z++) { 13450 TObjectName columnName = stmt.getUsingTable().getAliasClause().getColumns().getObjectName(z); 13451 TableColumn tableColumn = modelFactory.createTableColumn(usingTable, columnName, true); 13452 TResultColumn resultColumn = stmt.getUsingTable().getValueClause().getRows().get(0).getResultColumn(z); 13453 modelManager.bindModel(resultColumn, tableColumn); 13454 analyzeResultColumnExpressionRelation(tableColumn, resultColumn.getExpr()); } 13455 } 13456 else { 13457 modelFactory.createTable(stmt.getUsingTable()); 13458 } 13459 } 13460 13461 13462 if (stmt.getWhenClauses() != null && stmt.getWhenClauses().size() > 0) { 13463 for (int i = 0; i < stmt.getWhenClauses().size(); i++) { 13464 TMergeWhenClause clause = stmt.getWhenClauses().getElement(i); 13465 if (clause.getCondition() != null) { 13466 analyzeFilterCondition(null, clause.getCondition(), null, null, EffectType.merge_when); 13467 } 13468 if (clause.getUpdateClause() != null) { 13469 TResultColumnList columns = clause.getUpdateClause().getUpdateColumnList(); 13470 if (columns == null || columns.size() == 0) 13471 continue; 13472 13473 ResultSet resultSet = modelFactory.createResultSet(clause.getUpdateClause(), false); 13474 createPseudoImpactRelation(stmt, resultSet, EffectType.merge_update); 13475 13476 for (int j = 0; j < columns.size(); j++) { 13477 TResultColumn resultColumn = columns.getResultColumn(j); 13478 if (resultColumn.getExpr().getLeftOperand() 13479 .getExpressionType() == EExpressionType.simple_object_name_t) { 13480 TObjectName columnObject = resultColumn.getExpr().getLeftOperand().getObjectOperand(); 13481 13482 if (columnObject.getDbObjectType() == EDbObjectType.variable) { 13483 continue; 13484 } 13485 13486 if (columnObject.getColumnNameOnly().startsWith("@") 13487 && (option.getVendor() == EDbVendor.dbvmssql 13488 || option.getVendor() == EDbVendor.dbvazuresql)) { 13489 continue; 13490 } 13491 13492 if (columnObject.getColumnNameOnly().startsWith(":") 13493 && (option.getVendor() == EDbVendor.dbvhana 13494 || option.getVendor() == EDbVendor.dbvteradata)) { 13495 continue; 13496 } 13497 13498 ResultColumn updateColumn = modelFactory.createMergeResultColumn(resultSet, 13499 columnObject); 13500 13501 TExpression valueExpression = resultColumn.getExpr().getRightOperand(); 13502 if (valueExpression == null) 13503 continue; 13504 13505 columnsInExpr visitor = new columnsInExpr(); 13506 valueExpression.inOrderTraverse(visitor); 13507 List<TObjectName> objectNames = visitor.getObjectNames(); 13508 List<TParseTreeNode> functions = visitor.getFunctions(); 13509 13510 if (functions != null && !functions.isEmpty()) { 13511 analyzeFunctionDataFlowRelation(updateColumn, functions, EffectType.merge_update); 13512 } 13513 13514 List<TSelectSqlStatement> subquerys = visitor.getSubquerys(); 13515 if (subquerys != null && !subquerys.isEmpty()) { 13516 analyzeSubqueryDataFlowRelation(updateColumn, subquerys, EffectType.merge_update); 13517 } 13518 13519 analyzeDataFlowRelation(updateColumn, objectNames, EffectType.merge_update, functions); 13520 13521 List<TParseTreeNode> constants = visitor.getConstants(); 13522 analyzeConstantDataFlowRelation(updateColumn, constants, EffectType.merge_update, 13523 functions); 13524 13525 if (tableModel instanceof Table) { 13526 TObjectName targetColumn = getCteBaseColumn(cteTargetResolution, columnObject); 13527 TableColumn tableColumn = targetColumn == null ? null 13528 : modelFactory.createTableColumn((Table)tableModel, targetColumn, false); 13529 13530 if (tableColumn != null) { 13531 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 13532 relation.setEffectType(EffectType.merge_update); 13533 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 13534 relation.addSource(new ResultColumnRelationshipElement(updateColumn)); 13535 relation.setProcess(process); 13536 } 13537 } 13538 } 13539 } 13540 } 13541 if (clause.getInsertClause() != null) { 13542 TExpression insertValue = clause.getInsertClause().getInsertValue(); 13543 if (insertValue != null 13544 && insertValue.getExpressionType() == EExpressionType.objectConstruct_t) { 13545 ResultSet resultSet = modelFactory.createResultSet(clause.getInsertClause(), false); 13546 13547 createPseudoImpactRelation(stmt, resultSet, EffectType.merge_insert); 13548 13549 TObjectConstruct objectConstruct = insertValue.getObjectConstruct(); 13550 for (int z = 0; z < objectConstruct.getPairs().size(); z++) { 13551 TPair pair = objectConstruct.getPairs().getElement(z); 13552 13553 if (pair.getKeyName().getExpressionType() == EExpressionType.simple_constant_t) { 13554 TObjectName columnObject = new TObjectName(); 13555 TConstant constant = pair.getKeyName().getConstantOperand(); 13556 TSourceToken newSt = new TSourceToken( 13557 constant.getValueToken().getTextWithoutQuoted()); 13558 columnObject.setPartToken(newSt); 13559 columnObject.setSourceTable(stmt.getTargetTable()); 13560 columnObject.setStartToken(constant.getStartToken()); 13561 columnObject.setEndToken(constant.getEndToken()); 13562 13563 ResultColumn insertColumn = modelFactory.createMergeResultColumn(resultSet, 13564 columnObject); 13565 13566 TExpression valueExpression = pair.getKeyValue(); 13567 if (valueExpression == null) 13568 continue; 13569 13570 columnsInExpr visitor = new columnsInExpr(); 13571 valueExpression.inOrderTraverse(visitor); 13572 List<TObjectName> objectNames = visitor.getObjectNames(); 13573 List<TParseTreeNode> functions = visitor.getFunctions(); 13574 13575 if (functions != null && !functions.isEmpty()) { 13576 analyzeFunctionDataFlowRelation(insertColumn, functions, 13577 EffectType.merge_insert); 13578 } 13579 13580 List<TSelectSqlStatement> subquerys = visitor.getSubquerys(); 13581 if (subquerys != null && !subquerys.isEmpty()) { 13582 analyzeSubqueryDataFlowRelation(insertColumn, subquerys, 13583 EffectType.merge_insert); 13584 } 13585 13586 analyzeDataFlowRelation(insertColumn, objectNames, EffectType.merge_insert, 13587 functions); 13588 13589 List<TParseTreeNode> constants = visitor.getConstants(); 13590 analyzeConstantDataFlowRelation(insertColumn, constants, EffectType.merge_insert, 13591 functions); 13592 13593 TObjectName targetColumn = getCteBaseColumn(cteTargetResolution, columnObject); 13594 TableColumn tableColumn = !(tableModel instanceof Table) || targetColumn == null ? null 13595 : modelFactory.createTableColumn((Table)tableModel, targetColumn, false); 13596 13597 if (tableColumn != null) { 13598 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 13599 relation.setEffectType(EffectType.merge_insert); 13600 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 13601 relation.addSource(new ResultColumnRelationshipElement(insertColumn)); 13602 relation.setProcess(process); 13603 } 13604 } 13605 } 13606 } else { 13607 TObjectNameList columns = clause.getInsertClause().getColumnList(); 13608 TResultColumnList values = clause.getInsertClause().getValuelist(); 13609 if (values == null || values.size() == 0) { 13610 if (clause.getInsertClause().toString().toLowerCase().indexOf("row") != -1) { 13611 if (stmt.getUsingTable().getSubquery() != null && tableModel instanceof Table) { 13612 ResultSet sourceResultSet = modelFactory.createQueryTable(stmt.getUsingTable()); 13613 TObjectName targetStarColumn = new TObjectName(); 13614 targetStarColumn.setString("*"); 13615 TableColumn targetTableColumn = modelFactory.createTableColumn((Table)tableModel, 13616 targetStarColumn, true); 13617 if (sourceResultSet.getColumns() == null 13618 || sourceResultSet.getColumns().isEmpty()) { 13619 TObjectName sourceStarColumn = new TObjectName(); 13620 sourceStarColumn.setString("*"); 13621 modelFactory.createResultColumn(sourceResultSet, sourceStarColumn); 13622 } 13623 for (ResultColumn sourceColumn : sourceResultSet.getColumns()) { 13624 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 13625 relation.setEffectType(EffectType.merge_insert); 13626 relation.setTarget(new TableColumnRelationshipElement(targetTableColumn)); 13627 relation.addSource(new ResultColumnRelationshipElement(sourceColumn)); 13628 relation.setProcess(process); 13629 } 13630 } else if (tableModel instanceof Table) { 13631 Table sourceTable = modelFactory.createTable(stmt.getUsingTable()); 13632 TObjectName sourceStarColumn = new TObjectName(); 13633 sourceStarColumn.setString("*"); 13634 TableColumn sourceTableColumn = modelFactory.createTableColumn(sourceTable, 13635 sourceStarColumn, true); 13636 TObjectName targetStarColumn = new TObjectName(); 13637 targetStarColumn.setString("*"); 13638 TableColumn targetTableColumn = modelFactory.createTableColumn(((Table)tableModel), 13639 targetStarColumn, true); 13640 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 13641 relation.setEffectType(EffectType.merge_insert); 13642 relation.setTarget(new TableColumnRelationshipElement(targetTableColumn)); 13643 relation.addSource(new TableColumnRelationshipElement(sourceTableColumn)); 13644 relation.setProcess(process); 13645 } 13646 13647 } 13648 continue; 13649 } 13650 13651 List<TObjectName> tableColumns = new ArrayList<TObjectName>(); 13652 boolean unresolvedProjectedColumns = false; 13653 if (columns == null || columns.size() == 0) { 13654 List<TObjectName> inferredColumns = inferProjectedInsertColumns( 13655 stmt.getTargetTable(), values); 13656 unresolvedProjectedColumns = inferredColumns == null 13657 && cteTargetResolution != null; 13658 if (inferredColumns != null) { 13659 tableColumns.addAll(inferredColumns); 13660 } else { 13661// if (!((Table)tableModel).getColumns().isEmpty()) { 13662// for (int j = 0; j < ((Table)tableModel).getColumns().size(); j++) { 13663// if (((Table)tableModel).getColumns().get(j).getColumnObject() == null) { 13664// continue; 13665// } 13666// tableColumns.add(((Table)tableModel).getColumns().get(j).getColumnObject()); 13667// } 13668// } else { 13669 for (int j = 0; j < values.size(); j++) { 13670 TResultColumn column = values.getResultColumn(j); 13671 if (column.getAliasClause() != null) { 13672 tableColumns.add(column.getAliasClause().getAliasName()); 13673 } else if (column.getFieldAttr() != null) { 13674 tableColumns.add(column.getFieldAttr()); 13675 } else { 13676 TObjectName columnName = new TObjectName(); 13677 columnName.setString(column.toString()); 13678 tableColumns.add(columnName); 13679 } 13680// } 13681 } 13682 } 13683 } else { 13684 for (int j = 0; j < columns.size(); j++) { 13685 tableColumns.add(columns.getObjectName(j)); 13686 } 13687 } 13688 13689 ResultSet resultSet = modelFactory.createResultSet(clause.getInsertClause(), false); 13690 13691 createPseudoImpactRelation(stmt, resultSet, EffectType.merge_insert); 13692 13693 for (int j = 0; j < tableColumns.size() && j < values.size(); j++) { 13694 TObjectName columnObject = tableColumns.get(j); 13695 13696 ResultColumn insertColumn = modelFactory.createMergeResultColumn(resultSet, 13697 columnObject); 13698 13699 TExpression valueExpression = values.getResultColumn(j).getExpr(); 13700 if (valueExpression == null) 13701 continue; 13702 13703 columnsInExpr visitor = new columnsInExpr(); 13704 valueExpression.inOrderTraverse(visitor); 13705 List<TObjectName> objectNames = visitor.getObjectNames(); 13706 List<TParseTreeNode> functions = visitor.getFunctions(); 13707 13708 if (functions != null && !functions.isEmpty()) { 13709 analyzeFunctionDataFlowRelation(insertColumn, functions, EffectType.merge_insert); 13710 } 13711 13712 List<TSelectSqlStatement> subquerys = visitor.getSubquerys(); 13713 if (subquerys != null && !subquerys.isEmpty()) { 13714 analyzeSubqueryDataFlowRelation(insertColumn, subquerys, EffectType.merge_insert); 13715 } 13716 13717 analyzeDataFlowRelation(insertColumn, objectNames, EffectType.merge_insert, functions); 13718 13719 List<TParseTreeNode> constants = visitor.getConstants(); 13720 analyzeConstantDataFlowRelation(insertColumn, constants, EffectType.merge_insert, 13721 functions); 13722 13723 TObjectName targetColumn = unresolvedProjectedColumns ? null 13724 : getCteBaseColumn(cteTargetResolution, columnObject); 13725 TableColumn tableColumn = !(tableModel instanceof Table) || targetColumn == null ? null 13726 : modelFactory.createTableColumn(((Table)tableModel), targetColumn, false); 13727 if(tableColumn == null) { 13728 if (cteTargetResolution == null && tableModel instanceof Table 13729 && ((Table) tableModel).isCreateTable()) { 13730 tableColumn = ((Table) tableModel).getColumns().get(j); 13731 } 13732 else { 13733 continue; 13734 } 13735 } 13736 13737 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 13738 relation.setEffectType(EffectType.merge_insert); 13739 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 13740 relation.addSource(new ResultColumnRelationshipElement(insertColumn)); 13741 relation.setProcess(process); 13742 } 13743 } 13744 } 13745 } 13746 13747 } 13748 13749 if (stmt.getCondition() != null) { 13750 analyzeFilterCondition(null, stmt.getCondition(), null, JoinClauseType.on, EffectType.merge); 13751 } 13752 } 13753 } 13754 13755 private List<TableColumn> bindInsertTableColumn(Table tableModel, TInsertIntoValue value, List<TObjectName> keyMap, 13756 List<TResultColumn> valueMap) { 13757 List<TableColumn> tableColumns = new ArrayList<TableColumn>(); 13758 if (value.getColumnList() != null) { 13759 for (int z = 0; z < value.getColumnList().size(); z++) { 13760 TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel, 13761 value.getColumnList().getObjectName(z)); 13762 tableColumns.add(tableColumn); 13763 keyMap.add(tableColumn.getColumnObject()); 13764 } 13765 } 13766 13767 if (value.getTargetList() != null) { 13768 for (int z = 0; z < value.getTargetList().size(); z++) { 13769 TMultiTarget target = value.getTargetList().getMultiTarget(z); 13770 TResultColumnList columns = target.getColumnList(); 13771 for (int i = 0; i < columns.size(); i++) { 13772 if (value.getColumnList() == null) { 13773 TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel, 13774 columns.getResultColumn(i).getFieldAttr()); 13775 tableColumns.add(tableColumn); 13776 } 13777 valueMap.add(columns.getResultColumn(i)); 13778 } 13779 } 13780 } 13781 13782 return tableColumns; 13783 } 13784 13785 private TableColumn matchColumn(List<TableColumn> tableColumns, TableColumn targetColumn) { 13786 String columnName = targetColumn.getName(); 13787 if (tableColumns == null) { 13788 return null; 13789 } 13790 for (int i = 0; i < tableColumns.size(); i++) { 13791 TableColumn column = tableColumns.get(i); 13792 if (column.getColumnObject() == null) { 13793 continue; 13794 } 13795 if(column.isStruct() && targetColumn.isStruct()) { 13796 List<String> names = SQLUtil.parseNames(column.getName()); 13797 List<String> targetNames = SQLUtil 13798 .parseNames(targetColumn.getName()); 13799 if (!getColumnName(targetNames.get(0)) 13800 .equals(getColumnName(names.get(0)))) { 13801 continue; 13802 } 13803 } 13804 if (getColumnName(column.getColumnObject().toString()).equals(getColumnName(columnName))) { 13805 return column; 13806 } 13807 } 13808 return null; 13809 } 13810 13811 private TableColumn matchColumn(List<TableColumn> tableColumns, TObjectName columnName) { 13812 if (tableColumns == null) { 13813 return null; 13814 } 13815 for (int i = 0; i < tableColumns.size(); i++) { 13816 TableColumn column = tableColumns.get(i); 13817 if (column.getColumnObject() == null) { 13818 continue; 13819 } 13820 if (DlineageUtil.sameColumnName(column.getColumnObject(), columnName)) 13821 return column; 13822 } 13823 return null; 13824 } 13825 13826 private ResultColumn matchResultColumn(List<ResultColumn> resultColumns, ResultColumn resultColumn) { 13827 if (resultColumns == null) { 13828 return null; 13829 } 13830 13831 TObjectName columnName = getObjectName(resultColumn); 13832 if (columnName == null) { 13833 return null; 13834 } 13835 13836 for (int i = 0; i < resultColumns.size(); i++) { 13837 ResultColumn column = resultColumns.get(i); 13838 if (column.getAlias() != null 13839 && DlineageUtil.sameColumnName(column.getAlias(), columnName)) 13840 return column; 13841 if (column.getName() != null && DlineageUtil.sameColumnName(column.getName(), columnName)) 13842 return column; 13843 if (column.getName() != null && column.getName().endsWith("*")) { 13844 if ("*".equals(column.getColumnObject().toString())) { 13845 return column; 13846 } else { 13847 TObjectName columnObjectName = getObjectName(column); 13848 if (columnObjectName.getTableString() != null 13849 && columnObjectName.getTableString().equals(getResultSetAlias(resultColumn))) { 13850 return column; 13851 } 13852 } 13853 } 13854 } 13855 return null; 13856 } 13857 13858 private String getResultSetAlias(ResultColumn resultColumn) { 13859 ResultSet resultSet = resultColumn.getResultSet(); 13860 if (resultSet instanceof QueryTable) { 13861 return ((QueryTable) resultSet).getAlias(); 13862 } 13863 return null; 13864 } 13865 13866 private ResultColumn matchResultColumn(List<ResultColumn> resultColumns, TObjectName columnName) { 13867 if (resultColumns == null) { 13868 return null; 13869 } 13870 for (int i = 0; i < resultColumns.size(); i++) { 13871 ResultColumn column = resultColumns.get(i); 13872 if (column.getAlias() != null 13873 && DlineageUtil.sameColumnName(column.getAlias(), columnName)) 13874 return column; 13875 if (column.getName() != null && DlineageUtil.sameColumnName(column.getName(), columnName)) 13876 return column; 13877 if (column.getName() != null && column.getName().endsWith("*")) { 13878 if ("*".equals(column.getColumnObject().toString())) { 13879 return column; 13880 } else { 13881 TObjectName columnObjectName = getObjectName(column); 13882 if (columnObjectName.getTableString() != null 13883 && columnObjectName.getTableString().equals(columnName.getTableString())) { 13884 return column; 13885 } 13886 } 13887 } 13888 } 13889 return null; 13890 } 13891 13892 private void analyzeInsertStmt(TInsertSqlStatement stmt) { 13893 Map<Table, List<TObjectName>> insertTableKeyMap = new LinkedHashMap<Table, List<TObjectName>>(); 13894 Map<Table, List<TResultColumn>> insertTableValueMap = new LinkedHashMap<Table, List<TResultColumn>>(); 13895 Map<String, List<TableColumn>> tableColumnMap = new LinkedHashMap<String, List<TableColumn>>(); 13896 List<Table> inserTables = new ArrayList<Table>(); 13897 List<TExpression> expressions = new ArrayList<TExpression>(); 13898 boolean hasInsertColumns = false; 13899 13900 EffectType effectType = EffectType.insert; 13901 if(stmt.getInsertToken()!=null && stmt.getInsertToken().toString().toLowerCase().startsWith("replace")) { 13902 effectType = EffectType.replace; 13903 } 13904 13905 if (stmt.getInsertConditions() != null && stmt.getInsertConditions().size() > 0) { 13906 for (int i = 0; i < stmt.getInsertConditions().size(); i++) { 13907 TInsertCondition condition = stmt.getInsertConditions().getElement(i); 13908 if (condition.getCondition() != null) { 13909 expressions.add(condition.getCondition()); 13910 } 13911 for (int j = 0; j < condition.getInsertIntoValues().size(); j++) { 13912 TInsertIntoValue value = condition.getInsertIntoValues().getElement(j); 13913 TTable table = value.getTable(); 13914 Table tableModel = modelFactory.createTable(table); 13915 13916 inserTables.add(tableModel); 13917 List<TObjectName> keyMap = new ArrayList<TObjectName>(); 13918 List<TResultColumn> valueMap = new ArrayList<TResultColumn>(); 13919 insertTableKeyMap.put(tableModel, keyMap); 13920 insertTableValueMap.put(tableModel, valueMap); 13921 13922 List<TableColumn> tableColumns = bindInsertTableColumn(tableModel, value, keyMap, valueMap); 13923 if (tableColumnMap.get(DlineageUtil.getIdentifierNormalTableName(table.getFullName())) == null 13924 && !tableColumns.isEmpty()) { 13925 tableColumnMap.put(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName()), 13926 tableColumns); 13927 } 13928 13929 // if (stmt.getSubQuery() != null) 13930 { 13931 Process process = modelFactory.createProcess(stmt); 13932 tableModel.addProcess(process); 13933 } 13934 } 13935 } 13936 hasInsertColumns = true; 13937 } else if (stmt.getInsertIntoValues() != null && stmt.getInsertIntoValues().size() > 0) { 13938 for (int i = 0; i < stmt.getInsertIntoValues().size(); i++) { 13939 TInsertIntoValue value = stmt.getInsertIntoValues().getElement(i); 13940 TTable table = value.getTable(); 13941 Table tableModel = modelFactory.createTable(table); 13942 13943 inserTables.add(tableModel); 13944 List<TObjectName> keyMap = new ArrayList<TObjectName>(); 13945 List<TResultColumn> valueMap = new ArrayList<TResultColumn>(); 13946 insertTableKeyMap.put(tableModel, keyMap); 13947 insertTableValueMap.put(tableModel, valueMap); 13948 13949 List<TableColumn> tableColumns = bindInsertTableColumn(tableModel, value, keyMap, valueMap); 13950 if (tableColumnMap.get(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName())) == null && !tableColumns.isEmpty()) { 13951 tableColumnMap.put(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName()), tableColumns); 13952 } 13953 13954 // if (stmt.getSubQuery() != null) 13955 { 13956 Process process = modelFactory.createProcess(stmt); 13957 tableModel.addProcess(process); 13958 } 13959 } 13960 hasInsertColumns = true; 13961 } else if (stmt.getColumnList() != null && stmt.getColumnList().size() > 0) { 13962 TTable table = stmt.getTargetTable(); 13963 Table tableModel = modelFactory.createTable(table); 13964 13965 inserTables.add(tableModel); 13966 List<TObjectName> keyMap = new ArrayList<TObjectName>(); 13967 insertTableKeyMap.put(tableModel, keyMap); 13968 List<TableColumn> tableColumns = new ArrayList<TableColumn>(); 13969 for (int i = 0; i < stmt.getColumnList().size(); i++) { 13970 TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel, 13971 stmt.getColumnList().getObjectName(i)); 13972 tableColumns.add(tableColumn); 13973 } 13974 if (tableColumnMap.get(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName())) == null && !tableColumns.isEmpty()) { 13975 tableColumnMap.put(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName()), tableColumns); 13976 } 13977 13978 // if (stmt.getSubQuery() != null) 13979 { 13980 Process process = modelFactory.createProcess(stmt); 13981 tableModel.addProcess(process); 13982 } 13983 hasInsertColumns = true; 13984 } else if (stmt.getOutputClause() != null && stmt.getOutputClause().getSelectItemList().size() > 0) { 13985 TTable table = stmt.getTargetTable(); 13986 Table tableModel = modelFactory.createTable(table); 13987 13988 inserTables.add(tableModel); 13989 List<TObjectName> keyMap = new ArrayList<TObjectName>(); 13990 insertTableKeyMap.put(tableModel, keyMap); 13991 List<TableColumn> tableColumns = new ArrayList<TableColumn>(); 13992 for (int i = 0; i < stmt.getOutputClause().getSelectItemList().size(); i++) { 13993 TObjectName columnName = stmt.getOutputClause().getSelectItemList().getResultColumn(i).getFieldAttr(); 13994 if (columnName.getPseudoTableType() != EPseudoTableType.none) { 13995 // Phase 1 already swapped tokens; getColumnNameOnly() returns actual column name 13996 String column = columnName.getColumnNameOnly(); 13997 columnName = new TObjectName(); 13998 columnName.setString(column); 13999 } else { 14000 String column = columnName.toString().toLowerCase(); 14001 if ((column.startsWith("inserted.") || column.startsWith("deleted.")) 14002 && columnName.getPropertyToken() != null) { 14003 column = columnName.getPropertyToken().getAstext(); 14004 columnName = new TObjectName(); 14005 columnName.setString(column); 14006 } 14007 } 14008 TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel, columnName); 14009 tableColumns.add(tableColumn); 14010 } 14011 if (tableColumnMap.get(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName())) == null && !tableColumns.isEmpty()) { 14012 tableColumnMap.put(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName()), tableColumns); 14013 } 14014 14015 // if (stmt.getSubQuery() != null) 14016 { 14017 Process process = modelFactory.createProcess(stmt); 14018 tableModel.addProcess(process); 14019 } 14020 hasInsertColumns = true; 14021 } else { 14022 TTable table = stmt.getTargetTable(); 14023 Table tableModel; 14024 if (table != null) { 14025 tableModel = modelFactory.createTable(table); 14026 // if (stmt.getSubQuery() != null) 14027 { 14028 Process process = modelFactory.createProcess(stmt); 14029 tableModel.addProcess(process); 14030 } 14031 if (tableModel.getColumns() == null || tableModel.getColumns().isEmpty()) { 14032 tableModel.addColumnsFromSQLEnv(); 14033 } 14034 } else if (stmt.getDirectoryName() != null) { 14035 tableModel = modelFactory.createTableByName(stmt.getDirectoryName(), true); 14036 tableModel.setPath(true); 14037 tableModel.setCreateTable(true); 14038 TObjectName fileUri = new TObjectName(); 14039 fileUri.setString("uri=" + stmt.getDirectoryName()); 14040 TableColumn tableColumn = modelFactory.createFileUri(tableModel, fileUri); 14041 // if (stmt.getSubQuery() != null) 14042 { 14043 Process process = modelFactory.createProcess(stmt); 14044 tableModel.addProcess(process); 14045 } 14046 } else { 14047 ErrorInfo errorInfo = new ErrorInfo(); 14048 errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR); 14049 errorInfo.setErrorMessage("Can't get target table. InsertSqlStatement is " + stmt.toString()); 14050 errorInfo.setStartPosition(new Pair3<Long, Long, String>(stmt.getStartToken().lineNo, 14051 stmt.getStartToken().columnNo, ModelBindingManager.getGlobalHash())); 14052 errorInfo.setEndPosition(new Pair3<Long, Long, String>(stmt.getEndToken().lineNo, 14053 stmt.getEndToken().columnNo + stmt.getEndToken().getAstext().length(), 14054 ModelBindingManager.getGlobalHash())); 14055 errorInfo.fillInfo(this); 14056 errorInfos.add(errorInfo); 14057 return; 14058 } 14059 inserTables.add(tableModel); 14060 if (table != null 14061 && tableColumnMap.get(DlineageUtil.getIdentifierNormalTableName(table.getFullName())) == null) { 14062 if (tableModel.getColumns() != null && !tableModel.getColumns().isEmpty()) { 14063 tableColumnMap.put(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName()), 14064 tableModel.getColumns()); 14065 } else { 14066 tableColumnMap.put(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName()), null); 14067 } 14068 } 14069 } 14070 14071 if (stmt.getSubQuery() != null) { 14072 analyzeSelectStmt(stmt.getSubQuery()); 14073 } 14074 14075 Iterator<Table> tableIter = inserTables.iterator(); 14076 while (tableIter.hasNext()) { 14077 Table tableModel = tableIter.next(); 14078 List<TableColumn> tableColumns = tableColumnMap.get(DlineageUtil.getIdentifierNormalTableName(tableModel.getFullName())); 14079 List<TObjectName> keyMap = insertTableKeyMap.get(tableModel); 14080 List<TResultColumn> valueMap = insertTableValueMap.get(tableModel); 14081 boolean initColumn = (hasInsertColumns && tableColumns != null && !containStarColumn(tableColumns)); 14082 14083 if (stmt.getSubQuery() != null) { 14084 14085 List<TSelectSqlStatement> subquerys = new ArrayList<TSelectSqlStatement>(); 14086 if (stmt.getSubQuery().getResultColumnList() != null || stmt.getSubQuery().getTransformClause() != null) { 14087 subquerys.add(stmt.getSubQuery()); 14088 } else if (stmt.getSubQuery().getValueClause() != null 14089 && stmt.getSubQuery().getValueClause().getRows() != null) { 14090 for (TResultColumnList resultColumnList : stmt.getSubQuery().getValueClause().getRows()) { 14091 for(TResultColumn resultColumn: resultColumnList) { 14092 if(resultColumn.getExpr()!=null && resultColumn.getExpr().getSubQuery()!=null) { 14093 analyzeSelectStmt(resultColumn.getExpr().getSubQuery()); 14094 subquerys.add(resultColumn.getExpr().getSubQuery()); 14095 } 14096 } 14097 } 14098 } 14099 14100 for(TSelectSqlStatement subquery: subquerys) { 14101 if ((tableModel.isCreateTable() && tableModel.getColumns() != null) 14102 || (subquery.getSetOperatorType() == ESetOperatorType.none 14103 && stmt.getColumnList() != null && stmt.getColumnList().size() > 0)) { 14104 14105 ResultSet resultSetModel = null; 14106 14107 if (subquery != null) { 14108 resultSetModel = (ResultSet) modelManager.getModel(subquery); 14109 } 14110 14111 TResultColumnList resultset = subquery.getResultColumnList(); 14112 if (resultSetModel == null && resultset != null) { 14113 resultSetModel = (ResultSet) modelManager.getModel(resultset); 14114 } 14115 14116 if (resultSetModel == null) { 14117 ErrorInfo errorInfo = new ErrorInfo(); 14118 errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR); 14119 errorInfo.setErrorMessage("Can't get resultset model"); 14120 errorInfo.setStartPosition(new Pair3<Long, Long, String>(resultset.getStartToken().lineNo, 14121 resultset.getStartToken().columnNo, ModelBindingManager.getGlobalHash())); 14122 errorInfo.setEndPosition(new Pair3<Long, Long, String>(resultset.getEndToken().lineNo, 14123 resultset.getEndToken().columnNo + resultset.getEndToken().getAstext().length(), 14124 ModelBindingManager.getGlobalHash())); 14125 errorInfos.add(errorInfo); 14126 } 14127 14128 if (!resultSetModel.getRelationRows().getHoldRelations().isEmpty()) { 14129 ImpactRelationship impactRelation = modelFactory.createImpactRelation(); 14130 impactRelation.setEffectType(effectType); 14131 impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>( 14132 resultSetModel.getRelationRows())); 14133 impactRelation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>( 14134 tableModel.getRelationRows())); 14135 } 14136 14137 int resultSetSize = resultSetModel.getColumns().size(); 14138 int j = 0; 14139 int starIndex = 0; 14140 TObjectNameList items = stmt.getColumnList(); 14141 List<String> itemNames = new ArrayList<String>(); 14142 int starColumnCount = 0; 14143 for (ResultColumn item : resultSetModel.getColumns()) { 14144 if (item.getName().endsWith("*")) { 14145 starColumnCount += 1; 14146 } 14147 } 14148 if (items != null) { 14149 for (int i = 0; i < items.size() && j < resultSetSize; i++) { 14150 TObjectName column = items.getObjectName(i); 14151 14152 if (column.getDbObjectType() == EDbObjectType.variable) { 14153 continue; 14154 } 14155 14156 if (column.getColumnNameOnly().startsWith("@") 14157 && (option.getVendor() == EDbVendor.dbvmssql 14158 || option.getVendor() == EDbVendor.dbvazuresql)) { 14159 continue; 14160 } 14161 14162 if (column.getColumnNameOnly().startsWith(":") 14163 && (option.getVendor() == EDbVendor.dbvhana 14164 || option.getVendor() == EDbVendor.dbvteradata)) { 14165 continue; 14166 } 14167 14168 ResultColumn resultColumn = resultSetModel.getColumns().get(j); 14169 if (!resultSetModel.getColumns().get(j).getName().contains("*")) { 14170 j++; 14171 } else { 14172 starIndex++; 14173 if (resultSetSize - j == items.size() - i) { 14174 j++; 14175 14176 } 14177 } 14178 if (column != null) { 14179 TableColumn tableColumn; 14180 // if (!initColumn) { 14181 tableColumn = matchColumn(tableModel.getColumns(), column); 14182 if (tableColumn == null) { 14183 if (tableModel.isCreateTable() && !containStarColumn(tableModel.getColumns())) { 14184 if (tableModel.getColumns().size() <= i) { 14185 continue; 14186 } 14187 tableColumn = tableModel.getColumns().get(i); 14188 } else { 14189 tableColumn = modelFactory.createTableColumn(tableModel, column, false); 14190 if(tableColumn == null) { 14191 continue; 14192 } 14193 } 14194 } 14195// } else { 14196// tableColumn = matchColumn(tableColumns, column); 14197// if (tableColumn == null) { 14198// continue; 14199// } 14200// } 14201 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 14202 relation.setEffectType(effectType); 14203 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 14204 if (resultColumn.hasStarLinkColumn() 14205 && resultColumn.getStarLinkColumnNames().size() > starIndex - 1 && starColumnCount<=1) { 14206 boolean find = false; 14207 while (resultColumn.getStarLinkColumnNames().size() > starIndex - 1) { 14208 TObjectName name = resultColumn.getStarLinkColumnName(starIndex - 1); 14209 if (itemNames.contains(name.toString())) { 14210 starIndex++; 14211 continue; 14212 } 14213 ResultColumn expandStarColumn = modelFactory 14214 .createResultColumn(resultSetModel, name, false); 14215 relation.addSource(new ResultColumnRelationshipElement(expandStarColumn)); 14216 itemNames.add(resultColumn.getName()); 14217 find = true; 14218 break; 14219 } 14220 if (!find) { 14221 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 14222 itemNames.add(resultColumn.getName()); 14223 } 14224 } else { 14225 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 14226 itemNames.add(resultColumn.getName()); 14227 } 14228 Process process = modelFactory.createProcess(stmt); 14229 relation.setProcess(process); 14230 } 14231 } 14232 } else { 14233 List<TableColumn> columns = tableModel.getColumns(); 14234 if (columns.size() == 1 && tableModel.isPath()) { 14235 for(int i=0;i<resultSetSize;i++) { 14236 ResultColumn resultColumn = resultSetModel.getColumns().get(i); 14237 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 14238 relation.setEffectType(effectType); 14239 relation.setTarget(new TableColumnRelationshipElement(tableModel.getColumns().get(0))); 14240 if (resultColumn.hasStarLinkColumn() 14241 && resultColumn.getStarLinkColumnNames().size() > starIndex - 1) { 14242 ResultColumn expandStarColumn = modelFactory.createResultColumn( 14243 resultSetModel, resultColumn.getStarLinkColumnName(starIndex - 1), 14244 false); 14245 relation.addSource(new ResultColumnRelationshipElement(expandStarColumn)); 14246 } else { 14247 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 14248 } 14249 Process process = modelFactory.createProcess(stmt); 14250 relation.setProcess(process); 14251 } 14252 } else { 14253 boolean fromStruct = false; 14254 for (int i = 0; i < columns.size() && j < resultSetSize; i++) { 14255 String column = columns.get(i).getName(); 14256 ResultColumn resultColumn = resultSetModel.getColumns().get(j); 14257 if (!resultColumn.getName().contains("*")) { 14258 if (resultColumn.getName().equals(resultColumn.getRefColumnName()) 14259 && resultColumn.getColumnObject().toString().endsWith("*") 14260 && resultSetSize == 1) { 14261 starIndex++; 14262 if (resultSetSize - j == columns.size() - i) { 14263 j++; 14264 14265 } 14266 } 14267 else { 14268 j++; 14269 } 14270 } else { 14271 starIndex++; 14272 if (resultSetSize - j == columns.size() - i) { 14273 j++; 14274 14275 } 14276 } 14277 if (column != null) { 14278 TableColumn tableColumn; 14279 // if (!initColumn) { 14280 tableColumn = matchColumn(tableModel.getColumns(), columns.get(i)); 14281 if (tableColumn == null) { 14282 if (tableModel.isCreateTable() 14283 && !containStarColumn(tableModel.getColumns())) { 14284 if (tableModel.getColumns().size() <= i) { 14285 continue; 14286 } 14287 tableColumn = tableModel.getColumns().get(i); 14288 } else { 14289 TObjectName columnName = new TObjectName(); 14290 columnName.setString(column); 14291 tableColumn = modelFactory.createTableColumn(tableModel, columnName, 14292 false); 14293 } 14294 } 14295 else if (!resultColumn.isStruct() && tableColumn.isStruct() && columns.size() != resultSetSize) { 14296 j--; 14297 fromStruct = true; 14298 } 14299 if(fromStruct && !tableColumn.isStruct()) { 14300 fromStruct = false; 14301 resultColumn = resultSetModel.getColumns().get(j); 14302 j++; 14303 } 14304 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 14305 relation.setEffectType(effectType); 14306 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 14307 if (resultColumn.hasStarLinkColumn() 14308 && resultColumn.getStarLinkColumnNames().size() > starIndex - 1) { 14309 ResultColumn expandStarColumn = modelFactory.createResultColumn( 14310 resultSetModel, resultColumn.getStarLinkColumnName(starIndex - 1), 14311 false); 14312 relation.addSource(new ResultColumnRelationshipElement(expandStarColumn)); 14313 } else { 14314 relation.addSource(new ResultColumnRelationshipElement(resultColumn, starIndex - 1)); 14315 } 14316 Process process = modelFactory.createProcess(stmt); 14317 relation.setProcess(process); 14318 } 14319 } 14320 } 14321 } 14322 } else if (!subquery.isCombinedQuery()) { 14323 SelectResultSet resultSetModel = (SelectResultSet) modelManager 14324 .getModel(subquery.getResultColumnList() != null ? subquery.getResultColumnList() 14325 : subquery.getTransformClause()); 14326 14327 if (resultSetModel != null && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) { 14328 ImpactRelationship impactRelation = modelFactory.createImpactRelation(); 14329 impactRelation.setEffectType(effectType); 14330 impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>( 14331 resultSetModel.getRelationRows())); 14332 impactRelation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>( 14333 tableModel.getRelationRows())); 14334 } 14335 14336 List<ResultColumn> columnsSnapshot = new ArrayList<ResultColumn>(); 14337 columnsSnapshot.addAll(resultSetModel.getColumns()); 14338 if(resultSetModel.isDetermined() && stmt.getColumnList() == null) { 14339 tableModel.setDetermined(true); 14340 } 14341 for (int i = 0; i < columnsSnapshot.size(); i++) { 14342 ResultColumn resultColumn = columnsSnapshot.get(i); 14343 if (resultColumn.getColumnObject() instanceof TObjectName) { 14344 TableColumn tableColumn; 14345 if (!initColumn) { 14346 if (tableModel.isCreateTable() && !containStarColumn(tableModel.getColumns())) { 14347 if (tableModel.getColumns().size() <= i) { 14348 continue; 14349 } 14350 tableColumn = tableModel.getColumns().get(i); 14351 } else { 14352 tableColumn = modelFactory.createInsertTableColumn(tableModel, 14353 (TObjectName) resultColumn.getColumnObject()); 14354 } 14355 if (containStarColumn(tableColumns)) { 14356 getStarColumn(tableColumns) 14357 .bindStarLinkColumn((TObjectName) resultColumn.getColumnObject()); 14358 } 14359 } else { 14360 TObjectName matchedColumnName = (TObjectName) resultColumn.getColumnObject(); 14361 tableColumn = matchColumn(tableColumns, matchedColumnName); 14362 if (tableColumn == null) { 14363 if (!isEmptyCollection(valueMap)) { 14364 int index = indexOfColumn(valueMap, matchedColumnName); 14365 if (index != -1) { 14366 if (!isEmptyCollection(keyMap) && index < keyMap.size()) { 14367 tableColumn = matchColumn(tableColumns, keyMap.get(index)); 14368 } else if (isEmptyCollection(keyMap) && index < tableColumns.size()) { 14369 tableColumn = tableColumns.get(index); 14370 } else { 14371 continue; 14372 } 14373 } else { 14374 continue; 14375 } 14376 } else if (!isEmptyCollection(keyMap) && i < keyMap.size()) { 14377 tableColumn = matchColumn(tableColumns, keyMap.get(i)); 14378 } else if (isEmptyCollection(keyMap) && isEmptyCollection(valueMap) 14379 && i < tableColumns.size()) { 14380 tableColumn = tableColumns.get(i); 14381 } else { 14382 continue; 14383 } 14384 } 14385 } 14386 14387 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 14388 relation.setEffectType(effectType); 14389 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 14390 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 14391 Process process = modelFactory.createProcess(stmt); 14392 relation.setProcess(process); 14393 } else { 14394 TAliasClause alias = ((TResultColumn) resultColumn.getColumnObject()).getAliasClause(); 14395 if (alias != null && alias.getAliasName() != null) { 14396 TableColumn tableColumn; 14397 if (!initColumn) { 14398 if (tableModel.isCreateTable() && !containStarColumn(tableModel.getColumns())) { 14399 if (tableModel.getColumns().size() <= i) { 14400 if (tableModel.isPath()) { 14401 tableColumn = tableModel.getColumns().get(0); 14402 } else { 14403 continue; 14404 } 14405 } else { 14406 tableColumn = tableModel.getColumns().get(i); 14407 } 14408 } else { 14409 tableColumn = modelFactory.createInsertTableColumn(tableModel, 14410 alias.getAliasName()); 14411 if (containStarColumn(resultSetModel)) { 14412 tableColumn.notBindStarLinkColumn(true); 14413 } 14414 } 14415 if (containStarColumn(tableColumns)) { 14416 getStarColumn(tableColumns).bindStarLinkColumn(alias.getAliasName()); 14417 } 14418 } else { 14419 TObjectName matchedColumnName = alias.getAliasName(); 14420 tableColumn = matchColumn(tableColumns, matchedColumnName); 14421 if (tableColumn == null) { 14422 if (!isEmptyCollection(valueMap)) { 14423 int index = indexOfColumn(valueMap, matchedColumnName); 14424 if (index != -1) { 14425 if (!isEmptyCollection(keyMap) && index < keyMap.size()) { 14426 tableColumn = matchColumn(tableColumns, keyMap.get(index)); 14427 } else if (isEmptyCollection(keyMap) 14428 && index < tableColumns.size()) { 14429 tableColumn = tableColumns.get(index); 14430 } else { 14431 continue; 14432 } 14433 } else { 14434 continue; 14435 } 14436 } else if (!isEmptyCollection(keyMap) && i < keyMap.size()) { 14437 tableColumn = matchColumn(tableColumns, keyMap.get(i)); 14438 } else { 14439 tableColumn = modelFactory.createInsertTableColumn(tableModel, 14440 alias.getAliasName()); 14441 } 14442 } 14443 14444 } 14445 14446 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 14447 relation.setEffectType(effectType); 14448 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 14449 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 14450 Process process = modelFactory.createProcess(stmt); 14451 relation.setProcess(process); 14452 14453 } else if (((TResultColumn) resultColumn.getColumnObject()).getFieldAttr() != null) { 14454 TObjectName fieldAttr = ((TResultColumn) resultColumn.getColumnObject()) 14455 .getFieldAttr(); 14456 14457 Object model = modelManager.getModel( resultColumn.getColumnObject()); 14458 14459 TableColumn tableColumn; 14460 if (!initColumn) { 14461 if (tableModel.isCreateTable() && !containStarColumn(tableModel.getColumns())) { 14462 if (fieldAttr.toString().endsWith("*")) { 14463 int starIndex = 0; 14464 for (TableColumn column : tableModel.getColumns()) { 14465 starIndex++; 14466 DataFlowRelationship relation = modelFactory 14467 .createDataFlowRelation(); 14468 relation.setEffectType(effectType); 14469 relation.setTarget(new TableColumnRelationshipElement(column)); 14470 if (resultColumn.getStarLinkColumnList().size() == tableModel 14471 .getColumns().size()) { 14472 ResultColumn expandStarColumn = modelFactory.createResultColumn( 14473 resultSetModel, 14474 resultColumn.getStarLinkColumnList().get(starIndex - 1), 14475 false); 14476 relation.addSource( 14477 new ResultColumnRelationshipElement(expandStarColumn)); 14478 } else { 14479 relation.addSource( 14480 new ResultColumnRelationshipElement(resultColumn)); 14481 } 14482 Process process = modelFactory.createProcess(stmt); 14483 relation.setProcess(process); 14484 } 14485 continue; 14486 } 14487 if (tableModel.getColumns().size() <= i) { 14488 continue; 14489 } 14490 tableColumn = tableModel.getColumns().get(i); 14491 } else { 14492 if(model instanceof LinkedHashMap) { 14493 LinkedHashMap<String, ResultColumn> resultColumns = (LinkedHashMap<String, ResultColumn>)model; 14494 for(String key: resultColumns.keySet()) { 14495 tableColumn = modelFactory.createInsertTableColumn(tableModel, resultColumns.get(key).getName()); 14496 DataFlowRelationship relation = modelFactory 14497 .createDataFlowRelation(); 14498 relation.setEffectType(effectType); 14499 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 14500 relation.addSource( 14501 new ResultColumnRelationshipElement(resultColumns.get(key))); 14502 Process process = modelFactory.createProcess(stmt); 14503 relation.setProcess(process); 14504 } 14505 continue; 14506 } 14507 else { 14508 tableColumn = modelFactory.createInsertTableColumn(tableModel, fieldAttr); 14509 } 14510 } 14511 } else if (tableModel.isDetermined() && i < tableModel.getColumns().size()) { 14512 if (!isEmptyCollection(valueMap)) { 14513 TObjectName matchedColumnName = fieldAttr; 14514 int index = indexOfColumn(valueMap, matchedColumnName); 14515 if (index != -1) { 14516 if (!isEmptyCollection(keyMap) && index < keyMap.size()) { 14517 tableColumn = matchColumn(tableColumns, keyMap.get(index)); 14518 } else if (isEmptyCollection(keyMap) 14519 && index < tableColumns.size()) { 14520 tableColumn = tableColumns.get(index); 14521 } else { 14522 continue; 14523 } 14524 } else { 14525 continue; 14526 } 14527 } else if (!isEmptyCollection(keyMap) && i < keyMap.size()) { 14528 tableColumn = matchColumn(tableColumns, keyMap.get(i)); 14529 } else { 14530 tableColumn = tableModel.getColumns().get(i); 14531 } 14532 } else { 14533 TObjectName matchedColumnName = fieldAttr; 14534 tableColumn = matchColumn(tableColumns, matchedColumnName); 14535 if (tableColumn == null) { 14536 if (!isEmptyCollection(valueMap)) { 14537 int index = indexOfColumn(valueMap, matchedColumnName); 14538 if (index != -1) { 14539 if (!isEmptyCollection(keyMap) && index < keyMap.size()) { 14540 tableColumn = matchColumn(tableColumns, keyMap.get(index)); 14541 } else if (isEmptyCollection(keyMap) 14542 && index < tableColumns.size()) { 14543 tableColumn = tableColumns.get(index); 14544 } else { 14545 continue; 14546 } 14547 } else { 14548 continue; 14549 } 14550 } else if (!isEmptyCollection(keyMap) && i < keyMap.size()) { 14551 tableColumn = matchColumn(tableColumns, keyMap.get(i)); 14552 } else { 14553 tableColumn = modelFactory.createInsertTableColumn(tableModel, 14554 fieldAttr); 14555 } 14556 } 14557 } 14558 14559 if (!"*".equals(getColumnName(tableColumn.getColumnObject())) 14560 && "*".equals(getColumnName(fieldAttr))) { 14561 TObjectName columnObject = fieldAttr; 14562 TTable sourceTable = columnObject.getSourceTable(); 14563 if (columnObject.getTableToken() != null && sourceTable != null) { 14564 TObjectName[] columns = modelManager.getTableColumns(sourceTable); 14565 for (int j = 0; j < columns.length; j++) { 14566 TObjectName columnName = columns[j]; 14567 if (columnName == null || "*".equals(getColumnName(columnName))) { 14568 continue; 14569 } 14570 resultColumn.bindStarLinkColumn(columnName); 14571 } 14572 } else { 14573 TTableList tables = stmt.getTables(); 14574 for (int k = 0; k < tables.size(); k++) { 14575 TTable tableElement = tables.getTable(k); 14576 TObjectName[] columns = modelManager.getTableColumns(tableElement); 14577 for (int j = 0; j < columns.length; j++) { 14578 TObjectName columnName = columns[j]; 14579 if (columnName == null || "*".equals(getColumnName(columnName))) { 14580 continue; 14581 } 14582 resultColumn.bindStarLinkColumn(columnName); 14583 } 14584 } 14585 } 14586 } 14587 14588 if ("*".equals(getColumnName(tableColumn.getColumnObject())) && resultColumn != null 14589 && !resultColumn.getStarLinkColumns().isEmpty()) { 14590 tableColumn.bindStarLinkColumns(resultColumn.getStarLinkColumns()); 14591 } 14592 14593 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 14594 relation.setEffectType(effectType); 14595 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 14596 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 14597 14598 if (tableColumn.getName().endsWith("*") && resultColumn.getName().endsWith("*")) { 14599 tableColumn.getTable().setStarStmt("insert"); 14600 } 14601 14602 Process process = modelFactory.createProcess(stmt); 14603 relation.setProcess(process); 14604 } else if (((TResultColumn) resultColumn.getColumnObject()).getExpr() 14605 .getExpressionType() == EExpressionType.simple_constant_t) { 14606 if (!initColumn) { 14607 TableColumn tableColumn; 14608 if (tableModel.isCreateTable() && !containStarColumn(tableModel.getColumns())) { 14609 if (tableModel.getColumns().size() <= i) { 14610 continue; 14611 } 14612 tableColumn = tableModel.getColumns().get(i); 14613 } else { 14614 tableColumn = modelFactory.createInsertTableColumn(tableModel, 14615 ((TResultColumn) resultColumn.getColumnObject()).getExpr() 14616 .getConstantOperand(), 14617 i); 14618 } 14619 14620 if (DlineageUtil.isTempTable(tableModel, option.getVendor()) && sqlenv != null 14621 && tableModel.getDatabase() != null && tableModel.getSchema() != null) { 14622 TSQLSchema schema = sqlenv.getSQLSchema( 14623 tableModel.getDatabase() + "." + tableModel.getSchema(), true); 14624 if (schema != null) { 14625 TSQLTable tempTable = schema.createTable( 14626 DlineageUtil.getSimpleTableName(tableModel.getName())); 14627 tempTable.addColumn(tableColumn.getName()); 14628 } 14629 } 14630 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 14631 relation.setEffectType(effectType); 14632 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 14633 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 14634 Process process = modelFactory.createProcess(stmt); 14635 relation.setProcess(process); 14636 } else { 14637 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 14638 relation.setEffectType(effectType); 14639 relation.setTarget( 14640 new TableColumnRelationshipElement(tableModel.getColumns().get(i))); 14641 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 14642 Process process = modelFactory.createProcess(stmt); 14643 relation.setProcess(process); 14644 } 14645 } else { 14646 if (!initColumn) { 14647 TableColumn tableColumn; 14648 if (tableModel.isCreateTable() && !containStarColumn(tableModel.getColumns())) { 14649 if (tableModel.getColumns().size() <= i) { 14650 continue; 14651 } 14652 tableColumn = tableModel.getColumns().get(i); 14653 } else { 14654 tableColumn = modelFactory.createInsertTableColumn(tableModel, 14655 ((TResultColumn) resultColumn.getColumnObject()).getExpr(), i); 14656 } 14657 if (DlineageUtil.isTempTable(tableModel, option.getVendor()) && sqlenv != null 14658 && tableModel.getDatabase() != null && tableModel.getSchema() != null) { 14659 TSQLSchema schema = sqlenv.getSQLSchema( 14660 tableModel.getDatabase() + "." + tableModel.getSchema(), true); 14661 if (schema != null) { 14662 TSQLTable tempTable = schema.createTable( 14663 DlineageUtil.getSimpleTableName(tableModel.getName())); 14664 tempTable.addColumn(tableColumn.getName()); 14665 } 14666 } 14667 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 14668 relation.setEffectType(effectType); 14669 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 14670 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 14671 Process process = modelFactory.createProcess(stmt); 14672 relation.setProcess(process); 14673 } else { 14674 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 14675 relation.setEffectType(effectType); 14676 relation.setTarget( 14677 new TableColumnRelationshipElement(tableModel.getColumns().get(i))); 14678 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 14679 Process process = modelFactory.createProcess(stmt); 14680 relation.setProcess(process); 14681 } 14682 } 14683 } 14684 } 14685 } else if (stmt.getSubQuery() != null) { 14686 ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt.getSubQuery()); 14687 if (resultSetModel != null) { 14688 14689 if (!resultSetModel.getRelationRows().getHoldRelations().isEmpty()) { 14690 ImpactRelationship impactRelation = modelFactory.createImpactRelation(); 14691 impactRelation.setEffectType(effectType); 14692 impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>( 14693 resultSetModel.getRelationRows())); 14694 impactRelation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>( 14695 tableModel.getRelationRows())); 14696 } 14697 14698 if(stmt.getColumnList()!=null && stmt.getColumnList().size()>0) { 14699 for(int i=0;i<stmt.getColumnList().size();i++) { 14700 TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel, 14701 stmt.getColumnList().getObjectName(i)); 14702 } 14703 14704 List<TableColumn> columns = tableModel.getColumns(); 14705 int resultSetSize = resultSetModel.getColumns().size(); 14706 int starIndex = 0; 14707 int j = 0; 14708 boolean fromStruct = false; 14709 for (int i = 0; i < columns.size() && j < resultSetSize; i++) { 14710 String column = columns.get(i).getName(); 14711 ResultColumn resultColumn = resultSetModel.getColumns().get(j); 14712 if (!resultColumn.getName().contains("*")) { 14713 if (resultColumn.getName().equals(resultColumn.getRefColumnName()) 14714 && resultColumn.getColumnObject().toString().endsWith("*") 14715 && resultSetSize == 1) { 14716 starIndex++; 14717 if (resultSetSize - j == columns.size() - i) { 14718 j++; 14719 14720 } 14721 } 14722 else { 14723 j++; 14724 } 14725 } else { 14726 starIndex++; 14727 if (resultSetSize - j == columns.size() - i) { 14728 j++; 14729 14730 } 14731 } 14732 if (column != null) { 14733 TableColumn tableColumn; 14734 // if (!initColumn) { 14735 tableColumn = matchColumn(tableModel.getColumns(), columns.get(i)); 14736 if (tableColumn == null) { 14737 if (tableModel.isCreateTable() 14738 && !containStarColumn(tableModel.getColumns())) { 14739 if (tableModel.getColumns().size() <= i) { 14740 continue; 14741 } 14742 tableColumn = tableModel.getColumns().get(i); 14743 } else { 14744 TObjectName columnName = new TObjectName(); 14745 columnName.setString(column); 14746 tableColumn = modelFactory.createTableColumn(tableModel, columnName, 14747 false); 14748 } 14749 } 14750 else if (!resultColumn.isStruct() && tableColumn.isStruct() && columns.size() != resultSetSize) { 14751 j--; 14752 fromStruct = true; 14753 } 14754 if(fromStruct && !tableColumn.isStruct()) { 14755 fromStruct = false; 14756 resultColumn = resultSetModel.getColumns().get(j); 14757 j++; 14758 } 14759 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 14760 relation.setEffectType(effectType); 14761 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 14762 if (resultColumn.hasStarLinkColumn() 14763 && resultColumn.getStarLinkColumnNames().size() > starIndex - 1) { 14764 ResultColumn expandStarColumn = modelFactory.createResultColumn( 14765 resultSetModel, resultColumn.getStarLinkColumnName(starIndex - 1), 14766 false); 14767 relation.addSource(new ResultColumnRelationshipElement(expandStarColumn)); 14768 } else { 14769 relation.addSource(new ResultColumnRelationshipElement(resultColumn, starIndex - 1)); 14770 } 14771 Process process = modelFactory.createProcess(stmt); 14772 relation.setProcess(process); 14773 } 14774 } 14775 } 14776 else { 14777 for (int i = 0; i < resultSetModel.getColumns().size(); i++) { 14778 ResultColumn resultColumn = resultSetModel.getColumns().get(i); 14779 TAliasClause alias = null; 14780 if(resultColumn.getColumnObject() instanceof TResultColumn) { 14781 alias = ((TResultColumn) resultColumn.getColumnObject()).getAliasClause(); 14782 } 14783 if (stmt.getColumnList() != null) { 14784 if (i < stmt.getColumnList().size()) { 14785 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 14786 relation.setEffectType(effectType); 14787 TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel, 14788 stmt.getColumnList().getObjectName(i)); 14789 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 14790 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 14791 Process process = modelFactory.createProcess(stmt); 14792 relation.setProcess(process); 14793 } 14794 } else { 14795 if (alias != null && alias.getAliasName() != null) { 14796 TableColumn tableColumn; 14797 if (!initColumn) { 14798 if (tableModel.isCreateTable() 14799 && !containStarColumn(tableModel.getColumns())) { 14800 if (tableModel.getColumns().size() <= i) { 14801 continue; 14802 } 14803 tableColumn = tableModel.getColumns().get(i); 14804 } else { 14805 tableColumn = modelFactory.createInsertTableColumn(tableModel, 14806 alias.getAliasName()); 14807 } 14808 } else { 14809 tableColumn = matchColumn(tableColumns, alias.getAliasName()); 14810 if (tableColumn == null) { 14811 continue; 14812 } 14813 } 14814 14815 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 14816 relation.setEffectType(effectType); 14817 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 14818 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 14819 Process process = modelFactory.createProcess(stmt); 14820 relation.setProcess(process); 14821 } else if (resultColumn.getColumnObject() instanceof TObjectName 14822 || ( resultColumn.getColumnObject() instanceof TResultColumn && ((TResultColumn) resultColumn.getColumnObject()) 14823 .getFieldAttr() != null)) { 14824 TObjectName fieldAttr = null; 14825 if (resultColumn.getColumnObject() instanceof TObjectName) { 14826 fieldAttr = (TObjectName)resultColumn.getColumnObject(); 14827 } 14828 else if (resultColumn.getColumnObject() instanceof TResultColumn) { 14829 fieldAttr = ((TResultColumn) resultColumn.getColumnObject()) 14830 .getFieldAttr(); 14831 } 14832 14833 TableColumn tableColumn; 14834 if (!initColumn) { 14835 if (tableModel.isCreateTable() 14836 && !containStarColumn(tableModel.getColumns())) { 14837 if (tableModel.getColumns().size() <= i) { 14838 continue; 14839 } 14840 tableColumn = tableModel.getColumns().get(i); 14841 } else { 14842 tableColumn = modelFactory.createInsertTableColumn(tableModel, 14843 fieldAttr); 14844 } 14845 } else { 14846 tableColumn = matchColumn(tableColumns, fieldAttr); 14847 if (tableColumn == null) { 14848 continue; 14849 } 14850 } 14851 14852 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 14853 relation.setEffectType(effectType); 14854 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 14855 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 14856 Process process = modelFactory.createProcess(stmt); 14857 relation.setProcess(process); 14858 } else if (((TResultColumn) resultColumn.getColumnObject()).getExpr() 14859 .getExpressionType() == EExpressionType.simple_constant_t) { 14860 if (!initColumn) { 14861 TableColumn tableColumn; 14862 if (tableModel.isCreateTable() 14863 && !containStarColumn(tableModel.getColumns())) { 14864 if (tableModel.getColumns().size() <= i) { 14865 continue; 14866 } 14867 tableColumn = tableModel.getColumns().get(i); 14868 } else { 14869 tableColumn = modelFactory.createInsertTableColumn(tableModel, 14870 ((TResultColumn) resultColumn.getColumnObject()).getExpr() 14871 .getConstantOperand(), 14872 i); 14873 } 14874 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 14875 relation.setEffectType(effectType); 14876 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 14877 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 14878 Process process = modelFactory.createProcess(stmt); 14879 relation.setProcess(process); 14880 } 14881 } else { 14882 if (!initColumn) { 14883 TableColumn tableColumn; 14884 if (tableModel.isCreateTable() 14885 && !containStarColumn(tableModel.getColumns())) { 14886 if (tableModel.getColumns().size() <= i) { 14887 continue; 14888 } 14889 tableColumn = tableModel.getColumns().get(i); 14890 } else { 14891 tableColumn = modelFactory.createInsertTableColumn(tableModel, 14892 ((TResultColumn) resultColumn.getColumnObject()).getExpr(), i); 14893 } 14894 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 14895 relation.setEffectType(effectType); 14896 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 14897 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 14898 Process process = modelFactory.createProcess(stmt); 14899 relation.setProcess(process); 14900 } 14901 } 14902 } 14903 } 14904 } 14905 } 14906 } 14907 } 14908 } else if (stmt.getColumnList() != null && stmt.getColumnList().size() > 0) { 14909 TObjectNameList items = stmt.getColumnList(); 14910 TMultiTargetList values = stmt.getValues(); 14911 if (values != null) { 14912 for (int k = 0; values != null && k < values.size(); k++) { 14913 int j = 0; 14914 for (int i = 0; i < items.size(); i++) { 14915 TObjectName column = items.getObjectName(i); 14916 TableColumn tableColumn; 14917 if (!initColumn) { 14918 if (tableModel.isCreateTable() && !containStarColumn(tableModel.getColumns())) { 14919 if (tableModel.getColumns().size() <= i) { 14920 continue; 14921 } 14922 tableColumn = tableModel.getColumns().get(i); 14923 } else { 14924 tableColumn = modelFactory.createInsertTableColumn(tableModel, column); 14925 } 14926 } else { 14927 tableColumn = matchColumn(tableColumns, column); 14928 if (tableColumn == null) { 14929 continue; 14930 } 14931 } 14932 TResultColumn columnObject = values.getMultiTarget(k).getColumnList().getResultColumn(j); 14933 if (columnObject == null) { 14934 continue; 14935 } 14936 TExpression valueExpr = columnObject.getExpr(); 14937 columnsInExpr visitor = new columnsInExpr(); 14938 valueExpr.inOrderTraverse(visitor); 14939 List<TObjectName> objectNames = visitor.getObjectNames(); 14940 List<TParseTreeNode> constants = visitor.getConstants(); 14941 List<TParseTreeNode> functions = visitor.getFunctions(); 14942 List<TSelectSqlStatement> subquerys = visitor.getSubquerys(); 14943 14944 Process process = modelFactory.createProcess(stmt); 14945 14946 14947 14948 if (functions != null && !functions.isEmpty()) { 14949 analyzeFunctionDataFlowRelation(tableColumn, functions, effectType, process); 14950 } 14951 14952 if (subquerys != null && !subquerys.isEmpty()) { 14953 analyzeSubqueryDataFlowRelation(tableColumn, subquerys, effectType, process); 14954 } 14955 if (objectNames != null && !objectNames.isEmpty()) { 14956 analyzeDataFlowRelation(tableColumn, objectNames, null, effectType, functions, 14957 process, i); 14958 } 14959 //insert into values generate too many constant relations, ignore constant relations. 14960 if (constants != null && !constants.isEmpty()) { 14961 if (!option.isIgnoreInsertIntoValues() || stmt.getParentStmt() != null) { 14962 analyzeConstantDataFlowRelation(tableColumn, constants, effectType, 14963 functions, process); 14964 } 14965 } 14966 j++; 14967 } 14968 } 14969 } else if (stmt.getExecuteStmt() != null && stmt.getExecuteStmt().getModuleName() != null) { 14970 analyzeCustomSqlStmt(stmt.getExecuteStmt()); 14971 Procedure procedure = modelManager.getProcedureByName(DlineageUtil 14972 .getIdentifierNormalTableName(stmt.getExecuteStmt().getModuleName().toString())); 14973 if ((procedure != null && procedure.getProcedureObject() instanceof TStoredProcedureSqlStatement) 14974 || isSpExecuteExternalScript(stmt.getExecuteStmt().getModuleName())) { 14975 List<ResultSet> resultSetModels = calleeResultSetsForInsertExec(stmt.getExecuteStmt(), procedure); 14976 if (resultSetModels != null) { 14977 for (ResultSet resultSetModel : resultSetModels) { 14978 if (resultSetModel != null) { 14979 for (int i = 0; i < resultSetModel.getColumns().size(); i++) { 14980 ResultColumn resultColumn = resultSetModel.getColumns().get(i); 14981 Transform transform = new Transform(); 14982 transform.setType(Transform.FUNCTION); 14983 transform.setCode(stmt.getExecuteStmt().getModuleName()); 14984 resultColumn.setTransform(transform); 14985 14986 if (stmt.getColumnList() != null) { 14987 if (i < stmt.getColumnList().size()) { 14988 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 14989 relation.setEffectType(effectType); 14990 TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel, 14991 stmt.getColumnList().getObjectName(i)); 14992 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 14993 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 14994 Process process = modelFactory.createProcess(stmt); 14995 relation.setProcess(process); 14996 } 14997 } else { 14998 if (resultColumn.getColumnObject() instanceof TObjectName) { 14999 TObjectName fieldAttr = ((TObjectName) resultColumn.getColumnObject()); 15000 TableColumn tableColumn; 15001 if (!initColumn) { 15002 if (tableModel.isCreateTable() 15003 && !containStarColumn(tableModel.getColumns())) { 15004 if (tableModel.getColumns().size() <= i) { 15005 continue; 15006 } 15007 tableColumn = tableModel.getColumns().get(i); 15008 } else { 15009 tableColumn = modelFactory.createInsertTableColumn(tableModel, 15010 fieldAttr); 15011 } 15012 } else { 15013 tableColumn = matchColumn(tableColumns, fieldAttr); 15014 if (tableColumn == null) { 15015 continue; 15016 } 15017 } 15018 15019 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 15020 relation.setEffectType(effectType); 15021 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 15022 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 15023 Process process = modelFactory.createProcess(stmt); 15024 relation.setProcess(process); 15025 15026 15027 } else { 15028 TAliasClause alias = ((TResultColumn) resultColumn.getColumnObject()) 15029 .getAliasClause(); 15030 if (alias != null && alias.getAliasName() != null) { 15031 TableColumn tableColumn; 15032 if (!initColumn) { 15033 if (tableModel.isCreateTable() 15034 && !containStarColumn(tableModel.getColumns())) { 15035 if (tableModel.getColumns().size() <= i) { 15036 continue; 15037 } 15038 tableColumn = tableModel.getColumns().get(i); 15039 } else { 15040 tableColumn = modelFactory.createInsertTableColumn(tableModel, 15041 alias.getAliasName()); 15042 } 15043 } else { 15044 tableColumn = matchColumn(tableColumns, alias.getAliasName()); 15045 if (tableColumn == null) { 15046 continue; 15047 } 15048 } 15049 15050 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 15051 relation.setEffectType(effectType); 15052 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 15053 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 15054 Process process = modelFactory.createProcess(stmt); 15055 relation.setProcess(process); 15056 } else if (((TResultColumn) resultColumn.getColumnObject()) 15057 .getFieldAttr() != null) { 15058 TObjectName fieldAttr = ((TResultColumn) resultColumn.getColumnObject()) 15059 .getFieldAttr(); 15060 TableColumn tableColumn; 15061 if (!initColumn) { 15062 if (tableModel.isCreateTable() 15063 && !containStarColumn(tableModel.getColumns())) { 15064 if (tableModel.getColumns().size() <= i) { 15065 continue; 15066 } 15067 tableColumn = tableModel.getColumns().get(i); 15068 } else { 15069 tableColumn = modelFactory.createInsertTableColumn(tableModel, 15070 fieldAttr); 15071 } 15072 } else { 15073 tableColumn = matchColumn(tableColumns, fieldAttr); 15074 if (tableColumn == null) { 15075 continue; 15076 } 15077 } 15078 15079 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 15080 relation.setEffectType(effectType); 15081 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 15082 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 15083 Process process = modelFactory.createProcess(stmt); 15084 relation.setProcess(process); 15085 } else if (((TResultColumn) resultColumn.getColumnObject()).getExpr() 15086 .getExpressionType() == EExpressionType.simple_constant_t) { 15087 if (!initColumn) { 15088 TableColumn tableColumn; 15089 if (tableModel.isCreateTable() 15090 && !containStarColumn(tableModel.getColumns())) { 15091 if (tableModel.getColumns().size() <= i) { 15092 continue; 15093 } 15094 tableColumn = tableModel.getColumns().get(i); 15095 } else { 15096 tableColumn = modelFactory.createInsertTableColumn(tableModel, 15097 ((TResultColumn) resultColumn.getColumnObject()).getExpr() 15098 .getConstantOperand(), 15099 i); 15100 } 15101 15102 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 15103 relation.setEffectType(effectType); 15104 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 15105 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 15106 Process process = modelFactory.createProcess(stmt); 15107 relation.setProcess(process); 15108 } 15109 } else { 15110 if (!initColumn) { 15111 TableColumn tableColumn; 15112 if (tableModel.isCreateTable() 15113 && !containStarColumn(tableModel.getColumns())) { 15114 if (tableModel.getColumns().size() <= i) { 15115 continue; 15116 } 15117 tableColumn = tableModel.getColumns().get(i); 15118 } else { 15119 tableColumn = modelFactory.createInsertTableColumn(tableModel, 15120 ((TResultColumn) resultColumn.getColumnObject()).getExpr(), 15121 i); 15122 } 15123 15124 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 15125 relation.setEffectType(effectType); 15126 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 15127 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 15128 Process process = modelFactory.createProcess(stmt); 15129 relation.setProcess(process); 15130 } 15131 } 15132 } 15133 } 15134 } 15135 } 15136 } 15137 } 15138 } 15139 else if (procedure!=null && procedure.getProcedureObject() instanceof TObjectName) { 15140 TObjectName functionName = new TObjectName(); 15141 functionName.setString(procedure.getName()); 15142 Function function = (Function)createFunction(functionName); 15143 if (stmt.getColumnList() != null) { 15144 for (int i = 0; i < stmt.getColumnList().size(); i++) { 15145 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 15146 relation.setEffectType(effectType); 15147 TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel, 15148 stmt.getColumnList().getObjectName(i)); 15149 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 15150 relation.addSource(new ResultColumnRelationshipElement(function.getColumns().get(0))); 15151 Process process = modelFactory.createProcess(stmt); 15152 relation.setProcess(process); 15153 } 15154 } 15155 } 15156 } else if (stmt.getExecuteStmt() != null) { 15157 // INSERT INTO t EXEC(@sql) / EXEC('...'): the string form has no module name, 15158 // so the branch above never saw it - no dynamic site was recorded and the 15159 // inner statement was never analyzed. Analyze the EXEC like its top-level 15160 // twin (site + inner lineage); the INSERT target itself stays unlinked. 15161 analyzeCustomSqlStmt(stmt.getExecuteStmt()); 15162 } 15163 } else if (stmt.getValues() != null && stmt.getValues().size() > 0 && tableModel.isCreateTable() && !tableModel.getColumns().isEmpty()) { 15164 for (int k = 0; stmt.getValues() != null && k < stmt.getValues().size(); k++) { 15165 TResultColumnList columns = stmt.getValues().getMultiTarget(k).getColumnList(); 15166 boolean allConstant = true; 15167 Process process = modelFactory.createProcess(stmt); 15168 int columnSize = tableModel.getColumns().size(); 15169 for (int x = 0; x < columns.size(); x++) { 15170 if (x >= columnSize) { 15171 break; 15172 } 15173 TableColumn tableColumn = tableModel.getColumns().get(x); 15174 TResultColumn columnObject = columns.getResultColumn(x); 15175 if (columnObject == null) { 15176 continue; 15177 } 15178 TExpression valueExpr = columnObject.getExpr(); 15179 columnsInExpr visitor = new columnsInExpr(); 15180 valueExpr.inOrderTraverse(visitor); 15181 List<TObjectName> objectNames = visitor.getObjectNames(); 15182 List<TParseTreeNode> constants = visitor.getConstants(); 15183 List<TParseTreeNode> functions = visitor.getFunctions(); 15184 List<TSelectSqlStatement> subquerys = visitor.getSubquerys(); 15185 15186 if (functions != null && !functions.isEmpty()) { 15187 analyzeFunctionDataFlowRelation(tableColumn, functions, effectType, process); 15188 allConstant = false; 15189 } 15190 15191 if (subquerys != null && !subquerys.isEmpty()) { 15192 analyzeSubqueryDataFlowRelation(tableColumn, subquerys, effectType, process); 15193 allConstant = false; 15194 } 15195 if (objectNames != null && !objectNames.isEmpty()) { 15196 analyzeDataFlowRelation(tableColumn, objectNames, null, effectType, functions, 15197 process); 15198 allConstant = false; 15199 } 15200 //insert into values generate too many constant relations, ignore constant relations. 15201 if (constants != null && !constants.isEmpty() && stmt.getParentStmt() != null) { 15202 analyzeConstantDataFlowRelation(tableColumn, constants, effectType, functions, 15203 process); 15204 allConstant = false; 15205 } 15206 } 15207 15208 if(allConstant) { 15209 modelManager.unbindProcessModel(stmt); 15210 tableModel.removeProcess(process); 15211 } 15212 } 15213 } else if (stmt.getRecordName() != null) { 15214 String procedureName = DlineageUtil.getProcedureParentName(stmt); 15215 String variableString = stmt.getRecordName().toString(); 15216 if (variableString.startsWith(":")) { 15217 variableString = variableString.substring(variableString.indexOf(":") + 1); 15218 } 15219 if (!SQLUtil.isEmpty(procedureName)) { 15220 variableString = procedureName + "." + SQLUtil.getIdentifierNormalTableName(variableString); 15221 } 15222 15223 Table recordTable = modelManager 15224 .getTableByName(DlineageUtil.getTableFullName(variableString)); 15225 if (recordTable != null) { 15226 for (int i = 0; i < recordTable.getColumns().size(); i++) { 15227 TableColumn sourceTableColumn = recordTable.getColumns().get(i); 15228 TableColumn targetTableColumn = modelFactory.createTableColumn(tableModel, 15229 sourceTableColumn.getColumnObject(), false); 15230 if (targetTableColumn != null) { 15231 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 15232 relation.setEffectType(effectType); 15233 relation.setTarget(new TableColumnRelationshipElement(targetTableColumn)); 15234 relation.addSource(new TableColumnRelationshipElement(sourceTableColumn)); 15235 Process process = modelFactory.createProcess(stmt); 15236 relation.setProcess(process); 15237 } else if (sourceTableColumn.getName().endsWith("*") && tableModel.isCreateTable()) { 15238 for (TableColumn column : tableModel.getColumns()) { 15239 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 15240 relation.setEffectType(effectType); 15241 relation.setTarget(new TableColumnRelationshipElement(column)); 15242 relation.addSource(new TableColumnRelationshipElement(sourceTableColumn)); 15243 Process process = modelFactory.createProcess(stmt); 15244 relation.setProcess(process); 15245 } 15246 } 15247 } 15248 } 15249 } else if (stmt.getInsertSource() == EInsertSource.values_function && stmt.getFunctionCall() != null) { 15250 Table cursor = modelManager.getTableByName( 15251 DlineageUtil.getTableFullName(stmt.getFunctionCall().getFunctionName().toString())); 15252 if (cursor != null) { 15253 TObjectName starColumn = new TObjectName(); 15254 starColumn.setString("*"); 15255 TableColumn insertColumn = modelFactory.createTableColumn(tableModel, starColumn, true); 15256 insertColumn.setShowStar(false); 15257 insertColumn.setExpandStar(true); 15258 for (int j = 0; j < cursor.getColumns().size(); j++) { 15259 DataFlowRelationship dataflowRelation = modelFactory.createDataFlowRelation(); 15260 dataflowRelation.setEffectType(effectType); 15261 dataflowRelation.addSource(new TableColumnRelationshipElement(cursor.getColumns().get(j))); 15262 dataflowRelation.setTarget(new TableColumnRelationshipElement(insertColumn)); 15263 Process process = modelFactory.createProcess(stmt); 15264 dataflowRelation.setProcess(process); 15265 } 15266 } 15267 15268 } else if (stmt.getInsertSource() == EInsertSource.values && stmt.getValues() != null) { 15269 TObjectName starColumn = new TObjectName(); 15270 starColumn.setString("*"); 15271 TableColumn insertColumn = modelFactory.createTableColumn(tableModel, starColumn, true); 15272 insertColumn.setShowStar(false); 15273 insertColumn.setExpandStar(true); 15274 for (int k = 0; stmt.getValues() != null && k < stmt.getValues().size(); k++) { 15275 TResultColumnList columns = stmt.getValues().getMultiTarget(k).getColumnList(); 15276 for (int x = 0; x < columns.size(); x++) { 15277 TResultColumn columnObject = columns.getResultColumn(x); 15278 if (columnObject == null) { 15279 continue; 15280 } 15281 TExpression valueExpr = columnObject.getExpr(); 15282 columnsInExpr visitor = new columnsInExpr(); 15283 valueExpr.inOrderTraverse(visitor); 15284 List<TObjectName> objectNames = visitor.getObjectNames(); 15285 List<TParseTreeNode> constants = visitor.getConstants(); 15286 List<TParseTreeNode> functions = visitor.getFunctions(); 15287 List<TSelectSqlStatement> subquerys = visitor.getSubquerys(); 15288 15289 Process process = modelFactory.createProcess(stmt); 15290 if (functions != null && !functions.isEmpty()) { 15291 analyzeFunctionDataFlowRelation(insertColumn, functions, effectType, process); 15292 } 15293 15294 if (subquerys != null && !subquerys.isEmpty()) { 15295 analyzeSubqueryDataFlowRelation(insertColumn, subquerys, effectType, process); 15296 } 15297 if (objectNames != null && !objectNames.isEmpty()) { 15298 analyzeDataFlowRelation(insertColumn, objectNames, null, effectType, functions, 15299 process); 15300 } 15301 //insert into values generate too many constant relations, ignore constant relations. 15302 if (constants != null && !constants.isEmpty() && stmt.getParentStmt() != null) { 15303 analyzeConstantDataFlowRelation(insertColumn, constants, effectType, functions, 15304 process); 15305 } 15306 } 15307 } 15308 }else if (stmt.getExecuteStmt() != null && stmt.getExecuteStmt().getModuleName() != null) { 15309 analyzeCustomSqlStmt(stmt.getExecuteStmt()); 15310 Procedure procedure = modelManager.getProcedureByName(DlineageUtil 15311 .getIdentifierNormalTableName(stmt.getExecuteStmt().getModuleName().toString())); 15312 if ((procedure != null && procedure.getProcedureObject() instanceof TStoredProcedureSqlStatement) 15313 || isSpExecuteExternalScript(stmt.getExecuteStmt().getModuleName())) { 15314 List<ResultSet> resultSetModels = calleeResultSetsForInsertExec(stmt.getExecuteStmt(), procedure); 15315 if (resultSetModels != null) { 15316 for (ResultSet resultSetModel : resultSetModels) { 15317 if (resultSetModel != null) { 15318 for (int i = 0; i < resultSetModel.getColumns().size(); i++) { 15319 ResultColumn resultColumn = resultSetModel.getColumns().get(i); 15320 15321 Transform transform = new Transform(); 15322 transform.setType(Transform.FUNCTION); 15323 transform.setCode(stmt.getExecuteStmt().getModuleName()); 15324 resultColumn.setTransform(transform); 15325 15326 TAliasClause alias = null; 15327 15328 if (resultColumn.getColumnObject() instanceof TResultColumn) { 15329 alias = ((TResultColumn) resultColumn.getColumnObject()) 15330 .getAliasClause(); 15331 } 15332 15333 if (alias != null && alias.getAliasName() != null) { 15334 TableColumn tableColumn; 15335 if (!initColumn) { 15336 if (tableModel.isCreateTable() 15337 && !containStarColumn(tableModel.getColumns())) { 15338 if(resultColumn.getName().endsWith("*")) { 15339 for(TableColumn tableColumnItem: tableModel.getColumns()) { 15340 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 15341 relation.setEffectType(effectType); 15342 relation.setTarget(new TableColumnRelationshipElement(tableColumnItem)); 15343 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 15344 Process process = modelFactory.createProcess(stmt); 15345 relation.setProcess(process); 15346 } 15347 continue; 15348 } 15349 else { 15350 if (tableModel.getColumns().size() <= i) { 15351 continue; 15352 } 15353 tableColumn = tableModel.getColumns().get(i); 15354 } 15355 } else { 15356 tableColumn = modelFactory.createInsertTableColumn(tableModel, 15357 alias.getAliasName()); 15358 } 15359 } else { 15360 tableColumn = matchColumn(tableColumns, alias.getAliasName()); 15361 if (tableColumn == null) { 15362 continue; 15363 } 15364 } 15365 15366 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 15367 relation.setEffectType(effectType); 15368 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 15369 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 15370 Process process = modelFactory.createProcess(stmt); 15371 relation.setProcess(process); 15372 } else if (resultColumn.getColumnObject() instanceof TObjectName 15373 || (resultColumn.getColumnObject() instanceof TResultColumn 15374 && ((TResultColumn) resultColumn.getColumnObject()) 15375 .getFieldAttr() != null)) { 15376 TObjectName fieldAttr = null; 15377 if (resultColumn.getColumnObject() instanceof TObjectName) { 15378 fieldAttr = (TObjectName) resultColumn.getColumnObject(); 15379 } else { 15380 fieldAttr = ((TResultColumn) resultColumn.getColumnObject()).getFieldAttr(); 15381 } 15382 TableColumn tableColumn; 15383 if (!initColumn) { 15384 if (tableModel.isCreateTable() 15385 && !containStarColumn(tableModel.getColumns())) { 15386 if(resultColumn.getName().endsWith("*")) { 15387 for(TableColumn tableColumnItem: tableModel.getColumns()) { 15388 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 15389 relation.setEffectType(effectType); 15390 relation.setTarget(new TableColumnRelationshipElement(tableColumnItem)); 15391 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 15392 Process process = modelFactory.createProcess(stmt); 15393 relation.setProcess(process); 15394 } 15395 continue; 15396 } 15397 else { 15398 if (tableModel.getColumns().size() <= i) { 15399 continue; 15400 } 15401 tableColumn = tableModel.getColumns().get(i); 15402 } 15403 } else { 15404 tableColumn = modelFactory.createInsertTableColumn(tableModel, 15405 fieldAttr); 15406 } 15407 } else { 15408 tableColumn = matchColumn(tableColumns, fieldAttr); 15409 if (tableColumn == null) { 15410 continue; 15411 } 15412 } 15413 15414 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 15415 relation.setEffectType(effectType); 15416 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 15417 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 15418 Process process = modelFactory.createProcess(stmt); 15419 relation.setProcess(process); 15420 } else if (((TResultColumn) resultColumn.getColumnObject()).getExpr() 15421 .getExpressionType() == EExpressionType.simple_constant_t) { 15422 if (!initColumn) { 15423 TableColumn tableColumn; 15424 if (tableModel.isCreateTable() 15425 && !containStarColumn(tableModel.getColumns())) { 15426 if(resultColumn.getName().endsWith("*")) { 15427 for(TableColumn tableColumnItem: tableModel.getColumns()) { 15428 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 15429 relation.setEffectType(effectType); 15430 relation.setTarget(new TableColumnRelationshipElement(tableColumnItem)); 15431 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 15432 Process process = modelFactory.createProcess(stmt); 15433 relation.setProcess(process); 15434 } 15435 continue; 15436 } 15437 else { 15438 if (tableModel.getColumns().size() <= i) { 15439 continue; 15440 } 15441 tableColumn = tableModel.getColumns().get(i); 15442 } 15443 } else { 15444 tableColumn = modelFactory.createInsertTableColumn(tableModel, 15445 ((TResultColumn) resultColumn.getColumnObject()).getExpr() 15446 .getConstantOperand(), 15447 i); 15448 } 15449 15450 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 15451 relation.setEffectType(effectType); 15452 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 15453 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 15454 Process process = modelFactory.createProcess(stmt); 15455 relation.setProcess(process); 15456 } 15457 } else { 15458 if (!initColumn) { 15459 TableColumn tableColumn; 15460 if (tableModel.isCreateTable() 15461 && !containStarColumn(tableModel.getColumns())) { 15462 if(resultColumn.getName().endsWith("*")) { 15463 for(TableColumn tableColumnItem: tableModel.getColumns()) { 15464 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 15465 relation.setEffectType(effectType); 15466 relation.setTarget(new TableColumnRelationshipElement(tableColumnItem)); 15467 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 15468 Process process = modelFactory.createProcess(stmt); 15469 relation.setProcess(process); 15470 } 15471 continue; 15472 } 15473 else { 15474 if (tableModel.getColumns().size() <= i) { 15475 continue; 15476 } 15477 tableColumn = tableModel.getColumns().get(i); 15478 } 15479 } else { 15480 tableColumn = modelFactory.createInsertTableColumn(tableModel, 15481 ((TResultColumn) resultColumn.getColumnObject()).getExpr(), 15482 i); 15483 } 15484 15485 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 15486 relation.setEffectType(effectType); 15487 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 15488 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 15489 Process process = modelFactory.createProcess(stmt); 15490 relation.setProcess(process); 15491 } 15492 } 15493 } 15494 } 15495 } 15496 } 15497 } 15498 else if (procedure!=null && procedure.getProcedureObject() instanceof TObjectName) { 15499 TObjectName functionName = new TObjectName(); 15500 functionName.setString(procedure.getName()); 15501 Function function = (Function)createFunction(functionName); 15502 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 15503 relation.setEffectType(effectType); 15504 TObjectName starColumn = new TObjectName(); 15505 starColumn.setString("*"); 15506 TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel, 15507 starColumn); 15508 tableColumn.setExpandStar(false); 15509 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 15510 relation.addSource(new ResultColumnRelationshipElement(function.getColumns().get(0))); 15511 Process process = modelFactory.createProcess(stmt); 15512 relation.setProcess(process); 15513 } 15514 } else if (stmt.getExecuteStmt() != null) { 15515 // INSERT INTO t EXEC(@sql) / EXEC('...'): the string form has no module name, 15516 // so the branch above never saw it - no dynamic site was recorded and the 15517 // inner statement was never analyzed. Analyze the EXEC like its top-level 15518 // twin (site + inner lineage); the INSERT target itself stays unlinked. 15519 analyzeCustomSqlStmt(stmt.getExecuteStmt()); 15520 } 15521 } 15522 15523 if(stmt.getOnDuplicateKeyUpdate()!=null) { 15524 TTable table = stmt.getTargetTable(); 15525 Table tableModel = modelFactory.createTable(table); 15526 for(TResultColumn column: stmt.getOnDuplicateKeyUpdate()) { 15527 if(column.getExpr()==null || column.getExpr().getExpressionType() != EExpressionType.assignment_t) { 15528 continue; 15529 } 15530 TExpression left = column.getExpr().getLeftOperand(); 15531 TExpression right = column.getExpr().getRightOperand(); 15532 TObjectName columnObject = left.getObjectOperand(); 15533 if (columnObject != null) { 15534 TableColumn tableColumn = modelFactory.createTableColumn(tableModel, columnObject, false); 15535 if (tableColumn != null) { 15536 columnsInExpr visitor = new columnsInExpr(); 15537 right.inOrderTraverse(visitor); 15538 List<TObjectName> objectNames = visitor.getObjectNames(); 15539 List<TParseTreeNode> functions = visitor.getFunctions(); 15540 List<TParseTreeNode> constants = visitor.getConstants(); 15541 List<TSelectSqlStatement> subquerys = visitor.getSubquerys(); 15542 15543 if (functions != null && !functions.isEmpty()) { 15544 analyzeFunctionDataFlowRelation(tableColumn, functions, EffectType.update); 15545 } 15546 if (subquerys != null && !subquerys.isEmpty()) { 15547 analyzeSubqueryDataFlowRelation(tableColumn, subquerys, EffectType.update); 15548 } 15549 if (objectNames != null && !objectNames.isEmpty()) { 15550 analyzeDataFlowRelation(tableColumn, objectNames, EffectType.update, functions); 15551 } 15552 if (constants != null && !constants.isEmpty()) { 15553 analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.update, functions); 15554 } 15555 } 15556 } 15557 } 15558 } 15559 15560 if (!expressions.isEmpty() && stmt.getSubQuery() != null) { 15561 analyzeInsertImpactRelation(stmt.getSubQuery(), tableColumnMap, expressions, effectType); 15562 } 15563 } 15564 15565 /** 15566 * Check whether a SELECT statement actually returns a result set to the client. 15567 * A SELECT does NOT return a result set when: 15568 * <ul> 15569 * <li>It has an INTO clause ({@code SELECT ... INTO @var})</li> 15570 * <li>It has an INTO TABLE clause ({@code SELECT ... INTO TABLE t})</li> 15571 * <li>All result columns are variable assignments ({@code SELECT @var = expr}) — 15572 * MSSQL-only, detected via {@code EExpressionType.sqlserver_proprietary_column_alias_t}</li> 15573 * </ul> 15574 */ 15575 private static boolean isResultSetReturningSelect(TSelectSqlStatement select) { 15576 if (select.getIntoClause() != null || select.getIntoTableClause() != null) { 15577 return false; 15578 } 15579 if (select.getResultColumnList() != null && select.getResultColumnList().size() > 0) { 15580 boolean allAssignments = true; 15581 for (int i = 0; i < select.getResultColumnList().size(); i++) { 15582 TResultColumn col = select.getResultColumnList().getResultColumn(i); 15583 if (col.getExpr() == null 15584 || col.getExpr().getExpressionType() != EExpressionType.sqlserver_proprietary_column_alias_t) { 15585 allAssignments = false; 15586 break; 15587 } 15588 } 15589 if (allAssignments) { 15590 return false; 15591 } 15592 } 15593 return true; 15594 } 15595 15596 private List<TSelectSqlStatement> getLastSelectStmt(TStoredProcedureSqlStatement procedureStmt) { 15597 List<TSelectSqlStatement> stmts = new ArrayList<TSelectSqlStatement>(); 15598 if (procedureStmt.getBodyStatements().size() > 0) { 15599 for (int j = procedureStmt.getBodyStatements().size() - 1; j >= 0; j--) { 15600 TCustomSqlStatement stmtItem = procedureStmt.getBodyStatements().get(j); 15601 if (stmtItem instanceof TReturnStmt || stmtItem instanceof TMssqlReturn) { 15602 if (stmtItem.getStatements() != null) { 15603 List<TSelectSqlStatement> item = getLastSelectStmt(stmtItem); 15604 if (item != null && !item.isEmpty()) { 15605 stmts.addAll(item); 15606 if(option.getVendor()!=EDbVendor.dbvmssql && option.getVendor()!=EDbVendor.dbvazuresql) { 15607 break; 15608 } 15609 } 15610 } 15611 break; 15612 } 15613 if (stmtItem instanceof TSelectSqlStatement && !((TSelectSqlStatement) stmtItem).isQueryOfCTE()) { 15614 if (isResultSetReturningSelect((TSelectSqlStatement) stmtItem)) { 15615 if(!(DlineageUtil.getTopBasicStmt(stmtItem) instanceof TSelectSqlStatement)) { 15616 continue; 15617 } 15618 stmts.add((TSelectSqlStatement) stmtItem); 15619 if(option.getVendor()!=EDbVendor.dbvmssql && option.getVendor()!=EDbVendor.dbvazuresql) { 15620 break; 15621 } 15622 } 15623 } else if (stmtItem.getStatements() != null) { 15624 List<TSelectSqlStatement> item = getLastSelectStmt(stmtItem); 15625 if (item != null && !item.isEmpty()) { 15626 stmts.addAll(item); 15627 if(option.getVendor()!=EDbVendor.dbvmssql && option.getVendor()!=EDbVendor.dbvazuresql) { 15628 break; 15629 } 15630 } 15631 } 15632 } 15633 } 15634 return stmts; 15635 } 15636 15637 private List<TSelectSqlStatement> getLastSelectStmt(TCustomSqlStatement stmt) { 15638 List<TSelectSqlStatement> stmts = new ArrayList<TSelectSqlStatement>(); 15639 for (int j = stmt.getStatements().size() - 1; j >= 0; j--) { 15640 TCustomSqlStatement stmtItem = stmt.getStatements().get(j); 15641 if (stmtItem instanceof TReturnStmt || stmtItem instanceof TMssqlReturn) { 15642 if (stmtItem.getStatements() != null) { 15643 List<TSelectSqlStatement> item = getLastSelectStmt(stmtItem); 15644 if (item != null && !item.isEmpty()) { 15645 stmts.addAll(item); 15646 if(option.getVendor()!=EDbVendor.dbvmssql && option.getVendor()!=EDbVendor.dbvazuresql) { 15647 break; 15648 } 15649 } 15650 } 15651 break; 15652 } 15653 if (stmtItem instanceof TSelectSqlStatement && !((TSelectSqlStatement) stmtItem).isQueryOfCTE()) { 15654 if (isResultSetReturningSelect((TSelectSqlStatement) stmtItem)) { 15655 if(!(DlineageUtil.getTopBasicStmt(stmtItem) instanceof TSelectSqlStatement)) { 15656 continue; 15657 } 15658 stmts.add((TSelectSqlStatement) stmtItem); 15659 if(option.getVendor()!=EDbVendor.dbvmssql && option.getVendor()!=EDbVendor.dbvazuresql) { 15660 break; 15661 } 15662 } 15663 } else if (stmtItem.getStatements() != null) { 15664 List<TSelectSqlStatement> item = getLastSelectStmt(stmtItem); 15665 if (item != null && !item.isEmpty()) { 15666 stmts.addAll(item); 15667 if(option.getVendor()!=EDbVendor.dbvmssql && option.getVendor()!=EDbVendor.dbvazuresql) { 15668 break; 15669 } 15670 } 15671 } 15672 } 15673 return stmts; 15674 } 15675 15676 private TableColumn getStarColumn(List<TableColumn> columns) { 15677 for (TableColumn column : columns) { 15678 if (column.getName().endsWith("*")) { 15679 return column; 15680 } 15681 } 15682 return null; 15683 } 15684 15685 private boolean containStarColumn(List<TableColumn> columns) { 15686 if (columns == null) 15687 return false; 15688 for (TableColumn column : columns) { 15689 if (column.getName().endsWith("*")) { 15690 return true; 15691 } 15692 } 15693 return false; 15694 } 15695 15696 private boolean containStarColumn(ResultSet resultSet) { 15697 if (resultSet == null || resultSet.getColumns() == null) 15698 return false; 15699 for (ResultColumn column : resultSet.getColumns()) { 15700 if (column.getName().endsWith("*")) { 15701 return true; 15702 } 15703 } 15704 return false; 15705 } 15706 15707 private int indexOfColumn(List<TResultColumn> columns, TObjectName objectName) { 15708 for (int i = 0; i < columns.size(); i++) { 15709 if (columns.get(i).toString().trim().equalsIgnoreCase(objectName.toString().trim())) { 15710 return i; 15711 } 15712 } 15713 return -1; 15714 } 15715 15716 private boolean isEmptyCollection(Collection<?> keyMap) { 15717 return keyMap == null || keyMap.isEmpty(); 15718 } 15719 15720 private void analyzeInsertImpactRelation(TSelectSqlStatement stmt, Map<String, List<TableColumn>> insertMap, 15721 List<TExpression> expressions, EffectType effectType) { 15722 List<TObjectName> objectNames = new ArrayList<TObjectName>(); 15723 for (int i = 0; i < expressions.size(); i++) { 15724 TExpression condition = expressions.get(i); 15725 columnsInExpr visitor = new columnsInExpr(); 15726 condition.inOrderTraverse(visitor); 15727 objectNames.addAll(visitor.getObjectNames()); 15728 } 15729 15730 Iterator<String> iter = insertMap.keySet().iterator(); 15731 while (iter.hasNext()) { 15732 String table = iter.next(); 15733 List<TableColumn> tableColumns = insertMap.get(table); 15734 for (int i = 0; i < tableColumns.size(); i++) { 15735 15736 TableColumn column = tableColumns.get(i); 15737 ImpactRelationship relation = modelFactory.createImpactRelation(); 15738 relation.setEffectType(effectType); 15739 relation.setTarget(new TableColumnRelationshipElement(column)); 15740 15741 for (int j = 0; j < objectNames.size(); j++) { 15742 TObjectName columnName = objectNames.get(j); 15743 Object model = modelManager.getModel(stmt); 15744 if (model instanceof SelectResultSet) { 15745 SelectResultSet queryTable = (SelectResultSet) model; 15746 List<ResultColumn> columns = queryTable.getColumns(); 15747 for (int k = 0; k < columns.size(); k++) { 15748 ResultColumn resultColumn = columns.get(k); 15749 if (resultColumn.getAlias() != null 15750 && SQLUtil.compareIdentifier(option.getVendor(), ESQLDataObjectType.dotColumn, columnName.toString(), resultColumn.getAlias())) { 15751 relation.addSource( 15752 new ResultColumnRelationshipElement(resultColumn, columnName.getLocation(), columnName)); 15753 } else if (resultColumn.getName() != null 15754 && SQLUtil.compareIdentifier(option.getVendor(), ESQLDataObjectType.dotColumn, columnName.toString(), resultColumn.getName())) { 15755 relation.addSource( 15756 new ResultColumnRelationshipElement(resultColumn, columnName.getLocation(), columnName)); 15757 } 15758 } 15759 } 15760 } 15761 } 15762 } 15763 } 15764 15765 private void analyzeUpdateStmt(TUpdateSqlStatement stmt) { 15766 if (stmt.getResultColumnList() == null) 15767 return; 15768 15769 TTable targetTable = stmt.getTargetTable(); 15770 Process process = modelFactory.createProcess(stmt); 15771 Table tableModel = null; 15772 QueryTable cteTargetModel = null; 15773 CteWriteTargetResolution cteTargetResolution = null; 15774 if (targetTable.getCTE() != null) { 15775 cteTargetModel = analyzeCteWriteTarget(targetTable, process); 15776 cteTargetResolution = resolveCteWriteTarget(targetTable, collectUpdateCteWriteIntent(stmt)); 15777 if (cteTargetResolution.isWritable()) { 15778 tableModel = modelFactory.createTable(cteTargetResolution.baseTable); 15779 tableModel.addProcess(process); 15780 createProjectedWriteImpact(targetTable.getCTE().getSubquery(), tableModel, 15781 EffectType.update, process); 15782 } 15783 if (cteTargetResolution.failureReason != null) { 15784 addCteWriteTargetHint(targetTable, cteTargetResolution.failureReason, 15785 cteTargetResolution.isWritable()); 15786 } 15787 } else if (targetTable.getSubquery() != null 15788 || (targetTable.getLinkTable() != null 15789 && targetTable.getLinkTable().getSubquery() != null)) { 15790 TTable projectedTarget = targetTable.getSubquery() != null 15791 ? targetTable : targetTable.getLinkTable(); 15792 TSelectSqlStatement projectedSubquery = projectedTarget.getSubquery(); 15793 TObjectNameList exposedColumns = getProjectedTargetColumns(projectedTarget); 15794 cteTargetModel = analyzeProjectedWriteTarget(targetTable, projectedSubquery, 15795 exposedColumns, process); 15796 cteTargetResolution = resolveProjectedWriteTarget(projectedSubquery, exposedColumns, 15797 collectUpdateCteWriteIntent(stmt)); 15798 if (cteTargetResolution.isWritable()) { 15799 tableModel = modelFactory.createTable(cteTargetResolution.baseTable); 15800 tableModel.addProcess(process); 15801 createProjectedWriteImpact(projectedSubquery, tableModel, EffectType.update, process); 15802 } 15803 if (cteTargetResolution.failureReason != null) { 15804 addCteWriteTargetHint(targetTable, cteTargetResolution.failureReason, 15805 cteTargetResolution.isWritable()); 15806 } 15807 } else { 15808 tableModel = modelFactory.createTable(targetTable); 15809 tableModel.addProcess(process); 15810 } 15811 15812 for (int i = 0; i < stmt.tables.size(); i++) { 15813 TTable tableElement = stmt.tables.getTable(i); 15814 if (tableElement == targetTable && cteTargetModel != null) { 15815 continue; 15816 } 15817 if (tableElement.getSubquery() != null) { 15818 QueryTable queryTable = modelFactory.createQueryTable(tableElement); 15819 TSelectSqlStatement subquery = tableElement.getSubquery(); 15820 analyzeSelectStmt(subquery); 15821 15822 if (subquery.getSetOperatorType() != ESetOperatorType.none) { 15823 SelectSetResultSet selectSetResultSetModel = (SelectSetResultSet) modelManager.getModel(subquery); 15824 for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) { 15825 ResultColumn sourceColumn = selectSetResultSetModel.getColumns().get(j); 15826 ResultColumn targetColumn = modelFactory.createSelectSetResultColumn(queryTable, sourceColumn); 15827 DataFlowRelationship selectSetRalation = modelFactory.createDataFlowRelation(); 15828 selectSetRalation.setEffectType(EffectType.select); 15829 selectSetRalation.setTarget(new ResultColumnRelationshipElement(targetColumn)); 15830 selectSetRalation.addSource(new ResultColumnRelationshipElement(sourceColumn)); 15831 selectSetRalation.setProcess(process); 15832 } 15833 } 15834 15835 ResultSet resultSetModel = (ResultSet) modelManager.getModel(tableElement.getSubquery()); 15836 if (resultSetModel != null && resultSetModel != queryTable 15837 && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) { 15838 ImpactRelationship impactRelation = modelFactory.createImpactRelation(); 15839 impactRelation.setEffectType(EffectType.update); 15840 impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>( 15841 resultSetModel.getRelationRows())); 15842 impactRelation.setTarget( 15843 new RelationRowsRelationshipElement<ResultSetRelationRows>(queryTable.getRelationRows())); 15844 } 15845 15846 } else if (tableElement.getCTE() != null) { 15847 QueryTable queryTable = modelFactory.createQueryTable(tableElement); 15848 15849 TObjectNameList cteColumns = tableElement.getCTE().getColumnList(); 15850 if (cteColumns != null) { 15851 for (int j = 0; j < cteColumns.size(); j++) { 15852 modelFactory.createResultColumn(queryTable, cteColumns.getObjectName(j)); 15853 } 15854 } 15855 TSelectSqlStatement subquery = tableElement.getCTE().getSubquery(); 15856 if (subquery != null && !stmtStack.contains(subquery)) { 15857 analyzeSelectStmt(subquery); 15858 15859 ResultSet resultSetModel = (ResultSet) modelManager.getModel(subquery); 15860 if (resultSetModel != null && resultSetModel != queryTable 15861 && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) { 15862 ImpactRelationship impactRelation = modelFactory.createImpactRelation(); 15863 impactRelation.setEffectType(EffectType.select); 15864 impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>( 15865 resultSetModel.getRelationRows())); 15866 impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>( 15867 queryTable.getRelationRows())); 15868 } 15869 15870 if (subquery.getSetOperatorType() != ESetOperatorType.none) { 15871 SelectSetResultSet selectSetResultSetModel = (SelectSetResultSet) modelManager 15872 .getModel(subquery); 15873 for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) { 15874 ResultColumn sourceColumn = selectSetResultSetModel.getColumns().get(j); 15875 ResultColumn targetColumn = null; 15876 if (cteColumns != null) { 15877 targetColumn = queryTable.getColumns().get(j); 15878 } else { 15879 targetColumn = modelFactory.createSelectSetResultColumn(queryTable, sourceColumn); 15880 } 15881 for (Set<TObjectName> starLinkColumns : sourceColumn.getStarLinkColumns().values()) { 15882 for (TObjectName starLinkColumn : starLinkColumns) { 15883 targetColumn.bindStarLinkColumn(starLinkColumn); 15884 } 15885 } 15886 createCteSelectRelation(process, targetColumn, sourceColumn); 15887 } 15888 } else { 15889 for (int j = 0; j < resultSetModel.getColumns().size(); j++) { 15890 ResultColumn sourceColumn = resultSetModel.getColumns().get(j); 15891 ResultColumn targetColumn = null; 15892 if (cteColumns != null) { 15893 targetColumn = queryTable.getColumns().get(j); 15894 } else { 15895 targetColumn = modelFactory.createSelectSetResultColumn(queryTable, sourceColumn); 15896 } 15897 for (TObjectName starLinkColumn : sourceColumn.getStarLinkColumnList()) { 15898 targetColumn.bindStarLinkColumn(starLinkColumn); 15899 } 15900 createCteSelectRelation(process, targetColumn, sourceColumn); 15901 } 15902 } 15903 } else if (tableElement.getCTE().getUpdateStmt() != null) { 15904 analyzeCustomSqlStmt(tableElement.getCTE().getUpdateStmt()); 15905 } else if (tableElement.getCTE().getInsertStmt() != null) { 15906 analyzeCustomSqlStmt(tableElement.getCTE().getInsertStmt()); 15907 } else if (tableElement.getCTE().getDeleteStmt() != null) { 15908 analyzeCustomSqlStmt(tableElement.getCTE().getDeleteStmt()); 15909 } 15910 } else { 15911 modelFactory.createTable(stmt.tables.getTable(i)); 15912 } 15913 } 15914 15915 for (int i = 0; i < stmt.getResultColumnList().size(); i++) { 15916 TResultColumn field = stmt.getResultColumnList().getResultColumn(i); 15917 15918 if (field.getExpr().getExpressionType() == EExpressionType.function_t) { 15919 // Handle SQL Server XML modify() method for data lineage 15920 TFunctionCall funcCall = field.getExpr().getFunctionCall(); 15921 if (tableModel != null && funcCall != null 15922 && funcCall.getFunctionType() == EFunctionType.xmlmodify_t) { 15923 analyzeXmlModifyFunction(stmt, tableModel, process, funcCall); 15924 } 15925 continue; 15926 } 15927 15928 TExpression expression = field.getExpr().getLeftOperand(); 15929 if (expression == null) { 15930 ErrorInfo errorInfo = new ErrorInfo(); 15931 errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR); 15932 errorInfo.setErrorMessage( 15933 "Can't get result column expression. Expression is " + field.getExpr().toString()); 15934 errorInfo.setStartPosition(new Pair3<Long, Long, String>(field.getExpr().getStartToken().lineNo, 15935 field.getExpr().getStartToken().columnNo, ModelBindingManager.getGlobalHash())); 15936 errorInfo.setEndPosition(new Pair3<Long, Long, String>(field.getExpr().getEndToken().lineNo, 15937 field.getExpr().getEndToken().columnNo + field.getExpr().getEndToken().getAstext().length(), 15938 ModelBindingManager.getGlobalHash())); 15939 errorInfo.fillInfo(this); 15940 errorInfos.add(errorInfo); 15941 continue; 15942 } 15943 if (expression.getExpressionType() == EExpressionType.list_t) { 15944 TExpression setExpression = field.getExpr().getRightOperand(); 15945 if (setExpression != null && setExpression.getSubQuery() != null) { 15946 TSelectSqlStatement query = setExpression.getSubQuery(); 15947 analyzeSelectStmt(query); 15948 15949 SelectResultSet resultSetModel = (SelectResultSet) modelManager 15950 .getModel(query.getResultColumnList()); 15951 15952 if (!resultSetModel.getRelationRows().getHoldRelations().isEmpty()) { 15953 ImpactRelationship impactRelation = modelFactory.createImpactRelation(); 15954 impactRelation.setEffectType(EffectType.update); 15955 impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>( 15956 resultSetModel.getRelationRows())); 15957 if (tableModel != null) { 15958 impactRelation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>( 15959 tableModel.getRelationRows())); 15960 } else if (cteTargetModel != null) { 15961 impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>( 15962 cteTargetModel.getRelationRows())); 15963 } 15964 } 15965 15966 TExpressionList columnList = expression.getExprList(); 15967 for (int j = 0; j < columnList.size(); j++) { 15968 TObjectName column = columnList.getExpression(j).getObjectOperand(); 15969 15970 if (column.getDbObjectType() == EDbObjectType.variable) { 15971 continue; 15972 } 15973 15974 if (column.getColumnNameOnly().startsWith("@") && (option.getVendor() == EDbVendor.dbvmssql 15975 || option.getVendor() == EDbVendor.dbvazuresql)) { 15976 continue; 15977 } 15978 15979 if (column.getColumnNameOnly().startsWith(":") && (option.getVendor() == EDbVendor.dbvhana 15980 || option.getVendor() == EDbVendor.dbvteradata)) { 15981 continue; 15982 } 15983 15984 ResultColumn resultColumn = resultSetModel.getColumns().get(j); 15985 TObjectName targetColumn = getCteBaseColumn(cteTargetResolution, column); 15986 TableColumn tableColumn = tableModel == null || targetColumn == null ? null 15987 : modelFactory.createTableColumn(tableModel, targetColumn, false); 15988 if (tableColumn != null) { 15989 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 15990 relation.setEffectType(EffectType.update); 15991 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 15992 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 15993 relation.setProcess(process); 15994 } 15995 15996 } 15997 } 15998 } else if (expression.getExpressionType() == EExpressionType.simple_object_name_t) { 15999 TExpression setExpression = field.getExpr().getRightOperand(); 16000 if (setExpression != null && setExpression.getSubQuery() != null) { 16001 TSelectSqlStatement query = setExpression.getSubQuery(); 16002 analyzeSelectStmt(query); 16003 16004 SelectResultSet resultSetModel = (SelectResultSet) modelManager 16005 .getModel(query.getResultColumnList()); 16006 16007 if (!resultSetModel.getRelationRows().getHoldRelations().isEmpty()) { 16008 ImpactRelationship impactRelation = modelFactory.createImpactRelation(); 16009 impactRelation.setEffectType(EffectType.update); 16010 impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>( 16011 resultSetModel.getRelationRows())); 16012 if (tableModel != null) { 16013 impactRelation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>( 16014 tableModel.getRelationRows())); 16015 } else if (cteTargetModel != null) { 16016 impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>( 16017 cteTargetModel.getRelationRows())); 16018 } 16019 } 16020 16021 TObjectName column = expression.getObjectOperand(); 16022 ResultColumn resultColumn = resultSetModel.getColumns().get(0); 16023 TObjectName targetColumn = getCteBaseColumn(cteTargetResolution, column); 16024 TableColumn tableColumn = tableModel == null || targetColumn == null ? null 16025 : modelFactory.createTableColumn(tableModel, targetColumn, false); 16026 if (tableColumn != null) { 16027 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 16028 relation.setEffectType(EffectType.update); 16029 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 16030 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 16031 relation.setProcess(process); 16032 } 16033 } else if (setExpression != null) { 16034 // ResultSet resultSet = modelFactory.createResultSet(stmt, 16035 // true); 16036 16037 ResultSet resultSet = modelFactory.createResultSet(stmt, false); 16038 16039 createPseudoImpactRelation(stmt, resultSet, EffectType.update); 16040 16041 TObjectName columnObject = expression.getObjectOperand(); 16042 16043 ResultColumn updateColumn = modelFactory.createUpdateResultColumn(resultSet, columnObject); 16044 16045 columnsInExpr visitor = new columnsInExpr(); 16046 field.getExpr().getRightOperand().inOrderTraverse(visitor); 16047 16048 List<TObjectName> objectNames = visitor.getObjectNames(); 16049 List<TParseTreeNode> functions = visitor.getFunctions(); 16050 List<TSelectSqlStatement> subquerys = visitor.getSubquerys(); 16051 16052 if (functions != null && !functions.isEmpty()) { 16053 analyzeFunctionDataFlowRelation(updateColumn, functions, EffectType.update); 16054 } 16055 16056 if (subquerys != null && !subquerys.isEmpty()) { 16057 analyzeSubqueryDataFlowRelation(updateColumn, subquerys, EffectType.update); 16058 } 16059 16060 Transform transform = new Transform(); 16061 transform.setType(Transform.EXPRESSION); 16062 transform.setCode(setExpression); 16063 updateColumn.setTransform(transform); 16064 analyzeDataFlowRelation(updateColumn, objectNames, EffectType.update, functions); 16065 16066 List<TParseTreeNode> constants = visitor.getConstants(); 16067 analyzeConstantDataFlowRelation(updateColumn, constants, EffectType.update, functions); 16068 16069 TObjectName targetColumn = getCteBaseColumn(cteTargetResolution, columnObject); 16070 TableColumn tableColumn = tableModel == null || targetColumn == null ? null 16071 : modelFactory.createTableColumn(tableModel, targetColumn, false); 16072 if(tableColumn!=null) { 16073 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 16074 relation.setEffectType(EffectType.update); 16075 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 16076 relation.addSource(new ResultColumnRelationshipElement(updateColumn)); 16077 relation.setProcess(process); 16078 } 16079 16080 ImpactRelationship impactRelation = modelFactory.createImpactRelation(); 16081 impactRelation.setEffectType(EffectType.update); 16082 impactRelation.addSource( 16083 new RelationRowsRelationshipElement<ResultSetRelationRows>(resultSet.getRelationRows())); 16084 if (tableModel != null) { 16085 impactRelation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>( 16086 tableModel.getRelationRows())); 16087 } else if (cteTargetModel != null) { 16088 impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>( 16089 cteTargetModel.getRelationRows())); 16090 } 16091 } 16092 } 16093 } 16094 16095 if (stmt.getJoins() != null && stmt.getJoins().size() > 0) { 16096 for (int i = 0; i < stmt.getJoins().size(); i++) { 16097 TJoin join = stmt.getJoins().getJoin(i); 16098 if (join.getJoinItems() != null) { 16099 for (int j = 0; j < join.getJoinItems().size(); j++) { 16100 TJoinItem joinItem = join.getJoinItems().getJoinItem(j); 16101 TExpression expr = joinItem.getOnCondition(); 16102 analyzeFilterCondition(null, expr, joinItem.getJoinType(), JoinClauseType.on, 16103 EffectType.update); 16104 } 16105 } 16106 } 16107 } 16108 16109 if (stmt.getWhereClause() != null && stmt.getWhereClause().getCondition() != null) { 16110 analyzeFilterCondition(null, stmt.getWhereClause().getCondition(), null, JoinClauseType.where, 16111 EffectType.update); 16112 } 16113 16114 if (stmt.getOutputClause() != null) { 16115 TOutputClause outputClause = stmt.getOutputClause(); 16116 if (outputClause.getSelectItemList() != null) { 16117 ResultSet resultSet = modelFactory.createResultSet(outputClause, false); 16118 for (int j = 0; j < outputClause.getSelectItemList().size(); j++) { 16119 TResultColumn sourceColumn = outputClause.getSelectItemList().getResultColumn(j); 16120 ResultColumn sourceColumnModel = modelFactory.createResultColumn(resultSet, sourceColumn); 16121 analyzeResultColumn(sourceColumn, EffectType.select); 16122 16123 if (outputClause.getIntoTable() != null) { 16124 Table intoTableModel = modelFactory.createTableByName(outputClause.getIntoTable()); 16125 intoTableModel.addProcess(process); 16126 if (outputClause.getIntoColumnList() != null) { 16127 TableColumn intoTableColumn = modelFactory.createInsertTableColumn(intoTableModel, 16128 outputClause.getIntoColumnList().getObjectName(j)); 16129 16130 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 16131 relation.setEffectType(EffectType.insert); 16132 relation.setTarget(new TableColumnRelationshipElement(intoTableColumn)); 16133 relation.addSource(new ResultColumnRelationshipElement(sourceColumnModel)); 16134 } else if (sourceColumn.getAliasClause() != null 16135 || sourceColumn.getExpr().getObjectOperand() != null) { 16136 TObjectName tableColumnObject = null; 16137 if (sourceColumn.getAliasClause() != null) { 16138 tableColumnObject = sourceColumn.getAliasClause().getAliasName(); 16139 } else { 16140 tableColumnObject = sourceColumn.getExpr().getObjectOperand(); 16141 } 16142 16143 TableColumn intoTableColumn = modelFactory.createInsertTableColumn(intoTableModel, 16144 tableColumnObject); 16145 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 16146 relation.setEffectType(EffectType.insert); 16147 relation.setTarget(new TableColumnRelationshipElement(intoTableColumn)); 16148 relation.addSource(new ResultColumnRelationshipElement(sourceColumnModel)); 16149 } 16150 } 16151 } 16152 } 16153 } 16154 } 16155 16156 /** 16157 * Analyzes SQL Server XML modify() function to extract data lineage from sql:column() references 16158 * in XQuery expressions. 16159 * 16160 * Example SQL: 16161 * SET [Demographics].modify('... sql:column("deleted.LineTotal") ...') 16162 * 16163 * This extracts: 16164 * - Target column: Demographics (from the XML column being modified) 16165 * - Source column: deleted.LineTotal (from sql:column() reference in XQuery) 16166 */ 16167 private void analyzeXmlModifyFunction(TUpdateSqlStatement stmt, Table tableModel, Process process, TFunctionCall funcCall) { 16168 TObjectName funcName = funcCall.getFunctionName(); 16169 if (funcName == null || funcName.getPartToken() == null) { 16170 return; 16171 } 16172 16173 // Get the XML column being modified (e.g., [Demographics] from "[Demographics].modify") 16174 String xmlColumnName = funcName.getPartToken().astext; 16175 16176 // Get the XQuery argument 16177 if (funcCall.getArgs() == null || funcCall.getArgs().size() == 0) { 16178 return; 16179 } 16180 16181 String xqueryString = funcCall.getArgs().getExpression(0).toString(); 16182 16183 // Extract sql:column() references from the XQuery string 16184 List<String> sqlColumnRefs = extractSqlColumnReferences(xqueryString); 16185 if (sqlColumnRefs.isEmpty()) { 16186 return; 16187 } 16188 16189 // Extract XPath target from XQuery (e.g., /IndividualSurvey/TotalPurchaseYTD) 16190 String xpathTarget = extractXPathTarget(xqueryString); 16191 16192 // Build full target column name including XML path 16193 String fullTargetColumnName = xmlColumnName; 16194 if (xpathTarget != null && !xpathTarget.isEmpty()) { 16195 fullTargetColumnName = xmlColumnName + "." + xpathTarget; 16196 } 16197 16198 // Create the target table column for the XML column being modified 16199 TableColumn targetTableColumn = modelFactory.createInsertTableColumn(tableModel, fullTargetColumnName); 16200 16201 // Create source-to-target relationships for each sql:column reference 16202 for (String sqlColRef : sqlColumnRefs) { 16203 // Parse the column reference (e.g., "deleted.LineTotal" -> table="deleted", column="LineTotal") 16204 String[] parts = sqlColRef.split("\\.", 2); 16205 String sourceTableName = parts.length > 1 ? parts[0] : null; 16206 String sourceColumnName = parts.length > 1 ? parts[1] : parts[0]; 16207 16208 // Find the source table in the statement's tables 16209 TTable sourceTable = null; 16210 if (sourceTableName != null && stmt.tables != null) { 16211 for (int j = 0; j < stmt.tables.size(); j++) { 16212 TTable t = stmt.tables.getTable(j); 16213 String tableName = t.getTableName().toString(); 16214 String alias = t.getAliasName(); 16215 if (SQLUtil.compareIdentifier(option.getVendor(), ESQLDataObjectType.dotTable, tableName, sourceTableName) || 16216 (alias != null && SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotTable, alias, sourceTableName))) { 16217 sourceTable = t; 16218 break; 16219 } 16220 } 16221 } 16222 16223 if (sourceTable != null && targetTableColumn != null) { 16224 // Create a table model for the source if needed 16225 Table sourceTableModel = modelFactory.createTable(sourceTable); 16226 TableColumn sourceColumn = modelFactory.createInsertTableColumn(sourceTableModel, sourceColumnName); 16227 16228 if (sourceColumn != null) { 16229 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 16230 relation.setEffectType(EffectType.update); 16231 relation.setTarget(new TableColumnRelationshipElement(targetTableColumn)); 16232 relation.addSource(new TableColumnRelationshipElement(sourceColumn)); 16233 relation.setProcess(process); 16234 relation.setFunction("modify"); 16235 } 16236 } 16237 } 16238 } 16239 16240 /** 16241 * Extracts sql:column() references from an XQuery string. 16242 * Example: 'sql:column("deleted.LineTotal")' -> ["deleted.LineTotal"] 16243 */ 16244 private List<String> extractSqlColumnReferences(String xquery) { 16245 List<String> refs = new ArrayList<>(); 16246 if (xquery == null) { 16247 return refs; 16248 } 16249 16250 int startIdx = 0; 16251 while ((startIdx = xquery.indexOf("sql:column(", startIdx)) >= 0) { 16252 int parenStart = startIdx + "sql:column(".length(); 16253 int parenEnd = xquery.indexOf(")", parenStart); 16254 if (parenEnd < 0) { 16255 break; 16256 } 16257 16258 String arg = xquery.substring(parenStart, parenEnd).trim(); 16259 // Remove quotes (single or double) 16260 if ((arg.startsWith("\"") && arg.endsWith("\"")) || 16261 (arg.startsWith("'") && arg.endsWith("'"))) { 16262 arg = arg.substring(1, arg.length() - 1); 16263 } 16264 16265 if (!arg.isEmpty()) { 16266 refs.add(arg); 16267 } 16268 16269 startIdx = parenEnd + 1; 16270 } 16271 16272 return refs; 16273 } 16274 16275 /** 16276 * Extracts the XPath target from an XQuery modify expression. 16277 * Example: 'replace value of (/IndividualSurvey/TotalPurchaseYTD)[1]' -> "IndividualSurvey.TotalPurchaseYTD" 16278 */ 16279 private String extractXPathTarget(String xquery) { 16280 if (xquery == null) { 16281 return null; 16282 } 16283 16284 // Look for patterns like "(/path/to/element)" or "(/path/to/element)[1]" 16285 int replaceIdx = xquery.indexOf("replace value of"); 16286 if (replaceIdx < 0) { 16287 return null; 16288 } 16289 16290 int parenStart = xquery.indexOf("(/", replaceIdx); 16291 if (parenStart < 0) { 16292 return null; 16293 } 16294 16295 int parenEnd = xquery.indexOf(")", parenStart); 16296 if (parenEnd < 0) { 16297 return null; 16298 } 16299 16300 String xpath = xquery.substring(parenStart + 1, parenEnd); 16301 // Remove any predicates like [1] 16302 int bracketIdx = xpath.indexOf("["); 16303 if (bracketIdx > 0) { 16304 xpath = xpath.substring(0, bracketIdx); 16305 } 16306 16307 // Convert XPath to dot notation (e.g., /IndividualSurvey/TotalPurchaseYTD -> IndividualSurvey.TotalPurchaseYTD) 16308 if (xpath.startsWith("/")) { 16309 xpath = xpath.substring(1); 16310 } 16311 xpath = xpath.replace("/", "."); 16312 16313 return xpath; 16314 } 16315 16316 private void analyzeConstantDataFlowRelation(Object modelObject, List<TParseTreeNode> constants, 16317 EffectType effectType, List<TParseTreeNode> functions) { 16318 analyzeConstantDataFlowRelation(modelObject, constants, effectType, functions, null); 16319 } 16320 16321 private void analyzeConstantDataFlowRelation(Object modelObject, List<TParseTreeNode> constants, 16322 EffectType effectType, List<TParseTreeNode> functions, Process process) { 16323 if (constants == null || constants.size() == 0) 16324 return; 16325 16326 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 16327 relation.setEffectType(effectType); 16328 relation.setProcess(process); 16329 16330 if (functions != null && !functions.isEmpty()) { 16331 relation.setFunction(getFunctionName(functions.get(0))); 16332 } 16333 16334 if (modelObject instanceof ResultColumn) { 16335 relation.setTarget(new ResultColumnRelationshipElement((ResultColumn) modelObject)); 16336 16337 } else if (modelObject instanceof TableColumn) { 16338 relation.setTarget(new TableColumnRelationshipElement((TableColumn) modelObject)); 16339 16340 } else { 16341 throw new UnsupportedOperationException(); 16342 } 16343 16344 if (option.isShowConstantTable()) { 16345 Table constantTable = null; 16346 if(modelObject instanceof FunctionResultColumn && ((FunctionResultColumn)modelObject).getFunction() instanceof TFunctionCall) { 16347 TFunctionCall function = (TFunctionCall)((FunctionResultColumn)modelObject).getFunction(); 16348 if(function.getFunctionType() == EFunctionType.struct_t) { 16349 constantTable = modelFactory.createConstantsTable(String.valueOf(function.toString().hashCode())); 16350 } 16351 } 16352 if(constantTable == null) { 16353 constantTable = modelFactory.createConstantsTable(stmtStack.peek()); 16354 } 16355 for (int i = 0; i < constants.size(); i++) { 16356 TParseTreeNode constant = constants.get(i); 16357 if (constant instanceof TConstant) { 16358 TableColumn constantColumn = modelFactory.createTableColumn(constantTable, (TConstant) constant); 16359 relation.addSource(new ConstantRelationshipElement(constantColumn)); 16360 } else if (constant instanceof TObjectName) { 16361 TableColumn constantColumn = modelFactory.createTableColumn(constantTable, (TObjectName) constant, 16362 false); 16363 if(constantColumn == null) { 16364 continue; 16365 } 16366 relation.addSource(new ConstantRelationshipElement(constantColumn)); 16367 } 16368 } 16369 } 16370 16371 } 16372 16373 private String getFunctionName(TParseTreeNode parseTreeNode) { 16374 if (parseTreeNode instanceof TFunctionCall) { 16375 return ((TFunctionCall) parseTreeNode).getFunctionName().toString(); 16376 } 16377 if (parseTreeNode instanceof TCaseExpression) { 16378 return "case-when"; 16379 } 16380 return null; 16381 } 16382 16383 private void analyzeCreateViewStmt(TCustomSqlStatement stmt, TSelectSqlStatement subquery, 16384 TViewAliasClause viewAlias, TObjectName viewName) { 16385 16386 if (subquery != null) { 16387 TTableList tables = subquery.getTables(); 16388 if (tables != null) { 16389 for (int i = 0; i < tables.size(); i++) { 16390 TTable table = tables.getTable(i); 16391 TCustomSqlStatement createView = viewDDLMap 16392 .get(DlineageUtil.getTableFullName(table.getTableName().toString())); 16393 if (createView != null) { 16394 analyzeCustomSqlStmt(createView); 16395 } 16396 } 16397 } 16398 analyzeSelectStmt(subquery); 16399 } 16400 16401 16402 if (viewAlias != null && viewAlias.getViewAliasItemList() != null) { 16403 TViewAliasItemList viewItems = viewAlias.getViewAliasItemList(); 16404 Table viewModel = modelFactory.createView(stmt, viewName, true); 16405 viewModel.setFromDDL(true); 16406 viewModel.setDetermined(true); 16407 Process process = modelFactory.createProcess(stmt); 16408 viewModel.addProcess(process); 16409 ResultSet resultSetModel = (ResultSet) modelManager.getModel(subquery); 16410 if (resultSetModel != null) { 16411 int resultSetSize = resultSetModel.getColumns().size(); 16412 int viewItemSize = viewItems.size(); 16413 int j = 0; 16414 int viewColumnSize = viewItemSize; 16415 if (resultSetModel.isDetermined() && resultSetSize > viewColumnSize) { 16416 viewColumnSize = resultSetSize; 16417 } 16418 for (int i = 0; i < viewColumnSize && j < resultSetSize; i++) { 16419 ResultColumn resultColumn = resultSetModel.getColumns().get(j); 16420 if (i < viewItemSize) { 16421 TObjectName alias = viewItems.getViewAliasItem(i).getAlias(); 16422 16423 if (!resultSetModel.getColumns().get(j).getName().contains("*")) { 16424 j++; 16425 } else { 16426 if (resultSetSize - j == viewItems.size() - i) { 16427 j++; 16428 } 16429 } 16430 16431 if (alias != null) { 16432 TableColumn viewColumn = modelFactory.createViewColumn(viewModel, alias, i, true); 16433 appendTableColumnToSQLEnv(viewModel, viewColumn); 16434 if (resultColumn != null) { 16435 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 16436 relation.setEffectType(EffectType.create_view); 16437 relation.setTarget(new ViewColumnRelationshipElement(viewColumn)); 16438 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 16439 relation.setProcess(process); 16440 } 16441 } else if (resultColumn.getColumnObject() instanceof TObjectName) { 16442 TableColumn viewColumn = modelFactory.createViewColumn(viewModel, 16443 (TObjectName) resultColumn.getColumnObject(), i, true); 16444 appendTableColumnToSQLEnv(viewModel, viewColumn); 16445 if (resultColumn != null) { 16446 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 16447 relation.setEffectType(EffectType.create_view); 16448 relation.setTarget(new ViewColumnRelationshipElement(viewColumn)); 16449 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 16450 relation.setProcess(process); 16451 } 16452 } else if (resultColumn.getColumnObject() instanceof TResultColumn) { 16453 TableColumn viewColumn = modelFactory.createViewColumn(viewModel, 16454 ((TResultColumn) resultColumn.getColumnObject()).getFieldAttr(), i, true); 16455 appendTableColumnToSQLEnv(viewModel, viewColumn); 16456 ResultColumn column = (ResultColumn) modelManager.getModel(resultColumn.getColumnObject()); 16457 if (column != null && !column.getStarLinkColumns().isEmpty()) { 16458 viewColumn.bindStarLinkColumns(column.getStarLinkColumns()); 16459 } 16460 if (resultColumn != null) { 16461 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 16462 relation.setEffectType(EffectType.create_view); 16463 relation.setTarget(new ViewColumnRelationshipElement(viewColumn)); 16464 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 16465 relation.setProcess(process); 16466 } 16467 } 16468 } 16469 else if(resultSetModel.isDetermined()){ 16470 TObjectName viewColumnName = new TObjectName(); 16471 viewColumnName.setString(resultColumn.getName()); 16472 TableColumn viewColumn = modelFactory.createViewColumn(viewModel, viewColumnName, viewModel.getColumns().size(), true); 16473 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 16474 relation.setEffectType(EffectType.create_view); 16475 relation.setTarget(new ViewColumnRelationshipElement(viewColumn)); 16476 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 16477 relation.setProcess(process); 16478 j++; 16479 } 16480 } 16481 if (resultSetModel != null && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) { 16482 ImpactRelationship impactRelation = modelFactory.createImpactRelation(); 16483 impactRelation.setEffectType(EffectType.create_view); 16484 impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>( 16485 resultSetModel.getRelationRows())); 16486 impactRelation.setTarget( 16487 new RelationRowsRelationshipElement<TableRelationRows>(viewModel.getRelationRows())); 16488 } 16489 } 16490 16491 if (subquery.getResultColumnList() == null && subquery.getValueClause() != null 16492 && subquery.getValueClause().getValueRows().size() == viewItems.size()) { 16493 for (int i = 0; i < viewItems.size(); i++) { 16494 TObjectName alias = viewItems.getViewAliasItem(i).getAlias(); 16495 16496 if (alias != null) { 16497 TableColumn viewColumn = modelFactory.createViewColumn(viewModel, alias, i, true); 16498 appendTableColumnToSQLEnv(viewModel, viewColumn); 16499 TExpression expression = subquery.getValueClause().getValueRows().getValueRowItem(i).getExpr(); 16500 16501 columnsInExpr visitor = new columnsInExpr(); 16502 expression.inOrderTraverse(visitor); 16503 List<TObjectName> objectNames = visitor.getObjectNames(); 16504 List<TParseTreeNode> functions = visitor.getFunctions(); 16505 16506 if (functions != null && !functions.isEmpty()) { 16507 analyzeFunctionDataFlowRelation(viewColumn, functions, EffectType.select); 16508 16509 } 16510 16511 List<TSelectSqlStatement> subquerys = visitor.getSubquerys(); 16512 if (subquerys != null && !subquerys.isEmpty()) { 16513 analyzeSubqueryDataFlowRelation(viewColumn, subquerys, EffectType.select); 16514 } 16515 16516 analyzeDataFlowRelation(viewColumn, objectNames, EffectType.select, functions); 16517 List<TParseTreeNode> constants = visitor.getConstants(); 16518 analyzeConstantDataFlowRelation(viewColumn, constants, EffectType.select, functions); 16519 } 16520 } 16521 } 16522 16523 } else { 16524 if (viewName == null) { 16525 ErrorInfo errorInfo = new ErrorInfo(); 16526 errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR); 16527 errorInfo.setErrorMessage("Can't get view name. CreateView is " + stmt.toString()); 16528 errorInfo.setStartPosition(new Pair3<Long, Long, String>(stmt.getStartToken().lineNo, 16529 stmt.getStartToken().columnNo, ModelBindingManager.getGlobalHash())); 16530 errorInfo.setEndPosition(new Pair3<Long, Long, String>(stmt.getEndToken().lineNo, 16531 stmt.getEndToken().columnNo + stmt.getEndToken().getAstext().length(), 16532 ModelBindingManager.getGlobalHash())); 16533 errorInfo.fillInfo(this); 16534 errorInfos.add(errorInfo); 16535 return; 16536 } 16537 Table viewModel = modelFactory.createView(stmt, viewName); 16538 Process process = modelFactory.createProcess(stmt); 16539 viewModel.addProcess(process); 16540 if (subquery != null && !subquery.isCombinedQuery()) { 16541 SelectResultSet resultSetModel = (SelectResultSet) modelManager 16542 .getModel(subquery.getResultColumnList()); 16543 16544 boolean determined = false; 16545 if (!containStarColumn(resultSetModel)) { 16546 viewModel.setCreateTable(true); 16547 determined = true; 16548 viewModel.setFromDDL(true); 16549 } 16550 for (int i = 0; i < resultSetModel.getColumns().size(); i++) { 16551 ResultColumn resultColumn = resultSetModel.getColumns().get(i); 16552 if (resultColumn.getColumnObject() instanceof TResultColumn) { 16553 TResultColumn columnObject = ((TResultColumn) resultColumn.getColumnObject()); 16554 16555 TAliasClause alias = ((TResultColumn) resultColumn.getColumnObject()).getAliasClause(); 16556 if (alias != null && alias.getAliasName() != null) { 16557 TableColumn viewColumn = modelFactory.createViewColumn(viewModel, alias.getAliasName(), i, 16558 determined); 16559 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 16560 relation.setEffectType(EffectType.create_view); 16561 relation.setTarget(new ViewColumnRelationshipElement(viewColumn)); 16562 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 16563 relation.setProcess(process); 16564 if (determined) { 16565 appendTableColumnToSQLEnv(viewModel, viewColumn); 16566 } 16567 } else if (columnObject.getFieldAttr() != null) { 16568 TObjectName viewColumnObject = columnObject.getFieldAttr(); 16569 TableColumn viewColumn; 16570 Object model = modelManager.getModel(resultColumn.getColumnObject()); 16571 if ("*".equals(viewColumnObject.getColumnNameOnly())) { 16572 if (model instanceof LinkedHashMap) { 16573 String columnName = getColumnNameOnly(resultColumn.getName()); 16574 LinkedHashMap<String, ResultColumn> resultColumns = (LinkedHashMap<String, ResultColumn>) model; 16575 if ("*".equals(columnName)) { 16576 for (String key : resultColumns.keySet()) { 16577 ResultColumn column = resultColumns.get(key); 16578 viewColumn = modelFactory.createInsertTableColumn(viewModel, column.getName()); 16579 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 16580 relation.setEffectType(EffectType.create_view); 16581 relation.setTarget(new ViewColumnRelationshipElement(viewColumn)); 16582 relation.addSource(new ResultColumnRelationshipElement(column)); 16583 relation.setProcess(process); 16584 } 16585 continue; 16586 } else if (resultColumns.containsKey(columnName)) { 16587 ResultColumn column = resultColumns.get(columnName); 16588 viewColumn = modelFactory.createInsertTableColumn(viewModel, column.getName()); 16589 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 16590 relation.setEffectType(EffectType.create_view); 16591 relation.setTarget(new ViewColumnRelationshipElement(viewColumn)); 16592 relation.addSource(new ResultColumnRelationshipElement(column)); 16593 relation.setProcess(process); 16594 continue; 16595 } 16596 } 16597 } 16598 16599 if (!SQLUtil.isEmpty(viewColumnObject.getColumnNameOnly()) 16600 && viewColumnObject.getPropertyToken() != null) { 16601 TObjectName object = new TObjectName(); 16602 // object.setString(viewColumnObject.getPropertyToken().astext); 16603 object.setPartToken(viewColumnObject.getPropertyToken()); 16604 object.setStartToken(viewColumnObject.getPropertyToken()); 16605 object.setEndToken(viewColumnObject.getPropertyToken()); 16606 viewColumn = modelFactory.createViewColumn(viewModel, object, i, determined); 16607 } else { 16608 viewColumn = modelFactory.createViewColumn(viewModel, columnObject.getFieldAttr(), 16609 i, determined); 16610 } 16611 16612 if(determined) { 16613 appendTableColumnToSQLEnv(viewModel, viewColumn); 16614 } 16615 16616 if (model instanceof ResultColumn) { 16617 ResultColumn column = (ResultColumn) modelManager 16618 .getModel(resultColumn.getColumnObject()); 16619 if (column != null && !column.getStarLinkColumns().isEmpty()) { 16620 viewColumn.bindStarLinkColumns(column.getStarLinkColumns()); 16621 } 16622 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 16623 relation.setEffectType(EffectType.create_view); 16624 relation.setTarget(new ViewColumnRelationshipElement(viewColumn)); 16625 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 16626 if (sqlenv == null) { 16627 if (resultColumn.isShowStar()) { 16628 relation.setShowStarRelation(true); 16629 viewColumn.setShowStar(true); 16630 resultColumn.setShowStar(true); 16631 setSourceShowStar(resultColumn); 16632 } 16633 } 16634 if (viewColumn.getName().endsWith("*") && resultColumn.getName().endsWith("*")) { 16635 viewModel.setStarStmt("create_view"); 16636 } 16637 relation.setProcess(process); 16638 } 16639 } else if (resultColumn.getAlias() != null && columnObject.getExpr() 16640 .getExpressionType() == EExpressionType.sqlserver_proprietary_column_alias_t) { 16641 TableColumn viewColumn = modelFactory.createViewColumn(viewModel, 16642 columnObject.getExpr().getLeftOperand().getObjectOperand(), i, determined); 16643 if(determined) { 16644 appendTableColumnToSQLEnv(viewModel, viewColumn); 16645 } 16646 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 16647 relation.setEffectType(EffectType.create_view); 16648 relation.setTarget(new ViewColumnRelationshipElement(viewColumn)); 16649 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 16650 relation.setProcess(process); 16651 } else { 16652 TGSqlParser parser = columnObject.getGsqlparser(); 16653 TObjectName viewColumnName = parser 16654 .parseObjectName(generateQuotedName(parser, columnObject.toString())); 16655 if (viewColumnName == null) { 16656 ErrorInfo errorInfo = new ErrorInfo(); 16657 errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR); 16658 errorInfo.setErrorMessage( 16659 "Can't parse view column. Column is " + columnObject.toString()); 16660 errorInfo.setStartPosition(new Pair3<Long, Long, String>( 16661 columnObject.getStartToken().lineNo, columnObject.getStartToken().columnNo, 16662 ModelBindingManager.getGlobalHash())); 16663 errorInfo 16664 .setEndPosition(new Pair3<Long, Long, String>(columnObject.getEndToken().lineNo, 16665 columnObject.getEndToken().columnNo 16666 + columnObject.getEndToken().getAstext().length(), 16667 ModelBindingManager.getGlobalHash())); 16668 errorInfo.fillInfo(this); 16669 errorInfos.add(errorInfo); 16670 } else { 16671 TableColumn viewColumn = modelFactory.createViewColumn(viewModel, viewColumnName, i, 16672 determined); 16673 if(determined) { 16674 appendTableColumnToSQLEnv(viewModel, viewColumn); 16675 } 16676 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 16677 relation.setEffectType(EffectType.create_view); 16678 relation.setTarget(new ViewColumnRelationshipElement(viewColumn)); 16679 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 16680 relation.setProcess(process); 16681 } 16682 } 16683 } else if (resultColumn.getColumnObject() instanceof TObjectName) { 16684 TableColumn viewColumn = modelFactory.createViewColumn(viewModel, 16685 (TObjectName) resultColumn.getColumnObject(), i, determined); 16686 if(determined) { 16687 appendTableColumnToSQLEnv(viewModel, viewColumn); 16688 } 16689 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 16690 relation.setEffectType(EffectType.create_view); 16691 relation.setTarget(new ViewColumnRelationshipElement(viewColumn)); 16692 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 16693 relation.setProcess(process); 16694 } 16695 } 16696 if (!resultSetModel.getRelationRows().getHoldRelations().isEmpty()) { 16697 ImpactRelationship impactRelation = modelFactory.createImpactRelation(); 16698 impactRelation.setEffectType(EffectType.create_view); 16699 impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>( 16700 resultSetModel.getRelationRows())); 16701 impactRelation.setTarget( 16702 new RelationRowsRelationshipElement<TableRelationRows>(viewModel.getRelationRows())); 16703 } 16704 } else if (subquery != null && subquery.isCombinedQuery()) { 16705 SelectSetResultSet resultSetModel = (SelectSetResultSet) modelManager.getModel(subquery); 16706 16707 boolean determined = false; 16708 if (!containStarColumn(resultSetModel)) { 16709 viewModel.setCreateTable(true); 16710 viewModel.setFromDDL(true); 16711 determined = true; 16712 } 16713 16714 for (int i = 0; i < resultSetModel.getColumns().size(); i++) { 16715 ResultColumn resultColumn = resultSetModel.getColumns().get(i); 16716 16717 if (resultColumn.getColumnObject() instanceof TResultColumn) { 16718 TResultColumn columnObject = ((TResultColumn) resultColumn.getColumnObject()); 16719 16720 TAliasClause alias = columnObject.getAliasClause(); 16721 if (alias != null && alias.getAliasName() != null) { 16722 TableColumn viewColumn = modelFactory.createViewColumn(viewModel, alias.getAliasName(), i, 16723 determined); 16724 if(determined) { 16725 appendTableColumnToSQLEnv(viewModel, viewColumn); 16726 } 16727 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 16728 relation.setEffectType(EffectType.create_view); 16729 relation.setTarget(new ViewColumnRelationshipElement(viewColumn)); 16730 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 16731 relation.setProcess(process); 16732 } else if (columnObject.getFieldAttr() != null) { 16733 TableColumn viewColumn = modelFactory.createViewColumn(viewModel, 16734 columnObject.getFieldAttr(), i, determined); 16735 if(determined) { 16736 appendTableColumnToSQLEnv(viewModel, viewColumn); 16737 } 16738 ResultColumn column = (ResultColumn) modelManager.getModel(resultColumn.getColumnObject()); 16739 if (column != null && !column.getStarLinkColumns().isEmpty()) { 16740 viewColumn.bindStarLinkColumns(column.getStarLinkColumns()); 16741 } 16742 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 16743 relation.setEffectType(EffectType.create_view); 16744 relation.setTarget(new ViewColumnRelationshipElement(viewColumn)); 16745 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 16746 relation.setProcess(process); 16747 } else if (resultColumn.getAlias() != null && columnObject.getExpr() 16748 .getExpressionType() == EExpressionType.sqlserver_proprietary_column_alias_t) { 16749 TableColumn viewColumn = modelFactory.createViewColumn(viewModel, 16750 columnObject.getExpr().getLeftOperand().getObjectOperand(), i, determined); 16751 if(determined) { 16752 appendTableColumnToSQLEnv(viewModel, viewColumn); 16753 } 16754 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 16755 relation.setEffectType(EffectType.create_view); 16756 relation.setTarget(new ViewColumnRelationshipElement(viewColumn)); 16757 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 16758 relation.setProcess(process); 16759 } else { 16760 TGSqlParser parser = columnObject.getGsqlparser(); 16761 TObjectName viewColumnName = parser 16762 .parseObjectName(generateQuotedName(parser, columnObject.toString())); 16763 TableColumn viewColumn = modelFactory.createViewColumn(viewModel, viewColumnName, i, 16764 determined); 16765 if(determined) { 16766 appendTableColumnToSQLEnv(viewModel, viewColumn); 16767 } 16768 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 16769 relation.setEffectType(EffectType.create_view); 16770 relation.setTarget(new ViewColumnRelationshipElement(viewColumn)); 16771 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 16772 relation.setProcess(process); 16773 } 16774 } else if (resultColumn.getColumnObject() instanceof TObjectName) { 16775 TableColumn viewColumn = modelFactory.createViewColumn(viewModel, 16776 (TObjectName) resultColumn.getColumnObject(), i, determined); 16777 if(viewColumn!=null) { 16778 if(determined) { 16779 appendTableColumnToSQLEnv(viewModel, viewColumn); 16780 } 16781 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 16782 relation.setEffectType(EffectType.create_view); 16783 relation.setTarget(new ViewColumnRelationshipElement(viewColumn)); 16784 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 16785 relation.setProcess(process); 16786 } 16787 } 16788 } 16789 if (!resultSetModel.getRelationRows().getHoldRelations().isEmpty()) { 16790 ImpactRelationship impactRelation = modelFactory.createImpactRelation(); 16791 impactRelation.setEffectType(EffectType.create_view); 16792 impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>( 16793 resultSetModel.getRelationRows())); 16794 impactRelation.setTarget( 16795 new RelationRowsRelationshipElement<TableRelationRows>(viewModel.getRelationRows())); 16796 } 16797 } 16798 } 16799 } 16800 16801 private void setSourceShowStar(Object resultColumn) { 16802 for (Relationship relation : modelManager.getRelations()) { 16803 Collection<RelationshipElement<?>> sources = (Collection<RelationshipElement<?>>)relation.getSources(); 16804 if (relation.getTarget().getElement() == resultColumn && sources != null) { 16805 16806 ((AbstractRelationship) relation).setShowStarRelation(true); 16807 for (RelationshipElement<?> source : sources) { 16808 Object column = source.getElement(); 16809 if (column instanceof TableColumn) { 16810 if (((TableColumn) column).isShowStar()) 16811 continue; 16812 ((TableColumn) column).setShowStar(true); 16813 } 16814 if (column instanceof ResultColumn) { 16815 if (((ResultColumn) column).isShowStar()) 16816 continue; 16817 ((ResultColumn) column).setShowStar(true); 16818 } 16819 setSourceShowStar(column); 16820 } 16821 } 16822 } 16823 } 16824 16825 private String generateQuotedName(TGSqlParser parser, String name) { 16826 return "\"" + name + "\""; 16827 } 16828 16829 private void appendRelations(dataflow dataflow) { 16830 Relationship[] relations = modelManager.getRelations(); 16831 16832 appendRelation(dataflow, relations, DataFlowRelationship.class); 16833 appendRelation(dataflow, relations, IndirectImpactRelationship.class); 16834 appendRecordSetRelation(dataflow, relations); 16835 appendCallRelation(dataflow, relations); 16836 appendRelation(dataflow, relations, ImpactRelationship.class); 16837 appendRelation(dataflow, relations, JoinRelationship.class); 16838 appendRelation(dataflow, relations, ERRelationship.class); 16839 appendRelation(dataflow, relations, CrudRelationship.class); 16840 reconcileCitedColumns(dataflow); 16841 } 16842 16843 /** 16844 * Invariant enforcement (codex-696 F5): a relationship must never cite a 16845 * column its parent element does not declare. Star/variant columns of 16846 * record variables can be cited raw (a whole-record source in a filter, 16847 * an unexpanded fill) while the model writer only emitted their 16848 * star-linked expansions — declare exactly the cited leftovers, nothing 16849 * more. 16850 */ 16851 private void reconcileCitedColumns(dataflow dataflow) { 16852 // VARIABLES only: the undeclared-endpoint family is a record variable's 16853 // star/variant column cited raw. Declaring cited leftovers on tables or 16854 // resultsets changes what downstream consumers (JSON conversion, 16855 // normalized comparison) derive from the model — churn far beyond the 16856 // invariant this pass exists to keep. 16857 Map<String, table> parents = new HashMap<String, table>(); 16858 if (dataflow.getVariables() != null) { 16859 for (table t : dataflow.getVariables()) { 16860 if (t.getId() != null) { 16861 parents.put(t.getId(), t); 16862 } 16863 } 16864 } 16865 if (dataflow.getRelationships() == null) { 16866 return; 16867 } 16868 Set<String> declared = new HashSet<String>(); 16869 for (table t : parents.values()) { 16870 if (t.getColumns() == null) { 16871 continue; 16872 } 16873 for (column c : t.getColumns()) { 16874 declared.add(t.getId() + ":" + c.getId()); 16875 } 16876 } 16877 for (relationship relation : dataflow.getRelationships()) { 16878 if (relation.getSources() != null) { 16879 for (sourceColumn source : relation.getSources()) { 16880 declareCitedColumn(parents, declared, source.getParent_id(), source.getId(), 16881 source.getColumn(), source.getCoordinate()); 16882 } 16883 } 16884 targetColumn target = relation.getTarget(); 16885 if (target != null) { 16886 declareCitedColumn(parents, declared, target.getParent_id(), target.getId(), 16887 target.getColumn(), target.getCoordinate()); 16888 } 16889 } 16890 } 16891 16892 private void declareCitedColumn(Map<String, table> parents, Set<String> declared, 16893 String parentId, String columnId, String columnName, String coordinate) { 16894 if (parentId == null || columnId == null || columnName == null) { 16895 return; 16896 } 16897 String key = parentId + ":" + columnId; 16898 if (declared.contains(key)) { 16899 return; 16900 } 16901 table parent = parents.get(parentId); 16902 if (parent == null) { 16903 return; 16904 } 16905 column columnElement = new column(); 16906 columnElement.setId(columnId); 16907 columnElement.setName(columnName); 16908 if (coordinate != null) { 16909 columnElement.setCoordinate(coordinate); 16910 } 16911 parent.getColumns().add(columnElement); 16912 declared.add(key); 16913 } 16914 16915 16916 private relationship cloneRelationshipWithSingleSource(relationship orig, sourceColumn src) { 16917 relationship rel = new relationship(); 16918 rel.setType(orig.getType()); 16919 rel.setEffectType(orig.getEffectType()); 16920 rel.setId(orig.getId() + "_" + src.getColumn()); 16921 rel.setSqlHash(orig.getSqlHash()); 16922 rel.setSqlComment(orig.getSqlComment()); 16923 rel.setProcessId(orig.getProcessId()); 16924 rel.setProcessType(orig.getProcessType()); 16925 rel.setFunction(orig.getFunction()); 16926 rel.setProcedureId(orig.getProcedureId()); 16927 16928 targetColumn t = new targetColumn(); 16929 t.setId(orig.getTarget().getId()); 16930 t.setColumn(orig.getTarget().getColumn()); 16931 t.setParent_id(orig.getTarget().getParent_id()); 16932 t.setParent_name(orig.getTarget().getParent_name()); 16933 t.setParent_alias(orig.getTarget().getParent_alias()); 16934 t.setCoordinate(orig.getTarget().getCoordinate()); 16935 rel.setTarget(t); 16936 16937 sourceColumn s = new sourceColumn(); 16938 s.setId(src.getId()); 16939 s.setColumn(src.getColumn()); 16940 s.setParent_id(src.getParent_id()); 16941 s.setParent_name(src.getParent_name()); 16942 s.setCoordinate(src.getCoordinate()); 16943 rel.addSource(s); 16944 return rel; 16945 } 16946 16947 private Set<String> appendStarColumns = new HashSet<String>(); 16948 private void appendRelation(dataflow dataflow, Relationship[] relations, Class<? extends Relationship> clazz) { 16949 for (int i = 0; i < relations.length; i++) { 16950 AbstractRelationship relation = (AbstractRelationship) relations[i]; 16951 if (relation.getClass() == clazz) { 16952 if (relation.getSources() == null || relation.getTarget() == null) { 16953 continue; 16954 } 16955 Object targetElement = relation.getTarget().getElement(); 16956 TObjectName targetColumnName = null; 16957 if(relation.getTarget() instanceof ResultColumnRelationshipElement) { 16958 targetColumnName = ((ResultColumnRelationshipElement) relation.getTarget()).getColumnName(); 16959 } 16960 if (targetElement instanceof ResultColumn) { 16961 ResultColumn targetColumn = (ResultColumn) targetElement; 16962 if (!targetColumn.isPseduo()) 16963 { 16964 if (targetColumnName == null) 16965 { 16966 if ("*".equals(targetColumn.getName())) { 16967 updateResultColumnStarLinks(dataflow, relation, -1); 16968 } 16969 16970 if (targetColumn.hasStarLinkColumn()) { 16971 for (int j = 0; j < targetColumn.getStarLinkColumnNames().size(); j++) { 16972 appendStarRelation(dataflow, relation, j); 16973 } 16974 16975 if (!containsStar(relation.getSources())) { 16976 continue; 16977 } 16978 } 16979 } 16980 else { 16981 String columnName = DlineageUtil.getColumnName(targetColumnName); 16982 int index = targetColumn.getStarLinkColumnNames().indexOf(columnName); 16983 if (index != -1) { 16984 int size = targetColumn.getStarLinkColumnNames().size(); 16985 if ("*".equals(targetColumn.getName())) { 16986 if (appendStarColumns.contains(columnName)) { 16987 updateResultColumnStarLinks(dataflow, relation, index); 16988 } 16989 else { 16990 updateResultColumnStarLinks(dataflow, relation, -1); 16991 appendStarColumns.add(columnName); 16992 } 16993 } 16994 appendStarRelation(dataflow, relation, index); 16995 for(int j= size; j < targetColumn.getStarLinkColumnNames().size(); j++) { 16996 appendStarRelation(dataflow, relation, j); 16997 } 16998 if (!containsStar(relation.getSources())) { 16999 continue; 17000 } 17001 } 17002 } 17003 } 17004 } else if (targetElement instanceof TableColumn) { 17005 TableColumn targetColumn = (TableColumn) targetElement; 17006 if (!targetColumn.isPseduo()) { 17007 if ("*".equals(targetColumn.getName())) { 17008 updateTableColumnStarLinks(dataflow, relation); 17009 } 17010 17011 // Star expansion selected by HOLDER type (#695): a 17012 // VARIABLE's variant column is serialized as its 17013 // star-linked fields (appendVariableModel), so a 17014 // relation targeting it must expand onto those fields 17015 // or it cites a column the model never declares. A 17016 // physical table's variant column keeps the historical 17017 // refusal (appendTableModel refuses variant). 17018 boolean variantOfVariable = targetColumn.isVariant() 17019 && targetColumn.getTable() instanceof Variable; 17020 if (targetColumn.hasStarLinkColumn() 17021 && ((!targetColumn.isVariant() && targetColumn.isExpandStar()) 17022 || variantOfVariable)) { 17023 for (int j = 0; j < targetColumn.getStarLinkColumnNames().size(); j++) { 17024 appendStarRelation(dataflow, relation, j); 17025 } 17026 if (!containsStar(relation.getSources())) { 17027 continue; 17028 } 17029 } 17030 } 17031 } 17032 17033 relationship relationElement = new relationship(); 17034 relationElement.setType(relation.getRelationshipType().name()); 17035 if (relation.getEffectType() != null) { 17036 relationElement.setEffectType(relation.getEffectType().name()); 17037 } 17038 if (relation.getFunction() != null) { 17039 relationElement.setFunction(relation.getFunction()); 17040 } 17041 relationElement.setSqlHash(relation.getSqlHash()); 17042 relationElement.setSqlComment(relation.getSqlComment()); 17043 17044 if (relation.getProcedureId() != null) { 17045 relationElement.setProcedureId(String.valueOf(relation.getProcedureId())); 17046 } 17047 relationElement.setId(String.valueOf(relation.getId())); 17048 if (relation.getProcess() != null) { 17049 relationElement.setProcessId(String.valueOf(relation.getProcess().getId())); 17050 if (relation.getProcess().getGspObject() != null) { 17051 relationElement.setProcessType(relation.getProcess().getGspObject().sqlstatementtype.name()); 17052 } 17053 } 17054 17055 if (relation.getPartition() != null) { 17056 relationElement.setPartition(relation.getPartition()); 17057 } 17058 relationElement.setSqlHash(relation.getSqlHash()); 17059 relationElement.setSqlComment(relation.getSqlComment()); 17060 17061 if (relation.getProcedureId() != null) { 17062 relationElement.setProcedureId(String.valueOf(relation.getProcedureId())); 17063 } 17064 17065 if (relation instanceof JoinRelationship) { 17066 relationElement.setCondition(((JoinRelationship) relation).getJoinCondition()); 17067 relationElement.setJoinType(((JoinRelationship) relation).getJoinType().name()); 17068 relationElement.setClause(((JoinRelationship) relation).getJoinClauseType().name()); 17069 } 17070 17071 if(relation instanceof ImpactRelationship){ 17072 ImpactRelationship impactRelationship = (ImpactRelationship)relation; 17073 if(impactRelationship.getJoinClauseType()!=null){ 17074 relationElement.setClause(impactRelationship.getJoinClauseType().name()); 17075 } 17076 } 17077 17078 String targetName = null; 17079 Object columnObject = null; 17080 List<TObjectName> targetObjectNames = null; 17081 17082 if (targetElement instanceof ResultSetRelationRows) { 17083 ResultSetRelationRows targetColumn = (ResultSetRelationRows) targetElement; 17084 targetColumn target = new targetColumn(); 17085 target.setId(String.valueOf(targetColumn.getId())); 17086 target.setColumn(targetColumn.getName()); 17087 target.setParent_id(String.valueOf(targetColumn.getHolder().getId())); 17088 target.setParent_name(getResultSetName(targetColumn.getHolder())); 17089 if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) { 17090 target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + "," 17091 + convertCoordinate(targetColumn.getEndPosition())); 17092 } 17093 if (relation instanceof RecordSetRelationship) { 17094 target.setFunction(((RecordSetRelationship) relation).getAggregateFunction()); 17095 } 17096 target.setSource("system"); 17097 targetName = targetColumn.getName(); 17098 relationElement.setTarget(target); 17099 } else if (targetElement instanceof TableRelationRows) { 17100 TableRelationRows targetColumn = (TableRelationRows) targetElement; 17101 targetColumn target = new targetColumn(); 17102 target.setId(String.valueOf(targetColumn.getId())); 17103 target.setColumn(targetColumn.getName()); 17104 target.setParent_id(String.valueOf(targetColumn.getHolder().getId())); 17105 target.setParent_name(getTableName(targetColumn.getHolder())); 17106 if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) { 17107 target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + "," 17108 + convertCoordinate(targetColumn.getEndPosition())); 17109 } 17110 if (relation instanceof RecordSetRelationship) { 17111 target.setFunction(((RecordSetRelationship) relation).getAggregateFunction()); 17112 } 17113 target.setSource("system"); 17114 targetName = targetColumn.getName(); 17115 relationElement.setTarget(target); 17116 } else if (targetElement instanceof ResultColumn) { 17117 ResultColumn targetColumn = (ResultColumn) targetElement; 17118 targetColumn target = new targetColumn(); 17119 target.setId(String.valueOf(targetColumn.getId())); 17120 target.setColumn(targetColumn.getName()); 17121 target.setStruct(targetColumn.isStruct()); 17122 target.setParent_id(String.valueOf(targetColumn.getResultSet().getId())); 17123 target.setParent_name(getResultSetName(targetColumn.getResultSet())); 17124 if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) { 17125 target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + "," 17126 + convertCoordinate(targetColumn.getEndPosition())); 17127 } 17128 if (relation instanceof RecordSetRelationship) { 17129 target.setFunction(((RecordSetRelationship) relation).getAggregateFunction()); 17130 } 17131 targetName = targetColumn.getName(); 17132 if (((ResultColumn) targetColumn).getColumnObject() instanceof TResultColumn) { 17133 columnsInExpr visitor = new columnsInExpr(); 17134 ((TResultColumn) ((ResultColumn) targetColumn).getColumnObject()).getExpr() 17135 .inOrderTraverse(visitor); 17136 targetObjectNames = visitor.getObjectNames(); 17137 } 17138 17139 if (targetElement instanceof FunctionResultColumn) { 17140 columnObject = ((FunctionResultColumn) targetElement).getColumnObject(); 17141 } 17142 if(targetColumn.isPseduo()) { 17143 target.setSource("system"); 17144 } 17145 relationElement.setTarget(target); 17146 } else if (targetElement instanceof TableColumn) { 17147 TableColumn targetColumn = (TableColumn) targetElement; 17148 targetColumn target = new targetColumn(); 17149 target.setId(String.valueOf(targetColumn.getId())); 17150 target.setColumn(targetColumn.getName()); 17151 target.setStruct(targetColumn.isStruct()); 17152 target.setParent_id(String.valueOf(targetColumn.getTable().getId())); 17153 target.setParent_name(getTableName(targetColumn.getTable())); 17154 if (relation.getTarget() instanceof TableColumnRelationshipElement) { 17155 target.setParent_alias(((TableColumnRelationshipElement) relation.getTarget()).getTableAlias()); 17156 } 17157 if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) { 17158 target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + "," 17159 + convertCoordinate(targetColumn.getEndPosition())); 17160 } 17161 if (relation instanceof RecordSetRelationship) { 17162 target.setFunction(((RecordSetRelationship) relation).getAggregateFunction()); 17163 } 17164 if(targetColumn.isPseduo()) { 17165 target.setSource("system"); 17166 } 17167 targetName = targetColumn.getName(); 17168 relationElement.setTarget(target); 17169 } else if (targetElement instanceof Argument) { 17170 Argument targetColumn = (Argument) targetElement; 17171 targetColumn target = new targetColumn(); 17172 target.setId(String.valueOf(targetColumn.getId())); 17173 target.setColumn(targetColumn.getName()); 17174 target.setParent_id(String.valueOf(targetColumn.getProcedure().getId())); 17175 target.setParent_name(targetColumn.getProcedure().getName()); 17176 if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) { 17177 target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + "," 17178 + convertCoordinate(targetColumn.getEndPosition())); 17179 } 17180 if (relation instanceof RecordSetRelationship) { 17181 target.setFunction(((RecordSetRelationship) relation).getAggregateFunction()); 17182 } 17183 targetName = targetColumn.getName(); 17184 relationElement.setTarget(target); 17185 } else if (targetElement instanceof Table) { 17186 Table table = (Table) targetElement; 17187 targetColumn target = new targetColumn(); 17188 target.setTarget_id(String.valueOf(table.getId())); 17189 target.setTarget_name(getTableName(table)); 17190 if (table.getStartPosition() != null && table.getEndPosition() != null) { 17191 target.setCoordinate(convertCoordinate(table.getStartPosition()) + "," 17192 + convertCoordinate(table.getEndPosition())); 17193 } 17194 relationElement.setTarget(target); 17195 } else { 17196 continue; 17197 } 17198 17199 Collection<RelationshipElement<?>> sourceElements = relation.getSources(); 17200 if (sourceElements.size() == 0) { 17201 if(clazz == CrudRelationship.class && option.getAnalyzeMode() == AnalyzeMode.crud) { 17202 dataflow.getRelationships().add(relationElement); 17203 } 17204 continue; 17205 } 17206 17207 boolean append = false; 17208 for (RelationshipElement<?> sourceItem: relation.getSources()) { 17209 Object sourceElement = sourceItem.getElement(); 17210 TObjectName sourceColumnName = null; 17211 if (sourceItem instanceof ResultColumnRelationshipElement) { 17212 sourceColumnName = ((ResultColumnRelationshipElement) sourceItem).getColumnName(); 17213 } 17214// if (sourceItem instanceof TableColumnRelationElement 17215// && (((TableColumnRelationElement) sourceItem).getColumnIndex() != null)) { 17216// TableColumnRelationElement tableColumnRelationElement = (TableColumnRelationElement) sourceItem; 17217// sourceElement = tableColumnRelationElement.getElement().getTable().getColumns() 17218// .get(tableColumnRelationElement.getColumnIndex() + 1); 17219// } 17220 if (sourceElement instanceof ResultColumn) { 17221 ResultColumn sourceColumn = (ResultColumn) sourceElement; 17222 if (sourceColumn.hasStarLinkColumn() && !relation.isShowStarRelation() && !sourceColumn.isPseduo()) { 17223 List<String> sourceStarColumnNames = sourceColumn.getStarLinkColumnNames(); 17224 sourceColumn source = new sourceColumn(); 17225 if (targetObjectNames != null && !targetObjectNames.isEmpty()) { 17226 17227 for (int k = 0; k < targetObjectNames.size(); k++) { 17228 String targetObjectName = getColumnName(targetObjectNames.get(k)); 17229 if(sourceColumn.getResultSet()!=null && sourceColumn.getResultSet().getColumns()!=null) { 17230 boolean find = false; 17231 for(ResultColumn column: sourceColumn.getResultSet().getColumns()) { 17232 if(SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotColumn, column.getName(), targetObjectName)) { 17233 source = new sourceColumn(); 17234 source.setId(String.valueOf(column.getId())); 17235 source.setColumn(targetObjectName); 17236 source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId())); 17237 source.setParent_name(getResultSetName(sourceColumn.getResultSet())); 17238 setSourceCoordinate(source, sourceItem, sourceColumn); 17239 append = true; 17240 relationElement.addSource(source); 17241 find = true; 17242 break; 17243 } 17244 } 17245 if(find) { 17246 continue; 17247 } 17248 } 17249 if (sourceColumn.getStarLinkColumns().containsKey(targetObjectName)) { 17250 source = new sourceColumn(); 17251 source.setId(String.valueOf(sourceColumn.getId()) + "_" 17252 + sourceStarColumnNames.indexOf(targetObjectName)); 17253 source.setColumn(targetObjectName); 17254 source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId())); 17255 source.setParent_name(getResultSetName(sourceColumn.getResultSet())); 17256 setSourceCoordinate(source, sourceItem, sourceColumn); 17257 append = true; 17258 relationElement.addSource(source); 17259 } else { 17260 source = new sourceColumn(); 17261 source.setId(String.valueOf(sourceColumn.getId())); 17262 source.setColumn(relationElement.getTarget().getColumn()); 17263 source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId())); 17264 source.setParent_name(getResultSetName(sourceColumn.getResultSet())); 17265 setSourceCoordinate(source, sourceItem, sourceColumn); 17266 append = true; 17267 relationElement.addSource(source); 17268 } 17269 } 17270 } else { 17271 if (columnObject instanceof TWhenClauseItemList) { 17272 TCaseExpression expr = (TCaseExpression)((FunctionResultColumn) targetElement).getResultSet().getGspObject(); 17273 List<TExpression> directExpressions = new ArrayList<TExpression>(); 17274 17275// TExpression inputExpr = expr.getInput_expr(); 17276// if (inputExpr != null) { 17277// directExpressions.add(inputExpr); 17278// } 17279 TExpression defaultExpr = expr.getElse_expr(); 17280 if (defaultExpr != null) { 17281 directExpressions.add(defaultExpr); 17282 } 17283 TWhenClauseItemList list = expr.getWhenClauseItemList(); 17284 for (int k = 0; k < list.size(); k++) { 17285 TWhenClauseItem element = list.getWhenClauseItem(k); 17286 directExpressions.add(element.getReturn_expr()); 17287 } 17288 17289 for (int k = 0; k < directExpressions.size(); k++) { 17290 columnsInExpr visitor = new columnsInExpr(); 17291 directExpressions.get(k).inOrderTraverse(visitor); 17292 List<TObjectName> objectNames = visitor.getObjectNames(); 17293 if (objectNames == null) { 17294 continue; 17295 } 17296 for (int x = 0; x < objectNames.size(); x++) { 17297 String objectName = getColumnName(objectNames.get(x)); 17298 if (sourceColumn.getStarLinkColumns().containsKey(objectName)) { 17299 17300 if(sourceColumn.getResultSet()!=null && sourceColumn.getResultSet().getColumns()!=null) { 17301 boolean find = false; 17302 for(ResultColumn column: sourceColumn.getResultSet().getColumns()) { 17303 if(SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotColumn, column.getName(), objectName)) { 17304 source = new sourceColumn(); 17305 source.setId(String.valueOf(column.getId())); 17306 source.setColumn(objectName); 17307 source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId())); 17308 source.setParent_name(getResultSetName(sourceColumn.getResultSet())); 17309 setSourceCoordinate(source, sourceItem, sourceColumn); 17310 append = true; 17311 relationElement.addSource(source); 17312 find = true; 17313 break; 17314 } 17315 } 17316 if(find) { 17317 continue; 17318 } 17319 } 17320 17321 source.setId(String.valueOf(sourceColumn.getId()) + "_" 17322 + sourceStarColumnNames.indexOf(objectName)); 17323 source.setColumn(objectName); 17324 } else { 17325 source.setId(String.valueOf(sourceColumn.getId())); 17326 source.setColumn(sourceColumn.getName()); 17327 } 17328 source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId())); 17329 source.setParent_name(getResultSetName(sourceColumn.getResultSet())); 17330 setSourceCoordinate(source, sourceItem, sourceColumn); 17331 append = true; 17332 relationElement.addSource(source); 17333 } 17334 } 17335 } else { 17336 String objectName = getColumnName(targetName); 17337 boolean find = false; 17338 17339 if (!find && sourceColumn.getStarLinkColumns().containsKey(objectName)) { 17340 source.setId(String.valueOf(sourceColumn.getId()) + "_" 17341 + sourceStarColumnNames.indexOf(objectName)); 17342 source.setColumn(objectName); 17343 find = true; 17344 } 17345 17346 if (!find && sourceColumnName != null) { 17347 objectName = getColumnName(sourceColumnName); 17348 17349 if(sourceColumn.getResultSet()!=null && sourceColumn.getResultSet().getColumns()!=null) { 17350 for(ResultColumn column: sourceColumn.getResultSet().getColumns()) { 17351 if(SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotColumn, column.getName(), objectName)) { 17352 source = new sourceColumn(); 17353 source.setId(String.valueOf(column.getId())); 17354 source.setColumn(objectName); 17355 source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId())); 17356 source.setParent_name(getResultSetName(sourceColumn.getResultSet())); 17357 setSourceCoordinate(source, sourceItem, sourceColumn); 17358 append = true; 17359 relationElement.addSource(source); 17360 find = true; 17361 break; 17362 } 17363 } 17364 if(find) { 17365 continue; 17366 } 17367 } 17368 17369 if (sourceColumn.getStarLinkColumns().containsKey(objectName)) { 17370 source.setId(String.valueOf(sourceColumn.getId()) + "_" 17371 + sourceStarColumnNames.indexOf(objectName)); 17372 source.setColumn(objectName); 17373 find = true; 17374 } 17375 } 17376 17377 if (!find && sourceItem instanceof ResultColumnRelationshipElement) { 17378 int starIndex = ((ResultColumnRelationshipElement) sourceItem) 17379 .getStarIndex(); 17380 if (starIndex > -1 && sourceColumn.getStarLinkColumnList().size() > starIndex) { 17381 objectName = getColumnName(sourceColumn.getStarLinkColumnList().get(starIndex)); 17382 source.setId(String.valueOf(sourceColumn.getId()) + "_" 17383 + starIndex); 17384 source.setColumn(objectName); 17385 find = true; 17386 } 17387 } 17388 17389 if (!find) { 17390 source.setId(String.valueOf(sourceColumn.getId())); 17391 source.setColumn(sourceColumn.getName()); 17392 } 17393 17394 source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId())); 17395 source.setParent_name(getResultSetName(sourceColumn.getResultSet())); 17396 setSourceCoordinate(source, sourceItem, sourceColumn); 17397 append = true; 17398 relationElement.addSource(source); 17399 } 17400 } 17401 17402 } else { 17403 sourceColumn source = new sourceColumn(); 17404 source.setId(String.valueOf(sourceColumn.getId())); 17405 source.setColumn(sourceColumn.getName()); 17406 source.setStruct(sourceColumn.isStruct()); 17407 source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId())); 17408 source.setParent_name(getResultSetName(sourceColumn.getResultSet())); 17409 setSourceCoordinate(source, sourceItem, sourceColumn); 17410 append = true; 17411 if (sourceItem.getTransforms() != null) { 17412 for (Transform transform : sourceItem.getTransforms()) { 17413 source.addTransform(transform); 17414 } 17415 } 17416 if(sourceColumn.isPseduo()) { 17417 source.setSource("system"); 17418 } 17419 relationElement.addSource(source); 17420 } 17421 } else if (sourceElement instanceof TableColumn) { 17422 TableColumn sourceColumn = (TableColumn) sourceElement; 17423 if (!sourceColumn.isPseduo() && sourceColumn.hasStarLinkColumn() 17424 && sourceItem instanceof TableColumnRelationshipElement) { 17425 sourceColumn source = new sourceColumn(); 17426 boolean find = false; 17427 if (((TableColumnRelationshipElement) sourceItem).getColumnIndex() != null) { 17428 int columnIndex = ((TableColumnRelationshipElement) sourceItem).getColumnIndex(); 17429 if (sourceColumn.getStarLinkColumns().size() > columnIndex) { 17430 source = new sourceColumn(); 17431 source.setId(String.valueOf(sourceColumn.getId()) + "_" + columnIndex); 17432 String targetObjectName = getColumnName( 17433 sourceColumn.getStarLinkColumnList().get(columnIndex)); 17434 source.setColumn(targetObjectName); 17435 source.setParent_id(String.valueOf(sourceColumn.getTable().getId())); 17436 source.setParent_name(sourceColumn.getTable().getName()); 17437 if (sourceItem instanceof TableColumnRelationshipElement) { 17438 source.setParent_alias( 17439 ((TableColumnRelationshipElement) sourceItem).getTableAlias()); 17440 } 17441 setSourceCoordinate(source, sourceItem, sourceColumn); 17442 append = true; 17443 relationElement.addSource(source); 17444 find = true; 17445 } 17446 } 17447 17448 if (!find) { 17449 String objectName = getColumnName(targetName); 17450 17451 table tableElement = null; 17452 if (dataflow.getTables() != null) { 17453 for (table t : dataflow.getTables()) { 17454 if (t.getId().equals(String.valueOf(sourceColumn.getTable().getId()))) { 17455 tableElement = t; 17456 break; 17457 } 17458 } 17459 } 17460 if (tableElement == null && dataflow.getViews() != null) { 17461 for (table t : dataflow.getViews()) { 17462 if (t.getId().equals(String.valueOf(sourceColumn.getTable().getId()))) { 17463 tableElement = t; 17464 break; 17465 } 17466 } 17467 } 17468 if (tableElement == null && dataflow.getVariables() != null) { 17469 for (table t : dataflow.getVariables()) { 17470 if (t.getId().equals(String.valueOf(sourceColumn.getTable().getId()))) { 17471 tableElement = t; 17472 break; 17473 } 17474 } 17475 } 17476 17477 if(tableElement!=null) { 17478 for (column column : tableElement.getColumns()) { 17479 if (column.getName() != null && SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotColumn, column.getName(), objectName)) { 17480 source = new sourceColumn(); 17481 source.setId(String.valueOf(column.getId())); 17482 source.setColumn(objectName); 17483 source.setParent_id(String.valueOf(sourceColumn.getTable().getId())); 17484 source.setParent_name(sourceColumn.getTable().getName()); 17485 if (sourceItem instanceof TableColumnRelationshipElement) { 17486 source.setParent_alias( 17487 ((TableColumnRelationshipElement) sourceItem).getTableAlias()); 17488 } 17489 setSourceCoordinate(source, sourceItem, sourceColumn); 17490 if (sourceItem.getTransforms() != null) { 17491 for (Transform transform : sourceItem.getTransforms()) { 17492 source.addTransform(transform); 17493 } 17494 } 17495 if (sourceColumn.getCandidateParents() != null) { 17496 for(Object item: sourceColumn.getCandidateParents()) { 17497 candidateTable candidateParent = new candidateTable(); 17498 if(item instanceof Table) { 17499 candidateParent.setId(String.valueOf(((Table)item).getId())); 17500 candidateParent.setName(getTableName((Table)item)); 17501 source.addCandidateParent(candidateParent); 17502 } 17503 else if(item instanceof ResultSet) { 17504 candidateParent.setId(String.valueOf(((ResultSet)item).getId())); 17505 candidateParent.setName(getResultSetName((ResultSet)item)); 17506 source.addCandidateParent(candidateParent); 17507 } 17508 } 17509 } 17510 append = true; 17511 relationElement.addSource(source); 17512 find = true; 17513 break; 17514 } 17515 } 17516 } 17517 } 17518 17519 if(!find) { 17520 source = new sourceColumn(); 17521 source.setId(String.valueOf(sourceColumn.getId())); 17522 source.setColumn(sourceColumn.getName()); 17523 source.setParent_id(String.valueOf(sourceColumn.getTable().getId())); 17524 source.setParent_name(sourceColumn.getTable().getName()); 17525 if (sourceItem instanceof TableColumnRelationshipElement) { 17526 source.setParent_alias( 17527 ((TableColumnRelationshipElement) sourceItem).getTableAlias()); 17528 } 17529 setSourceCoordinate(source, sourceItem, sourceColumn); 17530 if (sourceItem.getTransforms() != null) { 17531 for (Transform transform : sourceItem.getTransforms()) { 17532 source.addTransform(transform); 17533 } 17534 } 17535 if (sourceColumn.getCandidateParents() != null) { 17536 for(Object item: sourceColumn.getCandidateParents()) { 17537 candidateTable candidateParent = new candidateTable(); 17538 if(item instanceof Table) { 17539 candidateParent.setId(String.valueOf(((Table)item).getId())); 17540 candidateParent.setName(getTableName((Table)item)); 17541 source.addCandidateParent(candidateParent); 17542 } 17543 else if(item instanceof ResultSet) { 17544 candidateParent.setId(String.valueOf(((ResultSet)item).getId())); 17545 candidateParent.setName(getResultSetName((ResultSet)item)); 17546 source.addCandidateParent(candidateParent); 17547 } 17548 } 17549 } 17550 append = true; 17551 relationElement.addSource(source); 17552 } 17553 17554 } else { 17555 sourceColumn source = new sourceColumn(); 17556 source.setId(String.valueOf(sourceColumn.getId())); 17557 source.setColumn(sourceColumn.getName()); 17558 source.setStruct(sourceColumn.isStruct()); 17559 source.setParent_id(String.valueOf(sourceColumn.getTable().getId())); 17560 source.setParent_name(getTableName(sourceColumn.getTable())); 17561 if (sourceItem instanceof TableColumnRelationshipElement) { 17562 source.setParent_alias( 17563 ((TableColumnRelationshipElement) sourceItem).getTableAlias()); 17564 } 17565 setSourceCoordinate(source, sourceItem, sourceColumn); 17566 if (sourceItem.getTransforms() != null) { 17567 for (Transform transform : sourceItem.getTransforms()) { 17568 source.addTransform(transform); 17569 } 17570 } 17571 if(sourceColumn.isPseduo()) { 17572 source.setSource("system"); 17573 } 17574 if (sourceColumn.getCandidateParents() != null) { 17575 for(Object item: sourceColumn.getCandidateParents()) { 17576 candidateTable candidateParent = new candidateTable(); 17577 if(item instanceof Table) { 17578 candidateParent.setId(String.valueOf(((Table)item).getId())); 17579 candidateParent.setName(getTableName((Table)item)); 17580 source.addCandidateParent(candidateParent); 17581 } 17582 else if(item instanceof ResultSet) { 17583 candidateParent.setId(String.valueOf(((ResultSet)item).getId())); 17584 candidateParent.setName(getResultSetName((ResultSet)item)); 17585 source.addCandidateParent(candidateParent); 17586 } 17587 } 17588 } 17589 append = true; 17590 relationElement.addSource(source); 17591 } 17592 } else if (sourceElement instanceof Argument) { 17593 Argument sourceColumn = (Argument) sourceElement; 17594 sourceColumn source = new sourceColumn(); 17595 source.setId(String.valueOf(sourceColumn.getId())); 17596 source.setColumn(sourceColumn.getName()); 17597 source.setParent_id(String.valueOf(sourceColumn.getProcedure().getId())); 17598 source.setParent_name(sourceColumn.getProcedure().getName()); 17599 if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) { 17600 source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + "," 17601 + convertCoordinate(sourceColumn.getEndPosition())); 17602 } 17603 append = true; 17604 relationElement.addSource(source); 17605 } else if (sourceElement instanceof TableRelationRows) { 17606 TableRelationRows sourceColumn = (TableRelationRows) sourceElement; 17607 sourceColumn source = new sourceColumn(); 17608 source.setId(String.valueOf(sourceColumn.getId())); 17609 source.setColumn(sourceColumn.getName()); 17610 source.setParent_id(String.valueOf(sourceColumn.getHolder().getId())); 17611 source.setParent_name(getTableName(sourceColumn.getHolder())); 17612 if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) { 17613 source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + "," 17614 + convertCoordinate(sourceColumn.getEndPosition())); 17615 } 17616 source.setSource("system"); 17617 append = true; 17618 relationElement.addSource(source); 17619 } else if (sourceElement instanceof ResultSetRelationRows) { 17620 ResultSetRelationRows sourceColumn = (ResultSetRelationRows) sourceElement; 17621 sourceColumn source = new sourceColumn(); 17622 source.setId(String.valueOf(sourceColumn.getId())); 17623 source.setColumn(sourceColumn.getName()); 17624 source.setParent_id(String.valueOf(sourceColumn.getHolder().getId())); 17625 source.setParent_name(getResultSetName(sourceColumn.getHolder())); 17626 if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) { 17627 source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + "," 17628 + convertCoordinate(sourceColumn.getEndPosition())); 17629 } 17630 source.setSource("system"); 17631 append = true; 17632 relationElement.addSource(source); 17633 } else if (sourceElement instanceof Constant) { 17634 Constant sourceColumn = (Constant) sourceElement; 17635 sourceColumn source = new sourceColumn(); 17636 source.setId(String.valueOf(sourceColumn.getId())); 17637 source.setColumn(sourceColumn.getName()); 17638 source.setColumn_type("constant"); 17639 if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) { 17640 source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + "," 17641 + convertCoordinate(sourceColumn.getEndPosition())); 17642 } 17643 append = true; 17644 relationElement.addSource(source); 17645 } else if (sourceElement instanceof Table) { 17646 Table table = (Table) sourceElement; 17647 sourceColumn source = new sourceColumn(); 17648 source.setSource_id(String.valueOf(table.getId())); 17649 source.setSource_name(getTableName(table)); 17650 if (table.getStartPosition() != null && table.getEndPosition() != null) { 17651 source.setCoordinate(convertCoordinate(table.getStartPosition()) + "," 17652 + convertCoordinate(table.getEndPosition())); 17653 } 17654 append = true; 17655 relationElement.addSource(source); 17656 } 17657 17658 if (relation instanceof ImpactRelationship) { 17659 ESqlClause clause = getSqlClause(sourceItem); 17660 if (clause != null 17661 && (relationElement.getSources() != null && !relationElement.getSources().isEmpty())) { 17662 relationElement.getSources().get(relationElement.getSources().size() - 1) 17663 .setClauseType(clause.name()); 17664 } 17665 } 17666 } 17667 if (append) 17668 dataflow.getRelationships().add(relationElement); 17669 } 17670 } 17671 } 17672 17673 private boolean containsStar(Collection<RelationshipElement<?>> elements) { 17674 if (elements == null || elements.size() == 0) 17675 return false; 17676 17677 for (RelationshipElement<?> element : elements) { 17678 if (element.getElement() instanceof ResultColumn) { 17679 ResultColumn object = (ResultColumn) element.getElement(); 17680 if (object.getName().endsWith("*") && object.isShowStar()) 17681 return true; 17682 } else if (element.getElement() instanceof TableColumn) { 17683 TableColumn object = (TableColumn) element.getElement(); 17684 if (object.getName().endsWith("*") && object.isShowStar()) 17685 return true; 17686 } 17687 } 17688 return false; 17689 } 17690 17691 private Map<String, Set<String>> appendTableStarColumns = new HashMap<String, Set<String>>(); 17692 17693 // Tracks column names that have explicit (non-star) sources per target star column. 17694 // Used across relationships to prevent phantom star expansions. 17695 // Key: target star column ID, Value: set of column names with explicit sources. 17696 private Map<Long, Set<String>> explicitStarTargetColumns = new HashMap<Long, Set<String>>(); 17697 private void updateResultColumnStarLinks(dataflow dataflow, AbstractRelationship relation, int index) { 17698 try { 17699 if (option.getAnalyzeMode() == AnalyzeMode.crud) { 17700 return; 17701 } 17702 17703 ResultColumn targetColumn = (ResultColumn) relation.getTarget().getElement(); 17704 17705 Collection<RelationshipElement<?>> sourceElements = (Collection<RelationshipElement<?>>) relation.getSources(); 17706 if (sourceElements == null || sourceElements.size() == 0) 17707 return; 17708 17709 for (RelationshipElement<?> sourceItem : sourceElements) { 17710 Object sourceElement = sourceItem.getElement(); 17711 if (sourceElement instanceof ResultColumn) { 17712 ResultColumn source = (ResultColumn) sourceElement; 17713 if (source.hasStarLinkColumn()) { 17714 for (Map.Entry<String, Set<TObjectName>> item : source.getStarLinkColumns().entrySet()) { 17715 if (!targetColumn.getStarLinkColumns().containsKey(item.getKey())) { 17716 targetColumn.getStarLinkColumns().put(item.getKey(), new LinkedHashSet<TObjectName>()); 17717 } 17718 targetColumn.getStarLinkColumns().get(item.getKey()).addAll(item.getValue()); 17719 } 17720 if (!source.isShowStar()) { 17721 targetColumn.setShowStar(false); 17722 relation.setShowStarRelation(false); 17723 } 17724 } else if (!"*".equals(source.getName())) { 17725 if (source.getColumnObject() instanceof TObjectName) { 17726 if (source instanceof FunctionResultColumn) { 17727 17728 } else { 17729 targetColumn.bindStarLinkColumn((TObjectName) source.getColumnObject()); 17730 } 17731 } else if (source.getColumnObject() instanceof TResultColumn) { 17732 TResultColumn sourceColumn = (TResultColumn) source.getColumnObject(); 17733 if (sourceColumn.getAliasClause() != null) { 17734 targetColumn.bindStarLinkColumn(sourceColumn.getAliasClause().getAliasName()); 17735 } else if (sourceColumn.getFieldAttr() != null) { 17736 targetColumn.bindStarLinkColumn(sourceColumn.getFieldAttr()); 17737 } else { 17738 TObjectName column = new TObjectName(); 17739 if (sourceColumn.getExpr().getExpressionType() == EExpressionType.typecast_t) { 17740 column.setString(sourceColumn.getExpr().getLeftOperand().toString()); 17741 } else { 17742 column.setString(sourceColumn.toString()); 17743 } 17744 targetColumn.bindStarLinkColumn(column); 17745 } 17746 } 17747 } 17748 } else if (sourceElement instanceof TableColumn && !targetColumn.isStruct()) { 17749 TableColumn source = (TableColumn) sourceElement; 17750 if (!source.isPseduo() && source.hasStarLinkColumn()) { 17751 for (Map.Entry<String, Set<TObjectName>> item : source.getStarLinkColumns().entrySet()) { 17752 if (!targetColumn.getStarLinkColumns().containsKey(item.getKey())) { 17753 targetColumn.getStarLinkColumns().put(item.getKey(), 17754 new LinkedHashSet<TObjectName>()); 17755 } 17756 targetColumn.getStarLinkColumns().get(item.getKey()).addAll(item.getValue()); 17757 } 17758 if ((source.getTable().isCreateTable() || source.getTable().hasSQLEnv()) 17759 && !source.isShowStar()) { 17760 targetColumn.setShowStar(false); 17761 relation.setShowStarRelation(false); 17762 } 17763 } else if (!"*".equals(source.getName())) { 17764 targetColumn.bindStarLinkColumn(source.getColumnObject()); 17765 } 17766 } 17767 } 17768 17769 if (targetColumn.hasStarLinkColumn()) { 17770 table resultSetElement = null; 17771 for (table t : dataflow.getResultsets()) { 17772 if (t.getId().equals(String.valueOf(targetColumn.getResultSet().getId()))) { 17773 resultSetElement = t; 17774 break; 17775 } 17776 } 17777 17778 int starColumnCount = 0; 17779 for (column item : resultSetElement.getColumns()) { 17780 if (item.getName() != null && item.getName().endsWith("*")) { 17781 starColumnCount += 1; 17782 } 17783 } 17784 17785 if (resultSetElement != null && starColumnCount <= 1) { 17786 List<String> columns = targetColumn.getStarLinkColumnNames(); 17787 if (index == -1) { 17788 for (int k = 0; k < columns.size(); k++) { 17789 String columnName = columns.get(k); 17790 String id = String.valueOf(targetColumn.getId()) + "_" + k; 17791 if (appendTableStarColumns.containsKey(resultSetElement.getId()) && appendTableStarColumns.get(resultSetElement.getId()).contains(id)) { 17792 continue; 17793 } 17794 column columnElement = new column(); 17795 columnElement.setId(id); 17796 columnElement.setName(columnName); 17797 if (targetColumn.isFunction()) { 17798 columnElement.setIsFunction(String.valueOf(targetColumn.isFunction())); 17799 } 17800 if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) { 17801 columnElement.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + "," 17802 + convertCoordinate(targetColumn.getEndPosition())); 17803 } 17804 17805 if (targetColumn.getResultSet() != null && targetColumn.getResultSet().getColumns() != null) { 17806 boolean find = false; 17807 for (int i = 0; i < targetColumn.getResultSet().getColumns().size(); i++) { 17808 ResultColumn columnModel = targetColumn.getResultSet().getColumns().get(i); 17809 if (DlineageUtil.getIdentifierNormalColumnName(columnModel.getName()) 17810 .equals(columnName)) { 17811 find = true; 17812 break; 17813 } 17814 } 17815 if (find) { 17816 continue; 17817 } 17818 } 17819 17820 if (!resultSetElement.getColumns().contains(columnElement)) { 17821 resultSetElement.getColumns().add(columnElement); 17822 appendTableStarColumns.putIfAbsent(resultSetElement.getId(), new HashSet<String>()); 17823 appendTableStarColumns.get(resultSetElement.getId()).add(id); 17824 } 17825 } 17826 } else { 17827 int k = index; 17828 String columnName = columns.get(k); 17829 column columnElement = new column(); 17830 columnElement.setId(String.valueOf(targetColumn.getId()) + "_" + k); 17831 columnElement.setName(columnName); 17832 if (targetColumn.isFunction()) { 17833 columnElement.setIsFunction(String.valueOf(targetColumn.isFunction())); 17834 } 17835 if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) { 17836 columnElement.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + "," 17837 + convertCoordinate(targetColumn.getEndPosition())); 17838 } 17839 17840 if (targetColumn.getResultSet() != null && targetColumn.getResultSet().getColumns() != null) { 17841 boolean find = false; 17842 for (int i = 0; i < targetColumn.getResultSet().getColumns().size(); i++) { 17843 ResultColumn columnModel = targetColumn.getResultSet().getColumns().get(i); 17844 if (DlineageUtil.getIdentifierNormalColumnName(columnModel.getName()).equals(columnName)) { 17845 find = true; 17846 break; 17847 } 17848 } 17849 if (find) { 17850 return; 17851 } 17852 } 17853 17854 if (!resultSetElement.getColumns().contains(columnElement)) { 17855 resultSetElement.getColumns().add(columnElement); 17856 } 17857 } 17858 } 17859 } 17860 } catch (Exception e) { 17861 logger.error("updateResultColumnStarLinks occurs unknown exceptions.", e); 17862 } 17863 } 17864 17865 private void updateTableColumnStarLinks(dataflow dataflow, AbstractRelationship relation) { 17866 TableColumn targetColumn = (TableColumn) relation.getTarget().getElement(); 17867 Collection<RelationshipElement<?>> sourceElements = (Collection<RelationshipElement<?>>) relation.getSources(); 17868 if (sourceElements == null || sourceElements.size() == 0) 17869 return; 17870 17871 TableColumn sourceStarColumn = null; 17872 17873 for (RelationshipElement<?> sourceItem: sourceElements) { 17874 Object sourceElement = sourceItem.getElement(); 17875 if (sourceElement instanceof ResultColumn) { 17876 ResultColumn source = (ResultColumn) sourceElement; 17877 if(source.getResultSet() instanceof Function) { 17878 continue; 17879 } 17880 if (source.hasStarLinkColumn()) { 17881 for (Map.Entry<String, Set<TObjectName>> item : source.getStarLinkColumns().entrySet()) { 17882 if (!targetColumn.getStarLinkColumns().containsKey(item.getKey())) { 17883 targetColumn.getStarLinkColumns().put(item.getKey(), new LinkedHashSet<TObjectName>()); 17884 } 17885 targetColumn.getStarLinkColumns().get(item.getKey()).addAll(item.getValue()); 17886 } 17887 if (!source.isShowStar()) { 17888 targetColumn.setShowStar(false); 17889 relation.setShowStarRelation(false); 17890 } 17891 } else if (!"*".equals(source.getName())) { 17892 if (source.getColumnObject() instanceof TObjectName) { 17893 targetColumn.bindStarLinkColumn((TObjectName) source.getColumnObject()); 17894 } else if (source.getColumnObject() instanceof TResultColumn) { 17895 if (((TResultColumn) source.getColumnObject()).getAliasClause() != null) { 17896 TObjectName field = ((TResultColumn) source.getColumnObject()).getAliasClause() 17897 .getAliasName(); 17898 if (field != null) { 17899 targetColumn.bindStarLinkColumn(field); 17900 } 17901 } else { 17902 TObjectName field = ((TResultColumn) source.getColumnObject()).getFieldAttr(); 17903 if (field != null) { 17904 targetColumn.bindStarLinkColumn(field); 17905 } else { 17906 TObjectName column = new TObjectName(); 17907 if (((TResultColumn) source.getColumnObject()).getExpr() 17908 .getExpressionType() == EExpressionType.typecast_t) { 17909 column.setString(((TResultColumn) source.getColumnObject()).getExpr() 17910 .getLeftOperand().toString()); 17911 } else { 17912 column.setString(((TResultColumn) source.getColumnObject()).toString()); 17913 } 17914 targetColumn.bindStarLinkColumn(column); 17915 } 17916 } 17917 } 17918 } 17919 } else if (sourceElement instanceof TableColumn) { 17920 TableColumn source = (TableColumn) sourceElement; 17921 if (source.hasStarLinkColumn()) { 17922 for (Map.Entry<String, Set<TObjectName>> item : source.getStarLinkColumns().entrySet()) { 17923 if (!targetColumn.getStarLinkColumns().containsKey(item.getKey())) { 17924 targetColumn.getStarLinkColumns().put(item.getKey(), new LinkedHashSet<TObjectName>()); 17925 } 17926 targetColumn.getStarLinkColumns().get(item.getKey()).addAll(item.getValue()); 17927 } 17928 for (Map.Entry<String, Set<TObjectName>> item : targetColumn.getStarLinkColumns().entrySet()) { 17929 if (!source.getStarLinkColumns().containsKey(item.getKey())) { 17930 source.getStarLinkColumns().put(item.getKey(), new LinkedHashSet<TObjectName>()); 17931 } 17932 source.getStarLinkColumns().get(item.getKey()).addAll(item.getValue()); 17933 } 17934 17935 sourceStarColumn = source; 17936 17937 if (source.getTable().isCreateTable() && !source.isShowStar()) { 17938 targetColumn.setShowStar(false); 17939 relation.setShowStarRelation(false); 17940 } 17941 } else if (!"*".equals(source.getName())) { 17942 if (source.isStruct()) { 17943 targetColumn.bindStarLinkColumn(source.getColumnObject()); 17944 } 17945 else { 17946 TObjectName objectName = new TObjectName(); 17947 objectName.setString(DlineageUtil.getColumnNameOnly(source.getName())); 17948 targetColumn.bindStarLinkColumn(objectName); 17949 } 17950 } 17951 } 17952 } 17953 17954 if (targetColumn.hasStarLinkColumn()) { 17955 table tableElement = null; 17956 if (dataflow.getTables() != null) { 17957 for (table t : dataflow.getTables()) { 17958 if (t.getId().equals(String.valueOf(targetColumn.getTable().getId()))) { 17959 tableElement = t; 17960 break; 17961 } 17962 } 17963 } 17964 if (tableElement == null && dataflow.getViews() != null) { 17965 for (table t : dataflow.getViews()) { 17966 if (t.getId().equals(String.valueOf(targetColumn.getTable().getId()))) { 17967 tableElement = t; 17968 break; 17969 } 17970 } 17971 } 17972 if (tableElement == null && dataflow.getVariables() != null) { 17973 for (table t : dataflow.getVariables()) { 17974 if (t.getId().equals(String.valueOf(targetColumn.getTable().getId()))) { 17975 tableElement = t; 17976 break; 17977 } 17978 } 17979 } 17980 17981 if (tableElement != null) { 17982 List<String> columns = new ArrayList<String>(targetColumn.getStarLinkColumns().keySet()); 17983 for (int k = 0; k < columns.size(); k++) { 17984 String columnName = columns.get(k); 17985 if (containStarColumn(targetColumn.getTable().getColumns(), columnName)) { 17986 continue; 17987 } 17988 column columnElement = new column(); 17989 columnElement.setId(String.valueOf(targetColumn.getId()) + "_" + k); 17990 columnElement.setName(columnName); 17991 if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) { 17992 columnElement.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + "," 17993 + convertCoordinate(targetColumn.getEndPosition())); 17994 } 17995 if (!tableElement.getColumns().contains(columnElement)) { 17996 tableElement.getColumns().add(columnElement); 17997 } 17998 } 17999 } 18000 } 18001 18002 if (sourceStarColumn != null) { 18003 table tableElement = null; 18004 if (dataflow.getTables() != null) { 18005 for (table t : dataflow.getTables()) { 18006 if (t.getId().equals(String.valueOf(sourceStarColumn.getTable().getId()))) { 18007 tableElement = t; 18008 break; 18009 } 18010 } 18011 } 18012 if (tableElement == null && dataflow.getViews() != null) { 18013 for (table t : dataflow.getViews()) { 18014 if (t.getId().equals(String.valueOf(sourceStarColumn.getTable().getId()))) { 18015 tableElement = t; 18016 break; 18017 } 18018 } 18019 } 18020 if (tableElement == null && dataflow.getVariables() != null) { 18021 for (table t : dataflow.getVariables()) { 18022 if (t.getId().equals(String.valueOf(sourceStarColumn.getTable().getId()))) { 18023 tableElement = t; 18024 break; 18025 } 18026 } 18027 } 18028 18029 if (tableElement != null) { 18030 List<String> columns = new ArrayList<String>(sourceStarColumn.getStarLinkColumns().keySet()); 18031 for (int k = 0; k < columns.size(); k++) { 18032 String columnName = columns.get(k); 18033 if (containStarColumn(sourceStarColumn.getTable().getColumns(), columnName)) { 18034 continue; 18035 } 18036 column columnElement = new column(); 18037 columnElement.setId(sourceStarColumn.getId() + "_" + k); 18038 columnElement.setName(columnName); 18039 if (sourceStarColumn.getStartPosition() != null && sourceStarColumn.getEndPosition() != null) { 18040 columnElement.setCoordinate(convertCoordinate(sourceStarColumn.getStartPosition()) + "," 18041 + convertCoordinate(sourceStarColumn.getEndPosition())); 18042 } 18043 if (!tableElement.getColumns().contains(columnElement)) { 18044 tableElement.getColumns().add(columnElement); 18045 } 18046 } 18047 } 18048 } 18049 } 18050 18051 private ESqlClause getSqlClause(RelationshipElement<?> relationshipElement) { 18052 if (relationshipElement instanceof TableColumnRelationshipElement) { 18053 return ((TableColumnRelationshipElement) relationshipElement).getRelationLocation(); 18054 } else if (relationshipElement instanceof ResultColumnRelationshipElement) { 18055 return ((ResultColumnRelationshipElement) relationshipElement).getRelationLocation(); 18056 } 18057 return null; 18058 } 18059 18060 /** 18061 * Emits a source coordinate, preferring the relationship element's own 18062 * reference site (where THIS relation referenced the column — e.g. the 18063 * ON/WHERE operand) over the shared column model's position. A created 18064 * (e.g. #temp) table reuses one TableColumn model for the definition and 18065 * every reference, so the model's position alone would report the 18066 * definition site for every operand. 18067 */ 18068 private void setSourceCoordinate(sourceColumn source, RelationshipElement<?> sourceItem, 18069 TableColumn sourceTableColumn) { 18070 Pair3<Long, Long, String> modelStart = sourceTableColumn.getStartPosition(); 18071 Pair3<Long, Long, String> modelEnd = sourceTableColumn.getEndPosition(); 18072 Pair3<Long, Long, String> start = modelStart; 18073 Pair3<Long, Long, String> end = modelEnd; 18074 if (sourceItem instanceof TableColumnRelationshipElement) { 18075 TableColumnRelationshipElement element = (TableColumnRelationshipElement) sourceItem; 18076 if (element.getReferenceStartPosition() != null && element.getReferenceEndPosition() != null) { 18077 start = element.getReferenceStartPosition(); 18078 end = element.getReferenceEndPosition(); 18079 if (modelStart != null && modelEnd != null) { 18080 // Keep the definition span for the merge pass's dedup steps. 18081 source.setDedupCoordinate(convertCoordinate(modelStart) + "," + convertCoordinate(modelEnd)); 18082 } 18083 } 18084 } 18085 if (start != null && end != null) { 18086 source.setCoordinate(convertCoordinate(start) + "," + convertCoordinate(end)); 18087 } 18088 } 18089 18090 /** 18091 * ResultColumn flavor of {@link #setSourceCoordinate}: a derived table's / 18092 * CTE's result column model carries its select-list definition span, so the 18093 * relationship element's reference site (the ON/WHERE operand) wins when it 18094 * was captured. 18095 */ 18096 private void setSourceCoordinate(sourceColumn source, RelationshipElement<?> sourceItem, 18097 ResultColumn sourceResultColumn) { 18098 Pair3<Long, Long, String> modelStart = sourceResultColumn.getStartPosition(); 18099 Pair3<Long, Long, String> modelEnd = sourceResultColumn.getEndPosition(); 18100 Pair3<Long, Long, String> start = modelStart; 18101 Pair3<Long, Long, String> end = modelEnd; 18102 if (sourceItem instanceof ResultColumnRelationshipElement) { 18103 ResultColumnRelationshipElement element = (ResultColumnRelationshipElement) sourceItem; 18104 if (element.getReferenceStartPosition() != null && element.getReferenceEndPosition() != null) { 18105 start = element.getReferenceStartPosition(); 18106 end = element.getReferenceEndPosition(); 18107 if (modelStart != null && modelEnd != null) { 18108 // Keep the definition span for the merge pass's dedup steps. 18109 source.setDedupCoordinate(convertCoordinate(modelStart) + "," + convertCoordinate(modelEnd)); 18110 } 18111 } 18112 } 18113 if (start != null && end != null) { 18114 source.setCoordinate(convertCoordinate(start) + "," + convertCoordinate(end)); 18115 } 18116 } 18117 18118 private void appendStarRelation(dataflow dataflow, AbstractRelationship relation, int index) { 18119 if(option.getAnalyzeMode() == AnalyzeMode.crud) { 18120 return; 18121 } 18122 18123 Object targetElement = relation.getTarget().getElement(); 18124 18125 relationship relationElement = new relationship(); 18126 relationElement.setType(relation.getRelationshipType().name()); 18127 if (relation.getEffectType() != null) { 18128 relationElement.setEffectType(relation.getEffectType().name()); 18129 } 18130 relationElement.setSqlHash(relation.getSqlHash()); 18131 relationElement.setSqlComment(relation.getSqlComment()); 18132 18133 if (relation.getProcedureId() != null) { 18134 relationElement.setProcedureId(String.valueOf(relation.getProcedureId())); 18135 } 18136 relationElement.setId(String.valueOf(relation.getId()) + "_" + index); 18137 if (relation.getProcess() != null) { 18138 relationElement.setProcessId(String.valueOf(relation.getProcess().getId())); 18139 if (relation.getProcess().getGspObject() != null) { 18140 relationElement.setProcessType(relation.getProcess().getGspObject().sqlstatementtype.name()); 18141 } 18142 } 18143 if (relation instanceof DataFlowRelationship) { 18144 relationElement.setSqlHash(((DataFlowRelationship) relation).getSqlHash()); 18145 relationElement.setSqlComment(((DataFlowRelationship) relation).getSqlComment()); 18146 18147 if (relation.getProcedureId() != null) { 18148 relationElement.setProcedureId(String.valueOf(relation.getProcedureId())); 18149 } 18150 } 18151 String targetName = ""; 18152 18153 if (targetElement instanceof ResultColumn) { 18154 ResultColumn targetColumn = (ResultColumn) targetElement; 18155 18156 targetName = targetColumn.getStarLinkColumnNames().get(index); 18157 18158 18159 targetColumn target = new targetColumn(); 18160 target.setId(String.valueOf(targetColumn.getId()) + "_" + index); 18161 if(targetColumn.getResultSet()!=null && targetColumn.getResultSet().getColumns()!=null) { 18162 Map<String, ResultColumn> lookup = getResultSetColumnLookup(targetColumn.getResultSet().getColumns()); 18163 ResultColumn matched = lookup.get(targetName); 18164 if (matched != null) { 18165 target.setId(String.valueOf(matched.getId())); 18166 } 18167 } 18168 target.setColumn(targetName); 18169 target.setStruct(targetColumn.isStruct()); 18170 target.setParent_id(String.valueOf(targetColumn.getResultSet().getId())); 18171 target.setParent_name(getResultSetName(targetColumn.getResultSet())); 18172 if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) { 18173 target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + "," 18174 + convertCoordinate(targetColumn.getEndPosition())); 18175 } 18176 relationElement.setTarget(target); 18177 } else if (targetElement instanceof TableColumn) { 18178 TableColumn targetColumn = (TableColumn) targetElement; 18179 18180 targetName = targetColumn.getStarLinkColumnNames().get(index); 18181 18182 Map<String, TableColumn> tableLookup = getTableColumnLookup(targetColumn.getTable().getColumns()); 18183 TableColumn tableColumn = tableLookup.get(targetName); 18184 18185 targetColumn target = new targetColumn(); 18186 if (tableColumn == null) { 18187 target.setId(targetColumn.getId() + "_" + index); 18188 } else { 18189 target.setId(String.valueOf(tableColumn.getId())); 18190 } 18191 target.setStruct(targetColumn.isStruct()); 18192 target.setColumn(targetName); 18193 target.setParent_id(String.valueOf(targetColumn.getTable().getId())); 18194 target.setParent_name(targetColumn.getTable().getName()); 18195 if (relation.getTarget() instanceof TableColumnRelationshipElement) { 18196 target.setParent_alias(((TableColumnRelationshipElement) relation.getTarget()).getTableAlias()); 18197 } 18198 if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) { 18199 target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + "," 18200 + convertCoordinate(targetColumn.getEndPosition())); 18201 } 18202 relationElement.setTarget(target); 18203 } else { 18204 return; 18205 } 18206 18207 Collection<RelationshipElement<?>> sourceElements = (Collection<RelationshipElement<?>>) relation.getSources(); 18208 if (sourceElements.size() == 0) { 18209 return; 18210 } 18211 18212 String targetIdentifierName = DlineageUtil.getIdentifierNormalColumnName(targetName); 18213 if (targetIdentifierName == null) { 18214 return; 18215 } 18216 18217 // Record-fill expansion (#695): when the star being expanded is a 18218 // record VARIABLE's own * (holder type decided at analysis, not 18219 // inferred here), every field is filled by the SAME producer, so the 18220 // name filters below — written for SELECT-* expansion, where source 18221 // and target columns pair by name — must not drop it. 18222 // Record-fill keep-all applies to a SINGLE-source fill only: one opaque 18223 // producer genuinely feeds every field. A multi-source relation fanned 18224 // across the fields would multiply unproven field-level claims. 18225 boolean recordFillTarget = targetElement instanceof TableColumn 18226 && ((TableColumn) targetElement).getTable() instanceof Variable 18227 && relation.getSources() != null && relation.getSources().size() == 1; 18228 18229 // Track explicit (non-star) source columns per target star column to prevent 18230 // phantom star expansions. Skip for UNION targets where branches contribute independently. 18231 long targetColumnId = -1; 18232 boolean isUnionTarget = false; 18233 if (targetElement instanceof ResultColumn) { 18234 ResultColumn tc = (ResultColumn) targetElement; 18235 targetColumnId = tc.getId(); 18236 isUnionTarget = (tc.getResultSet() instanceof SelectSetResultSet); 18237 } else if (targetElement instanceof TableColumn) { 18238 targetColumnId = ((TableColumn) targetElement).getId(); 18239 } 18240 18241 if (!isUnionTarget && targetColumnId != -1) { 18242 // Check if any source in this relationship is a non-star source matching the target 18243 // column name, AND whose parent does NOT also contribute a star source. 18244 // This distinguishes: 18245 // - "b.col_a" (definitive: b only provides explicit cols, not *) => suppress star expansion 18246 // - "aTab.id" (not definitive: aTab also provides *) => don't suppress 18247 boolean hasDefinitiveExplicitSource = hasDefinitiveNonStarSource(sourceElements, targetIdentifierName); 18248 if (hasDefinitiveExplicitSource) { 18249 if (!explicitStarTargetColumns.containsKey(targetColumnId)) { 18250 explicitStarTargetColumns.put(targetColumnId, new HashSet<String>()); 18251 } 18252 explicitStarTargetColumns.get(targetColumnId).add(targetIdentifierName); 18253 } 18254 } 18255 18256 // Column names that have explicit (non-star) sources for this target, across all relationships 18257 Set<String> targetExplicitNames = isUnionTarget ? null : explicitStarTargetColumns.get(targetColumnId); 18258 18259 for (RelationshipElement<?> sourceItem: sourceElements) { 18260 Object sourceElement = sourceItem.getElement(); 18261 if (sourceElement instanceof ResultColumn) { 18262 ResultColumn sourceColumn = (ResultColumn) sourceElement; 18263 if (sourceColumn.hasStarLinkColumn()) { 18264 List<String> linkColumnNames = sourceColumn.getStarLinkColumnNames(); 18265 int linkColumnNameSize = linkColumnNames.size(); 18266 for (int k = 0; k < linkColumnNameSize; k++) { 18267 String sourceName = linkColumnNames.get(k); 18268 if (relation.getRelationshipType() == RelationshipType.fdd) { 18269 if (!targetIdentifierName.equalsIgnoreCase(sourceName) && !"*".equals(sourceName)) 18270 continue; 18271 } 18272 // Skip star-expanded source when an explicit (non-star) source provides 18273 // the same column, either in this relationship or a prior one. 18274 if (targetExplicitNames != null && targetExplicitNames.contains(sourceName)) { 18275 continue; 18276 } 18277 sourceColumn source = new sourceColumn(); 18278 18279 boolean find = false; 18280 if(sourceColumn.getResultSet()!=null && sourceColumn.getResultSet().getColumns()!=null) { 18281 Map<String, ResultColumn> srcLookup = getResultSetColumnLookup(sourceColumn.getResultSet().getColumns()); 18282 ResultColumn matched = srcLookup.get(sourceName); 18283 if (matched != null) { 18284 source.setId(String.valueOf(matched.getId())); 18285 find = true; 18286 } 18287 } 18288 if(!find) { 18289 source.setId(String.valueOf(sourceColumn.getId()) + "_" + k); 18290 } 18291 source.setColumn(sourceName); 18292 source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId())); 18293 source.setParent_name(getResultSetName(sourceColumn.getResultSet())); 18294 if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) { 18295 source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + "," 18296 + convertCoordinate(sourceColumn.getEndPosition())); 18297 } 18298 if (sourceItem.getTransforms() != null) { 18299 for (Transform transform : sourceItem.getTransforms()) { 18300 source.addTransform(transform); 18301 } 18302 } 18303 relationElement.addSource(source); 18304 } 18305 if(relationElement.getSources().isEmpty() 18306 && !(targetExplicitNames != null && targetExplicitNames.contains(targetIdentifierName)) 18307 && sourceColumn.getResultSet()!=null && sourceColumn.getResultSet().getColumns()!=null) { 18308 Map<String, ResultColumn> srcLookup = getResultSetColumnLookup(sourceColumn.getResultSet().getColumns()); 18309 for (Map.Entry<String, ResultColumn> entry : srcLookup.entrySet()) { 18310 if (entry.getKey().equalsIgnoreCase(targetIdentifierName)) { 18311 ResultColumn columnModel = entry.getValue(); 18312 sourceColumn source = new sourceColumn(); 18313 source.setId(String.valueOf(columnModel.getId())); 18314 source.setColumn(columnModel.getName()); 18315 source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId())); 18316 source.setParent_name(getResultSetName(sourceColumn.getResultSet())); 18317 if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) { 18318 source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + "," 18319 + convertCoordinate(sourceColumn.getEndPosition())); 18320 } 18321 if (sourceItem.getTransforms() != null) { 18322 for (Transform transform : sourceItem.getTransforms()) { 18323 source.addTransform(transform); 18324 } 18325 } 18326 relationElement.addSource(source); 18327 break; 18328 } 18329 } 18330 } 18331 if (relationElement.getSources().isEmpty() 18332 && !(targetExplicitNames != null && targetExplicitNames.contains(targetIdentifierName))) { 18333 TObjectName sourceStarLinkColumn = new TObjectName(); 18334 sourceStarLinkColumn.setString(targetName); 18335 boolean newBinding = sourceColumn.bindStarLinkColumn(sourceStarLinkColumn); 18336 sourceColumn source = new sourceColumn(); 18337 String sourceName = DlineageUtil.getColumnName(sourceStarLinkColumn); 18338 if (!newBinding) { 18339 source.setId(String.valueOf(sourceColumn.getId()) + "_" 18340 + sourceColumn.indexOfStarLinkColumn(sourceStarLinkColumn)); 18341 } else { 18342 source.setId(String.valueOf(sourceColumn.getId()) + "_" + linkColumnNameSize); 18343 } 18344 source.setColumn(sourceName); 18345 source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId())); 18346 source.setParent_name(getResultSetName(sourceColumn.getResultSet())); 18347 if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) { 18348 source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + "," 18349 + convertCoordinate(sourceColumn.getEndPosition())); 18350 } 18351 if (sourceItem.getTransforms() != null) { 18352 for (Transform transform : sourceItem.getTransforms()) { 18353 source.addTransform(transform); 18354 } 18355 } 18356 relationElement.addSource(source); 18357 18358 if (newBinding) { 18359 table resultSetElement = null; 18360 for (table t : dataflow.getResultsets()) { 18361 if (t.getId().equals(String.valueOf(sourceColumn.getResultSet().getId()))) { 18362 resultSetElement = t; 18363 break; 18364 } 18365 } 18366 if (resultSetElement != null) { 18367 column columnElement = new column(); 18368 columnElement.setId(String.valueOf(sourceColumn.getId()) + "_" + linkColumnNameSize); 18369 columnElement.setName(sourceName); 18370 if (sourceColumn.isFunction()) { 18371 columnElement.setIsFunction(String.valueOf(sourceColumn.isFunction())); 18372 } 18373 if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) { 18374 columnElement.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + "," 18375 + convertCoordinate(sourceColumn.getEndPosition())); 18376 } 18377 resultSetElement.getColumns().add(columnElement); 18378 } 18379 } 18380 } 18381 } else { 18382 sourceColumn source = new sourceColumn(); 18383 source.setId(String.valueOf(sourceColumn.getId())); 18384 source.setColumn(sourceColumn.getName()); 18385 source.setStruct(sourceColumn.isStruct()); 18386 source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId())); 18387 source.setParent_name(getResultSetName(sourceColumn.getResultSet())); 18388 if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) { 18389 source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + "," 18390 + convertCoordinate(sourceColumn.getEndPosition())); 18391 } 18392 if (relation.getRelationshipType() == RelationshipType.fdd) { 18393 // Model display-name compares (can be constants/expressions) — see 18394 // the P0d.3c2 reclassification note above. 18395 if (!targetIdentifierName 18396 .equalsIgnoreCase(DlineageUtil.getIdentifierNormalColumnName(sourceColumn.getName()))) { 18397 if (!"*".equals(sourceColumn.getName())) { 18398 // Record-fill (#695): keep a producer only when it 18399 // is OPAQUE — a single-column result (function 18400 // value, collection element) genuinely feeds every 18401 // field. A source from a multi-column result shape 18402 // pairs its own fields by name; fanning it onto 18403 // fields of OTHER shapes asserts dependencies the 18404 // SQL never made. 18405 if (!recordFillTarget) { 18406 continue; 18407 } 18408 boolean opaqueProducer = sourceColumn.getResultSet() == null 18409 || sourceColumn.getResultSet().getColumns() == null 18410 || sourceColumn.getResultSet().getColumns().size() <= 1; 18411 if (!opaqueProducer) { 18412 continue; 18413 } 18414 } 18415 else { 18416 Map<String, ResultColumn> srcLookup = getResultSetColumnLookup(sourceColumn.getResultSet().getColumns()); 18417 boolean flag = false; 18418 for (String key : srcLookup.keySet()) { 18419 if (targetIdentifierName.equalsIgnoreCase(key)) { 18420 flag = true; 18421 break; 18422 } 18423 } 18424 if(flag) { 18425 continue; 18426 } 18427 } 18428 } 18429 } 18430 if (sourceItem.getTransforms() != null) { 18431 for (Transform transform : sourceItem.getTransforms()) { 18432 source.addTransform(transform); 18433 } 18434 } 18435 relationElement.addSource(source); 18436 } 18437 } else if (sourceElement instanceof TableColumn) { 18438 TableColumn sourceColumn = (TableColumn) sourceElement; 18439 if (!sourceColumn.isPseduo() && sourceColumn.hasStarLinkColumn()) { 18440 List<String> linkColumnNames = sourceColumn.getStarLinkColumnNames(); 18441 int linkColumnNameSize = linkColumnNames.size(); 18442 for (int k = 0; k < linkColumnNameSize; k++) { 18443 String sourceName = linkColumnNames.get(k); 18444 if (relation.getRelationshipType() == RelationshipType.fdd) { 18445 if (!targetIdentifierName.equalsIgnoreCase(sourceName) && !"*".equals(sourceName)) 18446 continue; 18447 } 18448 18449 Map<String, TableColumn> srcTableLookup = getTableColumnLookup(sourceColumn.getTable().getColumns()); 18450 TableColumn tableColumn = srcTableLookup.get(sourceName); 18451 18452 sourceColumn source = new sourceColumn(); 18453 if (tableColumn == null) { 18454 source.setId(sourceColumn.getId() + "_" + k); 18455 } else { 18456 source.setId(String.valueOf(tableColumn.getId())); 18457 } 18458 source.setColumn(sourceName); 18459 source.setStruct(sourceColumn.isStruct()); 18460 if (containStarColumn(sourceElements, sourceName)) { 18461 continue; 18462 } 18463 // Cross-relationship check: skip if explicit source was found in prior relationship 18464 if (targetExplicitNames != null && targetExplicitNames.contains(sourceName)) { 18465 continue; 18466 } 18467 if (sourceColumn.getTable().getColumns().size() > 1) { 18468 for (int y = 0; y < sourceColumn.getTable().getColumns().size(); y++) { 18469 if (SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotColumn, 18470 sourceColumn.getTable().getColumns().get(y).getName(), sourceName)) { 18471 source.setId(String.valueOf(sourceColumn.getTable().getColumns().get(y).getId())); 18472 break; 18473 } 18474 } 18475 } 18476 source.setParent_id(String.valueOf(sourceColumn.getTable().getId())); 18477 source.setParent_name(getTableName(sourceColumn.getTable())); 18478 if (sourceItem instanceof TableColumnRelationshipElement) { 18479 source.setParent_alias( 18480 ((TableColumnRelationshipElement) sourceItem).getTableAlias()); 18481 } 18482 if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) { 18483 source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + "," 18484 + convertCoordinate(sourceColumn.getEndPosition())); 18485 } 18486 if (sourceItem.getTransforms() != null) { 18487 for (Transform transform : sourceItem.getTransforms()) { 18488 source.addTransform(transform); 18489 } 18490 } 18491 relationElement.addSource(source); 18492 } 18493 } else { 18494 sourceColumn source = new sourceColumn(); 18495 source.setId(String.valueOf(sourceColumn.getId())); 18496 source.setColumn(sourceColumn.getName()); 18497 source.setStruct(sourceColumn.isStruct()); 18498 source.setParent_id(String.valueOf(sourceColumn.getTable().getId())); 18499 source.setParent_name(getTableName(sourceColumn.getTable())); 18500 if (sourceItem instanceof TableColumnRelationshipElement) { 18501 source.setParent_alias(((TableColumnRelationshipElement) sourceItem).getTableAlias()); 18502 } 18503 if (sourceColumn.getStartPosition() != null && sourceColumn.getEndPosition() != null) { 18504 source.setCoordinate(convertCoordinate(sourceColumn.getStartPosition()) + "," 18505 + convertCoordinate(sourceColumn.getEndPosition())); 18506 } 18507 if (relation.getRelationshipType() == RelationshipType.fdd) { 18508 // Model display-name compare — see the P0d.3c2 reclassification note. 18509 if (!targetIdentifierName 18510 .equalsIgnoreCase(DlineageUtil.getIdentifierNormalColumnName(sourceColumn.getName())) 18511 && !"*".equals(sourceColumn.getName())) { 18512 // Record-fill (#695): same keep-only-if-opaque rule as 18513 // the resultset-source branch above. 18514 if (!recordFillTarget) { 18515 continue; 18516 } 18517 boolean opaqueProducer = sourceColumn.getTable() == null 18518 || sourceColumn.getTable().getColumns() == null 18519 || sourceColumn.getTable().getColumns().size() <= 1; 18520 if (!opaqueProducer) { 18521 continue; 18522 } 18523 } 18524 } 18525 if (sourceItem.getTransforms() != null) { 18526 for (Transform transform : sourceItem.getTransforms()) { 18527 source.addTransform(transform); 18528 } 18529 } 18530 relationElement.addSource(source); 18531 } 18532 } 18533 } 18534 18535 if (relationElement.getTarget() != null && relationElement.getSources() != null 18536 && !relationElement.getSources().isEmpty()) { 18537 dataflow.getRelationships().add(relationElement); 18538 } 18539 } 18540 18541 private String getColumnName(TObjectName column) { 18542 if (column == null) { 18543 return null; 18544 } 18545 String name = column.getColumnNameOnly(); 18546 if (name == null || "".equals(name.trim())) { 18547 return DlineageUtil.getIdentifierNormalColumnName(column.toString().trim()); 18548 } else 18549 return DlineageUtil.getIdentifierNormalColumnName(name.trim()); 18550 } 18551 18552 /** 18553 * For BigQuery/Redshift struct field access, returns the full struct path 18554 * (e.g., "customer.name" from ColumnSource with exposedName="customer", fieldPath=["name"]). 18555 * Checks StructFieldHint first (for 3+ part no-alias), then ColumnSource (for 2-part/alias). 18556 * Returns null if this is not a struct field access. 18557 */ 18558 private String getStructFieldFullName(TObjectName column) { 18559 if (column == null) return null; 18560 if (getOption().getVendor() != EDbVendor.dbvbigquery 18561 && getOption().getVendor() != EDbVendor.dbvredshift) return null; 18562 // Priority 1: StructFieldHint (side-channel, for 3+ part no-alias deep struct access) 18563 gudusoft.gsqlparser.resolver2.model.StructFieldHint hint = column.getStructFieldHint(); 18564 if (hint != null && hint.getFieldPath() != null && !hint.getFieldPath().isEmpty()) { 18565 return hint.toFullReference(); 18566 } 18567 // Priority 2: ColumnSource (main resolution, for 2-part and alias cases) 18568 gudusoft.gsqlparser.resolver2.model.ColumnSource source = column.getColumnSource(); 18569 if (source != null && source.isStructFieldAccess() && source.hasFieldPath()) { 18570 return source.getFieldPath().toFullReference(source.getExposedName()); 18571 } 18572 return null; 18573 } 18574 18575 /** 18576 * Get the base column name for a struct field access column. 18577 * Checks StructFieldHint first (3+ part no-alias), then ColumnSource (2-part/alias). 18578 * Returns null if not a struct field access. 18579 */ 18580 private String getStructFieldBaseName(TObjectName column) { 18581 if (column == null) return null; 18582 gudusoft.gsqlparser.resolver2.model.StructFieldHint hint = column.getStructFieldHint(); 18583 if (hint != null && hint.getBaseColumn() != null) { 18584 return hint.getBaseColumn(); 18585 } 18586 gudusoft.gsqlparser.resolver2.model.ColumnSource source = column.getColumnSource(); 18587 if (source != null && source.isStructFieldAccess()) { 18588 return source.getExposedName(); 18589 } 18590 return null; 18591 } 18592 18593 private String getColumnName(String column) { 18594 if (column == null) { 18595 return null; 18596 } 18597 String name = column.substring(column.lastIndexOf(".") + 1).trim(); 18598 if (name.length() == 0) { 18599 name = column.trim(); 18600 } 18601 // Memoize the normalization. DlineageUtil.getIdentifierNormalColumnName(name) 18602 // resolves the vendor from ModelBindingManager.getGlobalVendor() internally, 18603 // so the result is a pure function of (vendor, name) — NOT of name alone. 18604 // The cache key therefore includes the vendor; passing it explicitly to the 18605 // 2-arg overload is identical to the original call for that same vendor. 18606 EDbVendor vendor = ModelBindingManager.getGlobalVendor(); 18607 String cacheKey = (vendor == null ? "" : vendor.name()) + ":" + name; 18608 String normalized = normalizedColumnNameCache.get(cacheKey); 18609 if (normalized == null) { 18610 normalized = DlineageUtil.getIdentifierNormalColumnName(name, vendor); 18611 if (normalized != null) { 18612 normalizedColumnNameCache.put(cacheKey, normalized); 18613 } 18614 } 18615 return normalized; 18616 } 18617 18618 private String getColumnNameOnly(String column) { 18619 if (column == null) { 18620 return null; 18621 } 18622 return DlineageUtil.getColumnNameOnly(column); 18623 } 18624 18625 private void appendRecordSetRelation(dataflow dataflow, Relationship[] relations) { 18626 for (int i = 0; i < relations.length; i++) { 18627 AbstractRelationship relation = (AbstractRelationship) relations[i]; 18628 relationship relationElement = new relationship(); 18629 relationElement.setType(relation.getRelationshipType().name()); 18630 if (relation.getFunction() != null) { 18631 relationElement.setFunction(relation.getFunction()); 18632 } 18633 if (relation.getEffectType() != null) { 18634 relationElement.setEffectType(relation.getEffectType().name()); 18635 } 18636 relationElement.setSqlHash(relation.getSqlHash()); 18637 relationElement.setSqlComment(relation.getSqlComment()); 18638 18639 if (relation.getProcedureId() != null) { 18640 relationElement.setProcedureId(String.valueOf(relation.getProcedureId())); 18641 } 18642 relationElement.setId(String.valueOf(relation.getId())); 18643 18644 if (relation instanceof RecordSetRelationship) { 18645 RecordSetRelationship recordCountRelation = (RecordSetRelationship) relation; 18646 18647 Object targetElement = recordCountRelation.getTarget().getElement(); 18648 if (targetElement instanceof ResultColumn) { 18649 ResultColumn targetColumn = (ResultColumn) targetElement; 18650 targetColumn target = new targetColumn(); 18651 target.setId(String.valueOf(targetColumn.getId())); 18652 target.setColumn(targetColumn.getName()); 18653 target.setStruct(targetColumn.isStruct()); 18654 target.setFunction(recordCountRelation.getAggregateFunction()); 18655 target.setParent_id(String.valueOf(targetColumn.getResultSet().getId())); 18656 target.setParent_name(getResultSetName(targetColumn.getResultSet())); 18657 if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) { 18658 target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + "," 18659 + convertCoordinate(targetColumn.getEndPosition())); 18660 } 18661 relationElement.setTarget(target); 18662 } else if (targetElement instanceof TableColumn) { 18663 TableColumn targetColumn = (TableColumn) targetElement; 18664 targetColumn target = new targetColumn(); 18665 target.setId(String.valueOf(targetColumn.getId())); 18666 target.setColumn(targetColumn.getName()); 18667 target.setStruct(targetColumn.isStruct()); 18668 target.setFunction(recordCountRelation.getAggregateFunction()); 18669 target.setParent_id(String.valueOf(targetColumn.getTable().getId())); 18670 target.setParent_name(getTableName(targetColumn.getTable())); 18671 if (recordCountRelation.getTarget() instanceof TableColumnRelationshipElement) { 18672 target.setParent_alias( 18673 ((TableColumnRelationshipElement) recordCountRelation.getTarget()).getTableAlias()); 18674 } 18675 if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) { 18676 target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + "," 18677 + convertCoordinate(targetColumn.getEndPosition())); 18678 } 18679 relationElement.setTarget(target); 18680 } else if (targetElement instanceof ResultSetRelationRows) { 18681 ResultSetRelationRows targetColumn = (ResultSetRelationRows) targetElement; 18682 targetColumn target = new targetColumn(); 18683 target.setId(String.valueOf(targetColumn.getId())); 18684 target.setColumn(targetColumn.getName()); 18685 target.setParent_id(String.valueOf(targetColumn.getHolder().getId())); 18686 target.setParent_name(getResultSetName(targetColumn.getHolder())); 18687 if (targetColumn.getStartPosition() != null && targetColumn.getEndPosition() != null) { 18688 target.setCoordinate(convertCoordinate(targetColumn.getStartPosition()) + "," 18689 + convertCoordinate(targetColumn.getEndPosition())); 18690 } 18691 target.setSource("system"); 18692 relationElement.setTarget(target); 18693 } else { 18694 continue; 18695 } 18696 18697 Collection<RelationshipElement<?>> sourceElements = (Collection<RelationshipElement<?>>)recordCountRelation.getSources(); 18698 if (sourceElements.size() == 0) { 18699 continue; 18700 } 18701 18702 boolean append = false; 18703 for (RelationshipElement<?> sourceItem: sourceElements) { 18704 Object sourceElement = sourceItem.getElement(); 18705 if (sourceElement instanceof Table) { 18706 Table table = (Table) sourceElement; 18707 sourceColumn source = new sourceColumn(); 18708 source.setSource_id(String.valueOf(table.getId())); 18709 source.setSource_name(getTableName(table)); 18710 if (table.getStartPosition() != null && table.getEndPosition() != null) { 18711 source.setCoordinate(convertCoordinate(table.getStartPosition()) + "," 18712 + convertCoordinate(table.getEndPosition())); 18713 } 18714 append = true; 18715 relationElement.addSource(source); 18716 } else if (sourceElement instanceof QueryTable) { 18717 QueryTable table = (QueryTable) sourceElement; 18718 sourceColumn source = new sourceColumn(); 18719 source.setSource_id(String.valueOf(table.getId())); 18720 source.setSource_name(getResultSetName(table)); 18721 if (table.getStartPosition() != null && table.getEndPosition() != null) { 18722 source.setCoordinate(convertCoordinate(table.getStartPosition()) + "," 18723 + convertCoordinate(table.getEndPosition())); 18724 } 18725 append = true; 18726 relationElement.addSource(source); 18727 } else if (sourceElement instanceof TableRelationRows) { 18728 TableRelationRows relationRows = (TableRelationRows) sourceElement; 18729 sourceColumn source = new sourceColumn(); 18730 source.setId(String.valueOf(relationRows.getId())); 18731 source.setColumn(relationRows.getName()); 18732 source.setParent_id(String.valueOf(relationRows.getHolder().getId())); 18733 source.setParent_name(getTableName(relationRows.getHolder())); 18734 if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) { 18735 source.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + "," 18736 + convertCoordinate(relationRows.getEndPosition())); 18737 } 18738 source.setSource("system"); 18739 append = true; 18740 relationElement.addSource(source); 18741 } else if (sourceElement instanceof ResultSetRelationRows) { 18742 ResultSetRelationRows relationRows = (ResultSetRelationRows) sourceElement; 18743 sourceColumn source = new sourceColumn(); 18744 source.setId(String.valueOf(relationRows.getId())); 18745 source.setColumn(relationRows.getName()); 18746 source.setParent_id(String.valueOf(relationRows.getHolder().getId())); 18747 source.setParent_name(getResultSetName(relationRows.getHolder())); 18748 if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) { 18749 source.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + "," 18750 + convertCoordinate(relationRows.getEndPosition())); 18751 } 18752 source.setSource("system"); 18753 append = true; 18754 relationElement.addSource(source); 18755 } else if (sourceElement instanceof TableColumn) { 18756 TableColumn sourceColumn = (TableColumn) sourceElement; 18757 sourceColumn source = new sourceColumn(); 18758 source.setId(String.valueOf(sourceColumn.getId())); 18759 source.setColumn(sourceColumn.getName()); 18760 source.setStruct(sourceColumn.isStruct()); 18761 source.setParent_id(String.valueOf(sourceColumn.getTable().getId())); 18762 source.setParent_name(getTableName(sourceColumn.getTable())); 18763 if (sourceItem instanceof TableColumnRelationshipElement) { 18764 source.setParent_alias( 18765 ((TableColumnRelationshipElement) sourceItem).getTableAlias()); 18766 } 18767 setSourceCoordinate(source, sourceItem, sourceColumn); 18768 append = true; 18769 relationElement.addSource(source); 18770 } 18771 if (sourceElement instanceof ResultColumn) { 18772 ResultColumn sourceColumn = (ResultColumn) sourceElement; 18773 sourceColumn source = new sourceColumn(); 18774 source.setId(String.valueOf(sourceColumn.getId())); 18775 source.setColumn(sourceColumn.getName()); 18776 source.setStruct(sourceColumn.isStruct()); 18777 source.setParent_id(String.valueOf(sourceColumn.getResultSet().getId())); 18778 source.setParent_name(getResultSetName(sourceColumn.getResultSet())); 18779 setSourceCoordinate(source, sourceItem, sourceColumn); 18780 append = true; 18781 relationElement.addSource(source); 18782 } 18783 } 18784 18785 if (append) 18786 dataflow.getRelationships().add(relationElement); 18787 } 18788 } 18789 } 18790 18791 private void appendCallRelation(dataflow dataflow, Relationship[] relations) { 18792 for (int i = 0; i < relations.length; i++) { 18793 AbstractRelationship relation = (AbstractRelationship) relations[i]; 18794 relationship relationElement = new relationship(); 18795 relationElement.setType(relation.getRelationshipType().name()); 18796 if (relation.getFunction() != null) { 18797 relationElement.setFunction(relation.getFunction()); 18798 } 18799 if (relation.getEffectType() != null) { 18800 relationElement.setEffectType(relation.getEffectType().name()); 18801 } 18802 relationElement.setSqlHash(relation.getSqlHash()); 18803 relationElement.setSqlComment(relation.getSqlComment()); 18804 18805 if (relation.getProcedureId() != null) { 18806 relationElement.setProcedureId(String.valueOf(relation.getProcedureId())); 18807 } 18808 relationElement.setId(String.valueOf(relation.getId())); 18809 18810 if (relation instanceof CallRelationship) { 18811 CallRelationship callRelation = (CallRelationship) relation; 18812 18813 if (callRelation.getCallObject() != null) { 18814 relationElement.setCallStmt(callRelation.getCallObject().toString()); 18815 if (callRelation.getStartPosition() != null && callRelation.getEndPosition() != null) { 18816 relationElement.setCallCoordinate(convertCoordinate(callRelation.getStartPosition()) + "," 18817 + convertCoordinate(callRelation.getEndPosition())); 18818 } 18819 } 18820 18821 if (Boolean.TRUE.equals(callRelation.getBuiltIn())) { 18822 relationElement.setBuiltIn(true); 18823 } 18824 Object targetElement = callRelation.getTarget().getElement(); 18825 if (targetElement instanceof Procedure) { 18826 Procedure procedure = (Procedure) targetElement; 18827 targetColumn target = new targetColumn(); 18828 target.setId(String.valueOf(procedure.getId())); 18829 target.setName(getProcedureName(procedure)); 18830 if (procedure.getStartPosition() != null && procedure.getEndPosition() != null) { 18831 target.setCoordinate(convertCoordinate(procedure.getStartPosition()) + "," 18832 + convertCoordinate(procedure.getEndPosition())); 18833 } 18834 String clazz = procedure.getProcedureObject().getClass().getSimpleName().toLowerCase(); 18835 if (clazz.indexOf("function") != -1) { 18836 target.setType("function"); 18837 } else if (clazz.indexOf("trigger") != -1) { 18838 target.setType("trigger"); 18839 } else if (clazz.indexOf("macro") != -1) { 18840 target.setType("macro"); 18841 } else { 18842 target.setType("procedure"); 18843 } 18844 relationElement.setCaller(target); 18845 } else { 18846 continue; 18847 } 18848 18849 Collection<RelationshipElement<?>> sourceElements = (Collection<RelationshipElement<?>>)callRelation.getSources(); 18850 if (sourceElements.size() == 0) { 18851 continue; 18852 } 18853 18854 boolean append = false; 18855 for (RelationshipElement<?> sourceItem: sourceElements) { 18856 Object sourceElement = sourceItem.getElement(); 18857 if (sourceElement instanceof Procedure) { 18858 Procedure procedure = (Procedure) sourceElement; 18859 sourceColumn source = new sourceColumn(); 18860 source.setId(String.valueOf(procedure.getId())); 18861 source.setName(getProcedureName(procedure)); 18862 if (procedure.getStartPosition() != null && procedure.getEndPosition() != null) { 18863 source.setCoordinate(convertCoordinate(procedure.getStartPosition()) + "," 18864 + convertCoordinate(procedure.getEndPosition())); 18865 } 18866 String clazz = procedure.getProcedureObject().getClass().getSimpleName().toLowerCase(); 18867 if (clazz.indexOf("function") != -1) { 18868 source.setType("function"); 18869 } else if (clazz.indexOf("trigger") != -1) { 18870 source.setType("trigger"); 18871 } else if (clazz.indexOf("macro") != -1) { 18872 source.setType("macro"); 18873 } else { 18874 source.setType("procedure"); 18875 } 18876 append = true; 18877 relationElement.getCallees().add(source); 18878 } else if (sourceElement instanceof Function) { 18879 Function function = (Function) sourceElement; 18880 sourceColumn source = new sourceColumn(); 18881 source.setId(String.valueOf(function.getId())); 18882 source.setName(getFunctionName(function.getFunctionObject())); 18883 if (function.getStartPosition() != null && function.getEndPosition() != null) { 18884 source.setCoordinate(convertCoordinate(function.getStartPosition()) + "," 18885 + convertCoordinate(function.getEndPosition())); 18886 } 18887 source.setType("function"); 18888 append = true; 18889 relationElement.getCallees().add(source); 18890 } 18891 } 18892 18893 if (append) 18894 dataflow.getRelationships().add(relationElement); 18895 } 18896 } 18897 } 18898 18899 private void appendResultSets(dataflow dataflow) { 18900 Set<ResultSet> resultSets = modelManager.getResultSets(); 18901 for (ResultSet resultSet: resultSets) { 18902 appendResultSet(dataflow, resultSet); 18903 } 18904 } 18905 18906 private void appendResultSet(dataflow dataflow, ResultSet resultSetModel) { 18907 if (!appendResultSets.contains(resultSetModel)) { 18908 appendResultSets.add(resultSetModel); 18909 } else { 18910 return; 18911 } 18912 18913 table resultSetElement = new table(); 18914 resultSetElement.setId(String.valueOf(resultSetModel.getId())); 18915 resultSetElement.setServer(resultSetModel.getServer()); 18916 if (!SQLUtil.isEmpty(resultSetModel.getDatabase())) { 18917 resultSetElement.setDatabase(resultSetModel.getDatabase()); 18918 } 18919 if (!SQLUtil.isEmpty(resultSetModel.getSchema())) { 18920 resultSetElement.setSchema(resultSetModel.getSchema()); 18921 } 18922 resultSetElement.setName(getResultSetName(resultSetModel)); 18923 resultSetElement.setType(getResultSetType(resultSetModel)); 18924 // if ((ignoreRecordSet || simpleOutput) && resultSetModel.isTarget()) { 18925 resultSetElement.setIsTarget(String.valueOf(resultSetModel.isTarget())); 18926 // } 18927 resultSetElement.setIsDetermined(String.valueOf(resultSetModel.isDetermined())); 18928 if (resultSetModel.getProcedureId() != null) { 18929 resultSetElement.setProcedureId(resultSetModel.getProcedureId()); 18930 } 18931 if (resultSetModel.getStartPosition() != null && resultSetModel.getEndPosition() != null) { 18932 resultSetElement.setCoordinate(convertCoordinate(resultSetModel.getStartPosition()) + "," 18933 + convertCoordinate(resultSetModel.getEndPosition())); 18934 } 18935 dataflow.getResultsets().add(resultSetElement); 18936 18937 List<ResultColumn> columns = resultSetModel.getColumns(); 18938 18939 Map<String, Integer> columnCounts = new HashMap<String, Integer>(); 18940 for (ResultColumn column : columns) { 18941 String columnName = DlineageUtil.getIdentifierNormalColumnName(column.getName()); 18942 if (!columnCounts.containsKey(columnName)) { 18943 columnCounts.put(columnName, 0); 18944 } 18945 columnCounts.put(columnName, columnCounts.get(columnName) + 1); 18946 // if (column.hasStarLinkColumn()) { 18947 // List<String> starLinkColumns = column.getStarLinkColumnNames(); 18948 // for (int k = 0; k < starLinkColumns.size(); k++) { 18949 // columnName = starLinkColumns.get(k); 18950 // if (!columnCounts.containsKey(columnName)) { 18951 // columnCounts.put(columnName, 0); 18952 // } 18953 // columnCounts.put(columnName, columnCounts.get(columnName) + 1); 18954 // } 18955 // } 18956 } 18957 18958 for (int j = 0; j < columns.size(); j++) { 18959 ResultColumn columnModel = columns.get(j); 18960 if (columnModel.hasStarLinkColumn()) { 18961 // List<String> starLinkColumns = 18962 // columnModel.getStarLinkColumnNames(); 18963 // for (int k = 0; k < starLinkColumns.size(); k++) { 18964 // column columnElement = new column(); 18965 // columnElement.setId( String.valueOf(columnModel.getId()) + 18966 // "_" + k); 18967 // String columnName = starLinkColumns.get(k); 18968 // columnElement.setName(columnName); 18969 // if(columnModel.isFunction()){ 18970 // columnElement.setIsFunction(String.valueOf(columnModel.isFunction())); 18971 // } 18972 // if (columnModel.getStartPosition() != null && 18973 // columnModel.getEndPosition() != null) { 18974 // columnElement.setCoordinate( 18975 // columnModel.getStartPosition() + "," + 18976 // columnModel.getEndPosition()); 18977 // } 18978 // String identifier = columnName; 18979 // if(columnCounts.containsKey(identifier) && 18980 // columnCounts.get(identifier)>1){ 18981 // TObjectName column = 18982 // columnModel.getStarLinkColumns().get(columnName).iterator().next(); 18983 // if(!SQLUtil.isEmpty(getQualifiedTable(column))){ 18984 // columnElement.setQualifiedTable(getQualifiedTable(column)); 18985 // } 18986 // } 18987 // resultSetElement.getColumns().add(columnElement); 18988 // } 18989 if (columnModel.isShowStar()) { 18990 column columnElement = new column(); 18991 columnElement.setId(String.valueOf(columnModel.getId())); 18992 columnElement.setName(columnModel.getName()); 18993 if (columnModel.isFunction()) { 18994 columnElement.setIsFunction(String.valueOf(columnModel.isFunction())); 18995 } 18996 if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) { 18997 columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + "," 18998 + convertCoordinate(columnModel.getEndPosition())); 18999 } 19000 19001 String identifier = DlineageUtil.getIdentifierNormalColumnName(columnModel.getName()); 19002 if (columnCounts.containsKey(identifier) && columnCounts.get(identifier) > 1) { 19003 String qualifiedTable = getQualifiedTable(columnModel); 19004 if (!SQLUtil.isEmpty(qualifiedTable)) { 19005 columnElement.setQualifiedTable(qualifiedTable); 19006 } 19007 } 19008 resultSetElement.getColumns().add(columnElement); 19009 } 19010 } else { 19011 column columnElement = new column(); 19012 columnElement.setId(String.valueOf(columnModel.getId())); 19013 columnElement.setName(columnModel.getName()); 19014 if (columnModel.isFunction()) { 19015 columnElement.setIsFunction(String.valueOf(columnModel.isFunction())); 19016 } 19017 if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) { 19018 columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + "," 19019 + convertCoordinate(columnModel.getEndPosition())); 19020 } 19021 19022 String identifier = DlineageUtil.getIdentifierNormalColumnName(columnModel.getName()); 19023 if (columnCounts.containsKey(identifier) && columnCounts.get(identifier) > 1) { 19024 String qualifiedTable = getQualifiedTable(columnModel); 19025 if (!SQLUtil.isEmpty(qualifiedTable)) { 19026 columnElement.setQualifiedTable(qualifiedTable); 19027 } 19028 } 19029 resultSetElement.getColumns().add(columnElement); 19030 } 19031 } 19032 19033 ResultSetRelationRows relationRows = resultSetModel.getRelationRows(); 19034 if (relationRows.hasRelation()) { 19035 column relationRowsElement = new column(); 19036 relationRowsElement.setId(String.valueOf(relationRows.getId())); 19037 relationRowsElement.setName(relationRows.getName()); 19038 if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) { 19039 relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + "," 19040 + convertCoordinate(relationRows.getEndPosition())); 19041 } 19042 relationRowsElement.setSource("system"); 19043 resultSetElement.getColumns().add(relationRowsElement); 19044 } 19045 } 19046 19047 private String getQualifiedTable(ResultColumn columnModel) { 19048 if (columnModel.getColumnObject() instanceof TObjectName) { 19049 return getQualifiedTable((TObjectName) columnModel.getColumnObject()); 19050 } 19051 if (columnModel.getColumnObject() instanceof TResultColumn) { 19052 TObjectName field = ((TResultColumn) columnModel.getColumnObject()).getFieldAttr(); 19053 if (field != null) { 19054 return getQualifiedTable(field); 19055 } 19056 } 19057 return null; 19058 } 19059 19060 private String getQualifiedTable(TObjectName column) { 19061 if (column == null) 19062 return null; 19063 String[] splits = column.toString().split("\\."); 19064 if (splits.length > 1) { 19065 return splits[splits.length - 2]; 19066 } 19067 return null; 19068 } 19069 19070 /** 19071 * Get the qualified prefix (schema.table) from a column name for 3-part names. 19072 * For example, for "sch.pk_constv2.c_cdsl", returns "sch.pk_constv2". 19073 * Returns null if the column doesn't have both schema and table parts. 19074 */ 19075 private String getQualifiedPrefixFromColumn(TObjectName column) { 19076 if (column == null) return null; 19077 19078 // Check if both schema and table tokens are present (3-part name) 19079 String schemaStr = column.getSchemaString(); 19080 String tableStr = column.getTableString(); 19081 19082 if (schemaStr != null && !schemaStr.isEmpty() && 19083 tableStr != null && !tableStr.isEmpty()) { 19084 return schemaStr + "." + tableStr; 19085 } 19086 19087 // Fallback: parse from toString() for complex cases 19088 String[] splits = column.toString().split("\\."); 19089 if (splits.length >= 3) { 19090 // Return all parts except the last one (column name) 19091 StringBuilder prefix = new StringBuilder(); 19092 for (int i = 0; i < splits.length - 1; i++) { 19093 if (i > 0) prefix.append("."); 19094 prefix.append(splits[i]); 19095 } 19096 return prefix.toString(); 19097 } 19098 19099 return null; 19100 } 19101 19102 private String getResultSetType(ResultSet resultSetModel) { 19103 if (resultSetModel instanceof QueryTable) { 19104 QueryTable table = (QueryTable) resultSetModel; 19105 if (table.getTableObject().getCTE() != null) { 19106 return "with_cte"; 19107 } 19108 } 19109 19110 if (resultSetModel instanceof SelectSetResultSet) { 19111 ESetOperatorType type = ((SelectSetResultSet) resultSetModel).getSetOperatorType(); 19112 return "select_" + type.name(); 19113 } 19114 19115 if (resultSetModel instanceof SelectResultSet) { 19116 if (((SelectResultSet) resultSetModel).getSelectStmt().getParentStmt() instanceof TInsertSqlStatement) { 19117 return "insert-select"; 19118 } 19119 if (((SelectResultSet) resultSetModel).getSelectStmt().getParentStmt() instanceof TUpdateSqlStatement) { 19120 return "update-select"; 19121 } 19122 } 19123 19124 if (resultSetModel.getGspObject() instanceof TMergeUpdateClause) { 19125 return "merge-update"; 19126 } 19127 19128 if (resultSetModel.getGspObject() instanceof TOutputClause) { 19129 return ResultSetType.output.name(); 19130 } 19131 19132 if (resultSetModel.getGspObject() instanceof TMergeInsertClause) { 19133 return "merge-insert"; 19134 } 19135 19136 if (resultSetModel.getGspObject() instanceof TUpdateSqlStatement) { 19137 return "update-set"; 19138 } 19139 19140 if (resultSetModel.getGspObject() instanceof TFunctionCall && ((TFunctionCall)resultSetModel.getGspObject()).getFunctionType() == EFunctionType.array_t) { 19141 return ResultSetType.array.name(); 19142 } 19143 19144 if (resultSetModel.getGspObject() instanceof TFunctionCall && ((TFunctionCall)resultSetModel.getGspObject()).getFunctionType() == EFunctionType.struct_t) { 19145 return ResultSetType.struct.name(); 19146 } 19147 19148 if (resultSetModel.getGspObject() instanceof TFunctionCall || resultSetModel instanceof Function) { 19149 return ResultSetType.function.name(); 19150 } 19151 19152 if (resultSetModel.getGspObject() instanceof TAliasClause) { 19153 return ResultSetType.alias.name(); 19154 } 19155 19156 if (resultSetModel.getGspObject() instanceof TCursorDeclStmt) { 19157 return ResultSetType.cursor.name(); 19158 } 19159 19160 if (resultSetModel instanceof PivotedTable) { 19161 if (((PivotedTable) resultSetModel).isUnpivoted()) { 19162 return ResultSetType.unpivot_table.name(); 19163 } 19164 return ResultSetType.pivot_table.name(); 19165 } 19166 19167 return "select_list"; 19168 } 19169 19170 private String getTableName(Table tableModel) { 19171 if (modelManager.DISPLAY_NAME.containsKey(tableModel.getId())) { 19172 return modelManager.DISPLAY_NAME.get(tableModel.getId()); 19173 } 19174 19175 String tableName; 19176 if (tableModel.getFullName() != null && tableModel.getFullName().trim().length() > 0) { 19177 return tableModel.getFullName(); 19178 } 19179 if (tableModel.getAlias() != null && tableModel.getAlias().trim().length() > 0) { 19180 tableName = getResultSetWithId("RESULT_OF_" + tableModel.getAlias()); 19181 19182 } else { 19183 tableName = getResultSetDisplayId("RS"); 19184 } 19185 modelManager.DISPLAY_NAME.put(tableModel.getId(), tableName); 19186 return tableName; 19187 } 19188 19189 private String getProcedureName(Procedure procedureModel) { 19190 if (modelManager.DISPLAY_NAME.containsKey(procedureModel.getId())) { 19191 return modelManager.DISPLAY_NAME.get(procedureModel.getId()); 19192 } 19193 19194 String procedureName = procedureModel.getFullName(); 19195 19196 modelManager.DISPLAY_NAME.put(procedureModel.getId(), procedureName); 19197 return procedureName; 19198 } 19199 19200 private String getProcessName(Process processModel) { 19201 if (modelManager.DISPLAY_NAME.containsKey(processModel.getId())) { 19202 return modelManager.DISPLAY_NAME.get(processModel.getId()); 19203 } else { 19204 if (processModel.getCustomType() != null) { 19205 String name = processModel.getCustomType(); 19206 modelManager.DISPLAY_NAME.put(processModel.getId(), name); 19207 return name; 19208 } 19209 String name = processModel.getType(); 19210 String procedureName = getProcedureParentName(processModel.getGspObject()); 19211 if (procedureName != null) { 19212 name = getResultSetDisplayId(procedureName + " " + name); 19213 } else { 19214 name = getResultSetDisplayId("Query " + name); 19215 } 19216 modelManager.DISPLAY_NAME.put(processModel.getId(), name); 19217 return name; 19218 } 19219 } 19220 19221 private String getDisplayIdByType(String type) { 19222 if (!modelManager.DISPLAY_ID.containsKey(type)) { 19223 modelManager.DISPLAY_ID.put(type, option.getStartId() + 1); 19224 return type + "-" + (option.getStartId() + 1); 19225 } else { 19226 long id = modelManager.DISPLAY_ID.get(type); 19227 modelManager.DISPLAY_ID.put(type, id + 1); 19228 return type + "-" + (id + 1); 19229 } 19230 } 19231 19232 private String getDisplayIdByTypeFromZero(String type) { 19233 if (!modelManager.DISPLAY_ID.containsKey(type)) { 19234 modelManager.DISPLAY_ID.put(type, option.getStartId()); 19235 if(option.getStartId() == 0) { 19236 return type; 19237 } 19238 return type + "-" + (option.getStartId() + 1); 19239 } else { 19240 long id = modelManager.DISPLAY_ID.get(type); 19241 modelManager.DISPLAY_ID.put(type, id + 1); 19242 return type + "-" + (id + 1); 19243 } 19244 } 19245 19246 private String getConstantName(Table tableModel) { 19247 if (modelManager.DISPLAY_NAME.containsKey(tableModel.getId())) { 19248 return modelManager.DISPLAY_NAME.get(tableModel.getId()); 19249 } else { 19250 String name = getDisplayIdByType("SQL_CONSTANTS"); 19251 modelManager.DISPLAY_NAME.put(tableModel.getId(), name); 19252 return name; 19253 } 19254 } 19255 19256 private String getTempTableName(TTable table) { 19257 if (modelManager.DISPLAY_NAME.containsKey((long)System.identityHashCode(table))) { 19258 return modelManager.DISPLAY_NAME.get((long)System.identityHashCode(table)); 19259 } else { 19260 String name = getDisplayIdByTypeFromZero(table.getName()); 19261 modelManager.DISPLAY_NAME.put((long)System.identityHashCode(table), name); 19262 return name; 19263 } 19264 } 19265 19266 private String getResultSetName(ResultSet resultSetModel) { 19267 19268 if (modelManager.DISPLAY_NAME.containsKey(resultSetModel.getId())) { 19269 return modelManager.DISPLAY_NAME.get(resultSetModel.getId()); 19270 } 19271 19272 if (resultSetModel.getGspObject() instanceof TFunctionCall && ((TFunctionCall)resultSetModel.getGspObject()).getFunctionType() == EFunctionType.array_t) { 19273 String name = getResultSetDisplayId("ARRAY"); 19274 modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name); 19275 if(option.containsResultSetType(ResultSetType.array)) { 19276 resultSetModel.setTarget(true); 19277 } 19278 return name; 19279 } 19280 19281 if (resultSetModel.getGspObject() instanceof TFunctionCall && ((TFunctionCall)resultSetModel.getGspObject()).getFunctionType() == EFunctionType.struct_t) { 19282 String name = getResultSetDisplayId("STRUCT"); 19283 modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name); 19284 if(option.containsResultSetType(ResultSetType.struct)) { 19285 resultSetModel.setTarget(true); 19286 } 19287 return name; 19288 } 19289 19290 if (resultSetModel instanceof QueryTable) { 19291 QueryTable table = (QueryTable) resultSetModel; 19292 if (table.getAlias() != null && table.getAlias().trim().length() > 0) { 19293 String name = getResultSetWithId("RESULT_OF_" + table.getAlias().trim()); 19294 if (table.getTableObject().getCTE() != null) { 19295 name = getResultSetWithId("RESULT_OF_" + table.getTableObject().getCTE().getTableName().toString() 19296 + "_" + table.getAlias().trim()); 19297 } 19298 modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name); 19299 if(option.containsResultSetType(ResultSetType.result_of)) { 19300 resultSetModel.setTarget(true); 19301 } 19302 return name; 19303 } else if (table.getTableObject().getCTE() != null) { 19304 String name = getResultSetWithId("CTE-" + table.getTableObject().getCTE().getTableName().toString()); 19305 modelManager.DISPLAY_NAME.put(table.getId(), name); 19306 if(option.containsResultSetType(ResultSetType.cte)) { 19307 resultSetModel.setTarget(true); 19308 } 19309 return name; 19310 } 19311 } 19312 19313 if (resultSetModel instanceof SelectResultSet) { 19314 if (((SelectResultSet) resultSetModel).getSelectStmt().getParentStmt() instanceof TInsertSqlStatement) { 19315 String name = getResultSetDisplayId("INSERT-SELECT"); 19316 modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name); 19317 if(option.containsResultSetType(ResultSetType.insert_select)) { 19318 resultSetModel.setTarget(true); 19319 } 19320 return name; 19321 } 19322 19323 if (((SelectResultSet) resultSetModel).getSelectStmt().getParentStmt() instanceof TUpdateSqlStatement) { 19324 String name = getResultSetDisplayId("UPDATE-SELECT"); 19325 modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name); 19326 if(option.containsResultSetType(ResultSetType.update_select)) { 19327 resultSetModel.setTarget(true); 19328 } 19329 return name; 19330 } 19331 } 19332 19333 if (resultSetModel instanceof SelectSetResultSet) { 19334 ESetOperatorType type = ((SelectSetResultSet) resultSetModel).getSetOperatorType(); 19335 String name = getResultSetDisplayId("RESULT_OF_" + type.name().toUpperCase()); 19336 modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name); 19337 if(option.containsResultSetType(ResultSetType.result_of)) { 19338 resultSetModel.setTarget(true); 19339 } 19340 return name; 19341 } 19342 19343 if (resultSetModel.getGspObject() instanceof TMergeUpdateClause) { 19344 String name = getResultSetDisplayId("MERGE-UPDATE"); 19345 modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name); 19346 if(option.containsResultSetType(ResultSetType.merge_update)) { 19347 resultSetModel.setTarget(true); 19348 } 19349 return name; 19350 } 19351 19352 if (resultSetModel.getGspObject() instanceof TOutputClause) { 19353 String name = getResultSetDisplayId("OUTPUT"); 19354 modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name); 19355 if(option.containsResultSetType(ResultSetType.output)) { 19356 resultSetModel.setTarget(true); 19357 } 19358 return name; 19359 } 19360 19361 if (resultSetModel.getGspObject() instanceof TMergeInsertClause) { 19362 String name = getResultSetDisplayId("MERGE-INSERT"); 19363 modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name); 19364 if(option.containsResultSetType(ResultSetType.merge_insert)) { 19365 resultSetModel.setTarget(true); 19366 } 19367 return name; 19368 } 19369 19370 if (resultSetModel.getGspObject() instanceof TUpdateSqlStatement) { 19371 String name = getResultSetDisplayId("UPDATE-SET"); 19372 modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name); 19373 if(option.containsResultSetType(ResultSetType.update_set)) { 19374 resultSetModel.setTarget(true); 19375 } 19376 return name; 19377 } 19378 19379 if (resultSetModel.getGspObject() instanceof TCaseExpression) { 19380 String name = ((Function) resultSetModel).getFunctionName(); 19381 modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name); 19382 if (option.containsResultSetType(ResultSetType.case_when) || option.containsResultSetType(ResultSetType.function)) { 19383 resultSetModel.setTarget(true); 19384 } 19385 return name; 19386 } 19387 19388 if (resultSetModel.getGspObject() instanceof TFunctionCall || resultSetModel instanceof Function) { 19389 String name = ((Function) resultSetModel).getFunctionName(); 19390 modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name); 19391 if(option.containsResultSetType(ResultSetType.function)) { 19392 resultSetModel.setTarget(true); 19393 } 19394 return name; 19395 } 19396 19397 if (resultSetModel instanceof PivotedTable) { 19398 String name = getResultSetDisplayId("PIVOT-TABLE"); 19399 if (((PivotedTable) resultSetModel).isUnpivoted()) { 19400 name = getResultSetDisplayId("UNPIVOT-TABLE"); 19401 if(option.containsResultSetType(ResultSetType.unpivot_table)) { 19402 resultSetModel.setTarget(true); 19403 } 19404 } 19405 else { 19406 if(option.containsResultSetType(ResultSetType.pivot_table)) { 19407 resultSetModel.setTarget(true); 19408 } 19409 } 19410 modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name); 19411 return name; 19412 } 19413 19414 if (resultSetModel instanceof Alias) { 19415 String name = getResultSetDisplayId("ALIAS"); 19416 modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name); 19417 if(option.containsResultSetType(ResultSetType.alias)) { 19418 resultSetModel.setTarget(true); 19419 } 19420 return name; 19421 } 19422 19423 String name = getResultSetDisplayId("RS"); 19424 modelManager.DISPLAY_NAME.put(resultSetModel.getId(), name); 19425 if(option.containsResultSetType(ResultSetType.select_list)) { 19426 resultSetModel.setTarget(true); 19427 } 19428 return name; 19429 } 19430 19431 private String getResultSetWithId(String type) { 19432 type = DlineageUtil.getIdentifierNormalTableName(type); 19433 if (!modelManager.DISPLAY_ID.containsKey(type)) { 19434 modelManager.DISPLAY_ID.put(type, option.getStartId() + 1); 19435 return type + "-" + (option.getStartId() + 1); 19436 } else { 19437 long id = modelManager.DISPLAY_ID.get(type); 19438 modelManager.DISPLAY_ID.put(type, id + 1); 19439 return type + "-" + (id + 1); 19440 } 19441 } 19442 19443 private String getResultSetDisplayId(String type) { 19444 if (!modelManager.DISPLAY_ID.containsKey(type)) { 19445 modelManager.DISPLAY_ID.put(type, option.getStartId() + 1); 19446 return type + "-" + (option.getStartId() + 1); 19447 } else { 19448 long id = modelManager.DISPLAY_ID.get(type); 19449 modelManager.DISPLAY_ID.put(type, id + 1); 19450 return type + "-" + (id + 1); 19451 } 19452 } 19453 19454 private void appendViews(dataflow dataflow) { 19455 List<TCustomSqlStatement> views = modelManager.getViews(); 19456 for (int i = 0; i < views.size(); i++) { 19457 Table viewModel = (Table) modelManager.getViewModel(views.get(i)); 19458 if (!tableIds.contains(viewModel.getId())) { 19459 appendViewModel(dataflow, viewModel); 19460 tableIds.add(viewModel.getId()); 19461 } 19462 } 19463 19464 List<TTable> tables = modelManager.getBaseTables(); 19465 for (int i = 0; i < tables.size(); i++) { 19466 Object model = modelManager.getModel(tables.get(i)); 19467 if (model instanceof Table) { 19468 Table tableModel = (Table) model; 19469 if (tableModel.isView()) { 19470 if (!tableIds.contains(tableModel.getId())) { 19471 appendViewModel(dataflow, tableModel); 19472 tableIds.add(tableModel.getId()); 19473 } 19474 } 19475 } 19476 } 19477 19478 List<Table> tableNames = modelManager.getTablesByName(); 19479 for (int i = 0; i < tableNames.size(); i++) { 19480 Table tableModel = tableNames.get(i); 19481 if (tableModel.isView()) { 19482 if (!tableIds.contains(tableModel.getId())) { 19483 appendViewModel(dataflow, tableModel); 19484 tableIds.add(tableModel.getId()); 19485 } 19486 } 19487 } 19488 } 19489 19490 private void appendViewModel(dataflow dataflow, Table viewModel) { 19491 table viewElement = new table(); 19492 viewElement.setId(String.valueOf(viewModel.getId())); 19493 if (!SQLUtil.isEmpty(viewModel.getDatabase())) { 19494 viewElement.setDatabase(viewModel.getDatabase()); 19495 } 19496 if (!SQLUtil.isEmpty(viewModel.getSchema())) { 19497 viewElement.setSchema(viewModel.getSchema()); 19498 } 19499 viewElement.setServer(viewModel.getServer()); 19500 viewElement.setName(viewModel.getName()); 19501 viewElement.setType("view"); 19502 // Propagate the view sub type (e.g. temp_table for CREATE TEMPORARY VIEW) 19503 // so consumers can tell a temporary view apart from a regular one. Mantis 4538. 19504 if (viewModel.getSubType() != null) { 19505 viewElement.setSubType(viewModel.getSubType().name()); 19506 } 19507 viewElement.setStarStmt(viewModel.getStarStmt()); 19508 19509 if(viewModel.isFromDDL()){ 19510 viewElement.setFromDDL(String.valueOf(viewModel.isFromDDL())); 19511 } 19512 19513 if(option.isTraceTablePosition()){ 19514 for (Pair<Pair3<Long, Long, String>, Pair3<Long, Long, String>> position:viewModel.getPositions()){ 19515 viewElement.setCoordinate(convertCoordinate(position.first)+","+convertCoordinate(position.second)); 19516 } 19517 } 19518 else { 19519 viewElement.setCoordinate(convertCoordinate(viewModel.getStartPosition()) + "," 19520 + convertCoordinate(viewModel.getEndPosition())); 19521 } 19522 19523 if (viewModel.getProcesses() != null) { 19524 List<String> processIds = new ArrayList<String>(); 19525 for (Process process : viewModel.getProcesses()) { 19526 processIds.add(String.valueOf(process.getId())); 19527 } 19528 viewElement.setProcessIds(processIds); 19529 } 19530 dataflow.getViews().add(viewElement); 19531 19532 List<TableColumn> columns = viewModel.getColumns(); 19533 19534 if (containStarColumn(columns)) { 19535 for (TableColumn column : columns) { 19536 if (column.getName().endsWith("*")) { 19537 for (TableColumn starElement : columns) { 19538 if (starElement == column) { 19539 continue; 19540 } 19541 TObjectName columnObject = starElement.getColumnObject(); 19542 column.bindStarLinkColumn(columnObject); 19543 } 19544 if (viewModel.isCreateTable()) { 19545 column.setShowStar(false); 19546 } 19547 } 19548 } 19549 } 19550 19551 for (int j = 0; j < columns.size(); j++) { 19552 TableColumn columnModel = (TableColumn) columns.get(j); 19553 if (!columnModel.isPseduo() && columnModel.hasStarLinkColumn()) { 19554 List<String> starLinkColumnList = columnModel.getStarLinkColumnNames(); 19555 for (int k = 0; k < starLinkColumnList.size(); k++) { 19556 column columnElement = new column(); 19557 columnElement.setId(String.valueOf(columnModel.getId()) + "_" + k); 19558 String columnName = starLinkColumnList.get(k); 19559 if (containStarColumn(columns, columnName)) { 19560 continue; 19561 } 19562 columnElement.setName(columnName); 19563 if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) { 19564 columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + "," 19565 + convertCoordinate(columnModel.getEndPosition())); 19566 } 19567 viewElement.getColumns().add(columnElement); 19568 } 19569 19570 if (columnModel.isShowStar()) { 19571 column columnElement = new column(); 19572 columnElement.setId(String.valueOf(columnModel.getId())); 19573 columnElement.setName(columnModel.getName()); 19574 if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) { 19575 columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + "," 19576 + convertCoordinate(columnModel.getEndPosition())); 19577 } 19578 viewElement.getColumns().add(columnElement); 19579 } 19580 19581 } else { 19582 column columnElement = new column(); 19583 columnElement.setId(String.valueOf(columnModel.getId())); 19584 columnElement.setName(columnModel.getName()); 19585 if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) { 19586 columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + "," 19587 + convertCoordinate(columnModel.getEndPosition())); 19588 } 19589 if(columnModel.isPseduo()) { 19590 columnElement.setSource("system"); 19591 } 19592 viewElement.getColumns().add(columnElement); 19593 } 19594 } 19595 19596 TableRelationRows relationRows = viewModel.getRelationRows(); 19597 if (relationRows.hasRelation()) { 19598 column relationRowsElement = new column(); 19599 relationRowsElement.setId(String.valueOf(relationRows.getId())); 19600 relationRowsElement.setName(relationRows.getName()); 19601 if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) { 19602 relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + "," 19603 + convertCoordinate(relationRows.getEndPosition())); 19604 } 19605 relationRowsElement.setSource("system"); 19606 viewElement.getColumns().add(relationRowsElement); 19607 } 19608 } 19609 19610 private void appendStreamModel(dataflow dataflow, Table streamModel) { 19611 table streamElement = new table(); 19612 streamElement.setId(String.valueOf(streamModel.getId())); 19613 if (!SQLUtil.isEmpty(streamModel.getDatabase())) { 19614 streamElement.setDatabase(streamModel.getDatabase()); 19615 } 19616 if (!SQLUtil.isEmpty(streamModel.getSchema())) { 19617 streamElement.setSchema(streamModel.getSchema()); 19618 } 19619 streamElement.setServer(streamModel.getServer()); 19620 streamElement.setName(streamModel.getName()); 19621 streamElement.setType("stream"); 19622 if (streamModel.getFileType() != null) { 19623 streamElement.setFileType(SQLUtil.trimColumnStringQuote(streamModel.getFileType())); 19624 } 19625 19626 if (streamModel.getStartPosition() != null && streamModel.getEndPosition() != null) { 19627 streamElement.setCoordinate(convertCoordinate(streamModel.getStartPosition()) + "," 19628 + convertCoordinate(streamModel.getEndPosition())); 19629 } 19630 19631 if (streamModel.getProcesses() != null) { 19632 List<String> processIds = new ArrayList<String>(); 19633 for (Process process : streamModel.getProcesses()) { 19634 processIds.add(String.valueOf(process.getId())); 19635 } 19636 streamElement.setProcessIds(processIds); 19637 } 19638 dataflow.getStreams().add(streamElement); 19639 19640 List<TableColumn> columns = streamModel.getColumns(); 19641 19642 for (int j = 0; j < columns.size(); j++) { 19643 TableColumn columnModel = (TableColumn) columns.get(j); 19644 column columnElement = new column(); 19645 columnElement.setId(String.valueOf(columnModel.getId())); 19646 columnElement.setName(SQLUtil.trimColumnStringQuote(columnModel.getName())); 19647 columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + "," 19648 + convertCoordinate(columnModel.getEndPosition())); 19649 streamElement.getColumns().add(columnElement); 19650 } 19651 19652 TableRelationRows relationRows = streamModel.getRelationRows(); 19653 if (relationRows.hasRelation()) { 19654 column relationRowsElement = new column(); 19655 relationRowsElement.setId(String.valueOf(relationRows.getId())); 19656 relationRowsElement.setName(relationRows.getName()); 19657 if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) { 19658 relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + "," 19659 + convertCoordinate(relationRows.getEndPosition())); 19660 } 19661 relationRowsElement.setSource("system"); 19662 streamElement.getColumns().add(relationRowsElement); 19663 } 19664 } 19665 19666 private void appendStageModel(dataflow dataflow, Table stageModel) { 19667 table stageElement = new table(); 19668 stageElement.setId(String.valueOf(stageModel.getId())); 19669 if (!SQLUtil.isEmpty(stageModel.getDatabase())) { 19670 stageElement.setDatabase(stageModel.getDatabase()); 19671 } 19672 if (!SQLUtil.isEmpty(stageModel.getSchema())) { 19673 stageElement.setSchema(stageModel.getSchema()); 19674 } 19675 stageElement.setServer(stageModel.getServer()); 19676 stageElement.setName(stageModel.getName()); 19677 stageElement.setType("stage"); 19678 stageElement.setLocation(stageModel.getLocation()); 19679 if (stageModel.getFileType() != null) { 19680 stageElement.setFileType(SQLUtil.trimColumnStringQuote(stageModel.getFileType())); 19681 } 19682 19683 if (stageModel.getStartPosition() != null && stageModel.getEndPosition() != null) { 19684 stageElement.setCoordinate(convertCoordinate(stageModel.getStartPosition()) + "," 19685 + convertCoordinate(stageModel.getEndPosition())); 19686 } 19687 19688 if (stageModel.getProcesses() != null) { 19689 List<String> processIds = new ArrayList<String>(); 19690 for (Process process : stageModel.getProcesses()) { 19691 processIds.add(String.valueOf(process.getId())); 19692 } 19693 stageElement.setProcessIds(processIds); 19694 } 19695 dataflow.getStages().add(stageElement); 19696 19697 List<TableColumn> columns = stageModel.getColumns(); 19698 19699 for (int j = 0; j < columns.size(); j++) { 19700 TableColumn columnModel = (TableColumn) columns.get(j); 19701 column columnElement = new column(); 19702 columnElement.setId(String.valueOf(columnModel.getId())); 19703 columnElement.setName(SQLUtil.trimColumnStringQuote(columnModel.getName())); 19704 columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + "," 19705 + convertCoordinate(columnModel.getEndPosition())); 19706 stageElement.getColumns().add(columnElement); 19707 } 19708 19709 TableRelationRows relationRows = stageModel.getRelationRows(); 19710 if (relationRows.hasRelation()) { 19711 column relationRowsElement = new column(); 19712 relationRowsElement.setId(String.valueOf(relationRows.getId())); 19713 relationRowsElement.setName(relationRows.getName()); 19714 if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) { 19715 relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + "," 19716 + convertCoordinate(relationRows.getEndPosition())); 19717 } 19718 relationRowsElement.setSource("system"); 19719 stageElement.getColumns().add(relationRowsElement); 19720 } 19721 } 19722 19723 private void appendSequenceModel(dataflow dataflow, Table sequenceModel) { 19724 table sequenceElement = new table(); 19725 sequenceElement.setId(String.valueOf(sequenceModel.getId())); 19726 if (!SQLUtil.isEmpty(sequenceModel.getDatabase())) { 19727 sequenceElement.setDatabase(sequenceModel.getDatabase()); 19728 } 19729 if (!SQLUtil.isEmpty(sequenceModel.getSchema())) { 19730 sequenceElement.setSchema(sequenceModel.getSchema()); 19731 } 19732 sequenceElement.setServer(sequenceModel.getServer()); 19733 sequenceElement.setName(sequenceModel.getName()); 19734 sequenceElement.setType("sequence"); 19735 sequenceElement.setLocation(sequenceModel.getLocation()); 19736 if (sequenceModel.getFileType() != null) { 19737 sequenceElement.setFileType(SQLUtil.trimColumnStringQuote(sequenceModel.getFileType())); 19738 } 19739 19740 if (sequenceModel.getStartPosition() != null && sequenceModel.getEndPosition() != null) { 19741 sequenceElement.setCoordinate(convertCoordinate(sequenceModel.getStartPosition()) + "," 19742 + convertCoordinate(sequenceModel.getEndPosition())); 19743 } 19744 19745 if (sequenceModel.getProcesses() != null) { 19746 List<String> processIds = new ArrayList<String>(); 19747 for (Process process : sequenceModel.getProcesses()) { 19748 processIds.add(String.valueOf(process.getId())); 19749 } 19750 sequenceElement.setProcessIds(processIds); 19751 } 19752 dataflow.getSequences().add(sequenceElement); 19753 19754 List<TableColumn> columns = sequenceModel.getColumns(); 19755 19756 for (int j = 0; j < columns.size(); j++) { 19757 TableColumn columnModel = (TableColumn) columns.get(j); 19758 column columnElement = new column(); 19759 columnElement.setId(String.valueOf(columnModel.getId())); 19760 columnElement.setName(SQLUtil.trimColumnStringQuote(columnModel.getName())); 19761 columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + "," 19762 + convertCoordinate(columnModel.getEndPosition())); 19763 sequenceElement.getColumns().add(columnElement); 19764 } 19765 19766 TableRelationRows relationRows = sequenceModel.getRelationRows(); 19767 if (relationRows.hasRelation()) { 19768 column relationRowsElement = new column(); 19769 relationRowsElement.setId(String.valueOf(relationRows.getId())); 19770 relationRowsElement.setName(relationRows.getName()); 19771 if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) { 19772 relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + "," 19773 + convertCoordinate(relationRows.getEndPosition())); 19774 } 19775 relationRowsElement.setSource("system"); 19776 sequenceElement.getColumns().add(relationRowsElement); 19777 } 19778 } 19779 19780 private void appendDataSourceModel(dataflow dataflow, Table datasourceModel) { 19781 table datasourceElement = new table(); 19782 datasourceElement.setId(String.valueOf(datasourceModel.getId())); 19783 if (!SQLUtil.isEmpty(datasourceModel.getDatabase())) { 19784 datasourceElement.setDatabase(datasourceModel.getDatabase()); 19785 } 19786 if (!SQLUtil.isEmpty(datasourceModel.getSchema())) { 19787 datasourceElement.setSchema(datasourceModel.getSchema()); 19788 } 19789 datasourceElement.setServer(datasourceModel.getServer()); 19790 datasourceElement.setName(datasourceModel.getName()); 19791 datasourceElement.setType("datasource"); 19792 datasourceElement.setLocation(datasourceModel.getLocation()); 19793 if (datasourceModel.getFileType() != null) { 19794 datasourceElement.setFileType(SQLUtil.trimColumnStringQuote(datasourceModel.getFileType())); 19795 } 19796 19797 if (datasourceModel.getStartPosition() != null && datasourceModel.getEndPosition() != null) { 19798 datasourceElement.setCoordinate(convertCoordinate(datasourceModel.getStartPosition()) + "," 19799 + convertCoordinate(datasourceModel.getEndPosition())); 19800 } 19801 19802 if (datasourceModel.getProcesses() != null) { 19803 List<String> processIds = new ArrayList<String>(); 19804 for (Process process : datasourceModel.getProcesses()) { 19805 processIds.add(String.valueOf(process.getId())); 19806 } 19807 datasourceElement.setProcessIds(processIds); 19808 } 19809 dataflow.getDatasources().add(datasourceElement); 19810 19811 List<TableColumn> columns = datasourceModel.getColumns(); 19812 19813 for (int j = 0; j < columns.size(); j++) { 19814 TableColumn columnModel = (TableColumn) columns.get(j); 19815 column columnElement = new column(); 19816 columnElement.setId(String.valueOf(columnModel.getId())); 19817 columnElement.setName(SQLUtil.trimColumnStringQuote(columnModel.getName())); 19818 columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + "," 19819 + convertCoordinate(columnModel.getEndPosition())); 19820 datasourceElement.getColumns().add(columnElement); 19821 } 19822 19823 TableRelationRows relationRows = datasourceModel.getRelationRows(); 19824 if (relationRows.hasRelation()) { 19825 column relationRowsElement = new column(); 19826 relationRowsElement.setId(String.valueOf(relationRows.getId())); 19827 relationRowsElement.setName(relationRows.getName()); 19828 if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) { 19829 relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + "," 19830 + convertCoordinate(relationRows.getEndPosition())); 19831 } 19832 relationRowsElement.setSource("system"); 19833 datasourceElement.getColumns().add(relationRowsElement); 19834 } 19835 } 19836 19837 private void appendDatabaseModel(dataflow dataflow, Table databaseModel) { 19838 table databaseElement = new table(); 19839 databaseElement.setId(String.valueOf(databaseModel.getId())); 19840 if (!SQLUtil.isEmpty(databaseModel.getDatabase())) { 19841 databaseElement.setDatabase(databaseModel.getDatabase()); 19842 } 19843 if (!SQLUtil.isEmpty(databaseModel.getSchema())) { 19844 databaseElement.setSchema(databaseModel.getSchema()); 19845 } 19846 databaseElement.setServer(databaseModel.getServer()); 19847 databaseElement.setName(databaseModel.getName()); 19848 databaseElement.setType("database"); 19849 if (databaseModel.getFileType() != null) { 19850 databaseElement.setFileType(SQLUtil.trimColumnStringQuote(databaseModel.getFileType())); 19851 } 19852 19853 if (databaseModel.getStartPosition() != null && databaseModel.getEndPosition() != null) { 19854 databaseElement.setCoordinate(convertCoordinate(databaseModel.getStartPosition()) + "," 19855 + convertCoordinate(databaseModel.getEndPosition())); 19856 } 19857 19858 if (databaseModel.getProcesses() != null) { 19859 List<String> processIds = new ArrayList<String>(); 19860 for (Process process : databaseModel.getProcesses()) { 19861 processIds.add(String.valueOf(process.getId())); 19862 } 19863 databaseElement.setProcessIds(processIds); 19864 } 19865 dataflow.getDatabases().add(databaseElement); 19866 19867 List<TableColumn> columns = databaseModel.getColumns(); 19868 19869 for (int j = 0; j < columns.size(); j++) { 19870 TableColumn columnModel = (TableColumn) columns.get(j); 19871 column columnElement = new column(); 19872 columnElement.setId(String.valueOf(columnModel.getId())); 19873 columnElement.setName(SQLUtil.trimColumnStringQuote(columnModel.getName())); 19874 columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + "," 19875 + convertCoordinate(columnModel.getEndPosition())); 19876 databaseElement.getColumns().add(columnElement); 19877 } 19878 19879 TableRelationRows relationRows = databaseModel.getRelationRows(); 19880 if (relationRows.hasRelation()) { 19881 column relationRowsElement = new column(); 19882 relationRowsElement.setId(String.valueOf(relationRows.getId())); 19883 relationRowsElement.setName(relationRows.getName()); 19884 if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) { 19885 relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + "," 19886 + convertCoordinate(relationRows.getEndPosition())); 19887 } 19888 relationRowsElement.setSource("system"); 19889 databaseElement.getColumns().add(relationRowsElement); 19890 } 19891 } 19892 19893 private void appendSchemaModel(dataflow dataflow, Table schemaModel) { 19894 table schemaElement = new table(); 19895 schemaElement.setId(String.valueOf(schemaModel.getId())); 19896 if (!SQLUtil.isEmpty(schemaModel.getDatabase())) { 19897 schemaElement.setDatabase(schemaModel.getDatabase()); 19898 } 19899 if (!SQLUtil.isEmpty(schemaModel.getSchema())) { 19900 schemaElement.setSchema(schemaModel.getSchema()); 19901 } 19902 schemaElement.setServer(schemaModel.getServer()); 19903 schemaElement.setName(schemaModel.getName()); 19904 schemaElement.setType("schema"); 19905 if (schemaModel.getFileType() != null) { 19906 schemaElement.setFileType(SQLUtil.trimColumnStringQuote(schemaModel.getFileType())); 19907 } 19908 19909 if (schemaModel.getStartPosition() != null && schemaModel.getEndPosition() != null) { 19910 schemaElement.setCoordinate(convertCoordinate(schemaModel.getStartPosition()) + "," 19911 + convertCoordinate(schemaModel.getEndPosition())); 19912 } 19913 19914 if (schemaModel.getProcesses() != null) { 19915 List<String> processIds = new ArrayList<String>(); 19916 for (Process process : schemaModel.getProcesses()) { 19917 processIds.add(String.valueOf(process.getId())); 19918 } 19919 schemaElement.setProcessIds(processIds); 19920 } 19921 dataflow.getSchemas().add(schemaElement); 19922 19923 List<TableColumn> columns = schemaModel.getColumns(); 19924 19925 for (int j = 0; j < columns.size(); j++) { 19926 TableColumn columnModel = (TableColumn) columns.get(j); 19927 column columnElement = new column(); 19928 columnElement.setId(String.valueOf(columnModel.getId())); 19929 columnElement.setName(SQLUtil.trimColumnStringQuote(columnModel.getName())); 19930 columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + "," 19931 + convertCoordinate(columnModel.getEndPosition())); 19932 schemaElement.getColumns().add(columnElement); 19933 } 19934 19935 TableRelationRows relationRows = schemaModel.getRelationRows(); 19936 if (relationRows.hasRelation()) { 19937 column relationRowsElement = new column(); 19938 relationRowsElement.setId(String.valueOf(relationRows.getId())); 19939 relationRowsElement.setName(relationRows.getName()); 19940 if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) { 19941 relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + "," 19942 + convertCoordinate(relationRows.getEndPosition())); 19943 } 19944 relationRowsElement.setSource("system"); 19945 schemaElement.getColumns().add(relationRowsElement); 19946 } 19947 } 19948 19949 private void appendPathModel(dataflow dataflow, Table pathModel) { 19950 table pathElement = new table(); 19951 pathElement.setId(String.valueOf(pathModel.getId())); 19952 if (!SQLUtil.isEmpty(pathModel.getDatabase())) { 19953 pathElement.setDatabase(pathModel.getDatabase()); 19954 } 19955 if (!SQLUtil.isEmpty(pathModel.getSchema())) { 19956 pathElement.setSchema(pathModel.getSchema()); 19957 } 19958 pathElement.setServer(pathModel.getServer()); 19959 pathElement.setName(pathModel.getName()); 19960 pathElement.setType("path"); 19961 if (pathModel.getFileFormat() != null) { 19962 pathElement.setFileFormat(SQLUtil.trimColumnStringQuote(pathModel.getFileFormat())); 19963 } 19964 19965 if (pathModel.getStartPosition() != null && pathModel.getEndPosition() != null) { 19966 pathElement.setCoordinate(convertCoordinate(pathModel.getStartPosition()) + "," 19967 + convertCoordinate(pathModel.getEndPosition())); 19968 } 19969 19970 if (pathModel.getProcesses() != null) { 19971 List<String> processIds = new ArrayList<String>(); 19972 for (Process process : pathModel.getProcesses()) { 19973 processIds.add(String.valueOf(process.getId())); 19974 } 19975 pathElement.setProcessIds(processIds); 19976 } 19977 19978 pathElement.setUri(pathModel.getName()); 19979 19980 dataflow.getPaths().add(pathElement); 19981 19982 List<TableColumn> columns = pathModel.getColumns(); 19983 19984 for (int j = 0; j < columns.size(); j++) { 19985 TableColumn columnModel = (TableColumn) columns.get(j); 19986 column columnElement = new column(); 19987 columnElement.setId(String.valueOf(columnModel.getId())); 19988 columnElement.setName(SQLUtil.trimColumnStringQuote(columnModel.getName())); 19989 columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + "," 19990 + convertCoordinate(columnModel.getEndPosition())); 19991 pathElement.getColumns().add(columnElement); 19992 } 19993 19994 TableRelationRows relationRows = pathModel.getRelationRows(); 19995 if (relationRows.hasRelation()) { 19996 column relationRowsElement = new column(); 19997 relationRowsElement.setId(String.valueOf(relationRows.getId())); 19998 relationRowsElement.setName(relationRows.getName()); 19999 if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) { 20000 relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + "," 20001 + convertCoordinate(relationRows.getEndPosition())); 20002 } 20003 relationRowsElement.setSource("system"); 20004 pathElement.getColumns().add(relationRowsElement); 20005 } 20006 } 20007 20008 private void appendVariableModel(dataflow dataflow, Table variableModel) { 20009 table variableElement = new table(); 20010 variableElement.setId(String.valueOf(variableModel.getId())); 20011 if (!SQLUtil.isEmpty(variableModel.getDatabase())) { 20012 variableElement.setDatabase(variableModel.getDatabase()); 20013 } 20014 if (!SQLUtil.isEmpty(variableModel.getSchema())) { 20015 variableElement.setSchema(variableModel.getSchema()); 20016 } 20017 variableElement.setServer(variableModel.getServer()); 20018 variableElement.setName(variableModel.getName()); 20019 variableElement.setType("variable"); 20020 variableElement.setParent(variableModel.getParent()); 20021 if (variableModel.getProcedureId() != null) { 20022 variableElement.setProcedureId(variableModel.getProcedureId()); 20023 } 20024 variableElement.setIsTarget(String.valueOf(variableModel.isTarget())); 20025 if (variableModel.getSubType() != null) { 20026 variableElement.setSubType(variableModel.getSubType().name()); 20027 } 20028 20029 if (variableModel.getStartPosition() != null && variableModel.getEndPosition() != null) { 20030 variableElement.setCoordinate(convertCoordinate(variableModel.getStartPosition()) + "," 20031 + convertCoordinate(variableModel.getEndPosition())); 20032 } 20033 dataflow.getVariables().add(variableElement); 20034 20035 List<TableColumn> columns = variableModel.getColumns(); 20036 20037 if (containStarColumn(columns)) { 20038 for (TableColumn column : columns) { 20039 if (column.getName().endsWith("*")) { 20040 for (TableColumn starElement : columns) { 20041 if (starElement == column) { 20042 continue; 20043 } 20044 TObjectName columnObject = starElement.getColumnObject(); 20045 column.bindStarLinkColumn(columnObject); 20046 } 20047// column.setShowStar(false); 20048 } 20049 } 20050 } 20051 20052 for (int j = 0; j < columns.size(); j++) { 20053 TableColumn columnModel = columns.get(j); 20054 if (columnModel.hasStarLinkColumn()) { 20055 List<String> starLinkColumnList = columnModel.getStarLinkColumnNames(); 20056 for (int k = 0; k < starLinkColumnList.size(); k++) { 20057 column columnElement = new column(); 20058 columnElement.setId(columnModel.getId() + "_" + k); 20059 String columnName = starLinkColumnList.get(k); 20060 if (containStarColumn(columns, columnName)) { 20061 continue; 20062 } 20063 columnElement.setName(columnName); 20064 if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) { 20065 columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + "," 20066 + convertCoordinate(columnModel.getEndPosition())); 20067 } 20068 variableElement.getColumns().add(columnElement); 20069 } 20070 20071 if (columnModel.isShowStar()) { 20072 column columnElement = new column(); 20073 columnElement.setId(String.valueOf(columnModel.getId())); 20074 columnElement.setName(columnModel.getName()); 20075 if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) { 20076 columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + "," 20077 + convertCoordinate(columnModel.getEndPosition())); 20078 } 20079 variableElement.getColumns().add(columnElement); 20080 } 20081 20082 } else { 20083 column columnElement = new column(); 20084 columnElement.setId(String.valueOf(columnModel.getId())); 20085 columnElement.setName(columnModel.getName()); 20086 if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) { 20087 columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + "," 20088 + convertCoordinate(columnModel.getEndPosition())); 20089 } 20090 variableElement.getColumns().add(columnElement); 20091 } 20092 } 20093 20094 TableRelationRows relationRows = variableModel.getRelationRows(); 20095 if (relationRows.hasRelation()) { 20096 column relationRowsElement = new column(); 20097 relationRowsElement.setId(String.valueOf(relationRows.getId())); 20098 relationRowsElement.setName(relationRows.getName()); 20099 if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) { 20100 relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + "," 20101 + convertCoordinate(relationRows.getEndPosition())); 20102 } 20103 relationRowsElement.setSource("system"); 20104 variableElement.getColumns().add(relationRowsElement); 20105 } 20106 } 20107 20108 private void appendCursorModel(dataflow dataflow, Table cursorModel) { 20109 table cursorElement = new table(); 20110 cursorElement.setId(String.valueOf(cursorModel.getId())); 20111 if (!SQLUtil.isEmpty(cursorModel.getDatabase())) { 20112 cursorElement.setDatabase(cursorModel.getDatabase()); 20113 } 20114 if (!SQLUtil.isEmpty(cursorModel.getSchema())) { 20115 cursorElement.setSchema(cursorModel.getSchema()); 20116 } 20117 cursorElement.setServer(cursorModel.getServer()); 20118 cursorElement.setName(cursorModel.getName()); 20119 cursorElement.setType("variable"); 20120 if (cursorElement.getSubType() != null) { 20121 cursorElement.setSubType(cursorModel.getSubType().name()); 20122 } 20123 20124 if (cursorModel.getStartPosition() != null && cursorModel.getEndPosition() != null) { 20125 cursorElement.setCoordinate(convertCoordinate(cursorModel.getStartPosition()) + "," 20126 + convertCoordinate(cursorModel.getEndPosition())); 20127 } 20128 dataflow.getVariables().add(cursorElement); 20129 20130 List<TableColumn> columns = cursorModel.getColumns(); 20131 20132 if (containStarColumn(columns)) { 20133 for (TableColumn column : columns) { 20134 if (column.getName().endsWith("*")) { 20135 for (TableColumn starElement : columns) { 20136 if (starElement == column) { 20137 continue; 20138 } 20139 TObjectName columnObject = starElement.getColumnObject(); 20140 column.bindStarLinkColumn(columnObject); 20141 } 20142 column.setShowStar(false); 20143 } 20144 } 20145 } 20146 20147 for (int j = 0; j < columns.size(); j++) { 20148 TableColumn columnModel = (TableColumn) columns.get(j); 20149 if (columnModel.hasStarLinkColumn()) { 20150 List<String> starLinkColumnList = columnModel.getStarLinkColumnNames(); 20151 for (int k = 0; k < starLinkColumnList.size(); k++) { 20152 column columnElement = new column(); 20153 columnElement.setId(String.valueOf(columnModel.getId()) + "_" + k); 20154 String columnName = starLinkColumnList.get(k); 20155 if (containStarColumn(columns, columnName)) { 20156 continue; 20157 } 20158 columnElement.setName(columnName); 20159 if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) { 20160 columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + "," 20161 + convertCoordinate(columnModel.getEndPosition())); 20162 } 20163 cursorElement.getColumns().add(columnElement); 20164 } 20165 20166 if (columnModel.isShowStar()) { 20167 column columnElement = new column(); 20168 columnElement.setId(String.valueOf(columnModel.getId())); 20169 columnElement.setName(columnModel.getName()); 20170 if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) { 20171 columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + "," 20172 + convertCoordinate(columnModel.getEndPosition())); 20173 } 20174 cursorElement.getColumns().add(columnElement); 20175 } 20176 20177 } else { 20178 column columnElement = new column(); 20179 columnElement.setId(String.valueOf(columnModel.getId())); 20180 columnElement.setName(columnModel.getName()); 20181 if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) { 20182 columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + "," 20183 + convertCoordinate(columnModel.getEndPosition())); 20184 } 20185 cursorElement.getColumns().add(columnElement); 20186 } 20187 } 20188 20189 TableRelationRows relationRows = cursorModel.getRelationRows(); 20190 if (relationRows.hasRelation()) { 20191 column relationRowsElement = new column(); 20192 relationRowsElement.setId(String.valueOf(relationRows.getId())); 20193 relationRowsElement.setName(relationRows.getName()); 20194 if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) { 20195 relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + "," 20196 + convertCoordinate(relationRows.getEndPosition())); 20197 } 20198 relationRowsElement.setSource("system"); 20199 cursorElement.getColumns().add(relationRowsElement); 20200 } 20201 } 20202 20203 private void appendErrors(dataflow dataflow) { 20204 List<ErrorInfo> errorInfos = this.getErrorMessages(); 20205 if (metadataErrors != null) { 20206 for (int i = 0; i < metadataErrors.size(); i++) { 20207 errorInfos.add(i, metadataErrors.get(i)); 20208 } 20209 } 20210 20211 for (int i = 0; i < errorInfos.size(); ++i) { 20212 ErrorInfo errorInfo = errorInfos.get(i); 20213 error error = new error(); 20214 if (!SQLUtil.isEmpty(errorInfo.getErrorMessage())) { 20215 error.setErrorMessage(errorInfo.getErrorMessage()); 20216 } 20217 if (!SQLUtil.isEmpty(errorInfo.getErrorType())) { 20218 error.setErrorType(errorInfo.getErrorType()); 20219 } 20220 if (errorInfo.getStartPosition() != null && errorInfo.getEndPosition() != null) { 20221 error.setCoordinate(convertCoordinate(errorInfo.getStartPosition()) + "," 20222 + convertCoordinate(errorInfo.getEndPosition())); 20223 } 20224 if (!SQLUtil.isEmpty(errorInfo.getFileName())) { 20225 error.setFile(errorInfo.getFileName()); 20226 } 20227 if (errorInfo.getOriginStartPosition() != null && errorInfo.getOriginEndPosition() != null) { 20228 error.setOriginCoordinate(errorInfo.getOriginStartPosition() + "," + errorInfo.getOriginEndPosition()); 20229 } 20230 dataflow.getErrors().add(error); 20231 } 20232 } 20233 20234 private void appendOraclePackages(dataflow dataflow) { 20235 List<OraclePackage> packages = this.modelManager.getOraclePackageModels(); 20236 20237 for (int i = 0; i < packages.size(); ++i) { 20238 OraclePackage model = packages.get(i); 20239 oraclePackage oraclePackage = new oraclePackage(); 20240 oraclePackage.setId(String.valueOf(model.getId())); 20241 if (!SQLUtil.isEmpty(model.getDatabase())) { 20242 oraclePackage.setDatabase(model.getDatabase()); 20243 } 20244 if (!SQLUtil.isEmpty(model.getSchema())) { 20245 oraclePackage.setSchema(model.getSchema()); 20246 } 20247 oraclePackage.setServer(model.getServer()); 20248 oraclePackage.setName(model.getName()); 20249 if (model.getType() != null) { 20250 oraclePackage.setType(model.getType().name().replace("sst", "")); 20251 } 20252 if (model.getStartPosition() != null && model.getEndPosition() != null) { 20253 oraclePackage.setCoordinate( 20254 convertCoordinate(model.getStartPosition()) + "," + convertCoordinate(model.getEndPosition())); 20255 } 20256 20257 dataflow.getPackages().add(oraclePackage); 20258 20259 List<Argument> arguments = model.getArguments(); 20260 20261 for (int j = 0; j < arguments.size(); ++j) { 20262 Argument argumentModel = (Argument) arguments.get(j); 20263 argument argumentElement = new argument(); 20264 argumentElement.setId(String.valueOf(argumentModel.getId())); 20265 argumentElement.setName(argumentModel.getName()); 20266 if (argumentModel.getStartPosition() != null && argumentModel.getEndPosition() != null) { 20267 argumentElement.setCoordinate(convertCoordinate(argumentModel.getStartPosition()) + "," 20268 + convertCoordinate(argumentModel.getEndPosition())); 20269 } 20270 20271 argumentElement.setDatatype(argumentModel.getDataType().getDataTypeName()); 20272 argumentElement.setInout(argumentModel.getMode().name()); 20273 oraclePackage.getArguments().add(argumentElement); 20274 } 20275 20276 for (int j = 0; j < model.getProcedures().size(); j++) { 20277 Procedure procedureModel = model.getProcedures().get(j); 20278 procedure procedure = new procedure(); 20279 procedure.setId(String.valueOf(procedureModel.getId())); 20280 if (!SQLUtil.isEmpty(procedureModel.getDatabase())) { 20281 procedure.setDatabase(procedureModel.getDatabase()); 20282 } 20283 if (!SQLUtil.isEmpty(procedureModel.getSchema())) { 20284 procedure.setSchema(procedureModel.getSchema()); 20285 } 20286 procedure.setServer(procedureModel.getServer()); 20287 procedure.setName(procedureModel.getName()); 20288 if (procedureModel.getType() != null) { 20289 procedure.setType(procedureModel.getType().name().replace("sst", "")); 20290 } 20291 if (procedureModel.getStartPosition() != null && procedureModel.getEndPosition() != null) { 20292 procedure.setCoordinate(convertCoordinate(procedureModel.getStartPosition()) + "," 20293 + convertCoordinate(procedureModel.getEndPosition())); 20294 } 20295 20296 oraclePackage.getProcedures().add(procedure); 20297 20298 List<Argument> procedureArguments = procedureModel.getArguments(); 20299 20300 for (int k = 0; k < procedureArguments.size(); ++k) { 20301 Argument argumentModel = (Argument) procedureArguments.get(k); 20302 argument argumentElement = new argument(); 20303 argumentElement.setId(String.valueOf(argumentModel.getId())); 20304 argumentElement.setName(argumentModel.getName()); 20305 if (argumentModel.getStartPosition() != null && argumentModel.getEndPosition() != null) { 20306 argumentElement.setCoordinate(convertCoordinate(argumentModel.getStartPosition()) + "," 20307 + convertCoordinate(argumentModel.getEndPosition())); 20308 } 20309 20310 argumentElement.setDatatype(argumentModel.getDataType().getDataTypeName()); 20311 argumentElement.setInout(argumentModel.getMode().name()); 20312 procedure.getArguments().add(argumentElement); 20313 } 20314 } 20315 } 20316 } 20317 20318 private void appendProcedures(dataflow dataflow) { 20319 List<Procedure> procedures = this.modelManager.getProcedureModels(); 20320 20321 for (int i = 0; i < procedures.size(); ++i) { 20322 Procedure model = procedures.get(i); 20323 if (model.getParentPackage() != null) { 20324 continue; 20325 } 20326 procedure procedure = new procedure(); 20327 procedure.setId(String.valueOf(model.getId())); 20328 if (!SQLUtil.isEmpty(model.getDatabase())) { 20329 procedure.setDatabase(model.getDatabase()); 20330 } 20331 if (!SQLUtil.isEmpty(model.getSchema())) { 20332 procedure.setSchema(model.getSchema()); 20333 } 20334 procedure.setServer(model.getServer()); 20335 procedure.setName(model.getName()); 20336 if (model.getType() != null) { 20337 procedure.setType(model.getType().name().replace("sst", "")); 20338 } 20339 if (model.getStartPosition() != null && model.getEndPosition() != null) { 20340 procedure.setCoordinate( 20341 convertCoordinate(model.getStartPosition()) + "," + convertCoordinate(model.getEndPosition())); 20342 } 20343 20344 dataflow.getProcedures().add(procedure); 20345 20346 List<Argument> arguments = model.getArguments(); 20347 20348 for (int j = 0; j < arguments.size(); ++j) { 20349 Argument argumentModel = (Argument) arguments.get(j); 20350 argument argumentElement = new argument(); 20351 argumentElement.setId(String.valueOf(argumentModel.getId())); 20352 argumentElement.setName(argumentModel.getName()); 20353 if (argumentModel.getStartPosition() != null && argumentModel.getEndPosition() != null) { 20354 argumentElement.setCoordinate(convertCoordinate(argumentModel.getStartPosition()) + "," 20355 + convertCoordinate(argumentModel.getEndPosition())); 20356 } 20357 20358 argumentElement.setDatatype(argumentModel.getDataType().getDataTypeName()); 20359 argumentElement.setInout(argumentModel.getMode().name()); 20360 procedure.getArguments().add(argumentElement); 20361 } 20362 } 20363 } 20364 20365 private void appendProcesses(dataflow dataflow) { 20366 List<Process> processes = this.modelManager.getProcessModels(); 20367 20368 for (int i = 0; i < processes.size(); ++i) { 20369 Process model = processes.get(i); 20370 process process = new process(); 20371 process.setId(String.valueOf(model.getId())); 20372 if (!SQLUtil.isEmpty(model.getDatabase())) { 20373 process.setDatabase(model.getDatabase()); 20374 } 20375 if (!SQLUtil.isEmpty(model.getSchema())) { 20376 process.setSchema(model.getSchema()); 20377 } 20378 process.setServer(model.getServer()); 20379 process.setName(getProcessName(model)); 20380 if (!SQLUtil.isEmpty(model.getProcedureName())) { 20381 process.setProcedureName(model.getProcedureName()); 20382 } 20383 if (model.getProcedureId() != null) { 20384 process.setProcedureId(String.valueOf(model.getProcedureId())); 20385 } 20386 if (!SQLUtil.isEmpty(model.getQueryHashId())) { 20387 process.setQueryHashId(model.getQueryHashId()); 20388 } 20389 if (model.getGspObject() != null) { 20390 process.setType(model.getGspObject().sqlstatementtype.name()); 20391 } 20392 if (model.getStartPosition() != null && model.getEndPosition() != null) { 20393 process.setCoordinate( 20394 convertCoordinate(model.getStartPosition()) + "," + convertCoordinate(model.getEndPosition())); 20395 } 20396 if (model.getTransforms() != null && !model.getTransforms().isEmpty()) { 20397 for (Transform transformItem : model.getTransforms()) { 20398 process.addTransform(transformItem); 20399 } 20400 } 20401 dataflow.getProcesses().add(process); 20402 } 20403 } 20404 20405 private void appendTables(dataflow dataflow) { 20406 List<TTable> tables = modelManager.getBaseTables(); 20407 Map<String, table> tableMap = new HashMap<String, table>(); 20408 Set<Long> tableModelIds = new HashSet<Long>(); 20409 for (int i = 0; i < tables.size(); i++) { 20410 Object model = modelManager.getModel(tables.get(i)); 20411 if (model instanceof Table) { 20412 Table tableModel = (Table) model; 20413 if(tableModelIds.contains(tableModel.getId())) { 20414 continue; 20415 } 20416 else { 20417 tableModelIds.add(tableModel.getId()); 20418 } 20419 if (tableModel.isView()) { 20420 continue; 20421 } 20422 if (tableModel.isStage()) { 20423 appendStageModel(dataflow, tableModel); 20424 continue; 20425 } 20426 if (tableModel.isSequence()) { 20427 appendSequenceModel(dataflow, tableModel); 20428 continue; 20429 } 20430 if (tableModel.isDataSource()) { 20431 appendDataSourceModel(dataflow, tableModel); 20432 continue; 20433 } 20434 if (tableModel.isDatabase()) { 20435 appendDatabaseModel(dataflow, tableModel); 20436 continue; 20437 } 20438 if (tableModel.isSchema()) { 20439 appendSchemaModel(dataflow, tableModel); 20440 continue; 20441 } 20442 if (tableModel.isStream()) { 20443 appendStreamModel(dataflow, tableModel); 20444 continue; 20445 } 20446 if (tableModel.isPath()) { 20447 appendPathModel(dataflow, tableModel); 20448 continue; 20449 } 20450 if (tableModel.isVariable() && !tableModel.isCursor()) { 20451 appendVariableModel(dataflow, tableModel); 20452 continue; 20453 } 20454 if (tableModel.isCursor()) { 20455 appendCursorModel(dataflow, tableModel); 20456 continue; 20457 } 20458 if (tableModel.isConstant()) { 20459 appendConstantModel(dataflow, tableModel); 20460 continue; 20461 } 20462 if (!tableIds.contains(tableModel.getId())) { 20463 appendTableModel(dataflow, tableModel, tableMap); 20464 tableIds.add(tableModel.getId()); 20465 } 20466 } else if (model instanceof QueryTable) { 20467 QueryTable queryTable = (QueryTable) model; 20468 if (!tableIds.contains(queryTable.getId())) { 20469 appendResultSet(dataflow, queryTable); 20470 tableIds.add(queryTable.getId()); 20471 } 20472 } 20473 } 20474 20475 List<Table> tableNames = modelManager.getTablesByName(); 20476 tableNames.addAll(modelManager.getDropTables()); 20477 20478 for (int i = 0; i < tableNames.size(); i++) { 20479 Table tableModel = tableNames.get(i); 20480 if(tableModelIds.contains(tableModel.getId())) { 20481 continue; 20482 } 20483 else { 20484 tableModelIds.add(tableModel.getId()); 20485 } 20486 if (tableModel.isView()) { 20487 continue; 20488 } 20489 if (tableModel.isDatabase()) { 20490 appendDatabaseModel(dataflow, tableModel); 20491 continue; 20492 } 20493 if (tableModel.isSchema()) { 20494 appendSchemaModel(dataflow, tableModel); 20495 continue; 20496 } 20497 if (tableModel.isStage()) { 20498 appendStageModel(dataflow, tableModel); 20499 continue; 20500 } 20501 if (tableModel.isSequence()) { 20502 appendSequenceModel(dataflow, tableModel); 20503 continue; 20504 } 20505 if (tableModel.isDataSource()) { 20506 appendDataSourceModel(dataflow, tableModel); 20507 continue; 20508 } 20509 if (tableModel.isStream()) { 20510 appendStreamModel(dataflow, tableModel); 20511 continue; 20512 } 20513 if (tableModel.isPath()) { 20514 appendPathModel(dataflow, tableModel); 20515 continue; 20516 } 20517 if (tableModel.isVariable()) { 20518 appendVariableModel(dataflow, tableModel); 20519 continue; 20520 } 20521 if (tableModel.isCursor()) { 20522 appendCursorModel(dataflow, tableModel); 20523 continue; 20524 } 20525 if (tableModel.isConstant()) { 20526 appendConstantModel(dataflow, tableModel); 20527 continue; 20528 } 20529 if (!tableIds.contains(tableModel.getId())) { 20530 appendTableModel(dataflow, tableModel, tableMap); 20531 tableIds.add(tableModel.getId()); 20532 } 20533 } 20534 } 20535 20536 private void appendConstantModel(dataflow dataflow, Table tableModel) { 20537 table constantElement = new table(); 20538 constantElement.setId(String.valueOf(tableModel.getId())); 20539 if (!SQLUtil.isEmpty(tableModel.getDatabase())) { 20540 constantElement.setDatabase(tableModel.getDatabase()); 20541 } 20542 if (!SQLUtil.isEmpty(tableModel.getSchema())) { 20543 constantElement.setSchema(tableModel.getSchema()); 20544 } 20545 constantElement.setServer(tableModel.getServer()); 20546 constantElement.setName(getConstantName(tableModel)); 20547 constantElement.setType("constantTable"); 20548 20549 if (tableModel.getStartPosition() != null && tableModel.getEndPosition() != null) { 20550 constantElement.setCoordinate(convertCoordinate(tableModel.getStartPosition()) + "," 20551 + convertCoordinate(tableModel.getEndPosition())); 20552 } 20553 dataflow.getTables().add(constantElement); 20554 20555 List<TableColumn> columns = tableModel.getColumns(); 20556 for (int j = 0; j < columns.size(); j++) { 20557 TableColumn columnModel = (TableColumn) columns.get(j); 20558 column columnElement = new column(); 20559 columnElement.setId(String.valueOf(columnModel.getId())); 20560 columnElement.setName(columnModel.getName()); 20561 if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) { 20562 columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + "," 20563 + convertCoordinate(columnModel.getEndPosition())); 20564 } 20565 constantElement.getColumns().add(columnElement); 20566 } 20567 } 20568 20569 /** 20570 * Copies the authoritative endpoint classification from the internal model onto the 20571 * XML element, emitting each field ONLY when it carries information (kind/intro not 20572 * UNKNOWN, createdInSql only when true, normalizedName only when it differs from the 20573 * raw name). This keeps existing XML byte-identical for endpoints that aren't 20574 * classified, honouring the strictly-additive contract (§9 of 20575 * docs/tmp/dlineage-authoritative-endpoint-classification.md). 20576 */ 20577 private void appendEndpointClassification(table tableElement, Table tableModel) { 20578 boolean created = tableModel.isCreatedInSql(); 20579 EndpointKind kind = tableModel.getEndpointKind(); 20580 // Emit endpointKind only when it carries signal: a non-catalog special kind 20581 // (temp / global-temp / table-variable / tempdb) always, or CATALOG_OBJECT only 20582 // when the object is created here. A plainly-referenced catalog table is the 20583 // default case and carries no attribute, so existing golden XML for ordinary 20584 // table references stays byte-identical. 20585 if (kind != null && kind != EndpointKind.UNKNOWN 20586 && (created || kind != EndpointKind.CATALOG_OBJECT)) { 20587 tableElement.setEndpointKind(kind.name()); 20588 } 20589 EndpointIntroduction intro = tableModel.getEndpointIntroduction(); 20590 if (intro != null && intro != EndpointIntroduction.UNKNOWN) { 20591 tableElement.setEndpointIntroduction(intro.name()); 20592 } 20593 if (created) { 20594 tableElement.setCreatedInSql("true"); 20595 } 20596 // Which name segments a dynamic fold assembled at runtime. Present only on endpoints 20597 // the analyzer marked inside a fold context, so ordinary output is untouched. 20598 if (tableModel.getTemplatedParts() != null) { 20599 tableElement.setTemplatedParts(tableModel.getTemplatedParts()); 20600 } 20601 // Emit normalizedName only when the raw name actually carries delimiters 20602 // (brackets / quotes / backticks) — that is when the normalized form is useful 20603 // and not trivially derivable. Avoids case-only churn on every plain name. 20604 // A templated endpoint always gets it: consumers match the delimiter-free skeleton 20605 // against later-resolved concrete names, and an unbracketed concatenation 20606 // ('RECON_' + @X + '_t') carries no delimiter to trigger the test above. 20607 String rawName = tableModel.getName(); 20608 if (rawName != null && (hasIdentifierDelimiter(rawName) || tableModel.isDynamicTemplate())) { 20609 String normalized = tableModel.getNormalizedName(); 20610 if (normalized != null && !normalized.equals(rawName)) { 20611 tableElement.setNormalizedName(normalized); 20612 } 20613 } 20614 } 20615 20616 private static boolean hasIdentifierDelimiter(String name) { 20617 return name.indexOf('[') != -1 || name.indexOf(']') != -1 20618 || name.indexOf('`') != -1 || name.indexOf('"') != -1; 20619 } 20620 20621 private void appendTableModel(dataflow dataflow, Table tableModel, Map<String, table> tableMap) { 20622 if(tableModel.getSubType() == SubType.unnest) {} 20623 table tableElement = new table(); 20624 tableElement.setId(String.valueOf(tableModel.getId())); 20625 // A one-part name that is ambiguous across two or more schemas (with no object 20626 // in the default schema) must NOT be bound to a fabricated default-schema 20627 // object: surface it unqualified and expose the candidate set instead. 20628 boolean ambiguousUnqualified = tableModel.isAmbiguousUnqualifiedTable() 20629 && tableModel.getCandidateTables() != null 20630 && !tableModel.getCandidateTables().isEmpty(); 20631 if (!ambiguousUnqualified) { 20632 if (!SQLUtil.isEmpty(tableModel.getDatabase())) { 20633 tableElement.setDatabase(tableModel.getDatabase()); 20634 } 20635 if (!SQLUtil.isEmpty(tableModel.getSchema())) { 20636 tableElement.setSchema(tableModel.getSchema()); 20637 } 20638 } 20639 tableElement.setServer(tableModel.getServer()); 20640 if (ambiguousUnqualified) { 20641 tableElement.setName(DlineageUtil.getSimpleTableName(tableModel.getName())); 20642 tableElement.setCandidateTables(new ArrayList<String>(tableModel.getCandidateTables())); 20643 } else { 20644 tableElement.setName(tableModel.getName()); 20645 } 20646 tableElement.setDisplayName(tableModel.getDisplayName()); 20647 tableElement.setStarStmt(tableModel.getStarStmt()); 20648 if(tableModel.isFromDDL()) { 20649 tableElement.setFromDDL(String.valueOf(tableModel.isFromDDL())); 20650 } 20651 // Authoritative endpoint classification (additive). Emit only non-default 20652 // values so existing XML stays byte-identical for unclassified endpoints. 20653 // See docs/tmp/dlineage-authoritative-endpoint-classification.md. 20654 appendEndpointClassification(tableElement, tableModel); 20655 20656 if (tableModel.isPseudo()) { 20657 tableElement.setType("pseudoTable"); 20658 } else { 20659 tableElement.setType("table"); 20660 } 20661 20662 if (tableModel.getSubType() != null) { 20663 if (tableModel.getSubType() == SubType.unnest) { 20664 tableElement.setType(SubType.unnest.name()); 20665 } 20666 else { 20667 tableElement.setSubType(tableModel.getSubType().name()); 20668 } 20669 } 20670 if (tableModel.getParent() != null) { 20671 tableElement.setParent(tableModel.getParent()); 20672 } 20673 if (tableModel.getAlias() != null && tableModel.getAlias().trim().length() > 0) { 20674 tableElement.setAlias(tableModel.getAlias()); 20675 } 20676 if (tableModel.getStartPosition() != null && tableModel.getEndPosition() != null) { 20677 if(option.isTraceTablePosition()){ 20678 for (Pair<Pair3<Long, Long, String>, Pair3<Long, Long, String>> position:tableModel.getPositions()){ 20679 tableElement.appendCoordinate(convertCoordinate(position.first)+","+convertCoordinate(position.second)); 20680 } 20681 } 20682 else { 20683 tableElement.setCoordinate(convertCoordinate(tableModel.getStartPosition()) + "," 20684 + convertCoordinate(tableModel.getEndPosition())); 20685 } 20686 } 20687 if (tableModel.getProcesses() != null) { 20688 List<String> processIds = new ArrayList<String>(); 20689 for (Process process : tableModel.getProcesses()) { 20690 processIds.add(String.valueOf(process.getId())); 20691 } 20692 tableElement.setProcessIds(processIds); 20693 } 20694 20695 table oldTableElement = null; 20696 String tableFullName = DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getQualifiedTableName(tableElement)); 20697 20698 if(tableMap.containsKey(tableFullName)) { 20699 oldTableElement = tableMap.get(tableFullName); 20700 } 20701 else { 20702 tableMap.put(tableFullName, tableElement); 20703 } 20704 20705 if (tableModel.getSubType() == SubType.unnest) { 20706 dataflow.getResultsets().add(tableElement); 20707 } else { 20708 dataflow.getTables().add(tableElement); 20709 } 20710 20711 List<TableColumn> columns = tableModel.getColumns(); 20712 20713 if (containStarColumn(columns)) { 20714 for (TableColumn column : columns) { 20715 if (column.getName().endsWith("*")) { 20716 for (TableColumn starElement : columns) { 20717 if (starElement == column) { 20718 continue; 20719 } 20720 if (starElement.isNotBindStarLinkColumn()) { 20721 continue; 20722 } 20723 TObjectName columnObject = starElement.getColumnObject(); 20724 column.bindStarLinkColumn(columnObject); 20725 } 20726 if (tableModel.isCreateTable() && column.isExpandStar()) { 20727 column.setShowStar(false); 20728 } 20729 } 20730 } 20731 } 20732 20733 for (int j = 0; j < columns.size(); j++) { 20734 TableColumn columnModel = columns.get(j); 20735 20736 if(oldTableElement != null){ 20737 if(!CollectionUtil.isEmpty(oldTableElement.getColumns())){ 20738 for(column oldColumnElement: oldTableElement.getColumns()){ 20739 if(SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotColumn, oldColumnElement.getName(), columnModel.getName())){ 20740 if(columnModel.getDataType() == null && oldColumnElement.getDataType() != null){ 20741 columnModel.setDataType(oldColumnElement.getDataType()); 20742 } 20743 if(columnModel.getPrimaryKey() == null && oldColumnElement.isPrimaryKey() != null){ 20744 columnModel.setPrimaryKey(oldColumnElement.isPrimaryKey()); 20745 } 20746 if(columnModel.getIndexKey() == null && oldColumnElement.isIndexKey() != null){ 20747 columnModel.setIndexKey(oldColumnElement.isIndexKey()); 20748 } 20749 if(columnModel.getUnqiueKey() == null && oldColumnElement.isUnqiueKey() != null){ 20750 columnModel.setUnqiueKey(oldColumnElement.isUnqiueKey()); 20751 } 20752 if(columnModel.getForeignKey() == null && oldColumnElement.isForeignKey() != null){ 20753 columnModel.setForeignKey(oldColumnElement.isForeignKey()); 20754 } 20755 20756 if(oldColumnElement.getDataType() == null && columnModel.getDataType() != null){ 20757 oldColumnElement.setDataType(columnModel.getDataType()); 20758 } 20759 if(oldColumnElement.isPrimaryKey() == null && columnModel.getPrimaryKey() != null){ 20760 oldColumnElement.setPrimaryKey(columnModel.getPrimaryKey()); 20761 } 20762 if(oldColumnElement.isIndexKey() == null && columnModel.getIndexKey() != null){ 20763 oldColumnElement.setIndexKey(columnModel.getIndexKey()); 20764 } 20765 if(oldColumnElement.isUnqiueKey() == null && columnModel.getUnqiueKey() != null){ 20766 oldColumnElement.setUnqiueKey(columnModel.getUnqiueKey()); 20767 } 20768 if(oldColumnElement.isForeignKey() == null && columnModel.getForeignKey() != null){ 20769 oldColumnElement.setForeignKey(columnModel.getForeignKey()); 20770 } 20771 break; 20772 } 20773 } 20774 } 20775 20776 } 20777 20778 if (!columnModel.isPseduo() && columnModel.hasStarLinkColumn() && !columnModel.isVariant()) { 20779 List<String> starLinkColumnList = columnModel.getStarLinkColumnNames(); 20780 for (int k = 0; k < starLinkColumnList.size(); k++) { 20781 column columnElement = new column(); 20782 columnElement.setId(String.valueOf(columnModel.getId()) + "_" + k); 20783 String columnName = starLinkColumnList.get(k); 20784 if (containStarColumn(columns, columnName)) { 20785 continue; 20786 } 20787 columnElement.setName(columnName); 20788 if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) { 20789 columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + "," 20790 + convertCoordinate(columnModel.getEndPosition())); 20791 } 20792 if (columnModel.getForeignKey()) { 20793 columnElement.setForeignKey(columnModel.getForeignKey()); 20794 } 20795 if (columnModel.getIndexKey()) { 20796 columnElement.setIndexKey(columnModel.getIndexKey()); 20797 } 20798 if (columnModel.getPrimaryKey()) { 20799 columnElement.setPrimaryKey(columnModel.getPrimaryKey()); 20800 } 20801 if (columnModel.getUnqiueKey()) { 20802 columnElement.setUnqiueKey(columnModel.getUnqiueKey()); 20803 } 20804 if (columnModel.getDataType() != null) { 20805 columnElement.setDataType(columnModel.getDataType()); 20806 } 20807 tableElement.getColumns().add(columnElement); 20808 } 20809 if (columnModel.isShowStar()) { 20810 column columnElement = new column(); 20811 columnElement.setId(String.valueOf(columnModel.getId())); 20812 columnElement.setName(columnModel.getName()); 20813 if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) { 20814 columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + "," 20815 + convertCoordinate(columnModel.getEndPosition())); 20816 } 20817 if (columnModel.getForeignKey()) { 20818 columnElement.setForeignKey(columnModel.getForeignKey()); 20819 } 20820 if (columnModel.getIndexKey()) { 20821 columnElement.setIndexKey(columnModel.getIndexKey()); 20822 } 20823 if (columnModel.getPrimaryKey()) { 20824 columnElement.setPrimaryKey(columnModel.getPrimaryKey()); 20825 } 20826 if (columnModel.getUnqiueKey()) { 20827 columnElement.setUnqiueKey(columnModel.getUnqiueKey()); 20828 } 20829 if (columnModel.getDataType() != null) { 20830 columnElement.setDataType(columnModel.getDataType()); 20831 } 20832 tableElement.getColumns().add(columnElement); 20833 } 20834 } else { 20835 column columnElement = new column(); 20836 columnElement.setId(String.valueOf(columnModel.getId())); 20837 columnElement.setName(columnModel.getName()); 20838 columnElement.setDisplayName(columnModel.getDisplayName()); 20839 if (columnModel.getStartPosition() != null && columnModel.getEndPosition() != null) { 20840 columnElement.setCoordinate(convertCoordinate(columnModel.getStartPosition()) + "," 20841 + convertCoordinate(columnModel.getEndPosition())); 20842 } 20843 if (columnModel.isPseduo()) { 20844 columnElement.setSource("system"); 20845 } 20846 if (columnModel.getForeignKey()) { 20847 columnElement.setForeignKey(columnModel.getForeignKey()); 20848 } 20849 if (columnModel.getIndexKey()) { 20850 columnElement.setIndexKey(columnModel.getIndexKey()); 20851 } 20852 if (columnModel.getPrimaryKey()) { 20853 columnElement.setPrimaryKey(columnModel.getPrimaryKey()); 20854 } 20855 if (columnModel.getUnqiueKey()) { 20856 columnElement.setUnqiueKey(columnModel.getUnqiueKey()); 20857 } 20858 if (columnModel.getDataType() != null) { 20859 columnElement.setDataType(columnModel.getDataType()); 20860 } 20861 tableElement.getColumns().add(columnElement); 20862 } 20863 } 20864 20865 TableRelationRows relationRows = tableModel.getRelationRows(); 20866 if (relationRows.hasRelation()) { 20867 column relationRowsElement = new column(); 20868 relationRowsElement.setId(String.valueOf(relationRows.getId())); 20869 relationRowsElement.setName(relationRows.getName()); 20870 if (relationRows.getStartPosition() != null && relationRows.getEndPosition() != null) { 20871 relationRowsElement.setCoordinate(convertCoordinate(relationRows.getStartPosition()) + "," 20872 + convertCoordinate(relationRows.getEndPosition())); 20873 } 20874 relationRowsElement.setSource("system"); 20875 tableElement.getColumns().add(relationRowsElement); 20876 } 20877 } 20878 20879 private boolean containStarColumn(List<TableColumn> columns, String qualifiedColumnName) { 20880 for (TableColumn tableColumn : columns) { 20881 if (DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()).equals(qualifiedColumnName)) { 20882 return true; 20883 } 20884 } 20885 return false; 20886 } 20887 20888 private TableColumn searchTableColumn(List<TableColumn> columns, String qualifiedColumnName) { 20889 for (TableColumn tableColumn : columns) { 20890 if (DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()).equals(qualifiedColumnName)) { 20891 return tableColumn; 20892 } 20893 } 20894 return null; 20895 } 20896 20897 private boolean containStarColumn(Collection<RelationshipElement<?>> elements, String qualifiedColumnName) { 20898 for (RelationshipElement element : elements) { 20899 if (element.getElement() instanceof TableColumn) { 20900 if (DlineageUtil.getIdentifierNormalColumnName(((TableColumn) element.getElement()).getName()) 20901 .equals(qualifiedColumnName)) { 20902 return true; 20903 } 20904 } else if (element.getElement() instanceof ResultColumn) { 20905 if (DlineageUtil.getIdentifierNormalColumnName(((ResultColumn) element.getElement()).getName()) 20906 .equals(qualifiedColumnName)) { 20907 return true; 20908 } 20909 } 20910 } 20911 return false; 20912 } 20913 20914 /** 20915 * Returns true if there is a non-star source element matching the column name 20916 * whose parent (table/resultset) does NOT also contribute a star source in 20917 * the same relationship. This identifies "definitive" explicit sources like 20918 * subquery columns (e.g., "coalesce(...) as col_a") vs incidental references 20919 * (e.g., "aTab.id" where aTab.* is also a source). 20920 */ 20921 private boolean hasDefinitiveNonStarSource(Collection<RelationshipElement<?>> elements, String qualifiedColumnName) { 20922 // First, collect parent IDs of all star (*) sources 20923 Set<Long> starSourceParentIds = new HashSet<Long>(); 20924 for (RelationshipElement<?> element : elements) { 20925 if (element.getElement() instanceof TableColumn) { 20926 TableColumn tc = (TableColumn) element.getElement(); 20927 if ("*".equals(tc.getName()) && tc.getTable() != null) { 20928 starSourceParentIds.add(tc.getTable().getId()); 20929 } 20930 } else if (element.getElement() instanceof ResultColumn) { 20931 ResultColumn rc = (ResultColumn) element.getElement(); 20932 if ("*".equals(rc.getName()) && rc.getResultSet() != null) { 20933 starSourceParentIds.add(rc.getResultSet().getId()); 20934 } 20935 } 20936 } 20937 20938 // Then check if any non-star source matching the column name has a parent 20939 // that does NOT also have a star source 20940 for (RelationshipElement<?> element : elements) { 20941 if (element.getElement() instanceof TableColumn) { 20942 TableColumn tc = (TableColumn) element.getElement(); 20943 if (!"*".equals(tc.getName()) 20944 && DlineageUtil.getIdentifierNormalColumnName(tc.getName()).equals(qualifiedColumnName)) { 20945 if (tc.getTable() != null && !starSourceParentIds.contains(tc.getTable().getId())) { 20946 return true; 20947 } 20948 } 20949 } else if (element.getElement() instanceof ResultColumn) { 20950 ResultColumn rc = (ResultColumn) element.getElement(); 20951 if (!"*".equals(rc.getName()) 20952 && DlineageUtil.getIdentifierNormalColumnName(rc.getName()).equals(qualifiedColumnName)) { 20953 if (rc.getResultSet() != null && !starSourceParentIds.contains(rc.getResultSet().getId())) { 20954 return true; 20955 } 20956 } 20957 } 20958 } 20959 return false; 20960 } 20961 20962 private void analyzeSelectStmt(TSelectSqlStatement stmt) { 20963 if (!accessedSubqueries.contains(stmt)) { 20964 accessedSubqueries.add(stmt); 20965 } else { 20966 if (modelManager.getModel(stmt) != null) { 20967 return; 20968 } 20969 } 20970 20971 // ClickHouse external table functions must be modelled BEFORE the 20972 // select-list relations are analyzed: the column path resolves its 20973 // source through modelManager.getModel(table), and without an early 20974 // binding it materializes a generic function node and the external 20975 // upstream silently vanishes. 20976 if (option.getVendor() == EDbVendor.dbvclickhouse && stmt.getRelations() != null) { 20977 for (TTable lcTable : stmt.getRelations()) { 20978 TFunctionCall lcFuncCall = lcTable.getFuncCall(); 20979 if (lcFuncCall == null && lcTable.getTableExpr() != null) { 20980 lcFuncCall = lcTable.getTableExpr().getFunctionCall(); 20981 } 20982 if (lcFuncCall != null) { 20983 modelClickhouseExternalTableFunction(lcTable, lcFuncCall); 20984 } 20985 } 20986 } 20987 20988 if (stmt.getParentStmt() == null && stmt.getIntoClause() == null && stmt.getIntoTableClause() == null) { 20989 if(option.isIgnoreTopSelect() && (option.isIgnoreRecordSet() || option.isSimpleOutput())){ 20990 if(option.getAnalyzeMode() == null || option.getAnalyzeMode() == AnalyzeMode.dataflow){ 20991 return; 20992 } 20993 } 20994 } 20995 20996 if (stmt.getSetOperatorType() != ESetOperatorType.none) { 20997 20998 // Iteratively analyze all descendant UNION branches before processing this node. 20999 // Uses iterative post-order traversal to preserve the original left-right-self 21000 // processing order, avoiding StackOverflow with deeply nested UNION trees. 21001 { 21002 Deque<TSelectSqlStatement> stack1 = new ArrayDeque<>(); 21003 List<TSelectSqlStatement> postOrder = new ArrayList<>(); 21004 stack1.push(stmt); 21005 while (!stack1.isEmpty()) { 21006 TSelectSqlStatement node = stack1.pop(); 21007 postOrder.add(node); 21008 if (node.getSetOperatorType() != ESetOperatorType.none) { 21009 // Push left first, then right, so that after reversal left comes first 21010 if (node.getLeftStmt() != null) stack1.push(node.getLeftStmt()); 21011 if (node.getRightStmt() != null) stack1.push(node.getRightStmt()); 21012 } 21013 } 21014 Collections.reverse(postOrder); 21015 // Process all descendants in post-order, skip stmt itself (processed below) 21016 for (int pi = 0; pi < postOrder.size() - 1; pi++) { 21017 TSelectSqlStatement node = postOrder.get(pi); 21018 if (!accessedStatements.contains(node)) { 21019 accessedStatements.add(node); 21020 analyzeSelectStmt(node); 21021 } 21022 } 21023 } 21024 21025 stmtStack.push(stmt); 21026 SelectSetResultSet resultSet = modelFactory.createSelectSetResultSet(stmt); 21027 21028 ResultSet leftResultSetModel = (ResultSet) modelManager.getModel(stmt.getLeftStmt()); 21029 if (leftResultSetModel != null && leftResultSetModel != resultSet 21030 && !leftResultSetModel.getRelationRows().getHoldRelations().isEmpty()) { 21031 ImpactRelationship impactRelation = modelFactory.createImpactRelation(); 21032 impactRelation.setEffectType(EffectType.select); 21033 impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>( 21034 leftResultSetModel.getRelationRows())); 21035 impactRelation.setTarget( 21036 new RelationRowsRelationshipElement<ResultSetRelationRows>(resultSet.getRelationRows())); 21037 } 21038 21039 ResultSet rightResultSetModel = (ResultSet) modelManager.getModel(stmt.getRightStmt()); 21040 if (rightResultSetModel != null && rightResultSetModel != resultSet 21041 && !rightResultSetModel.getRelationRows().getHoldRelations().isEmpty()) { 21042 ImpactRelationship impactRelation = modelFactory.createImpactRelation(); 21043 impactRelation.setEffectType(EffectType.select); 21044 impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>( 21045 rightResultSetModel.getRelationRows())); 21046 impactRelation.setTarget( 21047 new RelationRowsRelationshipElement<ResultSetRelationRows>(resultSet.getRelationRows())); 21048 } 21049 21050 if ((leftResultSetModel != null && leftResultSetModel.isDetermined()) 21051 || (rightResultSetModel != null && rightResultSetModel.isDetermined())) { 21052 resultSet.setDetermined(true); 21053 } 21054 21055 if (resultSet.getColumns() == null || resultSet.getColumns().isEmpty()) { 21056 if (getResultColumnList(stmt.getLeftStmt()) != null) { 21057 createSelectSetResultColumns(resultSet, stmt.getLeftStmt()); 21058 } else if (getResultColumnList(stmt.getRightStmt()) != null) { 21059 createSelectSetResultColumns(resultSet, stmt.getRightStmt()); 21060 } 21061 } 21062 21063 List<ResultColumn> columns = resultSet.getColumns(); 21064 for (int i = 0; i < columns.size(); i++) { 21065 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 21066 relation.setEffectType(EffectType.select); 21067 relation.setTarget(new ResultColumnRelationshipElement(columns.get(i))); 21068 21069 if (!stmt.getLeftStmt().isCombinedQuery()) { 21070 ResultSet sourceResultSet = (ResultSet) modelManager 21071 .getModel(stmt.getLeftStmt().getResultColumnList()); 21072 if (sourceResultSet!=null && sourceResultSet.getColumns().size() > i) { 21073 if (columns.get(i).getName().endsWith("*")) { 21074 for (ResultColumn column : sourceResultSet.getColumns()) { 21075 relation.addSource(new ResultColumnRelationshipElement(column)); 21076 } 21077 } else { 21078 relation.addSource( 21079 new ResultColumnRelationshipElement(sourceResultSet.getColumns().get(i))); 21080 } 21081 } 21082 } else { 21083 ResultSet sourceResultSet = (ResultSet) modelManager.getModel(stmt.getLeftStmt()); 21084 if (sourceResultSet != null && sourceResultSet.getColumns().size() > i) { 21085 if (columns.get(i).getName().endsWith("*")) { 21086 for (ResultColumn column : sourceResultSet.getColumns()) { 21087 relation.addSource(new ResultColumnRelationshipElement(column)); 21088 } 21089 } else { 21090 relation.addSource( 21091 new ResultColumnRelationshipElement(sourceResultSet.getColumns().get(i))); 21092 } 21093 } 21094 } 21095 21096 if (!stmt.getRightStmt().isCombinedQuery()) { 21097 ResultSet sourceResultSet = (ResultSet) modelManager 21098 .getModel(stmt.getRightStmt().getResultColumnList()); 21099 if (sourceResultSet != null && sourceResultSet.getColumns().size() > i) { 21100 if (columns.get(i).getName().endsWith("*")) { 21101 for (ResultColumn column : sourceResultSet.getColumns()) { 21102 relation.addSource(new ResultColumnRelationshipElement(column)); 21103 } 21104 } else { 21105 relation.addSource( 21106 new ResultColumnRelationshipElement(sourceResultSet.getColumns().get(i))); 21107 } 21108 } else if (sourceResultSet != null) { 21109 for (ResultColumn column : sourceResultSet.getColumns()) { 21110 if (column.hasStarLinkColumn()) { 21111 relation.addSource(new ResultColumnRelationshipElement(column)); 21112 } 21113 } 21114 } 21115 } else { 21116 ResultSet sourceResultSet = (ResultSet) modelManager.getModel(stmt.getRightStmt()); 21117 if (sourceResultSet != null && sourceResultSet.getColumns().size() > i) { 21118 relation.addSource(new ResultColumnRelationshipElement(sourceResultSet.getColumns().get(i))); 21119 } else if (sourceResultSet != null) { 21120 for (ResultColumn column : sourceResultSet.getColumns()) { 21121 if (column.hasStarLinkColumn()) { 21122 relation.addSource(new ResultColumnRelationshipElement(column)); 21123 } 21124 } 21125 } 21126 } 21127 } 21128 21129 analyzeSelectIntoClause(stmt); 21130 21131 stmtStack.pop(); 21132 } else { 21133 21134 // handle hive stmt, issue_id I3SGZB 21135 if (stmt.getHiveBodyList() != null && stmt.getHiveBodyList().size() > 0) { 21136 stmtStack.push(stmt); 21137 hiveFromTables = stmt.tables; 21138 if (hiveFromTables != null) { 21139 for (int i = 0; i < hiveFromTables.size(); i++) { 21140 modelFactory.createTable(hiveFromTables.getTable(i)); 21141 } 21142 } 21143 for (int i = 0; i < stmt.getHiveBodyList().size(); i++) { 21144 analyzeCustomSqlStmt(stmt.getHiveBodyList().get(i)); 21145 } 21146 stmtStack.pop(); 21147 return; 21148 } 21149 21150 if (stmt.getTransformClause() != null) { 21151 analyzeHiveTransformClause(stmt, stmt.getTransformClause()); 21152 return; 21153 } 21154 21155 if (stmt.getResultColumnList() == null) { 21156 return; 21157 } 21158 21159 stmtStack.push(stmt); 21160 21161 TTableList fromTables = stmt.tables; 21162 if ((fromTables == null || fromTables.size() == 0) 21163 && (hiveFromTables != null && hiveFromTables.size() > 0)) { 21164 fromTables = hiveFromTables; 21165 } 21166 21167 for (int i = 0; i < fromTables.size(); i++) { 21168 TTable table = fromTables.getTable(i); 21169 if (table.getLateralViewList() != null && !table.getLateralViewList().isEmpty()) { 21170 analyzeTableSubquery(table); 21171 analyzeLateralView(stmt, table, table.getLateralViewList()); 21172 stmtStack.pop(); 21173 return; 21174 } 21175 if (table.getUnnestClause() != null && option.getVendor() == EDbVendor.dbvpresto) { 21176 analyzeTableSubquery(table); 21177 analyzePrestoUnnest(stmt, table); 21178 stmtStack.pop(); 21179 return; 21180 } 21181 if (table.getUnnestClause() != null && option.getVendor() == EDbVendor.dbvbigquery) { 21182 analyzeBigQueryUnnest(stmt, table); 21183 } 21184 } 21185 21186 //用来获取 pivotedTable 对应的columns 21187 TPivotedTable pivotedTable = null; 21188 if (stmt.getJoins() != null && stmt.getJoins().size() > 0) { 21189 for (TJoin join : stmt.getJoins()) { 21190 if (join.getTable() != null && join.getTable().getPivotedTable() != null) { 21191 if (isUnPivotedTable(join.getTable().getPivotedTable())) { 21192 analyzeUnPivotedTable(stmt, join.getTable().getPivotedTable()); 21193 pivotedTable = join.getTable().getPivotedTable(); 21194 } else { 21195 analyzePivotedTable(stmt, join.getTable().getPivotedTable()); 21196 pivotedTable = join.getTable().getPivotedTable(); 21197 } 21198 } 21199 } 21200 } 21201 21202 for (int i = 0; i < fromTables.size(); i++) { 21203 TTable table = fromTables.getTable(i); 21204 // Handle TABLE(func()) which has tableType=tableExpr and funcCall=null 21205 if (table.getTableType() == ETableSource.tableExpr && table.getFuncCall() == null && pipelinedAnalyzer != null) { 21206 try { 21207 pipelinedAnalyzer.tryStitchTableExpr(table); 21208 } catch (Exception e) { 21209 // Don't let pipelined stitching failure break main flow 21210 } 21211 } 21212 21213 if (table.getFuncCall() != null || (table.getTableExpr()!=null && table.getTableExpr().getFunctionCall()!=null)) { 21214 TFunctionCall functionCall = table.getFuncCall(); 21215 if (functionCall == null) { 21216 functionCall = table.getTableExpr().getFunctionCall(); 21217 } 21218 Procedure callee = modelManager.getProcedureByName( 21219 DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall))); 21220 if (callee == null && procedureDDLMap.containsKey(DlineageUtil.getFunctionNameWithArgNum(functionCall))) { 21221 analyzeCustomSqlStmt(procedureDDLMap.get(DlineageUtil.getFunctionNameWithArgNum(functionCall))); 21222 callee = modelManager.getProcedureByName( 21223 DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionCall))); 21224 } 21225 21226 if(callee!=null) { 21227 if (callee.getArguments() != null) { 21228 for (int j = 0; j < callee.getArguments().size(); j++) { 21229 Argument argument = callee.getArguments().get(j); 21230 Variable variable = resolveFormalVariable(callee, argument); 21231 if(variable!=null) { 21232 if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) { 21233 Transform transform = new Transform(); 21234 transform.setType(Transform.FUNCTION); 21235 transform.setCode(functionCall); 21236 compositeBindingColumn(variable).setTransform(transform); 21237 } 21238 Process process = modelFactory.createProcess(functionCall); 21239 variable.addProcess(process); 21240 analyzeFunctionArgumentsDataFlowRelation(compositeBindingColumn(variable), functionCall, j, process); 21241 } 21242 } 21243 } 21244 Set<Object> functionTableModelObjs = modelManager.getFunctionTable(DlineageUtil 21245 .getIdentifierNormalTableName(functionCall.getFunctionName().toString())); 21246 if (functionTableModelObjs != null) { 21247 modelManager.bindModel(table, functionTableModelObjs.iterator().next()); 21248 } 21249 // Try pipelined function stitching 21250 if (pipelinedAnalyzer != null) { 21251 try { 21252 pipelinedAnalyzer.tryStitchCallSite(table, functionCall); 21253 } catch (Exception e) { 21254 // Don't let pipelined stitching failure break main flow 21255 } 21256 } 21257 continue; 21258 } else { 21259 Set<Object> functionTableModelObjs = modelManager.getFunctionTable(DlineageUtil 21260 .getIdentifierNormalTableName(functionCall.getFunctionName().toString())); 21261 if (functionTableModelObjs == null) { 21262 // ClickHouse external table functions expose real 21263 // external data; a generic function node silently 21264 // dropped their upstream (missing lineage is 21265 // invisible to the user — the correctness ruling). 21266 if (modelClickhouseExternalTableFunction(table, functionCall)) { 21267 continue; 21268 } 21269// Procedure procedure = modelManager.getProcedureByName(DlineageUtil 21270// .getIdentifierNormalTableName(functionCall.getFunctionName().toString())); 21271// if (procedure != null) { 21272 createFunction(functionCall); 21273 continue; 21274// } 21275 } 21276 if (functionTableModelObjs!=null && functionTableModelObjs.iterator().next() instanceof Table) { 21277 Table functionTableModel = (Table) functionTableModelObjs.iterator().next(); 21278 if (functionTableModel.getColumns() != null) { 21279 Table functionTable = modelFactory.createTableFromCreateDDL(table, true); 21280 if (functionTable == functionTableModel) 21281 continue; 21282 for (int j = 0; j < functionTableModel.getColumns().size(); j++) { 21283 TableColumn column = modelFactory.createTableColumn(functionTable, 21284 functionTableModel.getColumns().get(j).getColumnObject(), true); 21285 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 21286 relation.setEffectType(EffectType.select); 21287 relation.setTarget(new TableColumnRelationshipElement(column)); 21288 relation.addSource( 21289 new TableColumnRelationshipElement(functionTableModel.getColumns().get(j))); 21290 } 21291 } 21292 } else if (functionTableModelObjs!=null && functionTableModelObjs.iterator().next() instanceof ResultSet) { 21293 ResultSet functionTableModel = (ResultSet) functionTableModelObjs.iterator().next(); 21294 if (functionTableModel.getColumns() != null) { 21295 Table functionTable = modelFactory.createTableFromCreateDDL(table, true); 21296 for (int j = 0; j < functionTableModel.getColumns().size(); j++) { 21297 TParseTreeNode columnObj = functionTableModel.getColumns().get(j).getColumnObject(); 21298 if (columnObj instanceof TObjectName) { 21299 TableColumn column = modelFactory.createTableColumn(functionTable, 21300 (TObjectName) columnObj, true); 21301 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 21302 relation.setEffectType(EffectType.select); 21303 relation.setTarget(new TableColumnRelationshipElement(column)); 21304 relation.addSource(new ResultColumnRelationshipElement( 21305 functionTableModel.getColumns().get(j))); 21306 } else if (columnObj instanceof TResultColumn) { 21307 TableColumn column = modelFactory.createTableColumn(functionTable, 21308 (TResultColumn) columnObj); 21309 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 21310 relation.setEffectType(EffectType.select); 21311 relation.setTarget(new TableColumnRelationshipElement(column)); 21312 relation.addSource(new ResultColumnRelationshipElement( 21313 functionTableModel.getColumns().get(j))); 21314 } 21315 } 21316 } 21317 } 21318 // Try pipelined function stitching for unresolved function calls 21319 if (pipelinedAnalyzer != null) { 21320 try { 21321 pipelinedAnalyzer.tryStitchCallSite(table, functionCall); 21322 } catch (Exception e) { 21323 // Don't let pipelined stitching failure break main flow 21324 } 21325 } 21326 } 21327 } 21328 21329 if (table.getPartitionExtensionClause() != null 21330 && table.getPartitionExtensionClause().getKeyValues() != null) { 21331 TExpressionList values = table.getPartitionExtensionClause().getKeyValues(); 21332 Table tableModel = modelFactory.createTable(table); 21333 for (TExpression value : values) { 21334 if (value.getExpressionType() == EExpressionType.simple_constant_t) { 21335 ImpactRelationship impactRelation = modelFactory.createImpactRelation(); 21336 impactRelation.setEffectType(EffectType.select); 21337 Table constantTable = modelFactory.createConstantsTable(stmtStack.peek()); 21338 TableColumn constantColumn = modelFactory.createTableColumn(constantTable, value.getConstantOperand()); 21339 impactRelation.addSource(new TableColumnRelationshipElement(constantColumn)); 21340 impactRelation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>( 21341 tableModel.getRelationRows())); 21342 } 21343 } 21344 } 21345 21346 if (table.getSubquery() != null) { 21347 QueryTable queryTable = modelFactory.createQueryTable(table); 21348 TSelectSqlStatement subquery = table.getSubquery(); 21349 analyzeSelectStmt(subquery); 21350 21351 ResultSet resultSetModel = (ResultSet) modelManager.getModel(subquery); 21352 if (resultSetModel != null && resultSetModel.isDetermined()) { 21353 queryTable.setDetermined(true); 21354 } 21355 21356 if (resultSetModel != null && resultSetModel != queryTable 21357 && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) { 21358 ImpactRelationship impactRelation = modelFactory.createImpactRelation(); 21359 impactRelation.setEffectType(EffectType.select); 21360 impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>( 21361 resultSetModel.getRelationRows())); 21362 impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>( 21363 queryTable.getRelationRows())); 21364 } 21365 21366 if (resultSetModel != null && resultSetModel != queryTable 21367 && queryTable.getTableObject().getAliasClause() != null 21368 && queryTable.getTableObject().getAliasClause().getColumns() != null) { 21369 for (int j = 0; j < queryTable.getColumns().size() 21370 && j < resultSetModel.getColumns().size(); j++) { 21371 ResultColumn sourceColumn = resultSetModel.getColumns().get(j); 21372 ResultColumn targetColumn = queryTable.getColumns().get(j); 21373 21374 DataFlowRelationship queryRalation = modelFactory.createDataFlowRelation(); 21375 queryRalation.setEffectType(EffectType.select); 21376 queryRalation.setTarget(new ResultColumnRelationshipElement(targetColumn)); 21377 queryRalation.addSource(new ResultColumnRelationshipElement(sourceColumn)); 21378 } 21379 } else if (subquery.getSetOperatorType() != ESetOperatorType.none) { 21380 SelectSetResultSet selectSetResultSetModel = (SelectSetResultSet) modelManager 21381 .getModel(subquery); 21382 for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) { 21383 ResultColumn sourceColumn = selectSetResultSetModel.getColumns().get(j); 21384 ResultColumn targetColumn = modelFactory.createSelectSetResultColumn(queryTable, 21385 sourceColumn); 21386 for (TObjectName starLinkColumn : sourceColumn.getStarLinkColumnList()) { 21387 targetColumn.bindStarLinkColumn(starLinkColumn); 21388 } 21389 DataFlowRelationship selectSetRalation = modelFactory.createDataFlowRelation(); 21390 selectSetRalation.setEffectType(EffectType.select); 21391 selectSetRalation.setTarget(new ResultColumnRelationshipElement(targetColumn)); 21392 selectSetRalation.addSource(new ResultColumnRelationshipElement(sourceColumn)); 21393 } 21394 } 21395 } else if (table.getOutputMerge() != null) { 21396 QueryTable queryTable = modelFactory.createQueryTable(table); 21397 TMergeSqlStatement subquery = table.getOutputMerge(); 21398 analyzeMergeStmt(subquery); 21399 21400 for (TResultColumn column : subquery.getOutputClause().getSelectItemList()) { 21401 modelFactory.createResultColumn(queryTable, column); 21402 analyzeResultColumn(column, EffectType.select); 21403 } 21404 21405 ResultSet resultSetModel = (ResultSet) modelManager 21406 .getModel(subquery.getOutputClause().getSelectItemList()); 21407 21408 if (resultSetModel != null && resultSetModel != queryTable 21409 && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) { 21410 ImpactRelationship impactRelation = modelFactory.createImpactRelation(); 21411 impactRelation.setEffectType(EffectType.select); 21412 impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>( 21413 resultSetModel.getRelationRows())); 21414 impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>( 21415 queryTable.getRelationRows())); 21416 } 21417 21418 if (resultSetModel != null && resultSetModel != queryTable 21419 && queryTable.getTableObject().getAliasClause() != null 21420 && queryTable.getTableObject().getAliasClause().getColumns() != null) { 21421 for (int j = 0; j < queryTable.getColumns().size() 21422 && j < resultSetModel.getColumns().size(); j++) { 21423 ResultColumn sourceColumn = resultSetModel.getColumns().get(j); 21424 ResultColumn targetColumn = queryTable.getColumns().get(j); 21425 21426 DataFlowRelationship queryRalation = modelFactory.createDataFlowRelation(); 21427 queryRalation.setEffectType(EffectType.select); 21428 queryRalation.setTarget(new ResultColumnRelationshipElement(targetColumn)); 21429 queryRalation.addSource(new ResultColumnRelationshipElement(sourceColumn)); 21430 } 21431 } 21432 } else if (table.getTableExpr() != null && table.getTableExpr().getSubQuery() != null) { 21433 QueryTable queryTable = modelFactory.createQueryTable(table); 21434 TSelectSqlStatement subquery = table.getTableExpr().getSubQuery(); 21435 analyzeSelectStmt(subquery); 21436 21437 ResultSet resultSetModel = (ResultSet) modelManager.getModel(subquery); 21438 21439 if (resultSetModel != null && resultSetModel != queryTable 21440 && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) { 21441 ImpactRelationship impactRelation = modelFactory.createImpactRelation(); 21442 impactRelation.setEffectType(EffectType.select); 21443 impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>( 21444 resultSetModel.getRelationRows())); 21445 impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>( 21446 queryTable.getRelationRows())); 21447 } 21448 21449 if (resultSetModel != null && resultSetModel != queryTable 21450 && queryTable.getTableObject().getAliasClause() != null) { 21451 for (int j = 0; j < resultSetModel.getColumns().size(); j++) { 21452 ResultColumn sourceColumn = resultSetModel.getColumns().get(j); 21453 ResultColumn targetColumn = modelFactory.createSelectSetResultColumn(queryTable, 21454 sourceColumn); 21455 21456 DataFlowRelationship queryRalation = modelFactory.createDataFlowRelation(); 21457 queryRalation.setEffectType(EffectType.select); 21458 queryRalation.setTarget(new ResultColumnRelationshipElement(targetColumn)); 21459 queryRalation.addSource(new ResultColumnRelationshipElement(sourceColumn)); 21460 } 21461 } else if (subquery.getSetOperatorType() != ESetOperatorType.none) { 21462 SelectSetResultSet selectSetResultSetModel = (SelectSetResultSet) modelManager 21463 .getModel(subquery); 21464 for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) { 21465 ResultColumn sourceColumn = selectSetResultSetModel.getColumns().get(j); 21466 ResultColumn targetColumn = modelFactory.createSelectSetResultColumn(queryTable, 21467 sourceColumn); 21468 for (TObjectName starLinkColumn : sourceColumn.getStarLinkColumnList()) { 21469 targetColumn.bindStarLinkColumn(starLinkColumn); 21470 } 21471 DataFlowRelationship selectSetRalation = modelFactory.createDataFlowRelation(); 21472 selectSetRalation.setEffectType(EffectType.select); 21473 selectSetRalation.setTarget(new ResultColumnRelationshipElement(targetColumn)); 21474 selectSetRalation.addSource(new ResultColumnRelationshipElement(sourceColumn)); 21475 } 21476 } 21477 } else if (table.getCTE() != null) { 21478 QueryTable queryTable = modelFactory.createQueryTable(table); 21479 21480 TObjectNameList cteColumns = table.getCTE().getColumnList(); 21481 if (cteColumns != null) { 21482 for (int j = 0; j < cteColumns.size(); j++) { 21483 modelFactory.createResultColumn(queryTable, cteColumns.getObjectName(j)); 21484 } 21485 queryTable.setDetermined(true); 21486 } 21487 TSelectSqlStatement subquery = table.getCTE().getSubquery(); 21488 if (subquery != null && !stmtStack.contains(subquery) && subquery.getResultColumnList() != null) { 21489 analyzeSelectStmt(subquery); 21490 21491 ResultSet resultSetModel = (ResultSet) modelManager.getModel(subquery); 21492 if (resultSetModel != null && resultSetModel != queryTable 21493 && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) { 21494 ImpactRelationship impactRelation = modelFactory.createImpactRelation(); 21495 impactRelation.setEffectType(EffectType.select); 21496 impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>( 21497 resultSetModel.getRelationRows())); 21498 impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>( 21499 queryTable.getRelationRows())); 21500 } 21501 21502 if (subquery.getSetOperatorType() != ESetOperatorType.none) { 21503 SelectSetResultSet selectSetResultSetModel = (SelectSetResultSet) modelManager 21504 .getModel(subquery); 21505 int x = 0; 21506 for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) { 21507 ResultColumn sourceColumn = selectSetResultSetModel.getColumns().get(j); 21508 ResultColumn targetColumn = null; 21509 if (cteColumns != null) { 21510 if (queryTable.getColumns().size() <= j) { 21511 for (int k = 0; k < queryTable.getColumns().size(); k++) { 21512 if (queryTable.getColumns().get(k).getName().equals(sourceColumn.getName()) 21513 || queryTable.getColumns().get(k).getName() 21514 .equals(sourceColumn.getAlias())) { 21515 targetColumn = queryTable.getColumns().get(k); 21516 } 21517 } 21518 } else { 21519 if (x < j) { 21520 x = j; 21521 } 21522 targetColumn = queryTable.getColumns().get(x); 21523 x++; 21524 if (resultSetModel.getColumns().get(j).getName().contains("*") 21525 && x < cteColumns.size()) { 21526 j--; 21527 } 21528 } 21529 } else { 21530 targetColumn = modelFactory.createSelectSetResultColumn(queryTable, sourceColumn); 21531 } 21532 for (TObjectName starLinkColumn : sourceColumn.getStarLinkColumnList()) { 21533 targetColumn.bindStarLinkColumn(starLinkColumn); 21534 } 21535 createCteSelectRelation(null, targetColumn, sourceColumn); 21536 } 21537 if (!queryTable.isDetermined()) { 21538 queryTable.setDetermined(selectSetResultSetModel.isDetermined()); 21539 } 21540 } else { 21541 int x = 0; 21542 for (int j = 0; j < resultSetModel.getColumns().size(); j++) { 21543 ResultColumn sourceColumn = resultSetModel.getColumns().get(j); 21544 ResultColumn targetColumn = null; 21545 if (cteColumns != null) { 21546 if (queryTable.getColumns().size() <= j) { 21547 for (int k = 0; k < queryTable.getColumns().size(); k++) { 21548 if (queryTable.getColumns().get(k).getName().equals(sourceColumn.getName()) 21549 || queryTable.getColumns().get(k).getName() 21550 .equals(sourceColumn.getAlias())) { 21551 targetColumn = queryTable.getColumns().get(k); 21552 } 21553 } 21554 } else { 21555 if (x < j) { 21556 x = j; 21557 } 21558 targetColumn = queryTable.getColumns().get(x); 21559 x++; 21560 if (resultSetModel.getColumns().get(j).getName().contains("*") 21561 && x < cteColumns.size()) { 21562 j--; 21563 } 21564 } 21565 } else { 21566 targetColumn = modelFactory.createSelectSetResultColumn(queryTable, sourceColumn); 21567 } 21568 for (TObjectName starLinkColumn : sourceColumn.getStarLinkColumnList()) { 21569 targetColumn.bindStarLinkColumn(starLinkColumn); 21570 } 21571 createCteSelectRelation(null, targetColumn, sourceColumn); 21572 } 21573 if (!queryTable.isDetermined()) { 21574 queryTable.setDetermined(resultSetModel.isDetermined()); 21575 } 21576 } 21577 } else if (table.getCTE().getUpdateStmt() != null) { 21578 analyzeCustomSqlStmt(table.getCTE().getUpdateStmt()); 21579 } else if (table.getCTE().getInsertStmt() != null) { 21580 analyzeCustomSqlStmt(table.getCTE().getInsertStmt()); 21581 } else if (table.getCTE().getDeleteStmt() != null) { 21582 analyzeCustomSqlStmt(table.getCTE().getDeleteStmt()); 21583 } 21584 } else if (table.getTableType().name().startsWith("open")) { 21585 continue; 21586 } else if (table.getTableType() == ETableSource.jsonTable) { 21587 Table functionTable = modelFactory.createJsonTable(table); 21588 TJsonTable jsonTable = table.getJsonTable(); 21589 TColumnDefinitionList definitions = jsonTable.getColumnDefinitions(); 21590 if (definitions != null) { 21591 for (int j = 0; j < definitions.size(); j++) { 21592 TColumnDefinitionList nestDefinitions = definitions.getColumn(j).getNestedTableColumns(); 21593 if(nestDefinitions!=null) { 21594 for(int k=0;k<nestDefinitions.size();k++){ 21595 if (nestDefinitions.getColumn(k).getColumnName() == null) { 21596 continue; 21597 } 21598 TableColumn column = modelFactory.createTableColumn(functionTable, 21599 nestDefinitions.getColumn(k).getColumnName(), true); 21600 if (nestDefinitions.getColumn(k).getColumnPath() != null) { 21601 column.setDisplayName( 21602 column.getName() + ":" + nestDefinitions.getColumn(k).getColumnPath()); 21603 } 21604 } 21605 } 21606 else { 21607 TableColumn column = modelFactory.createTableColumn(functionTable, 21608 definitions.getColumn(j).getColumnName(), true); 21609 if (definitions.getColumn(j).getColumnPath() != null) { 21610 column.setDisplayName( 21611 column.getName() + ":" + definitions.getColumn(j).getColumnPath()); 21612 } 21613 } 21614 } 21615 } else { 21616 TObjectName keyColumn = new TObjectName(); 21617 keyColumn.setString("key"); 21618 modelFactory.createJsonTableColumn(functionTable, keyColumn); 21619 TObjectName valueColumn = new TObjectName(); 21620 valueColumn.setString("value"); 21621 modelFactory.createJsonTableColumn(functionTable, valueColumn); 21622 TObjectName typeColumn = new TObjectName(); 21623 typeColumn.setString("type"); 21624 modelFactory.createJsonTableColumn(functionTable, typeColumn); 21625 } 21626 21627 functionTable.setCreateTable(true); 21628 functionTable.setSubType(SubType.function); 21629 modelManager.bindCreateModel(table, functionTable); 21630 21631 if (jsonTable.getJsonExpression() == null) { 21632 ErrorInfo errorInfo = new ErrorInfo(); 21633 errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR); 21634 errorInfo.setErrorMessage("Can't handle the json table: " + jsonTable.toString()); 21635 errorInfo.setStartPosition(new Pair3<Long, Long, String>(jsonTable.getStartToken().lineNo, 21636 jsonTable.getStartToken().columnNo, ModelBindingManager.getGlobalHash())); 21637 errorInfo.setEndPosition(new Pair3<Long, Long, String>(jsonTable.getEndToken().lineNo, 21638 jsonTable.getEndToken().columnNo + jsonTable.getEndToken().getAstext().length(), 21639 ModelBindingManager.getGlobalHash())); 21640 errorInfo.fillInfo(this); 21641 errorInfos.add(errorInfo); 21642 } else { 21643 String jsonName = jsonTable.getJsonExpression().toString(); 21644 if (!jsonName.startsWith("@")) { 21645 columnsInExpr visitor = new columnsInExpr(); 21646 jsonTable.getJsonExpression().inOrderTraverse(visitor); 21647 List<TObjectName> objectNames = visitor.getObjectNames(); 21648 List<TParseTreeNode> functions = visitor.getFunctions(); 21649 List<TParseTreeNode> constants = visitor.getConstants(); 21650 List<TSelectSqlStatement> subquerys = visitor.getSubquerys(); 21651 21652 for (int j = 0; j < functionTable.getColumns().size(); j++) { 21653 TableColumn tableColumn = functionTable.getColumns().get(j); 21654 if (functions != null && !functions.isEmpty()) { 21655 analyzeFunctionDataFlowRelation(tableColumn, functions, EffectType.function); 21656 } 21657 if (subquerys != null && !subquerys.isEmpty()) { 21658 analyzeSubqueryDataFlowRelation(tableColumn, subquerys, EffectType.select); 21659 } 21660 if (objectNames != null && !objectNames.isEmpty()) { 21661 analyzeDataFlowRelation(tableColumn, objectNames, EffectType.select, functions); 21662 } 21663 if (constants != null && !constants.isEmpty()) { 21664 analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select, 21665 functions); 21666 } 21667 } 21668 } else { 21669 TStatementList stmts = stmt.getGsqlparser().getSqlstatements(); 21670 for (int j = 0; j < stmts.size(); j++) { 21671 TCustomSqlStatement item = stmts.get(j); 21672 if (item instanceof TMssqlDeclare) { 21673 if (analyzeMssqlJsonDeclare((TMssqlDeclare) item, jsonName, functionTable)) { 21674 break; 21675 } 21676 } 21677 } 21678 } 21679 } 21680 } else if (table.getTableType() == ETableSource.xmltable) { 21681 Table functionTable = modelFactory.createXmlTable(table); 21682 TXmlTable xmlTable = table.getXmlTable(); 21683 TXmlTableParameter param = xmlTable != null ? xmlTable.getArg() : null; 21684 21685 // Output columns from COLUMNS clause 21686 if (param != null && param.getXmlTableColumns() != null) { 21687 TColumnDefinitionList defs = param.getXmlTableColumns(); 21688 for (int j = 0; j < defs.size(); j++) { 21689 TColumnDefinition def = defs.getColumn(j); 21690 TableColumn column = modelFactory.createTableColumn( 21691 functionTable, def.getColumnName(), true); 21692 if (def.getXmlTableColumnPath() != null) { 21693 column.setDisplayName( 21694 column.getName() + ":" + def.getXmlTableColumnPath()); 21695 } 21696 } 21697 } 21698 functionTable.setCreateTable(true); 21699 functionTable.setSubType(SubType.function); 21700 modelManager.bindCreateModel(table, functionTable); 21701 21702 // Source columns from PASSING (v1: many-to-many) 21703 TResultColumnList passing = null; 21704 if (param != null && param.getXmlPassingClause() != null) { 21705 passing = param.getXmlPassingClause().getPassingList(); 21706 } 21707 if (passing != null && passing.size() > 0) { 21708 columnsInExpr visitor = new columnsInExpr(); 21709 for (int p = 0; p < passing.size(); p++) { 21710 TExpression e = passing.getResultColumn(p).getExpr(); 21711 if (e != null) e.inOrderTraverse(visitor); 21712 } 21713 List<TObjectName> objectNames = visitor.getObjectNames(); 21714 List<TParseTreeNode> functions = visitor.getFunctions(); 21715 List<TParseTreeNode> constants = visitor.getConstants(); 21716 List<TSelectSqlStatement> subquerys = visitor.getSubquerys(); 21717 21718 for (int j = 0; j < functionTable.getColumns().size(); j++) { 21719 TableColumn tc = functionTable.getColumns().get(j); 21720 if (functions != null && !functions.isEmpty()) 21721 analyzeFunctionDataFlowRelation(tc, functions, EffectType.function); 21722 if (subquerys != null && !subquerys.isEmpty()) 21723 analyzeSubqueryDataFlowRelation(tc, subquerys, EffectType.select); 21724 if (objectNames != null && !objectNames.isEmpty()) 21725 analyzeDataFlowRelation(tc, objectNames, EffectType.select, functions); 21726 if (constants != null && !constants.isEmpty()) 21727 analyzeConstantDataFlowRelation(tc, constants, EffectType.select, functions); 21728 } 21729 } else { 21730 ErrorInfo errorInfo = new ErrorInfo(); 21731 errorInfo.setErrorType(ErrorInfo.ANALYZE_ERROR); 21732 errorInfo.setErrorMessage("Can't analyze XMLTABLE: missing PASSING clause."); 21733 errorInfos.add(errorInfo); 21734 } 21735 } else if (getTableLinkedColumns(table) != null && getTableLinkedColumns(table).size() > 0) { 21736 if (table.getTableType() == ETableSource.rowList && table.getRowList() != null 21737 && table.getRowList().size() > 0) { 21738 Table tableModel = modelFactory.createTable(table); 21739 for (int j = 0; j < table.getRowList().size(); j++) { 21740 TMultiTarget rowList = table.getRowList().getMultiTarget(j); 21741 for (int k = 0; k < rowList.getColumnList().size(); k++) { 21742 TResultColumn column = rowList.getColumnList().getResultColumn(k); 21743 if (column.getFieldAttr() == null) 21744 continue; 21745 TableColumn tableColumn = modelFactory.createTableColumn(tableModel, 21746 column.getFieldAttr(), true); 21747 21748 columnsInExpr visitor = new columnsInExpr(); 21749 column.getExpr().inOrderTraverse(visitor); 21750 List<TObjectName> objectNames = visitor.getObjectNames(); 21751 List<TParseTreeNode> functions = visitor.getFunctions(); 21752 List<TParseTreeNode> constants = visitor.getConstants(); 21753 List<TSelectSqlStatement> subquerys = visitor.getSubquerys(); 21754 21755 if (functions != null && !functions.isEmpty()) { 21756 analyzeFunctionDataFlowRelation(tableColumn, functions, EffectType.function); 21757 } 21758 if (subquerys != null && !subquerys.isEmpty()) { 21759 analyzeSubqueryDataFlowRelation(tableColumn, subquerys, EffectType.select); 21760 } 21761 if (objectNames != null && !objectNames.isEmpty()) { 21762 analyzeDataFlowRelation(tableColumn, objectNames, EffectType.select, functions); 21763 } 21764 if (constants != null && !constants.isEmpty()) { 21765 analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select, 21766 functions); 21767 } 21768 } 21769 } 21770 } else { 21771 if(table.getTableType() == ETableSource.pivoted_table) { 21772 continue; 21773 } 21774 if (table.getTableType() == ETableSource.rowList) { 21775 List<TResultColumnList> rowList = table.getValueClause().getRows(); 21776 21777 QueryTable tableModel = modelFactory.createQueryTable(table); 21778 TAliasClause aliasClause = table.getAliasClause(); 21779 if (aliasClause != null && aliasClause.getColumns() != null) { 21780 for (TObjectName column : aliasClause.getColumns()) { 21781 modelFactory.createResultColumn(tableModel, column, true); 21782 } 21783 } else { 21784 int columnCount = rowList.get(0).size(); 21785 for (int j = 1; j <= columnCount; j++) { 21786 TObjectName columnName = new TObjectName(); 21787 columnName.setString("column" + j); 21788 modelFactory.createResultColumn(tableModel, columnName, true); 21789 } 21790 } 21791 tableModel.setDetermined(true); 21792 21793 for (TResultColumnList resultColumnList : rowList) { 21794 for (int j = 0; j < resultColumnList.size(); j++) { 21795 TResultColumn resultColumn = resultColumnList.getResultColumn(j); 21796 analyzeValueColumn(tableModel.getColumns().get(j), resultColumn, EffectType.select); 21797 } 21798 } 21799 21800 } else { 21801 Table tableModel = modelFactory.createTable(table); 21802 for (int j = 0; j < getTableLinkedColumns(table).size(); j++) { 21803 TObjectName object = getTableLinkedColumns(table).getObjectName(j); 21804 21805 if (object.getDbObjectType() == EDbObjectType.variable) { 21806 continue; 21807 } 21808 21809 if (object.getColumnNameOnly().startsWith("@") 21810 && (option.getVendor() == EDbVendor.dbvmssql 21811 || option.getVendor() == EDbVendor.dbvazuresql)) { 21812 continue; 21813 } 21814 21815 if (object.getColumnNameOnly().startsWith(":") 21816 && (option.getVendor() == EDbVendor.dbvhana 21817 || option.getVendor() == EDbVendor.dbvteradata)) { 21818 continue; 21819 } 21820 21821 if (isBuiltInFunctionName(object) && isFromFunction(object)) { 21822 continue; 21823 } 21824 21825 if (!"*".equals(getColumnName(object))) { 21826 if (isStructColumn(object)) { 21827 21828 } else { 21829 if (pivotedTable != null) { 21830 ResultColumn resultColumn = getPivotedTableColumn(pivotedTable, object); 21831 if (resultColumn != null) { 21832 continue; 21833 } 21834 } 21835 TableColumn tableColumn = modelFactory.createTableColumn(tableModel, object, 21836 false); 21837 if(tableColumn == null) { 21838 continue; 21839 } 21840 if (table.getUnnestClause() != null 21841 && table.getUnnestClause().getArrayExpr() != null) { 21842 columnsInExpr visitor = new columnsInExpr(); 21843 table.getUnnestClause().getArrayExpr().inOrderTraverse(visitor); 21844 21845 List<TObjectName> objectNames = visitor.getObjectNames(); 21846 List<TParseTreeNode> functions = visitor.getFunctions(); 21847 21848 if (functions != null && !functions.isEmpty()) { 21849 analyzeFunctionDataFlowRelation(tableColumn, functions, 21850 EffectType.select); 21851 21852 } 21853 21854 List<TSelectSqlStatement> subquerys = visitor.getSubquerys(); 21855 if (subquerys != null && !subquerys.isEmpty()) { 21856 analyzeSubqueryDataFlowRelation(tableColumn, subquerys, 21857 EffectType.select); 21858 } 21859 21860 analyzeDataFlowRelation(tableColumn, objectNames, EffectType.select, 21861 functions); 21862 21863 List<TParseTreeNode> constants = visitor.getConstants(); 21864 analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select, 21865 functions); 21866 } 21867 } 21868 } else { 21869 boolean flag = false; 21870 for (TObjectName column : getTableLinkedColumns(table)) { 21871 if ("*".equals(getColumnName(column))) { 21872 continue; 21873 } 21874 if (column.getLocation() != ESqlClause.where 21875 && column.getLocation() != ESqlClause.joinCondition) { 21876 flag = true; 21877 } 21878 } 21879 if (!flag) { 21880 TableColumn tableColumn = modelFactory.createTableColumn(tableModel, object, 21881 false); 21882 if (tableColumn != null && !tableModel.hasSQLEnv()) { 21883 tableColumn.setShowStar(true); 21884 } 21885 } 21886 } 21887 21888 } 21889 } 21890 } 21891 } 21892 else { 21893 modelFactory.createTable(table); 21894 } 21895 } 21896 21897 if (pivotedTable!=null) { 21898 for (TJoin join : stmt.getJoins()) { 21899 if (join.getTable() != null && join.getTable().getPivotedTable() != null) { 21900 if (isUnPivotedTable(join.getTable().getPivotedTable())) { 21901 analyzeUnPivotedTable(stmt, join.getTable().getPivotedTable()); 21902 } else { 21903 analyzePivotedTable(stmt, join.getTable().getPivotedTable()); 21904 } 21905 stmtStack.pop(); 21906 return; 21907 } 21908 } 21909 } 21910 21911 if (!stmt.isCombinedQuery()) { 21912 Object queryModel = modelManager.getModel(stmt.getResultColumnList()); 21913 21914 if (queryModel == null) { 21915 TSelectSqlStatement parentStmt = getParentSetSelectStmt(stmt); 21916 if (isTopResultSet(stmt) || parentStmt == null) { 21917 ResultSet resultSetModel = modelFactory.createResultSet(stmt, 21918 isTopResultSet(stmt) && isShowTopSelectResultSet() && stmt.getIntoClause() == null); 21919 21920 createPseudoImpactRelation(stmt, resultSetModel, EffectType.select); 21921 21922 boolean isDetermined = true; 21923 Map<String, AtomicInteger> keyMap = new HashMap<String, AtomicInteger>(); 21924 Set<String> columnNames = new HashSet<String>(); 21925 for (int i = 0; i < stmt.getResultColumnList().size(); i++) { 21926 TResultColumn column = stmt.getResultColumnList().getResultColumn(i); 21927 21928 if (column.getExpr().getComparisonType() == EComparisonType.equals 21929 && column.getExpr().getLeftOperand().getObjectOperand() != null) { 21930 TObjectName columnObject = column.getExpr().getLeftOperand().getObjectOperand(); 21931 21932 ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel, 21933 columnObject); 21934 if (columnObject.getDbObjectType() == EDbObjectType.variable) { 21935 Table variable = modelManager 21936 .getTableByName(DlineageUtil.getTableFullName(columnObject.toString())); 21937 if (variable != null) { 21938 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 21939 relation.setEffectType(EffectType.select); 21940 TableColumn columnModel = variable instanceof Variable 21941 ? compositeBindingColumn((Variable) variable) 21942 : variable.getColumns().get(0); 21943 relation.setTarget(new TableColumnRelationshipElement(columnModel)); 21944 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 21945 } else { 21946 variable = modelFactory.createVariable(columnObject); 21947 variable.setCreateTable(true); 21948 variable.setSubType(SubType.record); 21949 TObjectName variableProperties = new TObjectName(); 21950 variableProperties.setString("*"); 21951 TableColumn variableProperty = modelFactory.createTableColumn(variable, 21952 variableProperties, true); 21953 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 21954 relation.setEffectType(EffectType.select); 21955 relation.setTarget(new TableColumnRelationshipElement(variableProperty)); 21956 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 21957 } 21958 } 21959 21960 columnsInExpr visitor = new columnsInExpr(); 21961 column.getExpr().getRightOperand().inOrderTraverse(visitor); 21962 21963 List<TObjectName> objectNames = visitor.getObjectNames(); 21964 List<TParseTreeNode> functions = visitor.getFunctions(); 21965 21966 if (functions != null && !functions.isEmpty()) { 21967 analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.select); 21968 21969 } 21970 21971 List<TSelectSqlStatement> subquerys = visitor.getSubquerys(); 21972 if (subquerys != null && !subquerys.isEmpty()) { 21973 analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.select); 21974 } 21975 21976 analyzeDataFlowRelation(resultColumn, objectNames, EffectType.select, functions); 21977 21978 List<TParseTreeNode> constants = visitor.getConstants(); 21979 analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.select, functions); 21980 } else { 21981 if (column.getFieldAttr() != null && isStructColumn(column.getFieldAttr())) { 21982 Table table = modelFactory.createTable(column.getFieldAttr().getSourceTable()); 21983 for (int j = 0; j < table.getColumns().size(); j++) { 21984 TObjectName columnName = new TObjectName(); 21985 if (table.getColumns().get(j).getName().equals(table.getAlias())) { 21986 if (!SQLUtil.isEmpty(column.getColumnAlias())) { 21987 columnName.setString(column.getColumnAlias()); 21988 } else { 21989 columnName.setString(table.getColumns().get(j).getName()); 21990 } 21991 } else { 21992 columnName.setString( 21993 table.getAlias() + "." + table.getColumns().get(j).getName()); 21994 } 21995 ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel, 21996 columnName); 21997 DataFlowRelationship relationship = modelFactory.createDataFlowRelation(); 21998 relationship.setTarget(new ResultColumnRelationshipElement(resultColumn)); 21999 relationship.addSource( 22000 new TableColumnRelationshipElement(table.getColumns().get(j))); 22001 } 22002 } else if (column.getExpr().getFunctionCall() != null && column.getExpr().getFunctionCall().getFunctionType() == EFunctionType.struct_t) { 22003 Function function = (Function) createFunction(column.getExpr().getFunctionCall()); 22004 String functionName = getResultSetName(function); 22005 for (int j = 0; j < function.getColumns().size(); j++) { 22006 TObjectName columnName = new TObjectName(); 22007 if (column.getAliasClause() != null) { 22008 columnName.setString(column.getAliasClause() + "." 22009 + function.getColumns().get(j).getName()); 22010 } 22011 else { 22012 columnName.setString(functionName + "." 22013 + function.getColumns().get(j).getName()); 22014 } 22015 ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel, 22016 columnName); 22017 resultColumn.setStruct(true); 22018 DataFlowRelationship relationship = modelFactory.createDataFlowRelation(); 22019 relationship.setTarget(new ResultColumnRelationshipElement(resultColumn)); 22020 relationship.addSource( 22021 new ResultColumnRelationshipElement(function.getColumns().get(j))); 22022 } 22023 } else if (column.getExpr().getFunctionCall() != null && column.getExpr().getFunctionCall().getFunctionType() == EFunctionType.array_t) { 22024 Function function = (Function) createFunction(column.getExpr().getFunctionCall()); 22025 String functionName = getResultSetName(function); 22026 if (function.getColumns().size() == 1) { 22027 // Scalar result inside ARRAY() - use just alias as column name 22028 TObjectName columnName = new TObjectName(); 22029 if (column.getAliasClause() != null) { 22030 columnName.setString(column.getAliasClause().toString()); 22031 } else { 22032 columnName.setString(functionName); 22033 } 22034 ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel, 22035 columnName); 22036 DataFlowRelationship relationship = modelFactory.createDataFlowRelation(); 22037 relationship.setTarget(new ResultColumnRelationshipElement(resultColumn)); 22038 relationship.addSource( 22039 new ResultColumnRelationshipElement(function.getColumns().get(0))); 22040 } else { 22041 for (int j = 0; j < function.getColumns().size(); j++) { 22042 TObjectName columnName = new TObjectName(); 22043 if (column.getAliasClause() != null) { 22044 columnName.setString(column.getAliasClause() + "." 22045 + getColumnNameOnly(function.getColumns().get(j).getName())); 22046 } 22047 else { 22048 columnName.setString(functionName + "." 22049 + getColumnNameOnly(function.getColumns().get(j).getName())); 22050 } 22051 ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel, 22052 columnName); 22053 resultColumn.setStruct(true); 22054 DataFlowRelationship relationship = modelFactory.createDataFlowRelation(); 22055 relationship.setTarget(new ResultColumnRelationshipElement(resultColumn)); 22056 relationship.addSource( 22057 new ResultColumnRelationshipElement(function.getColumns().get(j))); 22058 } 22059 } 22060 } else if (column.getExpr().getExprList() != null && column.getExpr().getExpressionType() == EExpressionType.array_t) { 22061 if(column.getExpr().getObjectOperand()!=null) { 22062 TObjectName exprColumnName = column.getExpr().getObjectOperand(); 22063 Table table = modelFactory.createTable(exprColumnName.getSourceTable()); 22064 for (int z = 0; z < table.getColumns().size(); z++) { 22065 TableColumn tableColumn = table.getColumns().get(z); 22066 if(getColumnName(exprColumnName.toString()).equals(getColumnName(tableColumn.getName()))) { 22067 TObjectName columnName = new TObjectName(); 22068 if (column.getAliasClause() != null) { 22069 columnName.setString(column.getAliasClause().toString()); 22070 } else { 22071 columnName 22072 .setString(getColumnNameOnly(columnName.toString())); 22073 } 22074 ResultColumn resultColumn = modelFactory 22075 .createResultColumn(resultSetModel, columnName); 22076 resultColumn.setStruct(true); 22077 DataFlowRelationship relationship = modelFactory 22078 .createDataFlowRelation(); 22079 relationship.setTarget( 22080 new ResultColumnRelationshipElement(resultColumn)); 22081 relationship.addSource(new TableColumnRelationshipElement( 22082 tableColumn)); 22083 } 22084 } 22085 } 22086 else { 22087 for (int j = 0; j < column.getExpr().getExprList().size(); j++) { 22088 TExpression expression = column.getExpr().getExprList().getExpression(j); 22089 if(expression.getExpressionType() == EExpressionType.function_t) { 22090 Function function = (Function)createFunction(expression.getFunctionCall()); 22091 String functionName = getResultSetName(function); 22092 if (function != null && function.getColumns() != null) { 22093 if (function.getColumns().size() == 1) { 22094 // Scalar function inside ARRAY[] (e.g., ARRAY[TO_JSON_STRING(col)]) 22095 // Use just the alias as column name, not alias.functionName 22096 TObjectName columnName = new TObjectName(); 22097 if (column.getAliasClause() != null) { 22098 columnName.setString(column.getAliasClause().toString()); 22099 } else { 22100 columnName.setString(functionName); 22101 } 22102 ResultColumn resultColumn = modelFactory 22103 .createResultColumn(resultSetModel, columnName); 22104 DataFlowRelationship relationship = modelFactory 22105 .createDataFlowRelation(); 22106 relationship.setTarget( 22107 new ResultColumnRelationshipElement(resultColumn)); 22108 relationship.addSource(new ResultColumnRelationshipElement( 22109 function.getColumns().get(0))); 22110 } else { 22111 // Multi-column (struct-like) function - use dotted naming 22112 for (int x = 0; x < function.getColumns().size(); x++) { 22113 TObjectName columnName = new TObjectName(); 22114 if (column.getAliasClause() != null) { 22115 columnName.setString(column.getAliasClause() + "." 22116 + getColumnNameOnly( 22117 function.getColumns().get(x).getName())); 22118 } else { 22119 columnName.setString( 22120 functionName + "." + getColumnNameOnly( 22121 function.getColumns().get(x).getName())); 22122 } 22123 ResultColumn resultColumn = modelFactory 22124 .createResultColumn(resultSetModel, columnName); 22125 resultColumn.setStruct(true); 22126 DataFlowRelationship relationship = modelFactory 22127 .createDataFlowRelation(); 22128 relationship.setTarget( 22129 new ResultColumnRelationshipElement(resultColumn)); 22130 relationship.addSource(new ResultColumnRelationshipElement( 22131 function.getColumns().get(x))); 22132 } 22133 } 22134 } 22135 } 22136 else if(expression.getExpressionType() == EExpressionType.simple_object_name_t) { 22137 TObjectName exprColumnName = expression.getObjectOperand(); 22138 Table table = modelFactory.createTable(exprColumnName.getSourceTable()); 22139 for (int z = 0; z < table.getColumns().size(); z++) { 22140 TableColumn tableColumn = table.getColumns().get(z); 22141 if(getColumnName(exprColumnName.toString()).equals(getColumnName(tableColumn.getName()))) { 22142 TObjectName columnName = new TObjectName(); 22143 if (column.getAliasClause() != null) { 22144 columnName.setString(column.getAliasClause().toString()); 22145 } else { 22146 columnName 22147 .setString(getColumnNameOnly(columnName.toString())); 22148 } 22149 ResultColumn resultColumn = modelFactory 22150 .createResultColumn(resultSetModel, columnName); 22151 resultColumn.setStruct(true); 22152 DataFlowRelationship relationship = modelFactory 22153 .createDataFlowRelation(); 22154 relationship.setTarget( 22155 new ResultColumnRelationshipElement(resultColumn)); 22156 relationship.addSource(new TableColumnRelationshipElement( 22157 tableColumn)); 22158 } 22159 } 22160 } 22161 } 22162 } 22163 } else { 22164 if ("*".equals(column.getColumnNameOnly())) { 22165 Map<String, Pair<String, TExpression>> replaceAsIdentifierMap = new HashMap<String, Pair<String, TExpression>>(); 22166 Map<String, TObjectName> replaceColumnMap = new HashMap<String, TObjectName>(); 22167 if(column.getReplaceExprAsIdentifiers()!=null && column.getReplaceExprAsIdentifiers().size()>0) { 22168 for(TReplaceExprAsIdentifier replace: column.getReplaceExprAsIdentifiers()) { 22169 replaceAsIdentifierMap.put(replace.getIdentifier().toString(), new Pair<String, TExpression>(column.getExpr().getExceptReplaceClause().toString(), replace.getExpr())); 22170 replaceColumnMap.put(replace.getIdentifier().toString(), replace.getIdentifier()); 22171 } 22172 } 22173 22174 TObjectName columnObject = column.getFieldAttr(); 22175 List<TTable> sourceTables = columnObject.getSourceTableList(); 22176 if (sourceTables != null && !sourceTables.isEmpty()) { 22177 boolean[] determine = new boolean[sourceTables.size()]; 22178 for (int k = 0; k < sourceTables.size(); k++) { 22179 TTable sourceTable = sourceTables.get(k); 22180 Object tableModel = modelManager.getModel(sourceTable); 22181 if (tableModel instanceof Table && ((Table) tableModel).isCreateTable()) { 22182 Table table = (Table) tableModel; 22183 for (int j = 0; j < table.getColumns().size(); j++) { 22184 TableColumn tableColumn = table.getColumns().get(j); 22185 if (column.getExceptColumnList() != null) { 22186 boolean except = false; 22187 for (TObjectName objectName : column.getExceptColumnList()) { 22188 if(getColumnName(objectName.toString()).equals(getColumnName(tableColumn.getName()))) { 22189 except = true; 22190 break; 22191 } 22192 } 22193 if (!except && tableColumn.isStruct()) { 22194 List<String> names = SQLUtil 22195 .parseNames(tableColumn.getName()); 22196 for (String name : names) { 22197 for (TObjectName objectName : column 22198 .getExceptColumnList()) { 22199 if (getColumnName(objectName.toString()) 22200 .equals(getColumnName(name))) { 22201 except = true; 22202 break; 22203 } 22204 } 22205 if (except) { 22206 break; 22207 } 22208 } 22209 } 22210 if(except){ 22211 continue; 22212 } 22213 } 22214 22215 if (replaceAsIdentifierMap.containsKey(tableColumn.getName())) { 22216 Pair<String, TExpression> expr = replaceAsIdentifierMap.get(tableColumn.getName()); 22217 ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel, replaceColumnMap.get(tableColumn.getName())); 22218 Transform transform = new Transform(); 22219 transform.setType(Transform.EXPRESSION); 22220 TObjectName expression = new TObjectName(); 22221 expression.setString(expr.first); 22222 transform.setCode(expression); 22223 resultColumn.setTransform(transform); 22224 analyzeResultColumnExpressionRelation(resultColumn, expr.second); 22225 } 22226 else { 22227 String columnName = DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()); 22228 //bigquery 22229 boolean exist = false; 22230 if (!columnName.matches("(?i)f\\d+_")) { 22231 if (!keyMap.containsKey(columnName)) { 22232 keyMap.put(columnName, new AtomicInteger(0)); 22233 } else { 22234 while (columnNames.contains(columnName)) { 22235 if(columnName.indexOf("*") == -1 && stmt.toString().matches("(?is).*using\\s*\\(\\s*"+columnName+"\\s*\\).*")) { 22236 exist = true; 22237 break; 22238 } else if (keyMap.containsKey(columnName)) { 22239 int index = keyMap.get(columnName) 22240 .incrementAndGet(); 22241 columnName = columnName + index; 22242 } 22243 } 22244 } 22245 columnNames.add(columnName); 22246 } 22247 if (exist) { 22248 String targetColumn = columnName; 22249 DataFlowRelationship relation = modelFactory 22250 .createDataFlowRelation(); 22251 relation.setEffectType(EffectType.select); 22252 relation.setTarget(new ResultColumnRelationshipElement( 22253 resultSetModel.getColumns().stream() 22254 .filter(t -> SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotColumn, 22255 t.getName(), targetColumn)) 22256 .findFirst().get())); 22257 relation.addSource(new TableColumnRelationshipElement( 22258 tableColumn)); 22259 continue; 22260 } 22261 // Normal-form bookkeeping over synthetic collision-renamed keys 22262 // (columnName may carry an appended index) — NOT identifier 22263 // equality; stays on normalized-key compare by design (P0d.3c2). 22264 if (DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()) 22265 .equalsIgnoreCase(DlineageUtil.getIdentifierNormalColumnName(columnName))) { 22266 columnName = tableColumn.getName(); 22267 } 22268 ResultColumn resultColumn = modelFactory.createStarResultColumn(resultSetModel, column, columnName); 22269 resultColumn.setStruct(tableColumn.isStruct()); 22270 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 22271 relation.setEffectType(EffectType.select); 22272 relation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 22273 relation.addSource(new TableColumnRelationshipElement(tableColumn)); 22274 } 22275 } 22276 determine[k] = true; 22277 continue; 22278 } 22279 else if (tableModel instanceof ResultSet && ((ResultSet) tableModel).isDetermined()) { 22280 ResultSet table = (ResultSet) tableModel; 22281 for (int j = 0; j < table.getColumns().size(); j++) { 22282 ResultColumn tableColumn = table.getColumns().get(j); 22283 if (column.getExceptColumnList() != null) { 22284 boolean except = false; 22285 for (TObjectName objectName : column.getExceptColumnList()) { 22286 if(getColumnName(objectName.toString()).equals(getColumnName(tableColumn.getName()))) { 22287 except = true; 22288 break; 22289 } 22290 } 22291 if (!except && tableColumn.isStruct()) { 22292 List<String> names = SQLUtil 22293 .parseNames(tableColumn.getName()); 22294 for (String name : names) { 22295 for (TObjectName objectName : column 22296 .getExceptColumnList()) { 22297 if (getColumnName(objectName.toString()) 22298 .equals(getColumnName(name))) { 22299 except = true; 22300 break; 22301 } 22302 } 22303 if (except) { 22304 break; 22305 } 22306 } 22307 } 22308 if(except){ 22309 continue; 22310 } 22311 } 22312 if (replaceAsIdentifierMap.containsKey(tableColumn.getName())) { 22313 Pair<String, TExpression> expr = replaceAsIdentifierMap.get(tableColumn.getName()); 22314 ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel, replaceColumnMap.get(tableColumn.getName())); 22315 Transform transform = new Transform(); 22316 transform.setType(Transform.EXPRESSION); 22317 TObjectName expression = new TObjectName(); 22318 expression.setString(expr.first); 22319 transform.setCode(expression); 22320 resultColumn.setTransform(transform); 22321 analyzeResultColumnExpressionRelation(resultColumn, expr.second); 22322 } 22323 else if(tableColumn.getRefColumnName()!=null) { 22324 ResultColumn resultColumn = modelFactory.createStarResultColumn(resultSetModel, column, tableColumn.getRefColumnName()); 22325 if(tableColumn.isStruct()) { 22326 resultColumn.setStruct(true); 22327 } 22328 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 22329 relation.setEffectType(EffectType.select); 22330 relation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 22331 relation.addSource(new ResultColumnRelationshipElement(tableColumn)); 22332 } 22333 else { 22334 String columnName = DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()); 22335 //bigquery 22336 if (!columnName.matches("(?i)f\\d+_")) { 22337 if (!keyMap.containsKey(columnName)) { 22338 keyMap.put(columnName, new AtomicInteger(0)); 22339 } else { 22340 while (columnNames.contains(columnName)) { 22341 int index = keyMap.get(columnName) 22342 .incrementAndGet(); 22343 columnName = columnName + index; 22344 } 22345 } 22346 columnNames.add(columnName); 22347 } 22348 // Normal-form bookkeeping over synthetic collision-renamed keys 22349 // (columnName may carry an appended index) — NOT identifier 22350 // equality; stays on normalized-key compare by design (P0d.3c2). 22351 if (DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()) 22352 .equalsIgnoreCase(DlineageUtil.getIdentifierNormalColumnName(columnName))) { 22353 columnName = tableColumn.getName(); 22354 } 22355 if (modelManager.getModel(column) instanceof ResultColumn) { 22356 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 22357 relation.setEffectType(EffectType.select); 22358 relation.setTarget(new ResultColumnRelationshipElement((ResultColumn)modelManager.getModel(column))); 22359 relation.addSource(new ResultColumnRelationshipElement(tableColumn)); 22360 } 22361 else { 22362 ResultColumn resultColumn = modelFactory.createStarResultColumn(resultSetModel, column, columnName); 22363 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 22364 relation.setEffectType(EffectType.select); 22365 relation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 22366 relation.addSource(new ResultColumnRelationshipElement(tableColumn)); 22367 } 22368 } 22369 } 22370 determine[k] = true; 22371 continue; 22372 } 22373 else { 22374 ResultColumn resultColumn = modelFactory 22375 .createResultColumn(resultSetModel, column); 22376 if (tableModel instanceof Function) { 22377 22378 } else { 22379 TObjectName[] columns = modelManager 22380 .getTableColumns(sourceTable); 22381 for (int j = 0; j < columns.length; j++) { 22382 TObjectName columnName = columns[j]; 22383 if (columnName == null 22384 || "*".equals(getColumnName(columnName))) { 22385 continue; 22386 } 22387 if (isStructColumn(columnName)) { 22388 continue; 22389 } 22390 22391 resultColumn.bindStarLinkColumn(columnName); 22392 if (column.getExceptColumnList() != null) { 22393 for (TObjectName objectName : column 22394 .getExceptColumnList()) { 22395 resultColumn.unbindStarLinkColumn(objectName); 22396 } 22397 } 22398 } 22399 if (tableModel instanceof ResultSet) { 22400 ResultSet queryTable = (ResultSet) tableModel; 22401 if (!containStarColumn(queryTable)) { 22402 resultColumn.setShowStar(false); 22403 } 22404 } 22405 if (tableModel instanceof Table) { 22406 Table table = (Table) tableModel; 22407 if (table.isCreateTable()) { 22408 resultColumn.setShowStar(false); 22409 } 22410 } 22411 } 22412 } 22413 } 22414 if(!Arrays.toString(determine).contains("false")) { 22415 continue; 22416 } 22417 } else { 22418 ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel, column); 22419 TTableList tables = stmt.getTables(); 22420 for (int k = 0; k < tables.size(); k++) { 22421 TTable table = tables.getTable(k); 22422 TObjectName[] columns = modelManager.getTableColumns(table); 22423 for (int j = 0; j < columns.length; j++) { 22424 TObjectName columnName = columns[j]; 22425 if (columnName == null) { 22426 continue; 22427 } 22428 if ("*".equals(getColumnName(columnName))) { 22429 if (modelManager.getModel(table) instanceof Table) { 22430 Table tableModel = (Table) modelManager.getModel(table); 22431 if (tableModel != null 22432 && !tableModel.getColumns().isEmpty()) { 22433 for (TableColumn item : tableModel.getColumns()) { 22434 resultColumn 22435 .bindStarLinkColumn(item.getColumnObject()); 22436 if (table.getSubquery() == null 22437 && table.getCTE() == null 22438 && !tableModel.isCreateTable()) { 22439 resultColumn.setShowStar(true); 22440 } 22441 } 22442 } 22443 } else if (modelManager.getModel(table) instanceof QueryTable) { 22444 QueryTable tableModel = (QueryTable) modelManager 22445 .getModel(table); 22446 if (tableModel != null 22447 && !tableModel.getColumns().isEmpty()) { 22448 for (ResultColumn item : tableModel.getColumns()) { 22449 if (item.hasStarLinkColumn()) { 22450 for (TObjectName starLinkColumn : item 22451 .getStarLinkColumnList()) { 22452 resultColumn 22453 .bindStarLinkColumn(starLinkColumn); 22454 } 22455 } else if (item 22456 .getColumnObject() instanceof TObjectName) { 22457 resultColumn.bindStarLinkColumn( 22458 (TObjectName) item.getColumnObject()); 22459 } else if (item 22460 .getColumnObject() instanceof TResultColumn) { 22461 TResultColumn queryTableColumn = (TResultColumn) item 22462 .getColumnObject(); 22463 TObjectName tableColumnObject = queryTableColumn 22464 .getFieldAttr(); 22465 if (tableColumnObject != null) { 22466 resultColumn.bindStarLinkColumn( 22467 tableColumnObject); 22468 } else if (queryTableColumn 22469 .getAliasClause() != null && !item.isStruct()) { 22470 resultColumn.bindStarLinkColumn( 22471 queryTableColumn.getAliasClause() 22472 .getAliasName()); 22473 } 22474 } 22475 } 22476 } 22477 } 22478 continue; 22479 } 22480 resultColumn.bindStarLinkColumn(columnName); 22481 } 22482 } 22483 } 22484 isDetermined = false; 22485 } 22486 else { 22487 if(column.getAliasClause()!=null && column.getAliasClause().getColumns()!=null) { 22488 for(TObjectName aliasColumn: column.getAliasClause().getColumns()) { 22489 modelFactory.createResultColumn(resultSetModel, aliasColumn); 22490 } 22491 } 22492 else { 22493 modelFactory.createResultColumn(resultSetModel, column); 22494 } 22495 } 22496 analyzeResultColumn(column, EffectType.select); 22497 } 22498 } 22499 } 22500 if (isDetermined) { 22501 resultSetModel.setDetermined(isDetermined); 22502 } 22503 } 22504 22505 TSelectSqlStatement parent = getParentSetSelectStmt(stmt); 22506 if (parent != null && parent.getSetOperatorType() != ESetOperatorType.none) { 22507 ResultSet resultSetModel = modelFactory.createResultSet(stmt, false); 22508 if(queryModel == null) { 22509 queryModel = resultSetModel; 22510 } 22511 22512 createPseudoImpactRelation(stmt, resultSetModel, EffectType.select); 22513 22514 boolean isDetermined = true; 22515 for (int i = 0; i < stmt.getResultColumnList().size(); i++) { 22516 TResultColumn column = stmt.getResultColumnList().getResultColumn(i); 22517 if ("*".equals(column.getColumnNameOnly())) { 22518 22519 Map<String, Pair<String, TExpression>> replaceAsIdentifierMap = new HashMap<String, Pair<String, TExpression>>(); 22520 Map<String, TObjectName> replaceColumnMap = new HashMap<String, TObjectName>(); 22521 if(column.getReplaceExprAsIdentifiers()!=null && column.getReplaceExprAsIdentifiers().size()>0) { 22522 for(TReplaceExprAsIdentifier replace: column.getReplaceExprAsIdentifiers()) { 22523 replaceAsIdentifierMap.put(replace.getIdentifier().toString(), new Pair<String, TExpression>(column.getExpr().getExceptReplaceClause().toString(), replace.getExpr())); 22524 replaceColumnMap.put(replace.getIdentifier().toString(), replace.getIdentifier()); 22525 } 22526 } 22527 22528 TObjectName columnObject = column.getFieldAttr(); 22529 TTable sourceTable = columnObject.getSourceTable(); 22530 if (sourceTable != null) { 22531 Object tableModel = modelManager.getModel(sourceTable); 22532 if (tableModel instanceof Table && ((Table) tableModel).isCreateTable()) { 22533 Table table = (Table) tableModel; 22534 for (int j = 0; j < table.getColumns().size(); j++) { 22535 TableColumn tableColumn = table.getColumns().get(j); 22536 if (column.getExceptColumnList() != null) { 22537 boolean except = false; 22538 for (TObjectName objectName : column.getExceptColumnList()) { 22539 if (getColumnName(objectName.toString()) 22540 .equals(getColumnName(tableColumn.getName()))) { 22541 except = true; 22542 break; 22543 } 22544 } 22545 if (!except && tableColumn.isStruct()) { 22546 List<String> names = SQLUtil 22547 .parseNames(tableColumn.getName()); 22548 for (String name : names) { 22549 for (TObjectName objectName : column 22550 .getExceptColumnList()) { 22551 if (getColumnName(objectName.toString()) 22552 .equals(getColumnName(name))) { 22553 except = true; 22554 break; 22555 } 22556 } 22557 if (except) { 22558 break; 22559 } 22560 } 22561 } 22562 if (except) { 22563 continue; 22564 } 22565 } 22566 22567 if (replaceAsIdentifierMap.containsKey(tableColumn.getName())) { 22568 Pair<String, TExpression> expr = replaceAsIdentifierMap.get(tableColumn.getName()); 22569 ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel, replaceColumnMap.get(tableColumn.getName())); 22570 Transform transform = new Transform(); 22571 transform.setType(Transform.EXPRESSION); 22572 TObjectName expression = new TObjectName(); 22573 expression.setString(expr.first); 22574 transform.setCode(expression); 22575 resultColumn.setTransform(transform); 22576 analyzeResultColumnExpressionRelation(resultColumn, expr.second); 22577 } 22578 else { 22579 ResultColumn resultColumn = modelFactory.createStarResultColumn( 22580 resultSetModel, column, tableColumn.getName()); 22581 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 22582 relation.setEffectType(EffectType.select); 22583 relation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 22584 relation.addSource(new TableColumnRelationshipElement(tableColumn)); 22585 } 22586 } 22587 continue; 22588 } else if (tableModel instanceof ResultSet 22589 && ((ResultSet) tableModel).isDetermined()) { 22590 ResultSet table = (ResultSet) tableModel; 22591 for (int j = 0; j < table.getColumns().size(); j++) { 22592 ResultColumn tableColumn = table.getColumns().get(j); 22593 if (column.getExceptColumnList() != null) { 22594 boolean except = false; 22595 for (TObjectName objectName : column.getExceptColumnList()) { 22596 if (getColumnName(objectName.toString()) 22597 .equals(getColumnName(tableColumn.getName()))) { 22598 except = true; 22599 break; 22600 } 22601 } 22602 if (!except && tableColumn.isStruct()) { 22603 List<String> names = SQLUtil 22604 .parseNames(tableColumn.getName()); 22605 for (String name : names) { 22606 for (TObjectName objectName : column 22607 .getExceptColumnList()) { 22608 if (getColumnName(objectName.toString()) 22609 .equals(getColumnName(name))) { 22610 except = true; 22611 break; 22612 } 22613 } 22614 if (except) { 22615 break; 22616 } 22617 } 22618 } 22619 if (except) { 22620 continue; 22621 } 22622 } 22623 22624 if (replaceAsIdentifierMap.containsKey(tableColumn.getName())) { 22625 Pair<String, TExpression> expr = replaceAsIdentifierMap.get(tableColumn.getName()); 22626 ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel, replaceColumnMap.get(tableColumn.getName())); 22627 Transform transform = new Transform(); 22628 transform.setType(Transform.EXPRESSION); 22629 TObjectName expression = new TObjectName(); 22630 expression.setString(expr.first); 22631 transform.setCode(expression); 22632 resultColumn.setTransform(transform); 22633 analyzeResultColumnExpressionRelation(resultColumn, expr.second); 22634 } 22635 else if (tableColumn.getRefColumnName() != null) { 22636 ResultColumn resultColumn = modelFactory.createStarResultColumn( 22637 (ResultSet)queryModel, column, tableColumn.getRefColumnName()); 22638 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 22639 relation.setEffectType(EffectType.select); 22640 relation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 22641 relation.addSource(new ResultColumnRelationshipElement(tableColumn)); 22642 } else { 22643 ResultColumn resultColumn = modelFactory.createStarResultColumn( 22644 resultSetModel, column, tableColumn.getName()); 22645 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 22646 relation.setEffectType(EffectType.select); 22647 relation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 22648 relation.addSource(new ResultColumnRelationshipElement(tableColumn)); 22649 } 22650 } 22651 continue; 22652 } 22653 else { 22654 isDetermined = false; 22655 } 22656 } 22657 22658 ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel, column); 22659 22660 if (columnObject.getTableToken() != null && sourceTable != null) { 22661 TObjectName[] columns = modelManager.getTableColumns(sourceTable); 22662 for (int j = 0; j < columns.length; j++) { 22663 TObjectName columnName = columns[j]; 22664 if (columnName == null || "*".equals(getColumnName(columnName))) { 22665 continue; 22666 } 22667 resultColumn.bindStarLinkColumn(columnName); 22668 } 22669 22670 if (modelManager.getModel(sourceTable) instanceof Table) { 22671 Table tableModel = (Table) modelManager.getModel(sourceTable); 22672 if (tableModel != null && !tableModel.getColumns().isEmpty()) { 22673 for (TableColumn item : tableModel.getColumns()) { 22674 if ("*".equals(getColumnName(item.getColumnObject()))) { 22675 continue; 22676 } 22677 resultColumn.bindStarLinkColumn(item.getColumnObject()); 22678 } 22679 } 22680 } else if (modelManager.getModel(sourceTable) instanceof QueryTable) { 22681 QueryTable tableModel = (QueryTable) modelManager.getModel(sourceTable); 22682 if (tableModel != null && !tableModel.getColumns().isEmpty()) { 22683 for (ResultColumn item : tableModel.getColumns()) { 22684 if (item.hasStarLinkColumn()) { 22685 for (TObjectName starLinkColumn : item.getStarLinkColumnList()) { 22686 if ("*".equals(getColumnName(starLinkColumn))) { 22687 continue; 22688 } 22689 resultColumn.bindStarLinkColumn(starLinkColumn); 22690 } 22691 } else if (item.getColumnObject() instanceof TObjectName) { 22692 TObjectName starLinkColumn = (TObjectName) item.getColumnObject(); 22693 if ("*".equals(getColumnName(starLinkColumn))) { 22694 continue; 22695 } 22696 resultColumn.bindStarLinkColumn(starLinkColumn); 22697 } 22698 } 22699 } 22700 } 22701 22702 } else { 22703 TTableList tables = stmt.getTables(); 22704 for (int k = 0; k < tables.size(); k++) { 22705 TTable table = tables.getTable(k); 22706 TObjectName[] columns = modelManager.getTableColumns(table); 22707 for (int j = 0; j < columns.length; j++) { 22708 TObjectName columnName = columns[j]; 22709 if (columnName == null) { 22710 continue; 22711 } 22712 if ("*".equals(getColumnName(columnName))) { 22713 if (modelManager.getModel(table) instanceof Table) { 22714 Table tableModel = (Table) modelManager.getModel(table); 22715 if (tableModel != null && !tableModel.getColumns().isEmpty()) { 22716 for (TableColumn item : tableModel.getColumns()) { 22717 resultColumn.bindStarLinkColumn(item.getColumnObject()); 22718 } 22719 } 22720 } else if (modelManager.getModel(table) instanceof QueryTable) { 22721 QueryTable tableModel = (QueryTable) modelManager.getModel(table); 22722 if (tableModel != null && !tableModel.getColumns().isEmpty()) { 22723 for (ResultColumn item : tableModel.getColumns()) { 22724 if (item.hasStarLinkColumn()) { 22725 for (TObjectName starLinkColumn : item 22726 .getStarLinkColumnList()) { 22727 resultColumn.bindStarLinkColumn(starLinkColumn); 22728 } 22729 } else if (item.getColumnObject() instanceof TObjectName) { 22730 resultColumn.bindStarLinkColumn( 22731 (TObjectName) item.getColumnObject()); 22732 } else if (item 22733 .getColumnObject() instanceof TResultColumn) { 22734 TResultColumn queryTableColumn = (TResultColumn) item 22735 .getColumnObject(); 22736 TObjectName tableColumnObject = queryTableColumn 22737 .getFieldAttr(); 22738 if (tableColumnObject != null) { 22739 resultColumn.bindStarLinkColumn(tableColumnObject); 22740 } else if (queryTableColumn.getAliasClause() != null) { 22741 resultColumn.bindStarLinkColumn(queryTableColumn 22742 .getAliasClause().getAliasName()); 22743 } 22744 } 22745 } 22746 } 22747 } 22748 22749 continue; 22750 } 22751 resultColumn.bindStarLinkColumn(columnName); 22752 } 22753 } 22754 } 22755 } 22756 else { 22757 ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel, column); 22758 } 22759 analyzeResultColumn(column, EffectType.select); 22760 22761 } 22762 22763 resultSetModel.setDetermined(isDetermined); 22764 } 22765 } else { 22766 for (int i = 0; i < stmt.getResultColumnList().size(); i++) { 22767 TResultColumn column = stmt.getResultColumnList().getResultColumn(i); 22768 22769 if (!(queryModel instanceof ResultSet)) { 22770 continue; 22771 } 22772 22773 ResultSet resultSetModel = (ResultSet)queryModel; 22774 22775 if ("*".equals(column.getColumnNameOnly())) { 22776 TObjectName columnObject = column.getFieldAttr(); 22777 TTable sourceTable = columnObject.getSourceTable(); 22778 if (column.toString().indexOf(".") == -1 && stmt.getTables().size() > 1) { 22779 sourceTable = null; 22780 } 22781 if (sourceTable != null) { 22782 { 22783 Object tableModel = modelManager.getModel(sourceTable); 22784 if (tableModel instanceof Table && ((Table) tableModel).isCreateTable()) { 22785 Table table = (Table) tableModel; 22786 for (int j = 0; j < table.getColumns().size(); j++) { 22787 TableColumn tableColumn = table.getColumns().get(j); 22788 if (column.getExceptColumnList() != null) { 22789 boolean except = false; 22790 for (TObjectName objectName : column.getExceptColumnList()) { 22791 if (getColumnName(objectName.toString()) 22792 .equals(getColumnName(tableColumn.getName()))) { 22793 except = true; 22794 break; 22795 } 22796 } 22797 if (!except && tableColumn.isStruct()) { 22798 List<String> names = SQLUtil 22799 .parseNames(tableColumn.getName()); 22800 for (String name : names) { 22801 for (TObjectName objectName : column 22802 .getExceptColumnList()) { 22803 if (getColumnName(objectName.toString()) 22804 .equals(getColumnName(name))) { 22805 except = true; 22806 break; 22807 } 22808 } 22809 if (except) { 22810 break; 22811 } 22812 } 22813 } 22814 if (except) { 22815 continue; 22816 } 22817 } 22818 ResultColumn resultColumn = modelFactory.createStarResultColumn( 22819 resultSetModel, column, tableColumn.getName()); 22820 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 22821 relation.setEffectType(EffectType.select); 22822 relation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 22823 relation.addSource(new TableColumnRelationshipElement(tableColumn)); 22824 } 22825 if(stmt.getResultColumnList().size() == 1) { 22826 resultSetModel.setDetermined(true); 22827 } 22828 else { 22829 int starCount = 0; 22830 for (int j = 0; j < stmt.getResultColumnList().size(); j++) { 22831 if (stmt.getResultColumnList().getResultColumn(j).getColumnNameOnly() 22832 .endsWith("*")) { 22833 starCount += 1; 22834 } 22835 } 22836 if (starCount <= 1) { 22837 resultSetModel.setDetermined(true); 22838 } 22839 } 22840 continue; 22841 } else if (tableModel instanceof ResultSet 22842 && ((ResultSet) tableModel).isDetermined()) { 22843 ResultSet table = (ResultSet) tableModel; 22844 for (int j = 0; j < table.getColumns().size(); j++) { 22845 ResultColumn tableColumn = table.getColumns().get(j); 22846 if (column.getExceptColumnList() != null) { 22847 boolean except = false; 22848 for (TObjectName objectName : column.getExceptColumnList()) { 22849 if (getColumnName(objectName.toString()) 22850 .equals(getColumnName(tableColumn.getName()))) { 22851 except = true; 22852 break; 22853 } 22854 } 22855 if (!except && tableColumn.isStruct()) { 22856 List<String> names = SQLUtil 22857 .parseNames(tableColumn.getName()); 22858 for (String name : names) { 22859 for (TObjectName objectName : column 22860 .getExceptColumnList()) { 22861 if (getColumnName(objectName.toString()) 22862 .equals(getColumnName(name))) { 22863 except = true; 22864 break; 22865 } 22866 } 22867 if (except) { 22868 break; 22869 } 22870 } 22871 } 22872 if (except) { 22873 continue; 22874 } 22875 } 22876 if (tableColumn.getRefColumnName() != null) { 22877 ResultColumn resultColumn = modelFactory.createStarResultColumn( 22878 (ResultSet)queryModel, column, tableColumn.getRefColumnName()); 22879 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 22880 relation.setEffectType(EffectType.select); 22881 relation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 22882 relation.addSource(new ResultColumnRelationshipElement(tableColumn)); 22883 } else { 22884 ResultColumn resultColumn = modelFactory.createStarResultColumn( 22885 resultSetModel, column, tableColumn.getName()); 22886 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 22887 relation.setEffectType(EffectType.select); 22888 relation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 22889 relation.addSource(new ResultColumnRelationshipElement(tableColumn)); 22890 } 22891 } 22892 if(stmt.getResultColumnList().size() == 1) { 22893 resultSetModel.setDetermined(true); 22894 } 22895 else { 22896 int starCount = 0; 22897 for (int j = 0; j < stmt.getResultColumnList().size(); j++) { 22898 if (stmt.getResultColumnList().getResultColumn(j).getColumnNameOnly() 22899 .endsWith("*")) { 22900 starCount += 1; 22901 } 22902 } 22903 if (starCount <= 1) { 22904 resultSetModel.setDetermined(true); 22905 } 22906 } 22907 continue; 22908 } 22909 } 22910 22911 ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel, column); 22912 if (modelManager.getModel(sourceTable) instanceof Table) { 22913 Table tableModel = (Table) modelManager.getModel(sourceTable); 22914 if (tableModel != null) { 22915 modelFactory.createTableColumn(tableModel, columnObject, false); 22916 } 22917 TObjectName[] columns = modelManager.getTableColumns(sourceTable); 22918 for (int j = 0; j < columns.length; j++) { 22919 TObjectName columnName = columns[j]; 22920 if (columnName == null || "*".equals(getColumnName(columnName))) { 22921 continue; 22922 } 22923 resultColumn.bindStarLinkColumn(columnName); 22924 } 22925 22926 if (tableModel.getColumns() != null) { 22927 for (int j = 0; j < tableModel.getColumns().size(); j++) { 22928 TableColumn tableColumn = tableModel.getColumns().get(j); 22929 TObjectName columnName = tableColumn.getColumnObject(); 22930 if (columnName == null || "*".equals(getColumnName(columnName))) { 22931 continue; 22932 } 22933 resultColumn.bindStarLinkColumn(columnName); 22934 } 22935 } 22936 } else if (modelManager.getModel(sourceTable) instanceof QueryTable) { 22937 QueryTable tableModel = (QueryTable) modelManager.getModel(sourceTable); 22938 if (tableModel != null && !tableModel.getColumns().isEmpty()) { 22939 for (ResultColumn item : tableModel.getColumns()) { 22940 if (item.hasStarLinkColumn()) { 22941 for (TObjectName starLinkColumn : item.getStarLinkColumnList()) { 22942 resultColumn.bindStarLinkColumn(starLinkColumn); 22943 } 22944 } else if (item.getColumnObject() instanceof TObjectName) { 22945 resultColumn.bindStarLinkColumn((TObjectName) item.getColumnObject()); 22946 } else if (item.getColumnObject() instanceof TResultColumn) { 22947 TResultColumn queryTableColumn = (TResultColumn) item.getColumnObject(); 22948 TObjectName tableColumnObject = queryTableColumn.getFieldAttr(); 22949 if (tableColumnObject != null) { 22950 resultColumn.bindStarLinkColumn(tableColumnObject); 22951 } else if (queryTableColumn.getAliasClause() != null) { 22952 resultColumn.bindStarLinkColumn( 22953 queryTableColumn.getAliasClause().getAliasName()); 22954 } 22955 } 22956 } 22957 } 22958 } 22959 } else { 22960 ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel, column); 22961 TTableList tables = stmt.getTables(); 22962 for (int k = 0; k < tables.size(); k++) { 22963 TTable table = tables.getTable(k); 22964 TObjectName[] columns = modelManager.getTableColumns(table); 22965 for (int j = 0; j < columns.length; j++) { 22966 TObjectName columnName = columns[j]; 22967 if (columnName == null) { 22968 continue; 22969 } 22970 if ("*".equals(getColumnName(columnName))) { 22971 if (modelManager.getModel(table) instanceof Table) { 22972 Table tableModel = (Table) modelManager.getModel(table); 22973 if (tableModel != null) { 22974 modelFactory.createTableColumn(tableModel, columnName, false); 22975 } 22976 if (tableModel != null && !tableModel.getColumns().isEmpty()) { 22977 for (int z = 0; z < tableModel.getColumns().size(); z++) { 22978 resultColumn.bindStarLinkColumn( 22979 tableModel.getColumns().get(z).getColumnObject()); 22980 } 22981 } 22982 } else if (modelManager.getModel(table) instanceof QueryTable) { 22983 QueryTable tableModel = (QueryTable) modelManager.getModel(table); 22984 if (tableModel != null && !tableModel.getColumns().isEmpty()) { 22985 for (ResultColumn item : tableModel.getColumns()) { 22986 if (item.hasStarLinkColumn()) { 22987 for (TObjectName starLinkColumn : item 22988 .getStarLinkColumnList()) { 22989 resultColumn.bindStarLinkColumn(starLinkColumn); 22990 } 22991 } else if (item.getColumnObject() instanceof TObjectName) { 22992 resultColumn.bindStarLinkColumn( 22993 (TObjectName) item.getColumnObject()); 22994 } else if (item.getColumnObject() instanceof TResultColumn) { 22995 TResultColumn queryTableColumn = (TResultColumn) item 22996 .getColumnObject(); 22997 TObjectName tableColumnObject = queryTableColumn 22998 .getFieldAttr(); 22999 if (tableColumnObject != null) { 23000 resultColumn.bindStarLinkColumn(tableColumnObject); 23001 } else if (queryTableColumn.getAliasClause() != null) { 23002 resultColumn.bindStarLinkColumn(queryTableColumn 23003 .getAliasClause().getAliasName()); 23004 } 23005 } 23006 } 23007 } 23008 } 23009 continue; 23010 } 23011 resultColumn.bindStarLinkColumn(columnName); 23012 } 23013 } 23014 } 23015 } 23016 else { 23017 ResultColumn resultColumn = modelFactory.createResultColumn(resultSetModel, column); 23018 } 23019 23020 analyzeResultColumn(column, EffectType.select); 23021 23022 } 23023 23024 if (queryModel instanceof ResultSet) { 23025 boolean isDetermined = true; 23026 ResultSet resultSet = (ResultSet) queryModel; 23027 for (ResultColumn column : resultSet.getColumns()) { 23028 if (column.getName().endsWith("*")) { 23029 isDetermined = false; 23030 break; 23031 } 23032 } 23033 if (isDetermined) { 23034 resultSet.setDetermined(isDetermined); 23035 } 23036 } 23037 } 23038 } 23039 23040 23041 analyzeSelectIntoClause(stmt); 23042 23043 23044 if (stmt.getJoins() != null && stmt.getJoins().size() > 0) { 23045 for (int i = 0; i < stmt.getJoins().size(); i++) { 23046 TJoin join = stmt.getJoins().getJoin(i); 23047 ResultSet topResultSet = (ResultSet) modelManager.getModel(stmt); 23048 if (join.getJoinItems() != null && join.getJoinItems().size() > 0) { 23049 for (int k = 0; k < join.getJoinItems().size(); k++) { 23050 TTable table = join.getJoinItems().getJoinItem(k).getTable(); 23051 if (table != null && table.getSubquery() != null) { 23052 23053 ResultSet joinResultSet = (ResultSet) modelManager.getModel(table.getSubquery()); 23054 for (int x = 0; x < joinResultSet.getColumns().size(); x++) { 23055 ResultColumn sourceColumn = joinResultSet.getColumns().get(x); 23056 ResultColumn resultColumn = matchResultColumn(topResultSet.getColumns(), 23057 sourceColumn); 23058 if (resultColumn != null 23059 && resultColumn.getColumnObject() instanceof TResultColumn) { 23060 TResultColumn column = (TResultColumn) resultColumn.getColumnObject(); 23061 if (column.getAliasClause() == null && column.getFieldAttr() != null) { 23062 TObjectName resultObject = column.getFieldAttr(); 23063 if (resultObject.getSourceTable() == null 23064 || resultObject.getSourceTable().equals(table)) { 23065 DataFlowRelationship combinedQueryRelation = modelFactory 23066 .createDataFlowRelation(); 23067 combinedQueryRelation.setEffectType(EffectType.select); 23068 combinedQueryRelation 23069 .setTarget(new ResultColumnRelationshipElement(resultColumn)); 23070 combinedQueryRelation 23071 .addSource(new ResultColumnRelationshipElement(sourceColumn)); 23072 } 23073 } 23074 } 23075 } 23076 } 23077 23078 if(join.getJoinItems().getJoinItem(k).getJoin()!=null) { 23079 analyzeJoin(join.getJoinItems().getJoinItem(k).getJoin(), EffectType.select); 23080 } 23081 } 23082 } 23083 analyzeJoin(join, EffectType.select); 23084 } 23085 } 23086 23087 if (stmt.getWhereClause() != null) { 23088 TExpression expr = stmt.getWhereClause().getCondition(); 23089 if (expr != null) { 23090 analyzeFilterCondition(null, expr, null, JoinClauseType.where, EffectType.select); 23091 } 23092 } 23093 23094 stmtStack.pop(); 23095 } 23096 } 23097 23098 protected TObjectNameList getTableLinkedColumns(TTable table) { 23099 if(structObjectMap.containsKey(table)) { 23100 return structObjectMap.get(table); 23101 } 23102 return table.getLinkedColumns(); 23103 } 23104 23105 protected boolean isTopResultSet(TSelectSqlStatement stmt) { 23106 TCustomSqlStatement parent = stmt.getParentStmt(); 23107 if (parent == null) 23108 return true; 23109 if (parent instanceof TMssqlReturn) { 23110 return true; 23111 } 23112 if (parent instanceof TReturnStmt) { 23113 return true; 23114 } 23115 if (parent instanceof TCommonBlock) { 23116 TCommonBlock block = (TCommonBlock) parent; 23117 if (block.getStatements() != null) { 23118 for (int i = 0; i < block.getStatements().size(); i++) { 23119 TCustomSqlStatement child = block.getStatements().get(i); 23120 if(stmt == child) { 23121 return true; 23122 } 23123 } 23124 } 23125 } 23126 if (parent instanceof TMssqlBlock) { 23127 TMssqlBlock block = (TMssqlBlock) parent; 23128 if (block.getStatements() != null) { 23129 for (int i = 0; i < block.getStatements().size(); i++) { 23130 TCustomSqlStatement child = block.getStatements().get(i); 23131 if(stmt == child) { 23132 return true; 23133 } 23134 } 23135 } 23136 } 23137 if (parent instanceof TStoredProcedureSqlStatement) { 23138 TStoredProcedureSqlStatement block = (TStoredProcedureSqlStatement) parent; 23139 if (block.getStatements() != null) { 23140 for (int i = 0; i < block.getStatements().size(); i++) { 23141 TCustomSqlStatement child = block.getStatements().get(i); 23142 if(child == stmt) { 23143 return true; 23144 } 23145 if (child instanceof TReturnStmt) { 23146 TReturnStmt returnStmt = (TReturnStmt) child; 23147 if (returnStmt.getStatements() != null) { 23148 for (int j = 0; j < returnStmt.getStatements().size(); j++) { 23149 TCustomSqlStatement child1 = returnStmt.getStatements().get(j); 23150 if(child1 == stmt) { 23151 return true; 23152 } 23153 } 23154 } 23155 } 23156 if (child instanceof TMssqlReturn) { 23157 TMssqlReturn returnStmt = (TMssqlReturn) child; 23158 if (returnStmt.getStatements() != null) { 23159 for (int j = 0; j < returnStmt.getStatements().size(); j++) { 23160 TCustomSqlStatement child1 = returnStmt.getStatements().get(j); 23161 if(child1 == stmt) { 23162 return true; 23163 } 23164 } 23165 } 23166 } 23167 } 23168 } 23169 } 23170 return false; 23171 } 23172 23173 protected void analyzeTableSubquery(TTable table) { 23174 if(table.getSubquery()!=null) { 23175 QueryTable queryTable = modelFactory.createQueryTable(table); 23176 TSelectSqlStatement subquery = table.getSubquery(); 23177 analyzeSelectStmt(subquery); 23178 23179 ResultSet resultSetModel = (ResultSet) modelManager.getModel(subquery); 23180 23181 if (resultSetModel != null && resultSetModel != queryTable 23182 && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) { 23183 ImpactRelationship impactRelation = modelFactory.createImpactRelation(); 23184 impactRelation.setEffectType(EffectType.select); 23185 impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>( 23186 resultSetModel.getRelationRows())); 23187 impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>( 23188 queryTable.getRelationRows())); 23189 } 23190 23191 if (resultSetModel != null && resultSetModel != queryTable 23192 && queryTable.getTableObject().getAliasClause() != null 23193 && queryTable.getTableObject().getAliasClause().getColumns() != null) { 23194 for (int j = 0; j < queryTable.getColumns().size() 23195 && j < resultSetModel.getColumns().size(); j++) { 23196 ResultColumn sourceColumn = resultSetModel.getColumns().get(j); 23197 ResultColumn targetColumn = queryTable.getColumns().get(j); 23198 23199 DataFlowRelationship queryRalation = modelFactory.createDataFlowRelation(); 23200 queryRalation.setEffectType(EffectType.select); 23201 queryRalation.setTarget(new ResultColumnRelationshipElement(targetColumn)); 23202 queryRalation.addSource(new ResultColumnRelationshipElement(sourceColumn)); 23203 } 23204 } else if (subquery.getSetOperatorType() != ESetOperatorType.none) { 23205 SelectSetResultSet selectSetResultSetModel = (SelectSetResultSet) modelManager 23206 .getModel(subquery); 23207 for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) { 23208 ResultColumn sourceColumn = selectSetResultSetModel.getColumns().get(j); 23209 ResultColumn targetColumn = modelFactory.createSelectSetResultColumn(queryTable, 23210 sourceColumn); 23211 for (TObjectName starLinkColumn : sourceColumn.getStarLinkColumnList()) { 23212 targetColumn.bindStarLinkColumn(starLinkColumn); 23213 } 23214 DataFlowRelationship selectSetRalation = modelFactory.createDataFlowRelation(); 23215 selectSetRalation.setEffectType(EffectType.select); 23216 selectSetRalation.setTarget(new ResultColumnRelationshipElement(targetColumn)); 23217 selectSetRalation.addSource(new ResultColumnRelationshipElement(sourceColumn)); 23218 } 23219 } 23220 } 23221 } 23222 23223 private ResultColumn getPivotedTableColumn(TPivotedTable pivotedTable, TObjectName columnName) { 23224 List<TPivotClause> pivotClauses = new ArrayList<TPivotClause>(); 23225 if (pivotedTable.getPivotClause() != null) { 23226 pivotClauses.add(pivotedTable.getPivotClause()); 23227 } 23228 if (pivotedTable.getPivotClauseList() != null) { 23229 for (int i = 0; i < pivotedTable.getPivotClauseList().size(); i++) { 23230 pivotClauses.add(pivotedTable.getPivotClauseList().getElement(i)); 23231 } 23232 } 23233 for (TPivotClause clause : pivotClauses) { 23234 Object model = modelManager.getModel(clause); 23235 if (model instanceof PivotedTable) { 23236 PivotedTable pivotedTableModel = (PivotedTable) model; 23237 if (pivotedTableModel.getColumns() != null) { 23238 for (ResultColumn column : pivotedTableModel.getColumns()) { 23239 if (DlineageUtil.compareColumnIdentifier(getColumnName(columnName), 23240 getColumnName(SQLUtil.trimColumnStringQuote(column.getName())))) { 23241 return column; 23242 } 23243 } 23244 } 23245 } 23246 } 23247 return null; 23248 } 23249 23250 private void analyzeHiveTransformClause(TSelectSqlStatement stmt, THiveTransformClause transformClause) { 23251 Table mapSourceTable = null; 23252 QueryTable mapQueryTable = null; 23253 if(stmt.getTables()!=null) { 23254 for(int i=0;i<stmt.getTables().size();i++) { 23255 TTable table = stmt.getTables().getTable(i); 23256 if (table.getSubquery() != null) { 23257 if (transformClause.getTransformType() == ETransformType.ettReduce) { 23258 mapQueryTable = modelFactory.createQueryTable(table); 23259 } 23260 analyzeSelectStmt(table.getSubquery()); 23261 } 23262 else { 23263 mapSourceTable = modelFactory.createTable(table); 23264 } 23265 } 23266 } 23267 23268 if (transformClause.getTransformType() == ETransformType.ettReduce) { 23269 modelFactory.createResultSet(stmt, false); 23270 } 23271 23272 List<TableColumn> mapTableColumns = new ArrayList<TableColumn>(); 23273 List<ResultColumn> mapResultSetColumns = new ArrayList<ResultColumn>(); 23274 List<ResultColumn> redueResultSetColumns = new ArrayList<ResultColumn>(); 23275 23276 if(transformClause.getExpressionList()!=null) { 23277 for(TExpression expression: transformClause.getExpressionList()) { 23278 if(expression.getObjectOperand()!=null) { 23279 if (transformClause.getTransformType() == ETransformType.ettMap || transformClause.getTransformType() == ETransformType.ettSelect) { 23280 if (mapSourceTable != null) { 23281 TableColumn tableColumn = modelFactory.createTableColumn(mapSourceTable, 23282 expression.getObjectOperand(), false); 23283 if (tableColumn != null) { 23284 mapTableColumns.add(tableColumn); 23285 } 23286 } 23287 } 23288 else if (transformClause.getTransformType() == ETransformType.ettReduce) { 23289 if (mapQueryTable != null) { 23290 ResultColumn resultColumn = modelFactory.createResultColumn(mapQueryTable, 23291 expression.getObjectOperand(), false); 23292 if (resultColumn != null) { 23293 mapResultSetColumns.add(resultColumn); 23294 } 23295 } 23296 } 23297 } 23298 } 23299 } 23300 23301 if (transformClause.getAliasClause() != null) { 23302 Object model = modelManager.getModel(stmt); 23303 if (model instanceof ResultSet) { 23304 ResultSet result = (ResultSet) model; 23305 if (result!=null && transformClause.getAliasClause().getColumns() != null) { 23306 for (TObjectName column : transformClause.getAliasClause().getColumns()) { 23307 ResultColumn resultColumn = modelFactory.createResultColumn(result, column); 23308 if (resultColumn != null) { 23309 if (transformClause.getTransformType() == ETransformType.ettMap 23310 || transformClause.getTransformType() == ETransformType.ettSelect) { 23311 mapResultSetColumns.add(resultColumn); 23312 } 23313 else if (transformClause.getTransformType() == ETransformType.ettReduce) { 23314 redueResultSetColumns.add(resultColumn); 23315 } 23316 } 23317 } 23318 } 23319 } 23320 } 23321 23322 if (transformClause.getTransformType() == ETransformType.ettMap 23323 || transformClause.getTransformType() == ETransformType.ettSelect) { 23324 if (!mapTableColumns.isEmpty() && !mapResultSetColumns.isEmpty()) { 23325 for (ResultColumn resultColumn : mapResultSetColumns) { 23326 for (TableColumn tableColumn : mapTableColumns) { 23327 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 23328 relation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 23329 relation.addSource(new TableColumnRelationshipElement(tableColumn)); 23330 relation.setEffectType(EffectType.select); 23331 } 23332 } 23333 } 23334 } 23335 else if (transformClause.getTransformType() == ETransformType.ettReduce) { 23336 if (!redueResultSetColumns.isEmpty() && !mapResultSetColumns.isEmpty()) { 23337 for (ResultColumn reduceResultColumn : redueResultSetColumns) { 23338 for (ResultColumn mapResultColumn : mapResultSetColumns) { 23339 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 23340 relation.setTarget(new ResultColumnRelationshipElement(reduceResultColumn)); 23341 relation.addSource(new ResultColumnRelationshipElement(mapResultColumn)); 23342 relation.setEffectType(EffectType.select); 23343 } 23344 } 23345 } 23346 } 23347 } 23348 23349 protected boolean isStructColumn(TObjectName columnName) { 23350 return columnName.getSourceTable() != null && columnName.getSourceTable().getAliasClause() != null 23351 && columnName.getSourceTable().getUnnestClause() != null 23352 && DlineageUtil.compareColumnIdentifier(getColumnName(columnName), 23353 getColumnName(columnName.getSourceTable().getAliasClause().getAliasName())); 23354 } 23355 23356 private TObjectName getObjectName(ResultColumn column) { 23357 if (column.getColumnObject() instanceof TResultColumn) { 23358 TResultColumn resultColumn = (TResultColumn) column.getColumnObject(); 23359 if (resultColumn.getAliasClause() != null && resultColumn.getAliasClause().getAliasName() != null) { 23360 return resultColumn.getAliasClause().getAliasName(); 23361 } 23362 if (resultColumn.getFieldAttr() != null) { 23363 return resultColumn.getFieldAttr(); 23364 } 23365 if (resultColumn.getExpr() != null 23366 && resultColumn.getExpr().getExpressionType() == EExpressionType.simple_object_name_t) { 23367 return resultColumn.getExpr().getObjectOperand(); 23368 } 23369 } else if (column.getColumnObject() instanceof TObjectName) { 23370 return (TObjectName) column.getColumnObject(); 23371 } 23372 return null; 23373 } 23374 23375 private boolean isShowTopSelectResultSet() { 23376 if (option.isSimpleOutput() && !option.isSimpleShowTopSelectResultSet()) 23377 return false; 23378 return true; 23379 } 23380 23381 /** 23382 * The record field the i-th SELECT ... INTO item binds to (#695). 23383 * SELECT INTO a record is POSITIONAL: item k fills field k. Binding 23384 * get(0) put every select item on the first field once records expanded 23385 * to per-field columns — five sources asserted on one field, none on the 23386 * others. Items beyond the known fields land on the record's star 23387 * (an unresolved component), never on a wrong field. 23388 */ 23389 private TableColumn selectIntoRecordColumn(Table tableModel, TableColumn variableColumn, int ordinal) { 23390 if (variableColumn != null) { 23391 return variableColumn; 23392 } 23393 List<TableColumn> columns = tableModel.getColumns(); 23394 if (columns == null || columns.isEmpty()) { 23395 return null; 23396 } 23397 if (columns.size() == 1) { 23398 return columns.get(0); 23399 } 23400 if (!(tableModel instanceof Variable) || !((Variable) tableModel).isTypeOrderedFields()) { 23401 // Fields discovered from consumption carry no positional meaning, 23402 // and expanding a star here fans every select item onto every 23403 // consumed pseudo-field. Keep the historical whole-record binding. 23404 return columns.get(0); 23405 } 23406 int k = 0; 23407 for (TableColumn column : columns) { 23408 if ("*".equals(column.getName())) { 23409 continue; 23410 } 23411 if (k == ordinal) { 23412 return column; 23413 } 23414 k++; 23415 } 23416 return tableStarColumn(tableModel); 23417 } 23418 23419 private static boolean hasRealTokenPosition(Pair3<Long, Long, String> start) { 23420 return start != null && start.first != null && start.second != null && start.first > 0 23421 && !(start.first == 1L && start.second == 1L); 23422 } 23423 23424 /** 23425 * A SELECT INTO target column minted from a name-only object (star-expanded 23426 * columns of SELECT * INTO #t, set-operation and expanded-star branches) has 23427 * no source token, so its position renders as the synthetic 23428 * "[1,1,0],[1,n,0]" start-of-module coordinate. Give it the producing 23429 * result column's span instead (for a star column, the '*' token), so 23430 * definition-site consumers see an honest coordinate. 23431 */ 23432 private static void fillSyntheticColumnPosition(TableColumn tableColumn, ResultColumn producedBy) { 23433 if (tableColumn == null || producedBy == null) { 23434 return; 23435 } 23436 if (hasRealTokenPosition(tableColumn.getStartPosition())) { 23437 return; 23438 } 23439 if (!hasRealTokenPosition(producedBy.getStartPosition())) { 23440 return; 23441 } 23442 tableColumn.setStartPosition(producedBy.getStartPosition()); 23443 tableColumn.setEndPosition(producedBy.getEndPosition()); 23444 } 23445 23446 private void analyzeSelectIntoClause(TSelectSqlStatement stmt) { 23447 if (stmt.getParentStmt() instanceof TSelectSqlStatement) { 23448 return; 23449 } 23450 23451 // Keep Oracle trigger pseudo-record fields aligned with their INTO 23452 // expressions. A single shared slot made every source position target the 23453 // last trigger correlation field in a multi-column list (MantisBT 4707). 23454 List<TObjectName> oracleIntoColumnNames = new ArrayList<TObjectName>(); 23455 23456 TIntoClause intoClause = stmt.getIntoClause(); 23457 23458 TSelectSqlStatement leftStmt = DlineageUtil.getLeftStmt(stmt); 23459 23460 if (intoClause == null && leftStmt != null) { 23461 intoClause = leftStmt.getIntoClause(); 23462 } 23463 23464 if (intoClause != null) { 23465 List<TObjectName> tableNames = new ArrayList<TObjectName>(); 23466 if (intoClause.getExprList() != null) { 23467 for (int j = 0; j < intoClause.getExprList().size(); j++) { 23468 TObjectName tableName = intoClause.getExprList().getExpression(j).getObjectOperand(); 23469 if (tableName != null) { 23470 if (tableName.toString().startsWith(":") && option.getVendor() == EDbVendor.dbvoracle 23471 && tableName.getDbObjectType() == EDbObjectType.column) { 23472 TObjectName tableAlias = new TObjectName(); 23473 if (!SQLUtil.isEmpty(tableName.getTableString())) { 23474 tableAlias.setString(tableName.getTableString()); 23475 } 23476 else { 23477 tableAlias.setString(tableName.toString()); 23478 } 23479 tableNames.add(tableAlias); 23480 TTable sourceTable = tableName.getSourceTable(); 23481 Table boundTable = modelManager.getTableByName( 23482 DlineageUtil.getTableFullName(tableAlias.toString())); 23483 boolean sourceMatchesBinding = sourceTable != null && boundTable != null 23484 && SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotTable, 23485 boundTable.getName(), sourceTable.getTableName().toString()); 23486 if (sourceTable != null && (boundTable == null || sourceMatchesBinding)) { 23487 // Prefer a trigger-correlation binding registered by 23488 // bindTriggerCorrelationNames(). Resolver fallback can 23489 // otherwise misclassify a custom REFERENCING alias as the 23490 // SELECT's only FROM table. 23491 modelFactory.createTable(sourceTable, tableAlias); 23492 } 23493 // A bare host bind such as :first_name is a scalar 23494 // variable, not a pseudo-record field. Preserve its 23495 // historical result-column-derived name. 23496 oracleIntoColumnNames.add(SQLUtil.isEmpty(tableName.getTableString()) 23497 ? null : tableName); 23498 } else { 23499 if (tableName != null) { 23500 tableNames.add(tableName); 23501 oracleIntoColumnNames.add(null); 23502 } 23503 } 23504 } 23505 else if(intoClause.getExprList().getExpression(j).getFunctionCall()!=null) { 23506 TObjectName variableName = intoClause.getExprList().getExpression(j).getFunctionCall().getFunctionName(); 23507 tableNames.add(variableName); 23508 oracleIntoColumnNames.add(null); 23509 Variable variable = modelFactory.createVariable(variableName); 23510 variable.setSubType(SubType.record); 23511 TObjectName variableProperties = new TObjectName(); 23512 variableProperties.setString("*"); 23513 modelFactory.createTableColumn(variable, variableProperties, true); 23514 } 23515 } 23516 } else if (intoClause.getVariableList() != null) { 23517 for (int j = 0; j < intoClause.getVariableList().size(); j++) { 23518 TObjectName tableName = intoClause.getVariableList().getObjectName(j); 23519 if (tableName != null) { 23520 tableNames.add(tableName); 23521 oracleIntoColumnNames.add(null); 23522 } 23523 } 23524 } else if (intoClause.getIntoName() != null) { 23525 tableNames.add(intoClause.getIntoName()); 23526 oracleIntoColumnNames.add(null); 23527 } 23528 23529 ResultSet queryModel = (ResultSet) modelManager.getModel(stmt.getResultColumnList()); 23530 if (stmt.getSetOperatorType() != ESetOperatorType.none) { 23531 queryModel = (ResultSet) modelManager.getModel(stmt); 23532 } 23533 for (int j = 0; j < tableNames.size(); j++) { 23534 TObjectName tableName = tableNames.get(j); 23535 TObjectName oracleIntoColumnName = oracleIntoColumnNames.get(j); 23536 if (tableName.getColumnNameOnly().startsWith("@") 23537 && (option.getVendor() == EDbVendor.dbvmssql || option.getVendor() == EDbVendor.dbvazuresql)) { 23538 continue; 23539 } 23540 23541 if (tableName.getColumnNameOnly().startsWith(":") 23542 && (option.getVendor() == EDbVendor.dbvhana || option.getVendor() == EDbVendor.dbvteradata)) { 23543 continue; 23544 } 23545 23546 Table tableModel; 23547 TableColumn variableColumn = null; 23548 23549 if (tableName.getDbObjectType() == EDbObjectType.variable) { 23550 if (tableName.toString().indexOf(".") != -1) { 23551 List<String> splits = SQLUtil.parseNames(tableName.toString()); 23552 tableModel = modelFactory.createVariable(splits.get(splits.size() - 2)); 23553 // INTO rec.field names its target field explicitly (#695); 23554 // bind that field, never an ordinal of the record. 23555 variableColumn = modelFactory.createTableColumn(tableModel, tableName, true); 23556 } else { 23557 tableModel = modelFactory.createVariable(tableName); 23558 } 23559 if (tableModel.getSubType() == null) { 23560 tableModel.setSubType(SubType.record); 23561 } 23562 if(variableColumn == null 23563 && (tableModel.getColumns() == null || tableModel.getColumns().isEmpty())) { 23564 TObjectName variableProperties = new TObjectName(); 23565 variableProperties.setString("*"); 23566 variableColumn = modelFactory.createTableColumn(tableModel, variableProperties, true); 23567 } 23568 } else { 23569 tableModel = modelFactory.createTableByName(tableName); 23570 // SELECT ... INTO <table> creates the target. Parse fact, set 23571 // regardless of isDetermined() (unlike the gated setCreateTable 23572 // below). See dlineage-authoritative-endpoint-classification.md. 23573 tableModel.setCreatedInSql(true); 23574 tableModel.setEndpointIntroduction(EndpointIntroduction.SELECT_INTO); 23575 } 23576 TableColumn oracleIntoTableColumn = oracleIntoColumnName == null ? null 23577 : modelFactory.createTableColumn(tableModel, oracleIntoColumnName, true); 23578 23579 if (queryModel instanceof ResultSet && (stmt.getWhereClause() != null || hasJoin(stmt))) { 23580 ImpactRelationship impactRelation = modelFactory.createImpactRelation(); 23581 impactRelation.setEffectType(EffectType.insert); 23582 impactRelation.addSource( 23583 new RelationRowsRelationshipElement<ResultSetRelationRows>(((ResultSet)queryModel).getRelationRows())); 23584 impactRelation.setTarget( 23585 new RelationRowsRelationshipElement<TableRelationRows>(tableModel.getRelationRows())); 23586 } 23587 23588 Process process = modelFactory.createProcess(stmt); 23589 tableModel.addProcess(process); 23590 23591 if (stmt.getSetOperatorType() != ESetOperatorType.none) { 23592 for (int setopOrdinal = 0; setopOrdinal < queryModel.getColumns().size(); setopOrdinal++) { 23593 ResultColumn resultColumn = queryModel.getColumns().get(setopOrdinal); 23594 if (tableNames.size() > 1 23595 && (tableName.getDbObjectType() == EDbObjectType.variable 23596 || oracleIntoTableColumn != null) 23597 && setopOrdinal != j) { 23598 continue; 23599 } 23600 TableColumn tableColumn; 23601 if (oracleIntoTableColumn != null) { 23602 tableColumn = oracleIntoTableColumn; 23603 } else if (tableModel.isVariable()) { 23604 // SELECT ... INTO variables and typed records binds 23605 // positionally even through a set operation (codex-696 F4). 23606 tableColumn = selectIntoRecordColumn(tableModel, variableColumn, setopOrdinal); 23607 } else { 23608 tableColumn = modelFactory.createInsertTableColumn(tableModel, 23609 resultColumn.getName()); 23610 fillSyntheticColumnPosition(tableColumn, resultColumn); 23611 } 23612 23613 if (DlineageUtil.isTempTable(tableModel, option.getVendor()) && sqlenv != null 23614 && tableModel.getDatabase() != null && tableModel.getSchema() != null) { 23615 TSQLSchema schema = sqlenv 23616 .getSQLSchema(tableModel.getDatabase() + "." + tableModel.getSchema(), true); 23617 if (schema != null) { 23618 TSQLTable tempTable = schema 23619 .createTable(DlineageUtil.getSimpleTableName(tableModel.getName())); 23620 tempTable.addColumn(tableColumn.getName()); 23621 } 23622 } 23623 23624 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 23625 relation.setEffectType(EffectType.insert); 23626 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 23627 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 23628 relation.setProcess(process); 23629 } 23630 23631 tableModel.setDetermined(queryModel.isDetermined()); 23632 if(queryModel.isDetermined() && DlineageUtil.isTempTable(tableModel, option.getVendor())) { 23633 tableModel.setCreateTable(true, false); 23634 } 23635 if (j + 1 == tableNames.size()) { 23636 return; 23637 } 23638 continue; 23639 } 23640 23641 boolean isDetermined = true; 23642 for (int i = 0; i < stmt.getResultColumnList().size(); i++) { 23643 if (tableNames.size() > 1 23644 && (tableName.getDbObjectType() == EDbObjectType.variable 23645 || oracleIntoTableColumn != null)) { 23646 if (i != j) { 23647 continue; 23648 } 23649 } 23650 TResultColumn column = stmt.getResultColumnList().getResultColumn(i); 23651 23652 if ("*".equals(column.getColumnNameOnly()) && column.getFieldAttr() != null 23653 && column.getFieldAttr().getSourceTable() != null) { 23654 Object model = modelManager.getModel(column); 23655 if(model instanceof LinkedHashMap) { 23656 LinkedHashMap<String, ResultColumn> columns = (LinkedHashMap<String, ResultColumn>)model; 23657 int mapOrdinal = 0; 23658 for(String key: columns.keySet()) { 23659 ResultColumn sourceColumn = columns.get(key); 23660 TableColumn tableColumn; 23661 if (tableModel instanceof Variable 23662 && ((Variable) tableModel).isTypeOrderedFields()) { 23663 tableColumn = selectIntoRecordColumn(tableModel, null, mapOrdinal); 23664 } else { 23665 tableColumn = modelFactory.createInsertTableColumn(tableModel, 23666 sourceColumn.getName()); 23667 fillSyntheticColumnPosition(tableColumn, sourceColumn); 23668 } 23669 mapOrdinal++; 23670 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 23671 relation.setEffectType(EffectType.insert); 23672 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 23673 relation.addSource(new ResultColumnRelationshipElement(sourceColumn)); 23674 relation.setProcess(process); 23675 } 23676 } 23677 else if(model instanceof ResultColumn) { 23678 isDetermined = false; 23679 ResultColumn resultColumn = (ResultColumn) model; 23680 List<TObjectName> columns = resultColumn.getStarLinkColumnList(); 23681 if (columns.size() > 0) { 23682 for (int k = 0; k < columns.size(); k++) { 23683 23684 TableColumn tableColumn; 23685 if (tableModel instanceof Variable 23686 && ((Variable) tableModel).isTypeOrderedFields()) { 23687 // SELECT * INTO a typed record fills its 23688 // DECLARED fields by position — creating 23689 // fields from source names left the real 23690 // fields with no upstream (codex-696 F4). 23691 tableColumn = selectIntoRecordColumn(tableModel, null, k); 23692 } else { 23693 tableColumn = modelFactory.createInsertTableColumn(tableModel, 23694 columns.get(k)); 23695 // Star-link names are synthesized without 23696 // tokens; the '*' span is the honest 23697 // definition site for the expanded column. 23698 fillSyntheticColumnPosition(tableColumn, resultColumn); 23699 } 23700 23701 if (DlineageUtil.isTempTable(tableModel, option.getVendor()) && sqlenv != null 23702 && tableModel.getDatabase() != null && tableModel.getSchema() != null) { 23703 TSQLSchema schema = sqlenv.getSQLSchema( 23704 tableModel.getDatabase() + "." + tableModel.getSchema(), true); 23705 if (schema != null) { 23706 TSQLTable tempTable = schema.createTable(DlineageUtil.getSimpleTableName(tableModel.getName())); 23707 tempTable.addColumn(tableColumn.getName()); 23708 } 23709 } 23710 23711 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 23712 relation.setEffectType(EffectType.insert); 23713 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 23714 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 23715 relation.setProcess(process); 23716 } 23717 if (resultColumn.isShowStar()) { 23718 TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel, 23719 column.getFieldAttr()); 23720 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 23721 relation.setEffectType(EffectType.insert); 23722 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 23723 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 23724 relation.setProcess(process); 23725 } 23726 } else { 23727 TObjectName columnObject = column.getFieldAttr(); 23728 if (column.getAliasClause() != null) { 23729 columnObject = column.getAliasClause().getAliasName(); 23730 } 23731 if (columnObject != null) { 23732 TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel, 23733 columnObject); 23734 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 23735 relation.setEffectType(EffectType.insert); 23736 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 23737 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 23738 relation.setProcess(process); 23739 } else if (!SQLUtil.isEmpty(column.getColumnAlias())) { 23740 TableColumn tableColumn = modelFactory.createInsertTableColumn(tableModel, 23741 column.getAliasClause().getAliasName()); 23742 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 23743 relation.setEffectType(EffectType.insert); 23744 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 23745 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 23746 relation.setProcess(process); 23747 } 23748 } 23749 } 23750 } else { 23751 ResultColumn resultColumn = null; 23752 23753 if (queryModel instanceof QueryTable) { 23754 resultColumn = (ResultColumn) modelManager.getModel(column); 23755 } else if (queryModel instanceof ResultSet) { 23756 resultColumn = (ResultColumn) modelManager.getModel(column); 23757 } else { 23758 continue; 23759 } 23760 23761 if (resultColumn == null && column.getAliasClause() != null) { 23762 resultColumn = (ResultColumn) modelManager.getModel(column.getAliasClause().getAliasName()); 23763 } 23764 23765 if (resultColumn != null) { 23766 TObjectName columnObject = column.getFieldAttr(); 23767 if (column.getAliasClause() != null) { 23768 columnObject = column.getAliasClause().getAliasName(); 23769 } 23770 TableColumn tableColumn = null; 23771 if (columnObject != null) { 23772 if (tableModel.isVariable()) { 23773 tableColumn = selectIntoRecordColumn(tableModel, variableColumn, i); 23774 } 23775 else if (oracleIntoTableColumn != null) { 23776 tableColumn = oracleIntoTableColumn; 23777 } 23778 else { 23779 tableColumn = modelFactory.createInsertTableColumn(tableModel, columnObject); 23780 if (containStarColumn(queryModel)) { 23781 tableColumn.notBindStarLinkColumn(true); 23782 } 23783 } 23784 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 23785 relation.setEffectType(EffectType.insert); 23786 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 23787 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 23788 relation.setProcess(process); 23789 } else if (!SQLUtil.isEmpty(column.getColumnAlias())) { 23790 if (tableModel.isVariable()) { 23791 tableColumn = selectIntoRecordColumn(tableModel, variableColumn, i); 23792 } else { 23793 tableColumn = modelFactory.createInsertTableColumn(tableModel, 23794 column.getAliasClause().getAliasName()); 23795 } 23796 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 23797 relation.setEffectType(EffectType.insert); 23798 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 23799 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 23800 relation.setProcess(process); 23801 } else { 23802 if (tableModel.isVariable()) { 23803 tableColumn = selectIntoRecordColumn(tableModel, variableColumn, i); 23804 } 23805 if (tableColumn != null) { 23806 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 23807 relation.setEffectType(EffectType.insert); 23808 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 23809 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 23810 relation.setProcess(process); 23811 } 23812 } 23813 } 23814 } 23815 } 23816 tableModel.setDetermined(isDetermined); 23817 if(isDetermined && DlineageUtil.isTempTable(tableModel, option.getVendor())) { 23818 tableModel.setCreateTable(true, false); 23819 } 23820 } 23821 } 23822 } 23823 23824 private boolean isUnPivotedTable(TPivotedTable pivotedTable) { 23825 if (pivotedTable.getPivotClauseList() != null && pivotedTable.getPivotClauseList().size() > 0) { 23826 return pivotedTable.getPivotClauseList().getElement(0).getType() == TPivotClause.unpivot; 23827 } else { 23828 TPivotClause pivotClause = pivotedTable.getPivotClause(); 23829 return pivotClause.getType() == TPivotClause.unpivot; 23830 } 23831 } 23832 23833 private void analyzeUnPivotedTable(TSelectSqlStatement stmt, TPivotedTable pivotedTable) { 23834 List<Object> tables = new ArrayList<Object>(); 23835 Set<Object> pivotedColumns = new HashSet<Object>(); 23836 TTable fromTable = pivotedTable.getTableSource(); 23837 Object table = modelManager.getModel(fromTable); 23838 List<TPivotClause> pivotClauses = new ArrayList<TPivotClause>(); 23839 if (pivotedTable.getPivotClauseList() != null && pivotedTable.getPivotClauseList().size() > 0) { 23840 for (int i = 0; i < pivotedTable.getPivotClauseList().size(); i++) { 23841 pivotClauses.add(pivotedTable.getPivotClauseList().getElement(i)); 23842 } 23843 } else { 23844 TPivotClause pivotClause = pivotedTable.getPivotClause(); 23845 pivotClauses.add(pivotClause); 23846 } 23847 23848 for (int y = 0; y < pivotClauses.size(); y++) { 23849 TPivotClause pivotClause = pivotClauses.get(y); 23850 PivotedTable pivotTable = modelFactory.createPivotdTable(pivotClause); 23851 pivotTable.setUnpivoted(true); 23852 23853 if (pivotClause.getValueColumnList() != null) { 23854 for (int j = 0; j < pivotClause.getValueColumnList().size(); j++) { 23855 modelFactory.createResultColumn(pivotTable, pivotClause.getValueColumnList().getObjectName(j)); 23856 } 23857 } 23858 if (pivotClause.getPivotColumnList() != null) { 23859 for (int j = 0; j < pivotClause.getPivotColumnList().size(); j++) { 23860 modelFactory.createResultColumn(pivotTable, pivotClause.getPivotColumnList().getObjectName(j)); 23861 } 23862 } 23863 if (pivotClause.getUnpivotInClause()!=null && pivotClause.getUnpivotInClause().getItems() != null) { 23864 for (int j = 0; j < pivotClause.getUnpivotInClause().getItems().size(); j++) { 23865 TObjectName columnName = pivotClause.getUnpivotInClause().getItems().getElement(j).getColumn(); 23866 if (columnName != null) { 23867 if (table instanceof QueryTable) { 23868 for (ResultColumn tableColumn : ((QueryTable) table).getColumns()) { 23869 if (getColumnName(columnName) 23870 .equals(DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) { 23871 for (ResultColumn resultColumn : pivotTable.getColumns()) { 23872 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 23873 relation.setEffectType(EffectType.select); 23874 relation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 23875 relation.addSource(new ResultColumnRelationshipElement(tableColumn)); 23876 pivotedColumns.add(tableColumn); 23877 } 23878 break; 23879 } 23880 } 23881 } else if (table instanceof Table) { 23882 for (TableColumn tableColumn : ((Table) table).getColumns()) { 23883 if (getColumnName(columnName) 23884 .equals(DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) { 23885 for (ResultColumn resultColumn : pivotTable.getColumns()) { 23886 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 23887 relation.setEffectType(EffectType.select); 23888 relation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 23889 relation.addSource(new TableColumnRelationshipElement(tableColumn)); 23890 pivotedColumns.add(tableColumn); 23891 } 23892 break; 23893 } 23894 } 23895 } 23896 } else { 23897 TObjectNameList columnNames = pivotClause.getUnpivotInClause().getItems().getElement(j) 23898 .getColumnList(); 23899 for (TObjectName columnName1 : columnNames) { 23900 if (table instanceof QueryTable) { 23901 for (ResultColumn tableColumn : ((QueryTable) table).getColumns()) { 23902 if (getColumnName(columnName1).equals( 23903 DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) { 23904 for (ResultColumn resultColumn : pivotTable.getColumns()) { 23905 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 23906 relation.setEffectType(EffectType.select); 23907 relation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 23908 relation.addSource(new ResultColumnRelationshipElement(tableColumn)); 23909 pivotedColumns.add(tableColumn); 23910 } 23911 break; 23912 } 23913 } 23914 } else if (table instanceof Table) { 23915 for (TableColumn tableColumn : ((Table) table).getColumns()) { 23916 if (getColumnName(columnName1).equals( 23917 DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) { 23918 for (ResultColumn resultColumn : pivotTable.getColumns()) { 23919 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 23920 relation.setEffectType(EffectType.select); 23921 relation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 23922 relation.addSource(new TableColumnRelationshipElement(tableColumn)); 23923 pivotedColumns.add(tableColumn); 23924 } 23925 break; 23926 } 23927 } 23928 } 23929 } 23930 } 23931 } 23932 } 23933 tables.add(pivotTable); 23934 tables.add(table); 23935 } 23936 23937 ResultSet resultSet = modelFactory.createResultSet(stmt, 23938 isTopResultSet(stmt) && isShowTopSelectResultSet()); 23939 TResultColumnList columnList = stmt.getResultColumnList(); 23940 for (int i = 0; i < columnList.size(); i++) { 23941 TResultColumn column = columnList.getResultColumn(i); 23942 ResultColumn resultColumn = modelFactory.createAndBindingSelectSetResultColumn(resultSet, column, i); 23943 if (resultColumn.getColumnObject() instanceof TResultColumn) { 23944 TResultColumn columnObject = (TResultColumn) resultColumn.getColumnObject(); 23945 if (columnObject.getFieldAttr() != null) { 23946 if ("*".equals(getColumnName(columnObject.getFieldAttr()))) { 23947 resultColumn.setShowStar(false); 23948 int index = 0; 23949 for (int k = 0; k < tables.size(); k++) { 23950 Object tableItem = tables.get(k); 23951 if (tableItem instanceof ResultSet) { 23952 for (int x = 0; x < ((ResultSet) tableItem).getColumns().size(); x++) { 23953 ResultColumn tableColumn = ((ResultSet) tableItem).getColumns().get(x); 23954 if (pivotedColumns.contains(tableColumn)) { 23955 continue; 23956 } 23957 if (tableColumn.getColumnObject() instanceof TObjectName) { 23958 resultColumn.bindStarLinkColumn((TObjectName) tableColumn.getColumnObject()); 23959 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 23960 relation.setEffectType(EffectType.select); 23961 relation.setTarget(new ResultColumnRelationshipElement(resultColumn, (TObjectName) tableColumn.getColumnObject())); 23962 relation.addSource(new ResultColumnRelationshipElement(tableColumn)); 23963 } else if (tableColumn.getColumnObject() instanceof TResultColumn) { 23964 if (((TResultColumn) tableColumn.getColumnObject()).getFieldAttr() != null) { 23965 if (tableColumn.hasStarLinkColumn()) { 23966 for (int z = 0; z < tableColumn.getStarLinkColumnList().size(); z++) { 23967 ResultColumn resultColumn1 = modelFactory.createResultColumn( 23968 (ResultSet) tableItem, 23969 tableColumn.getStarLinkColumnList().get(z)); 23970 DataFlowRelationship relation = modelFactory 23971 .createDataFlowRelation(); 23972 relation.setEffectType(EffectType.select); 23973 relation.setTarget( 23974 new ResultColumnRelationshipElement(resultColumn)); 23975 relation.addSource( 23976 new ResultColumnRelationshipElement(resultColumn1)); 23977 tableColumn.getStarLinkColumns().remove( 23978 getColumnName(tableColumn.getStarLinkColumnList().get(z))); 23979 z--; 23980 } 23981 } else { 23982 resultColumn.bindStarLinkColumn( 23983 ((TResultColumn) tableColumn.getColumnObject()).getFieldAttr()); 23984 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 23985 relation.setEffectType(EffectType.select); 23986 relation.setTarget(new ResultColumnRelationshipElement(resultColumn, ((TResultColumn) tableColumn.getColumnObject()).getFieldAttr())); 23987 relation.addSource(new ResultColumnRelationshipElement(tableColumn)); 23988 } 23989 } else if (((TResultColumn) tableColumn.getColumnObject()).getExpr() != null) { 23990 TExpression expr = ((TResultColumn) tableColumn.getColumnObject()) 23991 .getExpr(); 23992 if (expr.getExpressionType() == EExpressionType.simple_constant_t) { 23993 TObjectName columnName = new TObjectName(); 23994 columnName.setString(expr.toString()); 23995 resultColumn.bindStarLinkColumn(columnName); 23996 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 23997 relation.setEffectType(EffectType.select); 23998 relation.setTarget(new ResultColumnRelationshipElement(resultColumn, columnName)); 23999 relation.addSource(new ResultColumnRelationshipElement(tableColumn)); 24000 } 24001 } 24002 } 24003 } 24004 } else if (tableItem instanceof Table) { 24005 for (TableColumn tableColumn : ((Table) tableItem).getColumns()) { 24006 if (pivotedColumns.contains(tableColumn)) { 24007 continue; 24008 } 24009 resultColumn.bindStarLinkColumn((TObjectName) tableColumn.getColumnObject(), index); 24010 index++; 24011 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 24012 relation.setEffectType(EffectType.select); 24013 relation.setTarget(new ResultColumnRelationshipElement(resultColumn, (TObjectName) tableColumn.getColumnObject())); 24014 relation.addSource(new TableColumnRelationshipElement(tableColumn)); 24015 } 24016 } 24017 } 24018 } else { 24019 for (int k = 0; k < tables.size(); k++) { 24020 Object tableItem = tables.get(k); 24021 if (tableItem instanceof ResultSet) { 24022 for (ResultColumn tableColumn : ((ResultSet) tableItem).getColumns()) { 24023 if (getColumnName(columnObject.getFieldAttr()).equals( 24024 DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) { 24025 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 24026 relation.setEffectType(EffectType.select); 24027 relation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 24028 relation.addSource(new ResultColumnRelationshipElement(tableColumn)); 24029 break; 24030 } 24031 } 24032 } else if (tableItem instanceof Table) { 24033 for (TableColumn tableColumn : ((Table) tableItem).getColumns()) { 24034 if (getColumnName(columnObject.getFieldAttr()).equals( 24035 DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) { 24036 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 24037 relation.setEffectType(EffectType.select); 24038 relation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 24039 relation.addSource(new TableColumnRelationshipElement(tableColumn)); 24040 break; 24041 } 24042 } 24043 } 24044 } 24045 } 24046 } else if (columnObject.getExpr() != null) { 24047 analyzeResultColumn(column, EffectType.select); 24048 } 24049 } 24050 } 24051 } 24052 24053 private void analyzePivotedTable(TSelectSqlStatement stmt, TPivotedTable pivotedTable) { 24054 List<Object> tables = new ArrayList<Object>(); 24055 Set<Object> pivotedColumns = new HashSet<Object>(); 24056 TTable fromTable = pivotedTable.getTableSource(); 24057 Object table = modelManager.getModel(fromTable); 24058 if(table == null && fromTable.getSubquery()!=null) { 24059 table = modelFactory.createQueryTable(fromTable); 24060 TSelectSqlStatement subquery = fromTable.getSubquery(); 24061 analyzeSelectStmt(subquery); 24062 } 24063 List<TPivotClause> pivotClauses = new ArrayList<TPivotClause>(); 24064 if (pivotedTable.getPivotClauseList() != null && pivotedTable.getPivotClauseList().size() > 0) { 24065 for (int i = 0; i < pivotedTable.getPivotClauseList().size(); i++) { 24066 pivotClauses.add(pivotedTable.getPivotClauseList().getElement(i)); 24067 } 24068 } else { 24069 TPivotClause pivotClause = pivotedTable.getPivotClause(); 24070 pivotClauses.add(pivotClause); 24071 } 24072 24073 for (int y = 0; y < pivotClauses.size(); y++) { 24074 TPivotClause pivotClause = pivotClauses.get(y); 24075 List<TFunctionCall> functionCalls = new ArrayList<TFunctionCall>(); 24076 if (pivotClause.getAggregation_function() != null || pivotClause.getAggregation_function_list() != null) { 24077 if (pivotClause.getAggregation_function() != null) { 24078 functionCalls.add((TFunctionCall) pivotClause.getAggregation_function()); 24079 } else if (pivotClause.getAggregation_function_list() != null) { 24080 for (int i = 0; i < pivotClause.getAggregation_function_list().size(); i++) { 24081 functionCalls.add((TFunctionCall) pivotClause.getAggregation_function_list().getResultColumn(i) 24082 .getExpr().getFunctionCall()); 24083 } 24084 } 24085 24086 if (functionCalls.isEmpty()) { 24087 return; 24088 } 24089 24090 if (pivotClause.getPivotColumnList() == null) { 24091 return; 24092 } 24093 24094 if (pivotClause.getPivotInClause() == null) { 24095 return; 24096 } 24097 24098 for (int x = 0; x < functionCalls.size(); x++) { 24099 TFunctionCall functionCall = functionCalls.get(x); 24100 Function function = modelFactory.createFunction(functionCall); 24101 ResultColumn column = modelFactory.createFunctionResultColumn(function, 24102 ((TFunctionCall) functionCall).getFunctionName()); 24103 24104 List<TExpression> expressions = new ArrayList<TExpression>(); 24105 getFunctionExpressions(expressions, new ArrayList<TExpression>(), functionCall); 24106 24107 for (int j = 0; j < expressions.size(); j++) { 24108 columnsInExpr visitor = new columnsInExpr(); 24109 expressions.get(j).inOrderTraverse(visitor); 24110 List<TObjectName> objectNames = visitor.getObjectNames(); 24111 if (objectNames == null) { 24112 continue; 24113 } 24114 for (TObjectName columnName : objectNames) { 24115 if (table instanceof QueryTable) { 24116 for (int i = 0; i < ((QueryTable) table).getColumns().size(); i++) { 24117 boolean find = false; 24118 ResultColumn tableColumn = ((QueryTable) table).getColumns().get(i); 24119 if (tableColumn.hasStarLinkColumn()) { 24120 for (int k = 0; k < tableColumn.getStarLinkColumnList().size(); k++) { 24121 TObjectName objectName = tableColumn.getStarLinkColumnList().get(k); 24122 if (getColumnName(columnName).equals(getColumnName(objectName))) { 24123 ResultColumn resultColumn = modelFactory 24124 .createResultColumn((QueryTable) table, objectName); 24125 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 24126 relation.setEffectType(EffectType.select); 24127 relation.setTarget(new ResultColumnRelationshipElement(column)); 24128 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 24129 pivotedColumns.add(resultColumn); 24130 tableColumn.getStarLinkColumns() 24131 .remove(DlineageUtil.getColumnName(objectName)); 24132 find = true; 24133 break; 24134 } 24135 } 24136 } else if (getColumnName(columnName).equals( 24137 DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) { 24138 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 24139 relation.setEffectType(EffectType.select); 24140 relation.setTarget(new ResultColumnRelationshipElement(column)); 24141 relation.addSource(new ResultColumnRelationshipElement(tableColumn)); 24142 pivotedColumns.add(tableColumn); 24143 find = true; 24144 break; 24145 } 24146 24147 if (!find && tableColumn.getName().endsWith("*")) { 24148 QueryTable queryTable = (QueryTable)table; 24149 ResultColumn resultColumn = modelFactory.createResultColumn(queryTable, columnName); 24150 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 24151 relation.setEffectType(EffectType.select); 24152 relation.setTarget(new ResultColumnRelationshipElement(column)); 24153 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 24154 tableColumn.bindStarLinkColumn(columnName); 24155 } 24156 } 24157 } else if (table instanceof Table) { 24158 for (TableColumn tableColumn : ((Table) table).getColumns()) { 24159 if (getColumnName(columnName).equals( 24160 DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) { 24161 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 24162 relation.setEffectType(EffectType.select); 24163 relation.setTarget(new ResultColumnRelationshipElement(column)); 24164 relation.addSource(new TableColumnRelationshipElement(tableColumn)); 24165 pivotedColumns.add(tableColumn); 24166 break; 24167 } 24168 } 24169 } 24170 } 24171 } 24172 24173 PivotedTable pivotTable = modelFactory.createPivotdTable(pivotClause); 24174 pivotTable.setUnpivoted(false); 24175 24176 if (pivotClause.getPivotInClause().getItems() != null) { 24177 for (int j = 0; j < pivotClause.getPivotInClause().getItems().size(); j++) { 24178 ResultColumn resultColumn = null; 24179 if (pivotClause.getAggregation_function_list() != null 24180 && pivotClause.getAggregation_function_list().size() > 1) { 24181 TResultColumn functionColumn = pivotClause.getAggregation_function_list() 24182 .getResultColumn(x); 24183 TObjectName tableColumn = new TObjectName(); 24184 if (option.getVendor() == EDbVendor.dbvbigquery) { 24185 tableColumn.setString(getResultColumnString(functionColumn) + "_" 24186 + SQLUtil.trimColumnStringQuote(getResultColumnString( 24187 pivotClause.getPivotInClause().getItems().getResultColumn(j)))); 24188 } 24189 else { 24190 tableColumn.setString(SQLUtil 24191 .trimColumnStringQuote(getResultColumnString( 24192 pivotClause.getPivotInClause().getItems().getResultColumn(j))) 24193 + "_" + getResultColumnString(functionColumn)); 24194 } 24195 resultColumn = modelFactory.createResultColumn(pivotTable, tableColumn); 24196 } else { 24197 resultColumn = modelFactory.createSelectSetResultColumn(pivotTable, 24198 pivotClause.getPivotInClause().getItems().getResultColumn(j), j); 24199 } 24200 { 24201 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 24202 relation.setEffectType(EffectType.select); 24203 relation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 24204 relation.addSource(new ResultColumnRelationshipElement(column)); 24205 } 24206 { 24207 for (TObjectName columnName : pivotClause.getPivotColumnList()) { 24208 if (table instanceof QueryTable) { 24209 for (int i = 0; i < ((QueryTable) table).getColumns().size(); i++) { 24210 ResultColumn tableColumn = ((QueryTable) table).getColumns().get(i); 24211 if (tableColumn.hasStarLinkColumn()) { 24212 for (int k = 0; k < tableColumn.getStarLinkColumnList().size(); k++) { 24213 TObjectName objectName = tableColumn.getStarLinkColumnList().get(k); 24214 if (getColumnName(columnName).equals(getColumnName(objectName))) { 24215 ResultColumn resultColumn1 = modelFactory 24216 .createResultColumn((QueryTable) table, objectName); 24217 DataFlowRelationship relation = modelFactory 24218 .createDataFlowRelation(); 24219 relation.setEffectType(EffectType.select); 24220 relation.setTarget(new ResultColumnRelationshipElement(column)); 24221 relation.addSource( 24222 new ResultColumnRelationshipElement(resultColumn1)); 24223 pivotedColumns.add(resultColumn1); 24224 tableColumn.getStarLinkColumns() 24225 .remove(DlineageUtil.getColumnName(objectName)); 24226 break; 24227 } 24228 } 24229 } else if (getColumnName(columnName).equals(DlineageUtil 24230 .getIdentifierNormalColumnName(tableColumn.getName()))) { 24231 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 24232 relation.setEffectType(EffectType.select); 24233 relation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 24234 relation.addSource(new ResultColumnRelationshipElement(tableColumn)); 24235 pivotedColumns.add(tableColumn); 24236 break; 24237 } 24238 } 24239 } else if (table instanceof Table) { 24240 for (TableColumn tableColumn : ((Table) table).getColumns()) { 24241 if (getColumnName(columnName).equals(DlineageUtil 24242 .getIdentifierNormalColumnName(tableColumn.getName()))) { 24243 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 24244 relation.setEffectType(EffectType.select); 24245 relation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 24246 relation.addSource(new TableColumnRelationshipElement(tableColumn)); 24247 pivotedColumns.add(tableColumn); 24248 break; 24249 } 24250 } 24251 } 24252 } 24253 } 24254 } 24255 } else if (pivotClause.getPivotInClause().getSubQuery() != null) { 24256 TSelectSqlStatement subquery = pivotClause.getPivotInClause().getSubQuery(); 24257 analyzeSelectStmt(subquery); 24258 ResultSet selectSetResultSetModel = (ResultSet) modelManager.getModel(subquery); 24259 for (int j = 0; j < subquery.getResultColumnList().size(); j++) { 24260 ResultColumn resultColumn = modelFactory.createSelectSetResultColumn(pivotTable, 24261 subquery.getResultColumnList().getResultColumn(j), j); 24262 { 24263 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 24264 relation.setEffectType(EffectType.select); 24265 relation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 24266 relation.addSource(new ResultColumnRelationshipElement(column)); 24267 relation.addSource(new ResultColumnRelationshipElement( 24268 selectSetResultSetModel.getColumns().get(j))); 24269 } 24270 { 24271 for (TObjectName columnName : pivotClause.getPivotColumnList()) { 24272 if (table instanceof QueryTable) { 24273 for (ResultColumn tableColumn : ((QueryTable) table).getColumns()) { 24274 if (getColumnName(columnName).equals(DlineageUtil 24275 .getIdentifierNormalColumnName(tableColumn.getName()))) { 24276 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 24277 relation.setEffectType(EffectType.select); 24278 relation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 24279 relation.addSource(new ResultColumnRelationshipElement(tableColumn)); 24280 pivotedColumns.add(tableColumn); 24281 break; 24282 } 24283 } 24284 } else if (table instanceof Table) { 24285 for (TableColumn tableColumn : ((Table) table).getColumns()) { 24286 if (getColumnName(columnName).equals(DlineageUtil 24287 .getIdentifierNormalColumnName(tableColumn.getName()))) { 24288 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 24289 relation.setEffectType(EffectType.select); 24290 relation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 24291 relation.addSource(new TableColumnRelationshipElement(tableColumn)); 24292 pivotedColumns.add(tableColumn); 24293 break; 24294 } 24295 } 24296 } 24297 } 24298 } 24299 } 24300 } 24301 tables.add(pivotTable); 24302 tables.add(table); 24303 } 24304 } 24305 } 24306 24307 TPivotClause pivotClause = pivotClauses.get(0); 24308 boolean hasAlias = pivotClause.getAliasClause() != null && pivotClause.getAliasClause().getColumns() != null 24309 && pivotClause.getAliasClause().getColumns().size() > 0; 24310 if (hasAlias) { 24311 Alias alias = modelFactory.createAlias(pivotClause.getAliasClause()); 24312 List<TObjectName> aliasColumns = new ArrayList<TObjectName>(); 24313 int index = 0; 24314 for (int k = 0; k < tables.size(); k++) { 24315 Object tableItem = tables.get(k); 24316 if (tableItem instanceof ResultSet) { 24317 for (ResultColumn tableColumn : ((ResultSet) tableItem).getColumns()) { 24318 if (pivotedColumns.contains(tableColumn)) { 24319 continue; 24320 } 24321 if (tableColumn.getColumnObject() instanceof TObjectName) { 24322 aliasColumns.add((TObjectName) tableColumn.getColumnObject()); 24323 } else if (tableColumn.getColumnObject() instanceof TResultColumn) { 24324 if (((TResultColumn) tableColumn.getColumnObject()).getFieldAttr() != null) { 24325 aliasColumns.add(((TResultColumn) tableColumn.getColumnObject()).getFieldAttr()); 24326 } else { 24327 TExpression expr = ((TResultColumn) tableColumn.getColumnObject()).getExpr(); 24328 if (expr.getExpressionType() == EExpressionType.simple_constant_t) { 24329 TObjectName columnName = new TObjectName(); 24330 columnName.setString(expr.toString()); 24331 aliasColumns.add(columnName); 24332 } 24333 } 24334 } 24335 } 24336 } else if (tableItem instanceof Table) { 24337 for (TableColumn tableColumn : ((Table) tableItem).getColumns()) { 24338 if (pivotedColumns.contains(tableColumn)) { 24339 continue; 24340 } 24341 aliasColumns.add(index, (TObjectName) tableColumn.getColumnObject()); 24342 index++; 24343 } 24344 } 24345 } 24346 24347 IndexedLinkedHashMap<String, ResultColumn> aliasColumnMap = new IndexedLinkedHashMap<String, ResultColumn>(); 24348 int diffCount = pivotClause.getAliasClause().getColumns().size() - aliasColumns.size(); 24349 for (int k = 0; k < pivotClause.getAliasClause().getColumns().size(); k++) { 24350 if (pivotClause.getAliasClause().getColumns().size() > aliasColumns.size()) { 24351 if (k < diffCount) { 24352 continue; 24353 } 24354 ResultColumn resultColumn = modelFactory.createResultColumn(alias, 24355 pivotClause.getAliasClause().getColumns().getObjectName(k)); 24356 if ((k - diffCount) < aliasColumns.size()) { 24357 aliasColumnMap.put(aliasColumns.get(k - diffCount).toString(), resultColumn); 24358 } 24359 } else { 24360 ResultColumn resultColumn = modelFactory.createResultColumn(alias, 24361 pivotClause.getAliasClause().getColumns().getObjectName(k)); 24362 if (k < aliasColumns.size()) { 24363 aliasColumnMap.put(aliasColumns.get(k).toString(), resultColumn); 24364 } 24365 } 24366 } 24367 24368 for (int k = 0; k < tables.size(); k++) { 24369 Object tableItem = tables.get(k); 24370 if (tableItem instanceof ResultSet) { 24371 int resultColumnSize = ((ResultSet) tableItem).getColumns().size(); 24372 for (int x = 0; x < resultColumnSize; x++) { 24373 ResultColumn tableColumn = ((ResultSet) tableItem).getColumns().get(x); 24374 if (pivotedColumns.contains(tableColumn)) { 24375 continue; 24376 } 24377 if (tableColumn.getColumnObject() instanceof TObjectName) { 24378 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 24379 relation.setEffectType(EffectType.select); 24380 relation.setTarget(new ResultColumnRelationshipElement( 24381 aliasColumnMap.get(tableColumn.getColumnObject().toString()))); 24382 relation.addSource(new ResultColumnRelationshipElement(tableColumn)); 24383 } else if (tableColumn.getColumnObject() instanceof TResultColumn) { 24384 if (((TResultColumn) tableColumn.getColumnObject()).getFieldAttr() != null) { 24385 ResultColumn targetColumn = aliasColumnMap.get( 24386 ((TResultColumn) tableColumn.getColumnObject()).getFieldAttr().toString()); 24387 if(targetColumn == null) { 24388 continue; 24389 } 24390 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 24391 relation.setEffectType(EffectType.select); 24392 relation.setTarget(new ResultColumnRelationshipElement(targetColumn)); 24393 relation.addSource(new ResultColumnRelationshipElement(tableColumn)); 24394 } else if (((TResultColumn) tableColumn.getColumnObject()).getExpr() != null) { 24395 TExpression expr = ((TResultColumn) tableColumn.getColumnObject()).getExpr(); 24396 if (expr.getExpressionType() == EExpressionType.simple_constant_t) { 24397 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 24398 relation.setEffectType(EffectType.select); 24399 ResultColumn targetColumn = (ResultColumn) aliasColumnMap 24400 .getValueAtIndex(aliasColumnMap.size() - resultColumnSize + x); 24401 relation.setTarget(new ResultColumnRelationshipElement(targetColumn)); 24402 relation.addSource(new ResultColumnRelationshipElement(tableColumn)); 24403 } 24404 } 24405 } 24406 } 24407 } else if (tableItem instanceof Table) { 24408 for (TableColumn tableColumn : ((Table) tableItem).getColumns()) { 24409 if (pivotedColumns.contains(tableColumn)) { 24410 continue; 24411 } 24412 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 24413 relation.setEffectType(EffectType.select); 24414 relation.setTarget(new ResultColumnRelationshipElement( 24415 aliasColumnMap.get(tableColumn.getColumnObject().toString()))); 24416 relation.addSource(new TableColumnRelationshipElement(tableColumn)); 24417 } 24418 } 24419 } 24420 24421 ResultSet resultSet = modelFactory.createResultSet(stmt, 24422 isTopResultSet(stmt) && isShowTopSelectResultSet()); 24423 TResultColumnList columnList = stmt.getResultColumnList(); 24424 for (int i = 0; i < columnList.size(); i++) { 24425 TResultColumn column = columnList.getResultColumn(i); 24426 ResultColumn resultColumn = modelFactory.createAndBindingSelectSetResultColumn(resultSet, column, i); 24427 if (resultColumn.getColumnObject() instanceof TResultColumn) { 24428 TResultColumn columnObject = (TResultColumn) resultColumn.getColumnObject(); 24429 if (columnObject.getFieldAttr() != null) { 24430 if ("*".equals(getColumnName(columnObject.getFieldAttr()))) { 24431 resultColumn.setShowStar(false); 24432 for (ResultColumn tableColumn : ((ResultSet) alias).getColumns()) { 24433 resultColumn.bindStarLinkColumn((TObjectName) tableColumn.getColumnObject()); 24434 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 24435 relation.setEffectType(EffectType.select); 24436 relation.setTarget(new ResultColumnRelationshipElement(resultColumn, (TObjectName) tableColumn.getColumnObject())); 24437 relation.addSource(new ResultColumnRelationshipElement(tableColumn)); 24438 } 24439 } else { 24440 for (ResultColumn tableColumn : ((ResultSet) alias).getColumns()) { 24441 if (getColumnName(columnObject.getFieldAttr()) 24442 .equals(DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) { 24443 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 24444 relation.setEffectType(EffectType.select); 24445 relation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 24446 relation.addSource(new ResultColumnRelationshipElement(tableColumn)); 24447 break; 24448 } 24449 } 24450 } 24451 } else if (columnObject.getExpr() != null && columnObject.getExpr() 24452 .getExpressionType() == EExpressionType.sqlserver_proprietary_column_alias_t) { 24453 for (ResultColumn tableColumn : ((ResultSet) alias).getColumns()) { 24454 if (getColumnName(columnObject.getExpr().getRightOperand().getObjectOperand()) 24455 .equals(DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) { 24456 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 24457 relation.setEffectType(EffectType.select); 24458 relation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 24459 relation.addSource(new ResultColumnRelationshipElement(tableColumn)); 24460 break; 24461 } 24462 } 24463 } else if (columnObject.getExpr() != null && columnObject.getExpr() 24464 .getExpressionType() == EExpressionType.function_t) { 24465 Function function = (Function) createFunction(columnObject.getExpr().getFunctionCall()); 24466 for (ResultColumn arg : function.getColumns()) { 24467 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 24468 relation.setEffectType(EffectType.select); 24469 relation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 24470 relation.addSource(new ResultColumnRelationshipElement(arg)); 24471 } 24472 } 24473 } 24474 } 24475 } else { 24476 ResultSet resultSet = modelFactory.createResultSet(stmt, isTopResultSet(stmt)); 24477 TResultColumnList columnList = stmt.getResultColumnList(); 24478 for (int i = 0; i < columnList.size(); i++) { 24479 TResultColumn column = columnList.getResultColumn(i); 24480 ResultColumn resultColumn = modelFactory.createAndBindingSelectSetResultColumn(resultSet, column, i); 24481 if (resultColumn.getColumnObject() instanceof TResultColumn) { 24482 boolean fromFunction = false; 24483 TResultColumn columnObject = (TResultColumn) resultColumn.getColumnObject(); 24484 TObjectName resultColumnFieldAttr = columnObject.getFieldAttr(); 24485 List<TObjectName> resultColumnNames = new ArrayList<TObjectName>(); 24486 if (resultColumnFieldAttr != null) { 24487 resultColumnNames.add(resultColumnFieldAttr); 24488 } else if (columnObject.getExpr() != null 24489 && column.getExpr().getExpressionType() == EExpressionType.function_t) { 24490 extractFunctionObjectNames(column.getExpr().getFunctionCall(), resultColumnNames); 24491 fromFunction = true; 24492 } 24493 24494 if (!resultColumnNames.isEmpty()) { 24495 for (TObjectName resultColumnName : resultColumnNames) { 24496 if ("*".equals(getColumnName(resultColumnName))) { 24497 resultColumn.setShowStar(false); 24498 int index = 0; 24499 for (int k = 0; k < tables.size(); k++) { 24500 Object tableItem = tables.get(k); 24501 if (tableItem instanceof ResultSet && !(tableItem instanceof QueryTable)) { 24502 for (int x = 0; x < ((ResultSet) tableItem).getColumns().size(); x++) { 24503 ResultColumn tableColumn = ((ResultSet) tableItem).getColumns().get(x); 24504 if (pivotedColumns.contains(tableColumn)) { 24505 continue; 24506 } 24507 if (tableColumn.getColumnObject() instanceof TObjectName) { 24508 resultColumn.bindStarLinkColumn( 24509 (TObjectName) tableColumn.getColumnObject()); 24510 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 24511 relation.setEffectType(EffectType.select); 24512 relation.setTarget(new ResultColumnRelationshipElement(resultColumn, (TObjectName) tableColumn.getColumnObject())); 24513 relation.addSource(new ResultColumnRelationshipElement(tableColumn)); 24514 } else if (tableColumn.getColumnObject() instanceof TResultColumn) { 24515 if (((TResultColumn) tableColumn.getColumnObject()) 24516 .getFieldAttr() != null) { 24517 if (tableColumn.hasStarLinkColumn()) { 24518 for (int z = 0; z < tableColumn.getStarLinkColumnList() 24519 .size(); z++) { 24520 ResultColumn resultColumn1 = modelFactory 24521 .createResultColumn((ResultSet) tableItem, 24522 tableColumn.getStarLinkColumnList().get(z)); 24523 DataFlowRelationship relation = modelFactory 24524 .createDataFlowRelation(); 24525 relation.setEffectType(EffectType.select); 24526 relation.setTarget( 24527 new ResultColumnRelationshipElement(resultColumn)); 24528 relation.addSource( 24529 new ResultColumnRelationshipElement(resultColumn1)); 24530 tableColumn.getStarLinkColumns().remove(getColumnName( 24531 tableColumn.getStarLinkColumnList().get(z))); 24532 z--; 24533 } 24534 } else { 24535 resultColumn.bindStarLinkColumn( 24536 ((TResultColumn) tableColumn.getColumnObject()) 24537 .getFieldAttr()); 24538 DataFlowRelationship relation = modelFactory 24539 .createDataFlowRelation(); 24540 relation.setEffectType(EffectType.select); 24541 relation.setTarget( 24542 new ResultColumnRelationshipElement(resultColumn, ((TResultColumn) tableColumn.getColumnObject()) 24543 .getFieldAttr())); 24544 relation.addSource( 24545 new ResultColumnRelationshipElement(tableColumn)); 24546 } 24547 } else if (((TResultColumn) tableColumn.getColumnObject()) 24548 .getExpr() != null) { 24549 TExpression expr = ((TResultColumn) tableColumn.getColumnObject()) 24550 .getExpr(); 24551 if (expr.getExpressionType() == EExpressionType.simple_constant_t) { 24552 TObjectName columnName = new TObjectName(); 24553 columnName.setString(expr.toString()); 24554 resultColumn.bindStarLinkColumn(columnName); 24555 DataFlowRelationship relation = modelFactory 24556 .createDataFlowRelation(); 24557 relation.setEffectType(EffectType.select); 24558 relation.setTarget( 24559 new ResultColumnRelationshipElement(resultColumn, columnName)); 24560 relation.addSource( 24561 new ResultColumnRelationshipElement(tableColumn)); 24562 } 24563 } 24564 } 24565 } 24566 } else if (tableItem instanceof Table) { 24567 for (TableColumn tableColumn : ((Table) tableItem).getColumns()) { 24568 if (pivotedColumns.contains(tableColumn)) { 24569 continue; 24570 } 24571 resultColumn.bindStarLinkColumn((TObjectName) tableColumn.getColumnObject(), 24572 index); 24573 index++; 24574 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 24575 relation.setEffectType(EffectType.select); 24576 relation.setTarget(new ResultColumnRelationshipElement(resultColumn, (TObjectName) tableColumn.getColumnObject())); 24577 relation.addSource(new TableColumnRelationshipElement(tableColumn)); 24578 } 24579 } else if (tableItem instanceof QueryTable) { 24580 for (ResultColumn tableColumn : ((QueryTable) tableItem).getColumns()) { 24581 if (pivotedColumns.contains(tableColumn)) { 24582 continue; 24583 } 24584 TObjectName column1 = new TObjectName(); 24585 column1.setString(tableColumn.getName()); 24586 resultColumn.bindStarLinkColumn(column1, index); 24587 index++; 24588 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 24589 relation.setEffectType(EffectType.select); 24590 relation.setTarget(new ResultColumnRelationshipElement(resultColumn, column1)); 24591 relation.addSource(new ResultColumnRelationshipElement(tableColumn), false); 24592 } 24593 } 24594 } 24595 } else { 24596 ResultColumn pivotedTableColumn = getPivotedTableColumn(pivotedTable, resultColumnName); 24597 if (pivotedTableColumn != null) { 24598 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 24599 relation.setEffectType(EffectType.select); 24600 relation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 24601 relation.addSource(new ResultColumnRelationshipElement(pivotedTableColumn)); 24602 } else { 24603 for (int k = 0; k < tables.size(); k++) { 24604 Object tableItem = tables.get(k); 24605 if (tableItem instanceof ResultSet) { 24606 for (ResultColumn tableColumn : ((ResultSet) tableItem).getColumns()) { 24607 if (DlineageUtil 24608 .getIdentifierNormalColumnName(tableColumn.getName()).equals(getColumnName(resultColumnName))) { 24609 if (fromFunction) { 24610 Function function = (Function)createPivotedFunction(column.getExpr().getFunctionCall(), tableColumn); 24611 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 24612 relation.setEffectType(EffectType.select); 24613 relation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 24614 if (function.getColumns() != null && !function.getColumns().isEmpty()) { 24615 for (ResultColumn functionColumn : function.getColumns()) { 24616 relation.addSource(new ResultColumnRelationshipElement(functionColumn)); 24617 } 24618 } 24619 } else { 24620 DataFlowRelationship relation = modelFactory 24621 .createDataFlowRelation(); 24622 relation.setEffectType(EffectType.select); 24623 relation.setTarget( 24624 new ResultColumnRelationshipElement(resultColumn)); 24625 relation.addSource( 24626 new ResultColumnRelationshipElement(tableColumn)); 24627 } 24628 break; 24629 } 24630 } 24631 } else if (tableItem instanceof Table) { 24632 for (TableColumn tableColumn : ((Table) tableItem).getColumns()) { 24633 if (getColumnName(resultColumnName).equals(DlineageUtil 24634 .getIdentifierNormalColumnName(tableColumn.getName()))) { 24635 DataFlowRelationship relation = modelFactory 24636 .createDataFlowRelation(); 24637 relation.setEffectType(EffectType.select); 24638 relation.setTarget( 24639 new ResultColumnRelationshipElement(resultColumn)); 24640 relation.addSource( 24641 new TableColumnRelationshipElement(tableColumn)); 24642 break; 24643 } 24644 } 24645 } 24646 } 24647 } 24648 } 24649 } 24650 } else if (columnObject.getExpr() != null && columnObject.getExpr() 24651 .getExpressionType() == EExpressionType.sqlserver_proprietary_column_alias_t) { 24652 for (int k = 0; k < tables.size(); k++) { 24653 Object tableItem = tables.get(k); 24654 if (tableItem instanceof ResultSet) { 24655 for (ResultColumn tableColumn : ((ResultSet) tableItem).getColumns()) { 24656 if (columnObject.getExpr().getRightOperand().getObjectOperand() != null 24657 && getColumnName( 24658 columnObject.getExpr().getRightOperand().getObjectOperand()) 24659 .equals(DlineageUtil.getIdentifierNormalColumnName( 24660 tableColumn.getName()))) { 24661 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 24662 relation.setEffectType(EffectType.select); 24663 relation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 24664 relation.addSource(new ResultColumnRelationshipElement(tableColumn)); 24665 break; 24666 } else if (columnObject.getExpr().getRightOperand() 24667 .getExpressionType() == EExpressionType.function_t) { 24668 List<TExpression> expressions = new ArrayList<TExpression>(); 24669 getFunctionExpressions(expressions, new ArrayList<TExpression>(), 24670 columnObject.getExpr().getRightOperand().getFunctionCall()); 24671 for (int j = 0; j < expressions.size(); j++) { 24672 columnsInExpr visitor = new columnsInExpr(); 24673 expressions.get(j).inOrderTraverse(visitor); 24674 List<TObjectName> objectNames = visitor.getObjectNames(); 24675 if (objectNames == null) { 24676 continue; 24677 } 24678 for (TObjectName columnName : objectNames) { 24679 if (getColumnName(columnName).equals(DlineageUtil 24680 .getIdentifierNormalColumnName(tableColumn.getName()))) { 24681 DataFlowRelationship relation = modelFactory 24682 .createDataFlowRelation(); 24683 relation.setEffectType(EffectType.select); 24684 relation.setTarget( 24685 new ResultColumnRelationshipElement(resultColumn)); 24686 relation.addSource( 24687 new ResultColumnRelationshipElement(tableColumn)); 24688 break; 24689 } 24690 } 24691 } 24692 } 24693 } 24694 } else if (tableItem instanceof Table) { 24695 for (TableColumn tableColumn : ((Table) tableItem).getColumns()) { 24696 if (getColumnName(columnObject.getExpr().getRightOperand().getObjectOperand()) 24697 .equals(DlineageUtil 24698 .getIdentifierNormalColumnName(tableColumn.getName()))) { 24699 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 24700 relation.setEffectType(EffectType.select); 24701 relation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 24702 relation.addSource(new TableColumnRelationshipElement(tableColumn)); 24703 break; 24704 } 24705 } 24706 } 24707 } 24708 } 24709 } 24710 } 24711 } 24712 24713 analyzeSelectIntoClause(stmt); 24714 } 24715 24716 private Function createPivotedFunction(TFunctionCall functionCall, ResultColumn sourceColumn) { 24717 Function function = modelFactory.createFunction((TFunctionCall) functionCall); 24718 ResultColumn column = modelFactory.createFunctionResultColumn(function, 24719 ((TFunctionCall) functionCall).getFunctionName()); 24720 if ("COUNT".equalsIgnoreCase(((TFunctionCall) functionCall).getFunctionName().toString())) { 24721 // @see https://e.gitee.com/gudusoft/issues/list?issue=I40NUP 24722 // COUNT特殊处理,不和参数关联 24723 if (option.isShowCountTableColumn()) { 24724 analyzePivotedFunctionArgumentsDataFlowRelation(column, functionCall, sourceColumn); 24725 } 24726 } else { 24727 analyzePivotedFunctionArgumentsDataFlowRelation(column, functionCall, sourceColumn); 24728 Set<Object> functionTableModelObjs = modelManager.getFunctionTable(getIdentifiedFunctionName(function)); 24729 if (functionTableModelObjs!=null && functionTableModelObjs.iterator().next() instanceof ResultSet) { 24730 ResultSet functionTableModel = (ResultSet) functionTableModelObjs.iterator().next(); 24731 if (functionTableModel.getColumns() != null) { 24732 for (int j = 0; j < functionTableModel.getColumns().size(); j++) { 24733 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 24734 relation.setEffectType(EffectType.select); 24735 relation.setTarget(new ResultColumnRelationshipElement(column)); 24736 relation.addSource(new ResultColumnRelationshipElement( 24737 functionTableModel.getColumns().get(j))); 24738 } 24739 } 24740 } 24741 } 24742 return function; 24743 } 24744 24745 private void analyzePivotedFunctionArgumentsDataFlowRelation(ResultColumn column, TFunctionCall functionCall, 24746 ResultColumn sourceColumn) { 24747 List<TExpression> directExpressions = new ArrayList<TExpression>(); 24748 List<TExpression> indirectExpressions = new ArrayList<TExpression>(); 24749 24750 getFunctionExpressions(directExpressions, indirectExpressions, functionCall); 24751 24752 for (int j = 0; j < directExpressions.size(); j++) { 24753 columnsInExpr visitor = new columnsInExpr(); 24754 directExpressions.get(j).inOrderTraverse(visitor); 24755 24756 List<TObjectName> objectNames = visitor.getObjectNames(); 24757 List<TParseTreeNode> constants = visitor.getConstants(); 24758 24759 if (objectNames != null) { 24760 for (TObjectName name : objectNames) { 24761 if (DlineageUtil.compareColumnIdentifier(name.toString(), sourceColumn.getName())) { 24762 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 24763 relation.setEffectType(EffectType.select); 24764 relation.setTarget(new ResultColumnRelationshipElement(column)); 24765 relation.addSource(new ResultColumnRelationshipElement(sourceColumn)); 24766 } 24767 } 24768 } 24769 if (constants != null) { 24770 for (TParseTreeNode name : constants) { 24771 if (name.toString().equals(sourceColumn.getName())) { 24772 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 24773 relation.setEffectType(EffectType.select); 24774 relation.setTarget(new ResultColumnRelationshipElement(column)); 24775 relation.addSource(new ResultColumnRelationshipElement(sourceColumn)); 24776 } 24777 } 24778 } 24779 } 24780 } 24781 24782 private void extractFunctionObjectNames(TFunctionCall functionCall, List<TObjectName> resultColumnNames) { 24783 List<TExpression> directExpressions = new ArrayList<TExpression>(); 24784 List<TExpression> indirectExpressions = new ArrayList<TExpression>(); 24785 24786 getFunctionExpressions(directExpressions, indirectExpressions, functionCall); 24787 24788 for (int j = 0; j < directExpressions.size(); j++) { 24789 columnsInExpr visitor = new columnsInExpr(); 24790 directExpressions.get(j).inOrderTraverse(visitor); 24791 24792 List<TObjectName> objectNames = visitor.getObjectNames(); 24793 List<TParseTreeNode> functions = visitor.getFunctions(); 24794 List<TParseTreeNode> constants = visitor.getConstants(); 24795 24796 if (objectNames != null) { 24797 resultColumnNames.addAll(objectNames); 24798 } 24799 24800 if (constants != null) { 24801 for(TParseTreeNode item: constants) { 24802 if(item instanceof TConstant && ((TConstant) item).getLiteralType().getText().equals(ELiteralType.string_et.getText())) { 24803 TObjectName object = new TObjectName(); 24804 object.setString(item.toString()); 24805 resultColumnNames.add(object); 24806 } 24807 } 24808 } 24809 24810 if (functions != null && !functions.isEmpty()) { 24811 for (TParseTreeNode function : functions) { 24812 if (function instanceof TFunctionCall) { 24813 extractFunctionObjectNames((TFunctionCall) function, resultColumnNames); 24814 } 24815 } 24816 } 24817 } 24818 } 24819 24820 private String getResultColumnString(TResultColumn resultColumn) { 24821 if (resultColumn.getAliasClause() != null) { 24822 return resultColumn.getAliasClause().toString(); 24823 } 24824 return resultColumn.toString(); 24825 } 24826 24827 private void analyzeBigQueryUnnest(TSelectSqlStatement stmt, TTable table) { 24828 Table unnestTable = modelFactory.createTableFromCreateDDL(table, false, getTempTableName(table)); 24829 unnestTable.setSubType(SubType.unnest); 24830 TUnnestClause clause = table.getUnnestClause(); 24831 TExpression arrayExpr = clause.getArrayExpr(); 24832 if (arrayExpr == null){ 24833 if (clause.getColumns() != null) { 24834 for (TObjectName column : clause.getColumns()) { 24835 if (clause.getDerivedColumnList() != null) { 24836 unnestTable.setCreateTable(true); 24837 for (int i = 0; i < clause.getDerivedColumnList().size(); i++) { 24838 TObjectName columnName = new TObjectName(); 24839 columnName.setString(column.getColumnNameOnly() + "." 24840 + clause.getDerivedColumnList().getObjectName(i).getColumnNameOnly()); 24841 TableColumn tableColumn = modelFactory.createTableColumn(unnestTable, columnName, true); 24842 List<TObjectName> columns = new ArrayList<TObjectName>(); 24843 columns.add(column); 24844 analyzeDataFlowRelation(tableColumn, columns, EffectType.select, null); 24845 } 24846 } 24847 else { 24848 unnestTable.setCreateTable(true); 24849 boolean find = false; 24850 if (column.getSourceTable() != null && modelManager.getModel(column.getSourceTable()) instanceof Table) { 24851 Table sourceTable = (Table)modelManager.getModel(column.getSourceTable()); 24852 if(sourceTable!=null) { 24853 for(TableColumn tableColumn: sourceTable.getColumns()) { 24854 if(tableColumn.isStruct()) { 24855 List<String> names = SQLUtil.parseNames(tableColumn.getName()); 24856 if (SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotColumn, names.get(0), column.getColumnNameOnly())) { 24857 TObjectName columnName = new TObjectName(); 24858 columnName.setString(tableColumn.getName()); 24859 TableColumn unnestTableColumn = modelFactory.createTableColumn(unnestTable, 24860 columnName, true); 24861 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 24862 relation.setEffectType(EffectType.select); 24863 relation.setTarget(new TableColumnRelationshipElement(unnestTableColumn)); 24864 relation.addSource(new TableColumnRelationshipElement(tableColumn)); 24865 find = true; 24866 } 24867 } 24868 } 24869 } 24870 } 24871 if (!find) { 24872 TObjectName colName = column; 24873 if (table.getAliasClause() != null && table.getAliasClause().getAliasName() != null) { 24874 colName = table.getAliasClause().getAliasName(); 24875 } 24876 TableColumn tableColumn = modelFactory.createTableColumn(unnestTable, colName, true); 24877 List<TObjectName> columns = new ArrayList<TObjectName>(); 24878 columns.add(column); 24879 analyzeDataFlowRelation(tableColumn, columns, EffectType.select, null); 24880 } 24881 } 24882 } 24883 } 24884 return; 24885 } 24886 List<TExpression> expressions = new ArrayList<TExpression>(); 24887 TExpressionList values = arrayExpr.getExprList(); 24888 if (values == null) { 24889 expressions.add(arrayExpr); 24890 } 24891 else { 24892 for(TExpression value: values) { 24893 expressions.add(value); 24894 } 24895 } 24896 for (TExpression value : expressions) { 24897 unnestTable.setCreateTable(true); 24898 if (value.getExpressionType() == EExpressionType.simple_object_name_t) { 24899 if (table.getAliasClause() != null) { 24900 TableColumn tableColumn = modelFactory.createTableColumn(unnestTable, 24901 table.getAliasClause().getAliasName(), true); 24902 columnsInExpr visitor = new columnsInExpr(); 24903 value.inOrderTraverse(visitor); 24904 List<TObjectName> columns = visitor.getObjectNames(); 24905 analyzeDataFlowRelation(tableColumn, columns, EffectType.select, null); 24906 } else { 24907 TResultColumnList resultColumnList = stmt.getResultColumnList(); 24908 for (int i = 0; i < resultColumnList.size(); i++) { 24909 TObjectName firstColumn = new TObjectName(); 24910 firstColumn.setString(resultColumnList.getResultColumn(0).getColumnNameOnly()); 24911 TableColumn tableColumn = modelFactory.createTableColumn(unnestTable, firstColumn, true); 24912 columnsInExpr visitor = new columnsInExpr(); 24913 value.inOrderTraverse(visitor); 24914 List<TObjectName> columns = visitor.getObjectNames(); 24915 analyzeDataFlowRelation(tableColumn, columns, EffectType.select, null); 24916 } 24917 } 24918 } else if (value.getExpressionType() == EExpressionType.function_t) { 24919 if (table.getAliasClause() != null) { 24920 TableColumn tableColumn = modelFactory.createTableColumn(unnestTable, 24921 table.getAliasClause().getAliasName(), true); 24922 columnsInExpr visitor = new columnsInExpr(); 24923 value.inOrderTraverse(visitor); 24924 List<TParseTreeNode> functions = visitor.getFunctions(); 24925 analyzeFunctionDataFlowRelation(tableColumn, functions, EffectType.select, null); 24926 } else { 24927 TResultColumnList resultColumnList = stmt.getResultColumnList(); 24928 for (int i = 0; i < resultColumnList.size(); i++) { 24929 TObjectName firstColumn = new TObjectName(); 24930 firstColumn.setString(resultColumnList.getResultColumn(0).getColumnNameOnly()); 24931 TableColumn tableColumn = modelFactory.createTableColumn(unnestTable, firstColumn, true); 24932 columnsInExpr visitor = new columnsInExpr(); 24933 value.inOrderTraverse(visitor); 24934 List<TObjectName> columns = visitor.getObjectNames(); 24935 analyzeDataFlowRelation(tableColumn, columns, EffectType.select, null); 24936 } 24937 } 24938 } else if (value.getExpressionType() == EExpressionType.simple_constant_t) { 24939 if (table.getAliasClause() != null) { 24940 TableColumn tableColumn = modelFactory.createTableColumn(unnestTable, 24941 table.getAliasClause().getAliasName(), true); 24942 columnsInExpr visitor = new columnsInExpr(); 24943 value.inOrderTraverse(visitor); 24944 List<TParseTreeNode> constants = visitor.getConstants(); 24945 analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select, null); 24946 } else { 24947 TObjectName firstColumn = new TObjectName(); 24948 firstColumn.setString("f0_"); 24949 TableColumn tableColumn = modelFactory.createTableColumn(unnestTable, firstColumn, true); 24950 columnsInExpr visitor = new columnsInExpr(); 24951 value.inOrderTraverse(visitor); 24952 List<TParseTreeNode> constants = visitor.getConstants(); 24953 analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select, null); 24954 } 24955 } else if (value.getExpressionType() == EExpressionType.list_t) { 24956 if (arrayExpr.getTypeName() != null && arrayExpr.getTypeName().getColumnDefList() != null) { 24957 for (int i = 0; i < arrayExpr.getTypeName().getColumnDefList().size(); i++) { 24958 TColumnDefinition column = arrayExpr.getTypeName().getColumnDefList().getColumn(i); 24959 if (column != null && column.getColumnName() != null) { 24960 TableColumn tableColumn = modelFactory.createTableColumn(unnestTable, 24961 column.getColumnName(), true); 24962 columnsInExpr visitor = new columnsInExpr(); 24963 value.getExprList().getExpression(i).inOrderTraverse(visitor); 24964 List<TParseTreeNode> constants = visitor.getConstants(); 24965 analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select, null); 24966 } 24967 } 24968 } else { 24969 for (int i = 0; i < value.getExprList().size(); i++) { 24970 TObjectName firstColumn = new TObjectName(); 24971 firstColumn.setString("f" + i + "_"); 24972 TableColumn tableColumn = modelFactory.createTableColumn(unnestTable, firstColumn, true); 24973 columnsInExpr visitor = new columnsInExpr(); 24974 value.getExprList().getExpression(i).inOrderTraverse(visitor); 24975 List<TParseTreeNode> constants = visitor.getConstants(); 24976 analyzeConstantDataFlowRelation(tableColumn, constants, EffectType.select, null); 24977 } 24978 } 24979 } else if (value.getExpressionType() == EExpressionType.subquery_t) { 24980 analyzeSelectStmt(value.getSubQuery()); 24981 ResultSet resultSet = (ResultSet) modelManager.getModel(value.getSubQuery()); 24982 if (resultSet != null) { 24983 for (int i = 0; i < resultSet.getColumns().size(); i++) { 24984 TObjectName columnName = new TObjectName(); 24985 if(resultSet.getColumns().get(i).getAlias()!=null) { 24986 columnName.setString(resultSet.getColumns().get(i).getAlias()); 24987 } 24988 else { 24989 columnName.setString(getColumnName(resultSet.getColumns().get(i).getName())); 24990 } 24991 TableColumn tableColumn = modelFactory.createTableColumn(unnestTable, columnName, true); 24992 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 24993 relation.setEffectType(EffectType.select); 24994 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 24995 relation.addSource( 24996 new ResultColumnRelationshipElement(resultSet.getColumns().get(i))); 24997 } 24998 } 24999 } 25000 } 25001 } 25002 25003 private void analyzePrestoUnnest(TSelectSqlStatement stmt, TTable table) { 25004 List<Table> tables = new ArrayList<Table>(); 25005 TTable targetTable = stmt.getTables().getTable(0); 25006 Table stmtTable = modelFactory.createTable(targetTable); 25007 Object sourceTable = null; 25008 if (targetTable.getSubquery() != null) { 25009 analyzeSelectStmt(targetTable.getSubquery()); 25010 if (targetTable.getSubquery().isCombinedQuery()) { 25011 ResultSet sourceResultSet = (ResultSet) modelManager.getModel(targetTable.getSubquery()); 25012 sourceTable = sourceResultSet; 25013 } else if (targetTable.getSubquery().getResultColumnList() != null) { 25014 ResultSet sourceResultSet = (ResultSet) modelManager 25015 .getModel(targetTable.getSubquery().getResultColumnList()); 25016 sourceTable = sourceResultSet; 25017 } else if (targetTable.getSubquery().getValueClause() != null) { 25018 List<TResultColumnList> rowList = targetTable.getSubquery().getValueClause().getRows(); 25019 if (rowList != null && rowList.size() > 0) { 25020 Table valuesTable = modelFactory.createTableByName("Values-Table", true); 25021 int columnCount = rowList.get(0).size(); 25022 for (int j = 1; j <= columnCount; j++) { 25023 TObjectName columnName = new TObjectName(); 25024 TResultColumn columnObject = rowList.get(0).getResultColumn(j - 1); 25025 if (columnObject.getExpr().getExpressionType() == EExpressionType.typecast_t) { 25026 columnName.setString(columnObject.getExpr().getLeftOperand().toString()); 25027 } else { 25028 columnName.setString(columnObject.getExpr().toString()); 25029 } 25030 modelFactory.createTableColumn(valuesTable, columnName, true); 25031 } 25032 valuesTable.setCreateTable(true); 25033 valuesTable.setSubType(SubType.values_table); 25034 sourceTable = valuesTable; 25035 } 25036 } 25037 } 25038 tables.add(stmtTable); 25039 Table unnestTable = modelFactory.createTable(table); 25040 unnestTable.setSubType(SubType.unnest); 25041 tables.add(unnestTable); 25042 if (table.getAliasClause() != null && table.getAliasClause().getColumns() != null 25043 && table.getUnnestClause().getColumns() != null) { 25044 int unnestTableSize = table.getUnnestClause().getColumns().size(); 25045 for (int i = 0; i < table.getAliasClause().getColumns().size(); i++) { 25046 TableColumn sourceColumn = null; 25047 if (unnestTableSize > i) { 25048 sourceColumn = modelFactory.createTableColumn(stmtTable, 25049 table.getUnnestClause().getColumns().getObjectName(i), true); 25050 if (sourceTable instanceof Table) { 25051 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 25052 relation.setEffectType(EffectType.select); 25053 relation.setTarget(new TableColumnRelationshipElement(sourceColumn)); 25054 relation.addSource( 25055 new TableColumnRelationshipElement(((Table) sourceTable).getColumns().get(i))); 25056 } else if (sourceTable instanceof ResultSet) { 25057 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 25058 relation.setEffectType(EffectType.select); 25059 relation.setTarget(new TableColumnRelationshipElement(sourceColumn)); 25060 relation.addSource( 25061 new ResultColumnRelationshipElement(((ResultSet) sourceTable).getColumns().get(i))); 25062 } 25063 } else { 25064 sourceColumn = modelFactory.createTableColumn(stmtTable, 25065 table.getUnnestClause().getColumns().getObjectName(unnestTableSize - 1), true); 25066 } 25067 TableColumn targetColumn = modelFactory.createTableColumn(unnestTable, 25068 table.getAliasClause().getColumns().getObjectName(i), true); 25069 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 25070 relation.setEffectType(EffectType.select); 25071 relation.setTarget(new TableColumnRelationshipElement(targetColumn)); 25072 relation.addSource(new TableColumnRelationshipElement(sourceColumn)); 25073 } 25074 } 25075 25076 ResultSet resultSet = modelFactory.createResultSet(stmt, 25077 isTopResultSet(stmt) && isShowTopSelectResultSet()); 25078 TResultColumnList columnList = stmt.getResultColumnList(); 25079 for (int i = 0; i < columnList.size(); i++) { 25080 TResultColumn column = columnList.getResultColumn(i); 25081 ResultColumn resultColumn = modelFactory.createSelectSetResultColumn(resultSet, column, i); 25082 if (resultColumn.getColumnObject() instanceof TResultColumn) { 25083 TResultColumn columnObject = (TResultColumn) resultColumn.getColumnObject(); 25084 if (columnObject.getFieldAttr() != null) { 25085 if ("*".equals(getColumnName(columnObject.getFieldAttr()))) { 25086 for (int k = 0; k < tables.size(); k++) { 25087 Table tableItem = tables.get(k); 25088 for (TableColumn tableColumn : tableItem.getColumns()) { 25089 resultColumn.bindStarLinkColumn(tableColumn.getColumnObject()); 25090 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 25091 relation.setEffectType(EffectType.select); 25092 relation.setTarget(new ResultColumnRelationshipElement(resultColumn, tableColumn.getColumnObject())); 25093 relation.addSource(new TableColumnRelationshipElement(tableColumn)); 25094 } 25095 } 25096 } else { 25097 boolean match = false; 25098 for (int k = 0; k < tables.size(); k++) { 25099 Table tableItem = tables.get(k); 25100 for (TableColumn tableColumn : tableItem.getColumns()) { 25101 if (getColumnName(columnObject.getFieldAttr()) 25102 .equals(DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) { 25103 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 25104 relation.setEffectType(EffectType.select); 25105 relation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 25106 relation.addSource(new TableColumnRelationshipElement(tableColumn)); 25107 match = true; 25108 } 25109 } 25110 } 25111 if (!match) { 25112 TableColumn tableColumn = modelFactory.createTableColumn(stmtTable, 25113 columnObject.getFieldAttr(), false); 25114 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 25115 relation.setEffectType(EffectType.select); 25116 relation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 25117 relation.addSource(new TableColumnRelationshipElement(tableColumn)); 25118 } 25119 } 25120 } 25121 } 25122 } 25123 } 25124 25125 private void analyzeLateralView(TSelectSqlStatement stmt, TTable table, ArrayList<TLateralView> lateralViews) { 25126 List<Object> tables = new ArrayList<Object>(); 25127 Object stmtTable = null; 25128 if(table.getSubquery()!=null) { 25129 stmtTable = modelFactory.createQueryTable(table); 25130 tables.add(stmtTable); 25131 } 25132 else { 25133 stmtTable = modelFactory.createTable(table); 25134 tables.add(stmtTable); 25135 } 25136 for (int i = 0; i < lateralViews.size(); i++) { 25137 TLateralView lateralView = lateralViews.get(i); 25138 TFunctionCall functionCall = lateralView.getUdtf(); 25139 List<TExpression> expressions = new ArrayList<TExpression>(); 25140 if (functionCall == null) { 25141 continue; 25142 } 25143 Function function = modelFactory.createFunction(functionCall); 25144 ResultColumn column = modelFactory.createFunctionResultColumn(function, 25145 ((TFunctionCall) functionCall).getFunctionName()); 25146 25147 Table lateralTable = null; 25148 if (lateralView.getTableAlias() != null) { 25149 lateralTable = modelFactory.createTableByName(lateralView.getTableAlias().getAliasName(), true); 25150 } else { 25151 lateralTable = modelFactory.createTableByName(functionCall.toString(), true); 25152 } 25153 25154 for (int j = 0; j < lateralView.getColumnAliasList().size(); j++) { 25155 TObjectName viewColumn = lateralView.getColumnAliasList().getObjectName(j); 25156 TableColumn tableColumn = modelFactory.createTableColumn(lateralTable, viewColumn, true); 25157 25158 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 25159 relation.setEffectType(EffectType.select); 25160 relation.setTarget(new TableColumnRelationshipElement(tableColumn)); 25161 relation.addSource(new ResultColumnRelationshipElement(column)); 25162 } 25163 25164 getFunctionExpressions(expressions, new ArrayList<TExpression>(), functionCall); 25165 for (int j = 0; j < expressions.size(); j++) { 25166 columnsInExpr visitor = new columnsInExpr(); 25167 expressions.get(j).inOrderTraverse(visitor); 25168 List<TObjectName> objectNames = visitor.getObjectNames(); 25169 if (objectNames == null) { 25170 continue; 25171 } 25172 for (TObjectName columnName : objectNames) { 25173 boolean match = false; 25174 for (int k = 0; k < tables.size(); k++) { 25175 Object item = tables.get(k); 25176 if(item instanceof Table) { 25177 Table tableItem = (Table)item; 25178 for (TableColumn tableColumn : tableItem.getColumns()) { 25179 if (getColumnName(columnName) 25180 .equals(DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) { 25181 match = true; 25182 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 25183 relation.setEffectType(EffectType.select); 25184 relation.setTarget(new ResultColumnRelationshipElement(column)); 25185 relation.addSource(new TableColumnRelationshipElement(tableColumn)); 25186 } 25187 } 25188 } 25189 else { 25190 ResultSet tableItem = (ResultSet)item; 25191 for (ResultColumn tableColumn : tableItem.getColumns()) { 25192 if (getColumnName(columnName) 25193 .equals(DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) { 25194 match = true; 25195 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 25196 relation.setEffectType(EffectType.select); 25197 relation.setTarget(new ResultColumnRelationshipElement(column)); 25198 relation.addSource(new ResultColumnRelationshipElement(tableColumn)); 25199 } 25200 } 25201 } 25202 } 25203 if (!match) { 25204 if(stmtTable instanceof Table) { 25205 TableColumn tableColumn = modelFactory.createTableColumn((Table)stmtTable, columnName, false); 25206 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 25207 relation.setEffectType(EffectType.select); 25208 relation.setTarget(new ResultColumnRelationshipElement(column)); 25209 relation.addSource(new TableColumnRelationshipElement(tableColumn)); 25210 } 25211 else { 25212 ResultColumn tableColumn = modelFactory.createResultColumn((ResultSet)stmtTable, columnName, false); 25213 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 25214 relation.setEffectType(EffectType.select); 25215 relation.setTarget(new ResultColumnRelationshipElement(column)); 25216 relation.addSource(new ResultColumnRelationshipElement(tableColumn)); 25217 } 25218 } 25219 } 25220 List<TParseTreeNode> constants = visitor.getConstants(); 25221 if (!constants.isEmpty()) { 25222 if (option.isShowConstantTable()) { 25223 Table constantTable = modelFactory.createConstantsTable(stmtStack.peek()); 25224 for (TParseTreeNode constant : constants) { 25225 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 25226 relation.setEffectType(EffectType.select); 25227 relation.setTarget(new ResultColumnRelationshipElement(column)); 25228 if (constant instanceof TConstant) { 25229 TableColumn constantColumn = modelFactory.createTableColumn(constantTable, 25230 (TConstant) constant); 25231 relation.addSource(new ConstantRelationshipElement(constantColumn)); 25232 } else if (constant instanceof TObjectName) { 25233 TableColumn constantColumn = modelFactory.createTableColumn(constantTable, 25234 (TObjectName) constant, false); 25235 relation.addSource(new ConstantRelationshipElement(constantColumn)); 25236 } 25237 } 25238 } 25239 } 25240 25241 List<TParseTreeNode> functions = visitor.getFunctions(); 25242 if (functions != null && !functions.isEmpty()) { 25243 analyzeFunctionDataFlowRelation(column, functions, EffectType.function); 25244 } 25245 } 25246 tables.add(lateralTable); 25247 } 25248 25249 ResultSet resultSet = modelFactory.createResultSet(stmt, 25250 isTopResultSet(stmt) && isShowTopSelectResultSet()); 25251 TResultColumnList columnList = stmt.getResultColumnList(); 25252 for (int i = 0; i < columnList.size(); i++) { 25253 TResultColumn column = columnList.getResultColumn(i); 25254 ResultColumn resultColumn = modelFactory.createSelectSetResultColumn(resultSet, column, i); 25255 if (resultColumn.getColumnObject() instanceof TResultColumn) { 25256 TResultColumn columnObject = (TResultColumn) resultColumn.getColumnObject(); 25257 if (columnObject.getFieldAttr() != null) { 25258 if ("*".equals(getColumnName(columnObject.getFieldAttr()))) { 25259 for (int k = 0; k < tables.size(); k++) { 25260 Object item = tables.get(k); 25261 if(item instanceof Table) { 25262 Table tableItem = (Table)item; 25263 for (TableColumn tableColumn : tableItem.getColumns()) { 25264 resultColumn.bindStarLinkColumn(tableColumn.getColumnObject()); 25265 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 25266 relation.setEffectType(EffectType.select); 25267 relation.setTarget(new ResultColumnRelationshipElement(resultColumn, tableColumn.getColumnObject())); 25268 relation.addSource(new TableColumnRelationshipElement(tableColumn)); 25269 } 25270 } 25271 else { 25272 ResultSet tableItem = (ResultSet)item; 25273 for (ResultColumn tableColumn : tableItem.getColumns()) { 25274 TObjectName linkColumn = new TObjectName(); 25275 linkColumn.setString(tableColumn.getName()); 25276 resultColumn.bindStarLinkColumn(linkColumn); 25277 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 25278 relation.setEffectType(EffectType.select); 25279 relation.setTarget(new ResultColumnRelationshipElement(resultColumn, linkColumn)); 25280 relation.addSource(new ResultColumnRelationshipElement(tableColumn)); 25281 } 25282 } 25283 } 25284 } else { 25285 boolean match = false; 25286 for (int k = 0; k < tables.size(); k++) { 25287 Object item = tables.get(k); 25288 if(item instanceof Table) { 25289 Table tableItem = (Table)item; 25290 for (TableColumn tableColumn : tableItem.getColumns()) { 25291 if (getColumnName(columnObject.getFieldAttr()) 25292 .equals(DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) { 25293 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 25294 relation.setEffectType(EffectType.select); 25295 relation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 25296 relation.addSource(new TableColumnRelationshipElement(tableColumn)); 25297 match = true; 25298 } 25299 } 25300 } 25301 else { 25302 ResultSet tableItem = (ResultSet)item; 25303 for (ResultColumn tableColumn : tableItem.getColumns()) { 25304 if (getColumnName(columnObject.getFieldAttr()) 25305 .equals(DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) { 25306 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 25307 relation.setEffectType(EffectType.select); 25308 relation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 25309 relation.addSource(new ResultColumnRelationshipElement(tableColumn)); 25310 match = true; 25311 } 25312 } 25313 } 25314 } 25315 if (!match) { 25316 if(stmtTable instanceof Table) { 25317 TableColumn tableColumn = modelFactory.createTableColumn((Table)stmtTable, 25318 columnObject.getFieldAttr(), false); 25319 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 25320 relation.setEffectType(EffectType.select); 25321 relation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 25322 relation.addSource(new TableColumnRelationshipElement(tableColumn)); 25323 } 25324 else { 25325 ResultColumn tableColumn = modelFactory.createResultColumn((ResultSet)stmtTable, 25326 columnObject.getFieldAttr(), false); 25327 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 25328 relation.setEffectType(EffectType.select); 25329 relation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 25330 relation.addSource(new ResultColumnRelationshipElement(tableColumn)); 25331 } 25332 } 25333 } 25334 } 25335 else if (columnObject.getExpr() != null 25336 && columnObject.getExpr().getExpressionType() == EExpressionType.function_t) { 25337 analyzeResultColumn(column, EffectType.select); 25338 for (int k = 0; k < tables.size(); k++) { 25339 Object item = tables.get(k); 25340 if (item instanceof Table) { 25341 Table tableItem = (Table) item; 25342 for (TableColumn tableColumn : tableItem.getColumns()) { 25343 if (getColumnName(resultColumn.getName()).equals( 25344 DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) { 25345 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 25346 relation.setEffectType(EffectType.select); 25347 relation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 25348 relation.addSource(new TableColumnRelationshipElement(tableColumn)); 25349 } 25350 } 25351 } else { 25352 ResultSet tableItem = (ResultSet) item; 25353 for (ResultColumn tableColumn : tableItem.getColumns()) { 25354 if (getColumnName(resultColumn.getName()).equals( 25355 DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()))) { 25356 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 25357 relation.setEffectType(EffectType.select); 25358 relation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 25359 relation.addSource(new ResultColumnRelationshipElement(tableColumn)); 25360 } 25361 } 25362 } 25363 } 25364 } 25365 } 25366 } 25367 } 25368 25369 private boolean isFromFunction(TObjectName object) { 25370 25371 Stack<TParseTreeNode> nodes = object.getStartToken().getNodesStartFromThisToken(); 25372 if (nodes != null) { 25373 for (int i = 0; i < nodes.size(); i++) { 25374 if (nodes.get(i) instanceof TFunctionCall) { 25375 return true; 25376 } 25377 } 25378 } 25379 return false; 25380 } 25381 25382 private TResultColumnList getResultColumnList(TSelectSqlStatement stmt) { 25383 // Iterative DFS (left-first) to find the first non-combined query's result column list. 25384 // Avoids StackOverflow with deeply nested UNION trees. 25385 Deque<TSelectSqlStatement> stack = new ArrayDeque<>(); 25386 stack.push(stmt); 25387 while (!stack.isEmpty()) { 25388 TSelectSqlStatement current = stack.pop(); 25389 if (current.isCombinedQuery()) { 25390 // Push right first so left is processed first (stack is LIFO) 25391 if (current.getRightStmt() != null) stack.push(current.getRightStmt()); 25392 if (current.getLeftStmt() != null) stack.push(current.getLeftStmt()); 25393 } else { 25394 if (current.getResultColumnList() != null) { 25395 return current.getResultColumnList(); 25396 } 25397 } 25398 } 25399 return null; 25400 } 25401 25402 private void createPseudoImpactRelation(TCustomSqlStatement stmt, ResultSet resultSetModel, EffectType effectType) { 25403 if (stmt.getTables() != null) { 25404 for (int i = 0; i < stmt.getTables().size(); i++) { 25405 TTable table = stmt.getTables().getTable(i); 25406 if (modelManager.getModel(table) instanceof ResultSet) { 25407 ResultSet tableModel = (ResultSet) modelManager.getModel(table); 25408 if (tableModel != resultSetModel && !tableModel.getRelationRows().getHoldRelations().isEmpty()) { 25409 ImpactRelationship impactRelation = modelFactory.createImpactRelation(); 25410 impactRelation.setEffectType(effectType); 25411 impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>( 25412 tableModel.getRelationRows())); 25413 impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>( 25414 resultSetModel.getRelationRows())); 25415 } 25416 } else if (modelManager.getModel(table) instanceof Table) { 25417 Table tableModel = (Table) modelManager.getModel(table); 25418 if (!tableModel.getRelationRows().getHoldRelations().isEmpty()) { 25419 ImpactRelationship impactRelation = modelFactory.createImpactRelation(); 25420 impactRelation.setEffectType(effectType); 25421 impactRelation.addSource( 25422 new RelationRowsRelationshipElement<TableRelationRows>(tableModel.getRelationRows())); 25423 impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>( 25424 resultSetModel.getRelationRows())); 25425 } 25426 } 25427 } 25428 } 25429 } 25430 25431 private void analyzeFunctionDataFlowRelation(Object gspObject, List<TParseTreeNode> functions, 25432 EffectType effectType) { 25433 for (int i = 0; i < functions.size(); i++) { 25434 TParseTreeNode functionCall = functions.get(i); 25435 if (functionCall instanceof TFunctionCall) { 25436 String functionName = DlineageUtil.getIdentifierNormalTableName( 25437 DlineageUtil.getFunctionNameWithArgNum((TFunctionCall) functionCall)); 25438 Procedure procedure = modelManager.getProcedureByName(functionName); 25439 if (procedure != null) { 25440 String procedureParent = getProcedureParentName(stmtStack.peek()); 25441 if (procedureParent != null) { 25442 Procedure caller = modelManager 25443 .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent)); 25444 if (caller != null) { 25445 CallRelationship callRelation = modelFactory.createCallRelation(); 25446 callRelation.setCallObject(functionCall); 25447 callRelation.setTarget(new ProcedureRelationshipElement(caller)); 25448 callRelation.addSource(new ProcedureRelationshipElement(procedure)); 25449 if (isBuiltInFunctionName(((TFunctionCall)functionCall).getFunctionName())||isKeyword(((TFunctionCall)functionCall).getFunctionName())) { 25450 callRelation.setBuiltIn(true); 25451 } 25452 } 25453 } 25454 if (procedure.getProcedureObject() instanceof TCreateFunctionStmt) { 25455 TCreateFunctionStmt createFunction = (TCreateFunctionStmt)procedure.getProcedureObject(); 25456 TTypeName dataType = createFunction.getReturnDataType(); 25457 if (dataType!=null && dataType.getTypeOfList() != null && dataType.getTypeOfList().getColumnDefList() != null) { 25458 Object modelObject = modelManager.getModel(gspObject); 25459 if(modelObject instanceof ResultColumn) { 25460 ResultColumn resultColumn = (ResultColumn)modelObject; 25461 ResultSet resultSet = resultColumn.getResultSet(); 25462 for (int j = 0; j < dataType.getTypeOfList().getColumnDefList().size(); j++) { 25463 TObjectName columnName = new TObjectName(); 25464 if( dataType.getDataType() == EDataType.array_t) { 25465// columnName.setString(resultColumn.getName() + ".array." 25466// + dataType.getTypeOfList().getColumnDefList().getColumn(j) 25467// .getColumnName().getColumnNameOnly()); 25468 columnName.setString(resultColumn.getName() + "." 25469 + dataType.getTypeOfList().getColumnDefList().getColumn(j) 25470 .getColumnName().getColumnNameOnly()); 25471 } 25472 else { 25473 columnName.setString(resultColumn.getName() + "." 25474 + dataType.getTypeOfList().getColumnDefList().getColumn(j) 25475 .getColumnName().getColumnNameOnly()); 25476 } 25477 ResultColumn sturctColumn = modelFactory.createResultColumn(resultSet, columnName, 25478 true); 25479 sturctColumn.setStruct(true); 25480 Function sourceFunction = (Function)createFunction(functionCall); 25481 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 25482 relation.setEffectType(effectType); 25483 relation.setTarget(new ResultColumnRelationshipElement(sturctColumn)); 25484 if (sourceFunction.getColumns() != null && !sourceFunction.getColumns().isEmpty()) { 25485 for (ResultColumn column : sourceFunction.getColumns()) { 25486 relation.addSource(new ResultColumnRelationshipElement(column)); 25487 } 25488 } 25489 } 25490 resultSet.getColumns().remove(resultColumn); 25491 } 25492 return; 25493 } 25494 } 25495 } 25496 } 25497 25498 if(gspObject instanceof TResultColumn) { 25499 TResultColumn resultColumn = (TResultColumn)gspObject; 25500 if(resultColumn.getAliasClause()!=null && resultColumn.getAliasClause().getColumns()!=null) { 25501 for(TObjectName columnName: resultColumn.getAliasClause().getColumns()) { 25502 analyzeFunctionDataFlowRelation(columnName, Arrays.asList(functionCall), effectType, null); 25503 } 25504 return; 25505 } 25506 } 25507 analyzeFunctionDataFlowRelation(gspObject, Arrays.asList(functionCall), effectType, null); 25508 } 25509 } 25510 25511 private void analyzeFunctionDataFlowRelation(Object gspObject, List<TParseTreeNode> functions, 25512 EffectType effectType, Process process) { 25513 25514 Object modelObject = modelManager.getModel(gspObject); 25515 if (modelObject == null) { 25516 if (gspObject instanceof ResultColumn || gspObject instanceof TableColumn) { 25517 modelObject = gspObject; 25518 } 25519 } 25520 25521 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 25522 relation.setEffectType(effectType); 25523 relation.setProcess(process); 25524 25525 if (modelObject instanceof ResultColumn) { 25526 relation.setTarget(new ResultColumnRelationshipElement((ResultColumn) modelObject)); 25527 25528 } else if (modelObject instanceof TableColumn) { 25529 relation.setTarget(new TableColumnRelationshipElement((TableColumn) modelObject)); 25530 25531 } else { 25532 throw new UnsupportedOperationException(); 25533 } 25534 25535 for (int i = 0; i < functions.size(); i++) { 25536 TParseTreeNode functionCall = functions.get(i); 25537 25538 if (functionCall instanceof TFunctionCall) { 25539 TFunctionCall call = (TFunctionCall) functionCall; 25540 Procedure callee = modelManager.getProcedureByName( 25541 DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(call))); 25542 if (callee == null && procedureDDLMap.containsKey(DlineageUtil.getFunctionNameWithArgNum(call))) { 25543 analyzeCustomSqlStmt(procedureDDLMap.get(DlineageUtil.getFunctionNameWithArgNum(call))); 25544 callee = modelManager.getProcedureByName( 25545 DlineageUtil.getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(call))); 25546 } 25547 25548 if (callee != null) { 25549 String procedureParent = getProcedureParentName(stmtStack.peek()); 25550 if (procedureParent != null) { 25551 Procedure caller = modelManager 25552 .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent)); 25553 if (caller != null) { 25554 CallRelationship callRelation = modelFactory.createCallRelation(); 25555 callRelation.setCallObject(functionCall); 25556 callRelation.setTarget(new ProcedureRelationshipElement(caller)); 25557 callRelation.addSource(new ProcedureRelationshipElement(callee)); 25558 if (isBuiltInFunctionName(call.getFunctionName()) || isKeyword(call.getFunctionName())) { 25559 callRelation.setBuiltIn(true); 25560 } 25561 } 25562 } 25563 if (callee.getArguments() != null) { 25564 for (int j = 0; j < callee.getArguments().size(); j++) { 25565 Argument argument = callee.getArguments().get(j); 25566 Variable variable = resolveFormalVariable(callee, argument); 25567 if(variable!=null) { 25568 if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) { 25569 Transform transform = new Transform(); 25570 transform.setType(Transform.FUNCTION); 25571 transform.setCode(call); 25572 compositeBindingColumn(variable).setTransform(transform); 25573 } 25574 Process callProcess = modelFactory.createProcess(call); 25575 variable.addProcess(callProcess); 25576 analyzeFunctionArgumentsDataFlowRelation(compositeBindingColumn(variable), call, j, callProcess); 25577 } 25578 } 25579 } 25580 Set<Object> functionTableModelObjs = modelManager.getFunctionTable(DlineageUtil 25581 .getIdentifierNormalTableName(call.getFunctionName().toString())); 25582 if (functionTableModelObjs != null) { 25583 modelManager.bindModel(call, functionTableModelObjs.iterator().next()); 25584 for (Object functionTableModelObj : functionTableModelObjs) { 25585 if (functionTableModelObj instanceof ResultSet) { 25586 ResultSet resultSet = (ResultSet) functionTableModelObj; 25587 for (ResultColumn column : resultSet.getColumns()) { 25588 relation.addSource(new ResultColumnRelationshipElement(column)); 25589 } 25590 } 25591 } 25592 } 25593 continue; 25594 } 25595 } 25596 25597 25598 Object functionModel = createFunction(functionCall); 25599 if (functionModel instanceof Function) { 25600 Function sourceFunction = (Function)functionModel; 25601 if (sourceFunction.getColumns() != null && !sourceFunction.getColumns().isEmpty()) { 25602 for (ResultColumn column : sourceFunction.getColumns()) { 25603 relation.addSource(new ResultColumnRelationshipElement(column)); 25604 } 25605 } 25606 else if (functionCall instanceof TFunctionCall) { 25607 relation.addSource(new ResultColumnRelationshipElement((FunctionResultColumn) modelManager 25608 .getModel(((TFunctionCall) functionCall).getFunctionName()))); 25609 } else if (functionCall instanceof TCaseExpression) { 25610 relation.addSource(new ResultColumnRelationshipElement((FunctionResultColumn) modelManager 25611 .getModel(((TCaseExpression) functionCall).getWhenClauseItemList()))); 25612 } 25613 25614 if (sourceFunction != null && !sourceFunction.getRelationRows().getHoldRelations().isEmpty()) { 25615 boolean find = false; 25616 if (modelObject instanceof ResultColumn) { 25617 ResultSetRelationRows targetRelationRows = ((ResultColumn) modelObject).getResultSet().getRelationRows(); 25618 if(targetRelationRows.hasRelation()) { 25619 for(Relationship relationship: targetRelationRows.getHoldRelations()) { 25620 if(relationship.getSources().contains(sourceFunction.getRelationRows())) { 25621 find = true; 25622 break; 25623 } 25624 } 25625 } 25626 } 25627 else if (modelObject instanceof TableColumn) { 25628 TableRelationRows targetRelationRows = ((TableColumn) modelObject).getTable().getRelationRows(); 25629 if(targetRelationRows.hasRelation()) { 25630 for(Relationship relationship: targetRelationRows.getHoldRelations()) { 25631 if(relationship.getSources().contains(sourceFunction.getRelationRows())) { 25632 find = true; 25633 break; 25634 } 25635 } 25636 } 25637 } 25638 25639 if (!find) { 25640 ImpactRelationship impactRelation = modelFactory.createImpactRelation(); 25641 impactRelation.setEffectType(EffectType.select); 25642 impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>( 25643 sourceFunction.getRelationRows())); 25644 if (modelObject instanceof ResultColumn) { 25645 impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>( 25646 ((ResultColumn) modelObject).getResultSet().getRelationRows())); 25647 } else if (modelObject instanceof TableColumn) { 25648 impactRelation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>( 25649 ((TableColumn) modelObject).getTable().getRelationRows())); 25650 } 25651 } 25652 } 25653 } else if (functionModel instanceof Table) { 25654 TFunctionCall call = (TFunctionCall) functionCall; 25655 String functionName = call.getFunctionName().toString(); 25656 boolean flag = false; 25657 if (functionName.indexOf(".") != -1) { 25658 String columnName = functionName.substring(functionName.indexOf(".") + 1); 25659 for (TableColumn tableColumn : ((Table) functionModel).getColumns()) { 25660 if (DlineageUtil.sameName(ESQLDataObjectType.dotColumn, DlineageUtil.columnSegmentOf(tableColumn.getName()), columnName)) { 25661 TableColumnRelationshipElement element = new TableColumnRelationshipElement(tableColumn); 25662 relation.addSource(element); 25663 flag = true; 25664 break; 25665 } 25666 } 25667 } 25668 25669 if (!flag) { 25670 TableColumn tableColumn = modelFactory.createTableColumn((Table) functionModel, 25671 ((TFunctionCall) functionCall)); 25672 TableColumnRelationshipElement element = new TableColumnRelationshipElement(tableColumn); 25673 relation.addSource(element); 25674 } 25675 } 25676 } 25677 25678 } 25679 25680 private void analyzeSubqueryDataFlowRelation(Object gspObject, List<TSelectSqlStatement> subquerys, 25681 EffectType effectType) { 25682 analyzeSubqueryDataFlowRelation(gspObject, subquerys, effectType, null); 25683 } 25684 25685 private void analyzeSubqueryDataFlowRelation(Object gspObject, List<TSelectSqlStatement> subquerys, 25686 EffectType effectType, Process process) { 25687 25688 Object modelObject = modelManager.getModel(gspObject); 25689 if (modelObject == null) { 25690 if (gspObject instanceof ResultColumn || gspObject instanceof TableColumn) { 25691 modelObject = gspObject; 25692 } 25693 } 25694 25695 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 25696 relation.setEffectType(effectType); 25697 relation.setProcess(process); 25698 25699 if (modelObject instanceof ResultColumn) { 25700 relation.setTarget(new ResultColumnRelationshipElement((ResultColumn) modelObject)); 25701 25702 } else if (modelObject instanceof TableColumn) { 25703 relation.setTarget(new TableColumnRelationshipElement((TableColumn) modelObject)); 25704 25705 } else { 25706 throw new UnsupportedOperationException(); 25707 } 25708 25709 for (int i = 0; i < subquerys.size(); i++) { 25710 TSelectSqlStatement subquery = subquerys.get(i); 25711 ResultSet resultSetModel = (ResultSet) modelManager.getModel(subquery); 25712 if (resultSetModel != null && resultSetModel.getColumns() != null) { 25713 for (ResultColumn column : resultSetModel.getColumns()) { 25714 relation.addSource(new ResultColumnRelationshipElement(column)); 25715 } 25716 } 25717 25718 if (resultSetModel != null && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) { 25719 boolean find = false; 25720 if (modelObject instanceof ResultColumn) { 25721 ResultSetRelationRows targetRelationRows = ((ResultColumn) modelObject).getResultSet().getRelationRows(); 25722 if(targetRelationRows.hasRelation()) { 25723 for(Relationship relationship: targetRelationRows.getHoldRelations()) { 25724 if(relationship.getSources().contains(resultSetModel.getRelationRows())) { 25725 find = true; 25726 break; 25727 } 25728 } 25729 } 25730 } 25731 else if (modelObject instanceof TableColumn) { 25732 TableRelationRows targetRelationRows = ((TableColumn) modelObject).getTable().getRelationRows(); 25733 if(targetRelationRows.hasRelation()) { 25734 for(Relationship relationship: targetRelationRows.getHoldRelations()) { 25735 if(relationship.getSources().contains(resultSetModel.getRelationRows())) { 25736 find = true; 25737 break; 25738 } 25739 } 25740 } 25741 } 25742 25743 if (!find) { 25744 ImpactRelationship impactRelation = modelFactory.createImpactRelation(); 25745 impactRelation.setEffectType(EffectType.select); 25746 impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>( 25747 resultSetModel.getRelationRows())); 25748 if (modelObject instanceof ResultColumn) { 25749 impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>( 25750 ((ResultColumn) modelObject).getResultSet().getRelationRows())); 25751 } else if (modelObject instanceof TableColumn) { 25752 impactRelation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>( 25753 ((TableColumn) modelObject).getTable().getRelationRows())); 25754 } 25755 } 25756 } 25757 } 25758 25759 } 25760 25761 private Object createFunction(TParseTreeNode functionCall) { 25762 if (!processingFunctions.add(functionCall)) { 25763 return modelManager.getModel(functionCall); 25764 } 25765 try { 25766 25767 if (functionCall instanceof TFunctionCall) { 25768 TFunctionCall functionObj = (TFunctionCall) functionCall; 25769 // Generic structured-dataflow dispatch. When a vendor adapter (e.g. 25770 // Spark from_json+explode) returns a descriptor, model the function 25771 // as a structured generator: per-field result columns linked to 25772 // exact structured source paths such as nodes[*].key. When no 25773 // adapter matches, fall through to the existing function logic 25774 // unchanged. 25775 StructuredAdapterContext sctx = new StructuredAdapterContext(option.getVendor()); 25776 StructuredDataflowDescriptor sdescriptor = 25777 StructuredDataflowRegistry.defaultRegistry().describe(functionObj, sctx); 25778 if (sdescriptor != null) { 25779 Function structuredFn = createStructuredDataflowFunction(sdescriptor); 25780 if (structuredFn != null) { 25781 return structuredFn; 25782 } 25783 } 25784 25785 if (!isBuiltInFunctionName(functionObj.getFunctionName())) { 25786 TCustomSqlStatement stmt = stmtStack.peek(); 25787 String procedureParent = getProcedureParentName(stmt); 25788 if (procedureParent != null) { 25789 Procedure procedureCallee = modelManager.getProcedureByName(DlineageUtil 25790 .getIdentifierNormalTableName(DlineageUtil.getFunctionNameWithArgNum(functionObj))); 25791 if (procedureCallee != null) { 25792 if (procedureParent != null) { 25793 Procedure caller = modelManager 25794 .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent)); 25795 if (caller != null) { 25796 CallRelationship callRelation = modelFactory.createCallRelation(); 25797 callRelation.setCallObject(functionCall); 25798 callRelation.setTarget(new ProcedureRelationshipElement(caller)); 25799 callRelation.addSource(new ProcedureRelationshipElement(procedureCallee)); 25800 if (isBuiltInFunctionName(functionObj.getFunctionName()) || isKeyword(functionObj.getFunctionName())) { 25801 callRelation.setBuiltIn(true); 25802 } 25803 } 25804 } 25805 if (procedureCallee.getArguments() != null) { 25806 for (int j = 0; j < procedureCallee.getArguments().size(); j++) { 25807 Argument argument = procedureCallee.getArguments().get(j); 25808 Variable variable = modelFactory.createVariable(procedureCallee, argument.getName(), false); 25809 if (variable != null) { 25810 if (argument.getMode() == EParameterMode.out || argument.getMode() == EParameterMode.output) { 25811 Transform transform = new Transform(); 25812 transform.setType(Transform.FUNCTION); 25813 transform.setCode(functionObj); 25814 compositeBindingColumn(variable).setTransform(transform); 25815 } 25816 Process process = modelFactory.createProcess(functionObj); 25817 variable.addProcess(process); 25818 analyzeFunctionArgumentsDataFlowRelation(compositeBindingColumn(variable), functionObj, j, process); 25819 } 25820 } 25821 } 25822 } else { 25823 TFunctionCall call = (TFunctionCall) functionCall; 25824 String functionName = call.getFunctionName().toString(); 25825 if (functionName.indexOf(".") != -1) { 25826 Table functionTable = modelManager 25827 .getTableByName(functionName.substring(0, functionName.indexOf("."))); 25828 if (functionTable != null) { 25829 String columnName = functionName.substring(functionName.indexOf(".") + 1); 25830 for (TableColumn tableColumn : functionTable.getColumns()) { 25831 if (DlineageUtil.sameName(ESQLDataObjectType.dotColumn, DlineageUtil.columnSegmentOf(tableColumn.getName()), columnName)) { 25832 return functionTable; 25833 } 25834 } 25835 } 25836 } 25837 Function function = modelFactory.createFunction(call); 25838 if (procedureParent != null) { 25839 Procedure caller = modelManager 25840 .getProcedureByName(DlineageUtil.getIdentifierNormalTableName(procedureParent)); 25841 if (caller != null) { 25842 CallRelationship callRelation = modelFactory.createCallRelation(); 25843 callRelation.setCallObject(functionCall); 25844 callRelation.setTarget(new ProcedureRelationshipElement(caller)); 25845 callRelation.addSource(new FunctionRelationshipElement(function)); 25846 if (isBuiltInFunctionName(functionObj.getFunctionName()) || isKeyword(functionObj.getFunctionName())) { 25847 callRelation.setBuiltIn(true); 25848 } 25849 } 25850 } 25851 } 25852 } 25853 } else if (isConstantFunction(functionObj.getFunctionName()) 25854 && (functionObj.getArgs() == null || functionObj.getArgs().size() == 0)) { 25855 if (option.isShowConstantTable()) { 25856 Table constantTable = modelFactory.createConstantsTable(stmtStack.peek()); 25857 modelFactory.createTableColumn(constantTable, functionObj); 25858 return constantTable; 25859 } else { 25860 return null; 25861 } 25862 } 25863 25864 if (functionObj.getFunctionType() == EFunctionType.struct_t) { 25865 Function function = modelFactory.createFunction((TFunctionCall) functionCall); 25866 if (functionObj instanceof TTableFunction) { 25867 TTableFunction tableFunction = (TTableFunction) functionObj; 25868 if (tableFunction.getFieldValues() != null) { 25869 for (int i = 0; i < tableFunction.getFieldValues().size(); i++) { 25870 TResultColumn resultColumn = tableFunction.getFieldValues().getResultColumn(i); 25871 if (resultColumn.getAliasClause() != null) { 25872 ResultColumn column = modelFactory.createFunctionResultColumn(function, 25873 resultColumn.getAliasClause().getAliasName()); 25874 columnsInExpr visitor = new columnsInExpr(); 25875 resultColumn.getExpr().inOrderTraverse(visitor); 25876 List<TObjectName> objectNames = visitor.getObjectNames(); 25877 List<TParseTreeNode> functions = visitor.getFunctions(); 25878 if (functions != null && !functions.isEmpty()) { 25879 analyzeFunctionDataFlowRelation(column, functions, EffectType.function); 25880 } 25881 List<TSelectSqlStatement> subquerys = visitor.getSubquerys(); 25882 if (subquerys != null && !subquerys.isEmpty()) { 25883 analyzeSubqueryDataFlowRelation(column, subquerys, EffectType.function); 25884 } 25885 analyzeDataFlowRelation(column, objectNames, EffectType.function, functions); 25886 List<TParseTreeNode> constants = visitor.getConstants(); 25887 analyzeConstantDataFlowRelation(column, constants, EffectType.function, functions); 25888 } else if (resultColumn.getFieldAttr() != null) { 25889 ResultColumn column = modelFactory.createFunctionResultColumn(function, 25890 resultColumn.getFieldAttr()); 25891 analyzeDataFlowRelation(column, Arrays.asList(resultColumn.getFieldAttr()), EffectType.function, null); 25892 } else if (resultColumn.getExpr() != null) { 25893 if (resultColumn.getExpr().getFunctionCall() != null) { 25894 Function resultColumnFunction = (Function) createFunction( 25895 resultColumn.getExpr().getFunctionCall()); 25896 String functionName = getResultSetName(function); 25897 for (int j = 0; j < resultColumnFunction.getColumns().size(); j++) { 25898 TObjectName columnName = new TObjectName(); 25899 if (resultColumn.getAliasClause() != null) { 25900 columnName.setString(resultColumn.getAliasClause() + "." + getColumnNameOnly( 25901 resultColumnFunction.getColumns().get(j).getName())); 25902 } else { 25903 columnName.setString(functionName + "." + getColumnNameOnly( 25904 resultColumnFunction.getColumns().get(j).getName())); 25905 } 25906 ResultColumn functionResultColumn = modelFactory.createResultColumn(function, 25907 columnName); 25908 DataFlowRelationship relationship = modelFactory.createDataFlowRelation(); 25909 relationship.setTarget(new ResultColumnRelationshipElement(functionResultColumn)); 25910 relationship.addSource(new ResultColumnRelationshipElement( 25911 resultColumnFunction.getColumns().get(j))); 25912 } 25913 } else if (resultColumn.getExpr().getCaseExpression() != null) { 25914 function = modelFactory.createFunction(resultColumn.getExpr().getCaseExpression()); 25915 ResultColumn column = modelFactory.createFunctionResultColumn(function, 25916 ((TCaseExpression) resultColumn.getExpr().getCaseExpression()).getWhenClauseItemList()); 25917 analyzeFunctionArgumentsDataFlowRelation(column, functionCall); 25918 } 25919 } 25920 } 25921 return function; 25922 } 25923 } else if (functionObj instanceof TFunctionCall) { 25924 25925 } 25926 } 25927 25928 // BigQuery array_agg(table_alias) row-reference expansion 25929 if (functionObj.getFunctionType() == EFunctionType.array_agg_t 25930 && option.getVendor() == EDbVendor.dbvbigquery 25931 && functionObj.getArgs() != null 25932 && functionObj.getArgs().size() == 1) { 25933 25934 TExpression arg0 = functionObj.getArgs().getExpression(0); 25935 if (arg0 != null 25936 && arg0.getExpressionType() == EExpressionType.simple_object_name_t) { 25937 25938 TObjectName on = arg0.getObjectOperand(); 25939 if (isArrayAggRowReference(on)) { 25940 Function function = (Function) modelManager.getModel(functionCall); 25941 if (function == null) { 25942 function = modelFactory.createFunction((TFunctionCall) functionCall); 25943 } 25944 25945 // Only bind once (avoid duplicate processing when 25946 // createFunction is called from analyzeFunctionDataFlowRelation) 25947 if (function.getColumns() == null || function.getColumns().isEmpty()) { 25948 TTable srcTable = on.getSourceTable(); 25949 String tableAlias = srcTable.getAliasName() != null 25950 ? srcTable.getAliasName().toString() : null; 25951 25952 Table sourceTable = (Table) modelManager.getModel(srcTable); 25953 sourceTable.removeColumn(tableAlias); 25954 25955 // Collect inferred column names from ORDER BY (in function) 25956 // and GROUP BY (in enclosing SELECT). Excludes the table alias. 25957 List<TObjectName> inferredColumns = collectRowReferenceInferredColumns( 25958 functionObj, tableAlias); 25959 25960 // Create a single * function result column carrying star-link 25961 // metadata for the inferred columns and direct source edges 25962 // for * and each inferred column. 25963 createRowReferenceStarColumn(function, srcTable, inferredColumns); 25964 } 25965 25966 return function; 25967 } 25968 } 25969 } 25970 25971 if (functionObj.getFunctionType() == EFunctionType.array_t || functionObj.getFunctionType() == EFunctionType.array_agg_t) { 25972 Function function = modelFactory.createFunction((TFunctionCall) functionCall); 25973 if (functionObj.getArgs() != null) { 25974 if (functionObj.getArgs().getExpression(0).getSubQuery() != null) { 25975 TSelectSqlStatement stmt = functionObj.getArgs().getExpression(0).getSubQuery(); 25976 analyzeSelectStmt(stmt); 25977 ResultSet resultset = (ResultSet) modelManager.getModel(stmt); 25978 for (int i = 0; i < resultset.getColumns().size(); i++) { 25979 ResultColumn sourceColumn = resultset.getColumns().get(i); 25980 TObjectName columnName = new TObjectName(); 25981 columnName.setString(sourceColumn.getName()); 25982 ResultColumn resultColumn = modelFactory.createFunctionResultColumn(function, 25983 columnName); 25984 DataFlowRelationship relationship = modelFactory.createDataFlowRelation(); 25985 relationship.setTarget(new ResultColumnRelationshipElement(resultColumn)); 25986 relationship.addSource( 25987 new ResultColumnRelationshipElement(sourceColumn)); 25988 } 25989 return function; 25990 } else if (functionObj.getArgs().getExpression(0).getExpressionType() == EExpressionType.function_t) { 25991 Object functionTableModelObj = createFunction(functionObj.getArgs().getExpression(0).getFunctionCall()); 25992 if (functionTableModelObj instanceof ResultSet) { 25993 ResultSet resultset = (ResultSet) functionTableModelObj; 25994 for (int i = 0; i < resultset.getColumns().size(); i++) { 25995 ResultColumn sourceColumn = resultset.getColumns().get(i); 25996 TObjectName columnName = new TObjectName(); 25997 columnName.setString(sourceColumn.getName()); 25998 ResultColumn resultColumn = modelFactory.createFunctionResultColumn(function, columnName); 25999 DataFlowRelationship relationship = modelFactory.createDataFlowRelation(); 26000 relationship.setTarget(new ResultColumnRelationshipElement(resultColumn)); 26001 relationship.addSource(new ResultColumnRelationshipElement(sourceColumn)); 26002 } 26003 return function; 26004 } 26005 } 26006 } 26007 } 26008 26009 Function function = modelFactory.createFunction((TFunctionCall) functionCall); 26010 ResultColumn column = modelFactory.createFunctionResultColumn(function, 26011 ((TFunctionCall) functionCall).getFunctionName()); 26012 if ("COUNT".equalsIgnoreCase(((TFunctionCall) functionCall).getFunctionName().toString())) { 26013 // @see https://e.gitee.com/gudusoft/issues/list?issue=I40NUP 26014 // COUNT特殊处理,不和参数关联 26015 if (option.isShowCountTableColumn()) { 26016 analyzeFunctionArgumentsDataFlowRelation(column, functionCall); 26017 } 26018 } else { 26019 boolean isCustomFunction = analyzeCustomFunctionCall((TFunctionCall) functionCall); 26020// if(!isCustomFunction) 26021 { 26022 analyzeFunctionArgumentsDataFlowRelation(column, functionCall); 26023 } 26024 Set<Object> functionTableModelObjs = modelManager.getFunctionTable(getIdentifiedFunctionName(function)); 26025 if (functionTableModelObjs != null) { 26026 for (Object functionTableModelObj : functionTableModelObjs) { 26027 if (functionTableModelObj instanceof ResultSet) { 26028 ResultSet functionTableModel = (ResultSet) functionTableModelObj; 26029 if (functionTableModel.getColumns() != null) { 26030 for (int j = 0; j < functionTableModel.getColumns().size(); j++) { 26031 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 26032 relation.setEffectType(EffectType.select); 26033 relation.setTarget(new ResultColumnRelationshipElement(column)); 26034 relation.addSource(new ResultColumnRelationshipElement( 26035 functionTableModel.getColumns().get(j))); 26036 } 26037 } 26038 } 26039 } 26040 } 26041 } 26042 return function; 26043 } else if (functionCall instanceof TCaseExpression) { 26044 Function function = modelFactory.createFunction((TCaseExpression) functionCall); 26045 ResultColumn column = modelFactory.createFunctionResultColumn(function, 26046 ((TCaseExpression) functionCall).getWhenClauseItemList()); 26047 analyzeFunctionArgumentsDataFlowRelation(column, functionCall); 26048 return function; 26049 } else if (functionCall instanceof TObjectName) { 26050 Function function = modelFactory.createFunction((TObjectName) functionCall); 26051 TObjectName columnName = new TObjectName(); 26052 columnName.setString(function.getFunctionName()); 26053 ResultColumn column = modelFactory.createResultColumn(function, 26054 columnName); 26055 analyzeFunctionArgumentsDataFlowRelation(column, functionCall); 26056 return function; 26057 } 26058 return null; 26059 } finally { 26060 processingFunctions.remove(functionCall); 26061 } 26062 } 26063 26064 protected String getIdentifiedFunctionName(Function function) { 26065 return DlineageUtil.getIdentifierNormalFunctionName(function.getFunctionName()); 26066 } 26067 26068 private boolean isConstantFunction(TObjectName functionName) { 26069 boolean result = CONSTANT_BUILTIN_FUNCTIONS.contains(functionName.toString().toUpperCase()); 26070 if (result) { 26071 return true; 26072 } 26073 return false; 26074 } 26075 26076 private void analyzeFunctionArgumentsDataFlowRelation(Object resultColumn, TParseTreeNode gspObject) { 26077 List<TExpression> directExpressions = new ArrayList<TExpression>(); 26078 List<TExpression> indirectExpressions = new ArrayList<TExpression>(); 26079 List<TExpression> conditionExpressions = new ArrayList<TExpression>(); 26080 if (gspObject instanceof TFunctionCall) { 26081 TFunctionCall functionCall = (TFunctionCall) gspObject; 26082 getFunctionExpressions(directExpressions, indirectExpressions, functionCall); 26083 } else if (gspObject instanceof TCaseExpression) { 26084 TCaseExpression expr = (TCaseExpression) gspObject; 26085 TExpression inputExpr = expr.getInput_expr(); 26086 if (inputExpr != null) { 26087 if(option.isShowCaseWhenAsDirect()){ 26088 directExpressions.add(inputExpr); 26089 } 26090 else { 26091 conditionExpressions.add(inputExpr); 26092 } 26093 } 26094 TExpression defaultExpr = expr.getElse_expr(); 26095 if (defaultExpr != null) { 26096 directExpressions.add(defaultExpr); 26097 } 26098 TWhenClauseItemList list = expr.getWhenClauseItemList(); 26099 for (int i = 0; i < list.size(); i++) { 26100 TWhenClauseItem element = list.getWhenClauseItem(i); 26101 if(option.isShowCaseWhenAsDirect()){ 26102 directExpressions.add(element.getComparison_expr()); 26103 } 26104 else { 26105 conditionExpressions.add(element.getComparison_expr()); 26106 } 26107 directExpressions.add(element.getReturn_expr()); 26108 } 26109 } 26110 26111 for (int j = 0; j < directExpressions.size(); j++) { 26112 columnsInExpr visitor = new columnsInExpr(); 26113 directExpressions.get(j).inOrderTraverse(visitor); 26114 26115 List<TObjectName> objectNames = visitor.getObjectNames(); 26116 List<TParseTreeNode> functions = visitor.getFunctions(); 26117 26118 if (functions != null && !functions.isEmpty()) { 26119 analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function); 26120 } 26121 26122 List<TSelectSqlStatement> subquerys = visitor.getSubquerys(); 26123 if (subquerys != null && !subquerys.isEmpty()) { 26124 analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.function); 26125 } 26126 26127 analyzeDataFlowRelation(resultColumn, objectNames, EffectType.function, functions); 26128 26129 List<TParseTreeNode> constants = visitor.getConstants(); 26130 analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.function, functions); 26131 } 26132 26133 conditionExpressions.addAll(indirectExpressions); 26134 for (int j = 0; j < conditionExpressions.size(); j++) { 26135 analyzeFilterCondition(resultColumn, conditionExpressions.get(j), null, null, EffectType.function); 26136 } 26137 } 26138 26139 private void analyzeFunctionArgumentsDataFlowRelation(Object resultColumn, TCallStatement callStatment, int argumentIndex, Process process) { 26140 List<TExpression> directExpressions = new ArrayList<TExpression>(); 26141 List<TExpression> indirectExpressions = new ArrayList<TExpression>(); 26142 List<TExpression> conditionExpressions = new ArrayList<TExpression>(); 26143 26144 getFunctionExpressions(directExpressions, indirectExpressions, callStatment, argumentIndex); 26145 26146 for (int j = 0; j < directExpressions.size(); j++) { 26147 columnsInExpr visitor = new columnsInExpr(); 26148 directExpressions.get(j).inOrderTraverse(visitor); 26149 26150 List<TObjectName> objectNames = visitor.getObjectNames(); 26151 List<TParseTreeNode> functions = visitor.getFunctions(); 26152 26153 if (functions != null && !functions.isEmpty()) { 26154 analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function); 26155 } 26156 26157 List<TSelectSqlStatement> subquerys = visitor.getSubquerys(); 26158 if (subquerys != null && !subquerys.isEmpty()) { 26159 analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.function); 26160 } 26161 26162 DataFlowRelationship relation = analyzeDataFlowRelation(resultColumn, objectNames, EffectType.function, functions); 26163 if (relation != null) { 26164 relation.setProcess(process); 26165 } 26166 26167 List<TParseTreeNode> constants = visitor.getConstants(); 26168 analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.function, functions); 26169 } 26170 26171 conditionExpressions.addAll(indirectExpressions); 26172 for (int j = 0; j < conditionExpressions.size(); j++) { 26173 analyzeFilterCondition(resultColumn, conditionExpressions.get(j), null, null, EffectType.function); 26174 } 26175 } 26176 26177 private void analyzeFunctionArgumentsDataFlowRelation(Object resultColumn, TDb2CallStmt callStatment, int argumentIndex, Process process) { 26178 List<TExpression> directExpressions = new ArrayList<TExpression>(); 26179 List<TExpression> indirectExpressions = new ArrayList<TExpression>(); 26180 List<TExpression> conditionExpressions = new ArrayList<TExpression>(); 26181 26182 getFunctionExpressions(directExpressions, indirectExpressions, callStatment, argumentIndex); 26183 26184 for (int j = 0; j < directExpressions.size(); j++) { 26185 columnsInExpr visitor = new columnsInExpr(); 26186 directExpressions.get(j).inOrderTraverse(visitor); 26187 26188 List<TObjectName> objectNames = visitor.getObjectNames(); 26189 List<TParseTreeNode> functions = visitor.getFunctions(); 26190 26191 if (functions != null && !functions.isEmpty()) { 26192 analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function); 26193 } 26194 26195 List<TSelectSqlStatement> subquerys = visitor.getSubquerys(); 26196 if (subquerys != null && !subquerys.isEmpty()) { 26197 analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.function); 26198 } 26199 26200 DataFlowRelationship relation = analyzeDataFlowRelation(resultColumn, objectNames, EffectType.function, functions); 26201 if (relation != null) { 26202 relation.setProcess(process); 26203 } 26204 26205 List<TParseTreeNode> constants = visitor.getConstants(); 26206 analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.function, functions); 26207 } 26208 26209 conditionExpressions.addAll(indirectExpressions); 26210 for (int j = 0; j < conditionExpressions.size(); j++) { 26211 analyzeFilterCondition(resultColumn, conditionExpressions.get(j), null, null, EffectType.function); 26212 } 26213 } 26214 26215 private void analyzeFunctionArgumentsDataFlowRelation(Object resultColumn, TFunctionCall functionCall, int argumentIndex, Process process) { 26216 List<TExpression> directExpressions = new ArrayList<TExpression>(); 26217 List<TExpression> indirectExpressions = new ArrayList<TExpression>(); 26218 List<TExpression> conditionExpressions = new ArrayList<TExpression>(); 26219 26220 getFunctionExpressions(directExpressions, indirectExpressions, functionCall, argumentIndex); 26221 26222 for (int j = 0; j < directExpressions.size(); j++) { 26223 columnsInExpr visitor = new columnsInExpr(); 26224 directExpressions.get(j).inOrderTraverse(visitor); 26225 26226 List<TObjectName> objectNames = visitor.getObjectNames(); 26227 List<TParseTreeNode> functions = visitor.getFunctions(); 26228 26229 if (functions != null && !functions.isEmpty()) { 26230 analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function); 26231 } 26232 26233 List<TSelectSqlStatement> subquerys = visitor.getSubquerys(); 26234 if (subquerys != null && !subquerys.isEmpty()) { 26235 analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.function); 26236 } 26237 26238 DataFlowRelationship relation = analyzeDataFlowRelation(resultColumn, objectNames, EffectType.function, functions); 26239 if (relation != null) { 26240 relation.setProcess(process); 26241 } 26242 26243 List<TParseTreeNode> constants = visitor.getConstants(); 26244 analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.function, functions); 26245 } 26246 26247 conditionExpressions.addAll(indirectExpressions); 26248 for (int j = 0; j < conditionExpressions.size(); j++) { 26249 analyzeFilterCondition(resultColumn, conditionExpressions.get(j), null, null, EffectType.function); 26250 } 26251 } 26252 26253 private void analyzeFunctionArgumentsDataFlowRelation(Object resultColumn, TMssqlExecute functionCall, String argumentName, int argumentIndex, Process process) { 26254 List<TExpression> directExpressions = new ArrayList<TExpression>(); 26255 List<TExpression> indirectExpressions = new ArrayList<TExpression>(); 26256 List<TExpression> conditionExpressions = new ArrayList<TExpression>(); 26257 26258 getFunctionExpressions(directExpressions, indirectExpressions, functionCall, argumentName, argumentIndex); 26259 26260 for (int j = 0; j < directExpressions.size(); j++) { 26261 columnsInExpr visitor = new columnsInExpr(); 26262 directExpressions.get(j).inOrderTraverse(visitor); 26263 26264 List<TObjectName> objectNames = visitor.getObjectNames(); 26265 List<TParseTreeNode> functions = visitor.getFunctions(); 26266 26267 if (functions != null && !functions.isEmpty()) { 26268 analyzeFunctionDataFlowRelation(resultColumn, functions, EffectType.function); 26269 } 26270 26271 List<TSelectSqlStatement> subquerys = visitor.getSubquerys(); 26272 if (subquerys != null && !subquerys.isEmpty()) { 26273 analyzeSubqueryDataFlowRelation(resultColumn, subquerys, EffectType.function); 26274 } 26275 26276 DataFlowRelationship relation = analyzeDataFlowRelation(resultColumn, objectNames, EffectType.function, functions); 26277 if (relation != null) { 26278 relation.setProcess(process); 26279 } 26280 26281 List<TParseTreeNode> constants = visitor.getConstants(); 26282 analyzeConstantDataFlowRelation(resultColumn, constants, EffectType.function, functions); 26283 } 26284 26285 conditionExpressions.addAll(indirectExpressions); 26286 for (int j = 0; j < conditionExpressions.size(); j++) { 26287 analyzeFilterCondition(resultColumn, conditionExpressions.get(j), null, null, EffectType.function); 26288 } 26289 } 26290 26291 26292 private static void addResultColumnListExpressions( 26293 TResultColumnList list, List<TExpression> target) { 26294 if (list == null) return; 26295 for (int k = 0; k < list.size(); k++) { 26296 TExpression e = list.getResultColumn(k).getExpr(); 26297 if (e != null) target.add(e); 26298 } 26299 } 26300 26301 private static void addPassingClauseExpressions( 26302 TFunctionCall fc, List<TExpression> target) { 26303 TResultColumnList list = null; 26304 if (fc.getXmlPassingClause() != null 26305 && fc.getXmlPassingClause().getPassingList() != null) { 26306 list = fc.getXmlPassingClause().getPassingList(); 26307 } else if (fc.getPassingClause() != null 26308 && fc.getPassingClause().getPassingList() != null) { 26309 list = fc.getPassingClause().getPassingList(); 26310 } 26311 addResultColumnListExpressions(list, target); 26312 } 26313 26314 private void collectOracleXmlFunctionExpressions( 26315 TFunctionCall fc, 26316 List<TExpression> direct, 26317 List<TExpression> indirect) { 26318 26319 if (option.getVendor() != EDbVendor.dbvoracle) return; 26320 26321 // XMLELEMENT: value exprs + XMLATTRIBUTES exprs (direct) 26322 addResultColumnListExpressions(fc.getXMLElementValueExprList(), direct); 26323 if (fc.getXMLAttributesClause() != null) { 26324 addResultColumnListExpressions( 26325 fc.getXMLAttributesClause().getValueExprList(), direct); 26326 } 26327 26328 // XMLFOREST (direct) 26329 addResultColumnListExpressions(fc.getXMLForestValueList(), direct); 26330 26331 // EXTRACT(XML) — AST-shape disambiguation (V3) 26332 // XML form populates getXMLType_Instance(); scalar EXTRACT(YEAR FROM d) does not. 26333 if (fc.getXMLType_Instance() != null) { 26334 direct.add(fc.getXMLType_Instance()); 26335 } 26336 26337 String name = fc.getFunctionName() == null 26338 ? "" : fc.getFunctionName().toString().toUpperCase(); 26339 26340 // XMLEXISTS indirect (filter) from PASSING expressions 26341 if ("XMLEXISTS".equals(name)) { 26342 addPassingClauseExpressions(fc, indirect); 26343 } 26344 26345 // XMLAGG / SYS_XMLAGG ORDER BY → indirect 26346 if (("XMLAGG".equals(name) || "SYS_XMLAGG".equals(name)) 26347 && fc.getSortClause() != null) { 26348 TOrderByItemList orderByList = fc.getSortClause().getItems(); 26349 if (orderByList != null) { 26350 for (int k = 0; k < orderByList.size(); k++) { 26351 TExpression e = orderByList.getOrderByItem(k).getSortKey(); 26352 if (e != null) indirect.add(e); 26353 } 26354 } 26355 } 26356 26357 // DELIBERATELY NOT HARVESTED (metadata, not data): 26358 // fc.getTypeExpression() — target type for XMLCAST 26359 // fc.getXMLElementNameExpr() — element tag identifier 26360 // XPath / XQuery literal strings that appear as function args 26361 // datatype / format / style tokens 26362 } 26363 26364 private void collectArrayAggOrderByExpressions( 26365 TFunctionCall fc, 26366 List<TExpression> direct, 26367 List<TExpression> indirect) { 26368 if (fc.getFunctionType() != EFunctionType.array_agg_t) return; 26369 26370 TOrderByItemList items = null; 26371 if (fc.getSortClause() != null) items = fc.getSortClause().getItems(); 26372 if (items == null || items.size() == 0) { 26373 items = fc.getOrderByList(); 26374 } 26375 if (items == null) return; 26376 26377 // Resolve source tables for ORDER BY columns that the parser 26378 // doesn't link (they're inside a function call, outside the 26379 // resolver's normal scope) 26380 TTable fallbackTable = null; 26381 if (!stmtStack.isEmpty() && stmtStack.peek() instanceof TSelectSqlStatement) { 26382 TTableList tables = ((TSelectSqlStatement) stmtStack.peek()).tables; 26383 if (tables != null && tables.size() > 0) { 26384 fallbackTable = tables.getTable(0); 26385 } 26386 } 26387 26388 for (int k = 0; k < items.size(); k++) { 26389 TExpression e = items.getOrderByItem(k).getSortKey(); 26390 if (e != null) { 26391 if (fallbackTable != null 26392 && e.getExpressionType() == EExpressionType.simple_object_name_t 26393 && e.getObjectOperand() != null 26394 && e.getObjectOperand().getSourceTable() == null) { 26395 e.getObjectOperand().setSourceTable(fallbackTable); 26396 } 26397 direct.add(e); 26398 } 26399 } 26400 } 26401 26402 private boolean isArrayAggRowReference(TObjectName on) { 26403 if (on == null) return false; 26404 TTable src = on.getSourceTable(); 26405 if (src == null) return false; 26406 26407 // Raw equivalent of the old getColumnName extraction (toString fallback preserved). 26408 String raw = on.getColumnNameOnly(); 26409 if (raw == null || raw.trim().isEmpty()) { 26410 raw = on.toString() == null ? "" : on.toString().trim(); 26411 } else { 26412 raw = raw.trim(); 26413 } 26414 if (raw.isEmpty()) return false; 26415 26416 // Check if identifier text matches the table alias or table name (canonical dotTable). 26417 boolean matchesAlias = 26418 (src.getAliasName() != null 26419 && SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotTable, raw, src.getAliasName().toString())) 26420 || (src.getTableName() != null 26421 && SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotTable, raw, src.getTableName().toString())); 26422 return matchesAlias; 26423 } 26424 26425 /** 26426 * Collects column names inferred from a row-reference array_agg: 26427 * - ORDER BY keys inside the function call 26428 * - GROUP BY keys in the enclosing SELECT 26429 * Excludes the table alias itself (so `original` in `array_agg(original)` 26430 * does not leak into the column list). 26431 */ 26432 private List<TObjectName> collectRowReferenceInferredColumns( 26433 TFunctionCall functionCall, String tableAlias) { 26434 List<TObjectName> columns = new ArrayList<TObjectName>(); 26435 26436 TOrderByItemList orderItems = null; 26437 if (functionCall.getSortClause() != null) { 26438 orderItems = functionCall.getSortClause().getItems(); 26439 } 26440 if (orderItems == null || orderItems.size() == 0) { 26441 orderItems = functionCall.getOrderByList(); 26442 } 26443 if (orderItems != null) { 26444 for (int k = 0; k < orderItems.size(); k++) { 26445 TExpression sortKey = orderItems.getOrderByItem(k).getSortKey(); 26446 collectInferredColumn(sortKey, tableAlias, columns); 26447 } 26448 } 26449 26450 if (!stmtStack.isEmpty() && stmtStack.peek() instanceof TSelectSqlStatement) { 26451 TSelectSqlStatement select = (TSelectSqlStatement) stmtStack.peek(); 26452 if (select.getGroupByClause() != null) { 26453 TGroupByItemList groupByList = select.getGroupByClause().getItems(); 26454 for (int k = 0; k < groupByList.size(); k++) { 26455 TExpression expr = groupByList.getGroupByItem(k).getExpr(); 26456 collectInferredColumn(expr, tableAlias, columns); 26457 } 26458 } 26459 } 26460 return columns; 26461 } 26462 26463 private void collectInferredColumn(TExpression expr, String tableAlias, 26464 List<TObjectName> columns) { 26465 if (expr == null) return; 26466 if (expr.getExpressionType() != EExpressionType.simple_object_name_t) return; 26467 if (expr.getObjectOperand() == null) return; 26468 String colName = DlineageUtil.getColumnName(expr.getObjectOperand()); 26469 if (colName == null || colName.isEmpty()) return; 26470 if (tableAlias != null && SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotTable, tableAlias, colName)) return; 26471 if(!columns.contains(expr.getObjectOperand())) { 26472 columns.add(expr.getObjectOperand()); 26473 } 26474 } 26475 26476 /** 26477 * Creates a `*` FunctionResultColumn plus one FunctionResultColumn per 26478 * inferred column. Each column gets its own source-edge back to 26479 * the source table (source.* -> function.*, source.id -> function.id, etc). 26480 * 26481 * Downstream, the `*` column collects all three edges into the inner 26482 * `unique` ResultColumn, and post-processing (rewriteArrayAggRowReferenceTargets) 26483 * splits the collapsed CTAS relation into one per column. 26484 */ 26485 private void createRowReferenceStarColumn(Function function, TTable src, 26486 List<TObjectName> inferredColumns) { 26487 Object tableModel = modelManager.getModel(src); 26488 26489 // Create function.* and bind source.* -> function.* 26490 TObjectName starName = new TObjectName(); 26491 starName.setString("*"); 26492 ResultColumn starColumn = modelFactory.createFunctionResultColumn(function, starName); 26493 starColumn.setStruct(true); 26494 starColumn.setShowStar(true); 26495 26496 if (tableModel instanceof Table) { 26497 Table sourceTable = (Table) tableModel; 26498 TableColumn sourceCol = findOrCreateTableColumn(sourceTable, "*"); 26499 26500 for (TObjectName colName : inferredColumns) { 26501 TableColumn tableColumn = modelFactory.createTableColumn(sourceTable, colName, false); 26502 if (tableColumn != null) { 26503 //order by and group by fdr relation 26504 AbstractRelationship relation = modelFactory.createImpactRelation(); 26505 relation.setEffectType(EffectType.function); 26506 relation.setTarget(new ResultColumnRelationshipElement(starColumn)); 26507 relation.addSource(new TableColumnRelationshipElement(tableColumn)); 26508 26509 // star column fdd relation 26510 relation = modelFactory.createDataFlowRelation(); 26511 relation.setEffectType(EffectType.function); 26512 relation.setTarget(new ResultColumnRelationshipElement(starColumn)); 26513 relation.addSource(new TableColumnRelationshipElement(tableColumn)); 26514 } 26515 } 26516 26517 26518 if (sourceCol != null) { 26519 DataFlowRelationship rel = modelFactory.createDataFlowRelation(); 26520 rel.setEffectType(EffectType.function); 26521 rel.setTarget(new ResultColumnRelationshipElement(starColumn)); 26522 rel.addSource(new TableColumnRelationshipElement(sourceCol)); 26523 } 26524 26525 } else if (tableModel instanceof ResultSet) { 26526 ResultSet sourceRS = (ResultSet) tableModel; 26527 ResultColumn sourceCol = findOrCreateResultColumn(sourceRS, "*"); 26528 for (TObjectName colName : inferredColumns) { 26529 sourceCol.bindStarLinkColumn(colName); 26530 } 26531 26532 if (sourceCol != null) { 26533 DataFlowRelationship rel = modelFactory.createDataFlowRelation(); 26534 rel.setEffectType(EffectType.function); 26535 rel.setTarget(new ResultColumnRelationshipElement(starColumn)); 26536 rel.addSource(new ResultColumnRelationshipElement(sourceCol)); 26537 } 26538 } 26539 } 26540 26541 /** 26542 * Builds the Function lineage model for a structured-dataflow descriptor. 26543 * Creates per-field FunctionResultColumns and links each to an exact 26544 * structured source TableColumn (e.g. {@code nodes[*].key}). Returns the 26545 * Function on success; returns {@code null} when the source column cannot 26546 * be resolved so the caller falls back to existing function lineage. 26547 */ 26548 private Function createStructuredDataflowFunction(StructuredDataflowDescriptor desc) { 26549 TParseTreeNode syntaxNode = desc.getSyntaxNode(); 26550 if (!(syntaxNode instanceof TFunctionCall)) { 26551 return null; 26552 } 26553 TFunctionCall fnCall = (TFunctionCall) syntaxNode; 26554 26555 Function function = null; 26556 Object existing = modelManager.getModel(fnCall); 26557 if (existing instanceof Function) { 26558 function = (Function) existing; 26559 } 26560 boolean alreadyBuilt = function != null 26561 && function.getColumns() != null 26562 && !function.getColumns().isEmpty(); 26563 if (alreadyBuilt) { 26564 return function; 26565 } 26566 if (function == null) { 26567 function = modelFactory.createFunction(fnCall); 26568 } 26569 26570 StructuredValueSource src = desc.getSource(); 26571 TObjectName sourceColumn = src.getSourceColumn(); 26572 TTable srcTable = sourceColumn != null ? sourceColumn.getSourceTable() : null; 26573 Object srcTableModel = srcTable != null ? modelManager.getModel(srcTable) : null; 26574 if (!(srcTableModel instanceof Table)) { 26575 return null; 26576 } 26577 Table sourceTable = (Table) srcTableModel; 26578 26579 for (StructuredFieldBinding fb : desc.getFieldBindings()) { 26580 String pathDisplay = fb.getSourcePath().toDisplayString(); 26581 TableColumn pathColumn = findOrCreateStructuredPathColumn(sourceTable, pathDisplay); 26582 if (pathColumn == null) { 26583 continue; 26584 } 26585 26586 TObjectName fieldName = new TObjectName(); 26587 fieldName.setString(fb.getOutputFieldName()); 26588 ResultColumn fnResult = modelFactory.createFunctionResultColumn(function, fieldName); 26589 26590 DataFlowRelationship rel = modelFactory.createDataFlowRelation(); 26591 rel.setEffectType(EffectType.function); 26592 rel.setTarget(new ResultColumnRelationshipElement(fnResult)); 26593 rel.addSource(new TableColumnRelationshipElement(pathColumn)); 26594 } 26595 return function; 26596 } 26597 26598 /** 26599 * Resolve a struct-field-style outer reference ({@code structAlias.field} 26600 * or {@code relAlias.structAlias.field}) against in-scope subquery 26601 * QueryTables that hold a struct-bearing result column. When found, 26602 * append a relation source pointing at the function's field result 26603 * column so the exact source path (e.g. {@code nodes[*].key}) is 26604 * preserved through the outer projection. Returns {@code true} when 26605 * the reference was bound and the caller should skip orphan-column 26606 * fallback. 26607 */ 26608 private boolean tryAppendStructuredFieldRelation(DataFlowRelationship relation, 26609 TObjectName columnName, TTableList tableList) { 26610 if (relation == null || columnName == null || tableList == null) { 26611 return false; 26612 } 26613 String fullText = columnName.toString(); 26614 if (fullText == null) return false; 26615 int firstDot = fullText.indexOf('.'); 26616 if (firstDot < 0) return false; 26617 26618 String structAlias; 26619 String fieldName; 26620 int lastDot = fullText.lastIndexOf('.'); 26621 if (firstDot == lastDot) { 26622 structAlias = fullText.substring(0, firstDot); 26623 fieldName = fullText.substring(firstDot + 1); 26624 } else { 26625 structAlias = fullText.substring(firstDot + 1, lastDot); 26626 fieldName = fullText.substring(lastDot + 1); 26627 } 26628 if (structAlias == null || structAlias.isEmpty()) return false; 26629 if (fieldName == null || fieldName.isEmpty()) return false; 26630 26631 for (int i = 0; i < tableList.size(); i++) { 26632 TTable tt = tableList.getTable(i); 26633 if (tt == null) continue; 26634 Object tm = modelManager.getModel(tt); 26635 if (!(tm instanceof QueryTable)) continue; 26636 QueryTable qt = (QueryTable) tm; 26637 if (qt.getColumns() == null) continue; 26638 for (ResultColumn rc : qt.getColumns()) { 26639 if (rc == null || rc.getName() == null) continue; 26640 if (!SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotColumn, structAlias, rc.getName())) continue; 26641 ResultColumn fieldColumn = findStructuredFieldColumn(rc, fieldName); 26642 if (fieldColumn != null) { 26643 relation.addSource(new ResultColumnRelationshipElement(fieldColumn)); 26644 return true; 26645 } 26646 } 26647 } 26648 return false; 26649 } 26650 26651 /** 26652 * Given a QueryTable result column that aliases a structured generator 26653 * (e.g. {@code nodes1} backed by Spark {@code explode(from_json(...))}), 26654 * locate the FunctionResultColumn named {@code fieldName}. The Function 26655 * model is discovered by walking the existing data-flow relations whose 26656 * target is the alias column. 26657 */ 26658 private ResultColumn findStructuredFieldColumn(ResultColumn aliasColumn, String fieldName) { 26659 if (aliasColumn == null || fieldName == null) return null; 26660 Relationship[] rels = modelManager.getRelations(); 26661 if (rels == null) return null; 26662 String normalField = fieldName.trim(); 26663 for (Relationship r : rels) { 26664 if (!(r instanceof DataFlowRelationship)) continue; 26665 DataFlowRelationship dfr = (DataFlowRelationship) r; 26666 if (dfr.getTarget() == null) continue; 26667 if (!(dfr.getTarget().getElement() == aliasColumn)) continue; 26668 if (dfr.getSources() == null) continue; 26669 for (RelationshipElement<?> se : dfr.getSources()) { 26670 Object srcElement = se.getElement(); 26671 if (!(srcElement instanceof ResultColumn)) continue; 26672 ResultColumn src = (ResultColumn) srcElement; 26673 if (SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotColumn, normalField, src.getName())) { 26674 return src; 26675 } 26676 } 26677 } 26678 return null; 26679 } 26680 26681 /** 26682 * Find-or-create a TableColumn carrying a structured display path 26683 * (e.g. {@code nodes[*].key}) on the given source table. Unlike the 26684 * standard {@link #findOrCreateTableColumn(Table, String)}, this does 26685 * not treat the dotted/bracketed text as a schema-qualified name; 26686 * the entire path is used as the column display, so existing flat 26687 * columns named after the leaf field (e.g. {@code key}) do not 26688 * collide with structured-path columns. 26689 */ 26690 private TableColumn findOrCreateStructuredPathColumn(Table table, String pathDisplay) { 26691 if (table == null || pathDisplay == null || pathDisplay.isEmpty()) return null; 26692 for (TableColumn tc : table.getColumns()) { 26693 if (pathDisplay.equalsIgnoreCase(tc.getName())) { 26694 return tc; 26695 } 26696 } 26697 return new TableColumn(table, pathDisplay); 26698 } 26699 26700 private TableColumn findOrCreateTableColumn(Table table, String colName) { 26701 String normalName = DlineageUtil.getIdentifierNormalColumnName(colName); 26702 for (TableColumn tc : table.getColumns()) { 26703 if ("*".equals(colName) && "*".equals(tc.getName())) return tc; 26704 if (DlineageUtil.compareColumnIdentifier(tc.getName(), normalName)) return tc; 26705 } 26706 TObjectName obj = new TObjectName(); 26707 obj.setString(colName); 26708 return modelFactory.createTableColumn(table, obj, "*".equals(colName)); 26709 } 26710 26711 private ResultColumn findOrCreateResultColumn(ResultSet resultSet, String colName) { 26712 String normalName = DlineageUtil.getIdentifierNormalColumnName(colName); 26713 for (ResultColumn rc : resultSet.getColumns()) { 26714 if ("*".equals(colName) && "*".equals(rc.getName())) return rc; 26715 if (DlineageUtil.compareColumnIdentifier(rc.getName(), normalName)) return rc; 26716 } 26717 TObjectName obj = new TObjectName(); 26718 obj.setString(colName); 26719 return modelFactory.createResultColumn(resultSet, obj); 26720 } 26721 26722 private void getFunctionExpressions(List<TExpression> directExpressions, List<TExpression> indirectExpressions, 26723 TFunctionCall functionCall) { 26724 if (functionCall.getArgs() != null) { 26725 for (int k = 0; k < functionCall.getArgs().size(); k++) { 26726 TExpression expr = functionCall.getArgs().getExpression(k); 26727 if(FunctionUtility.isDirectRelation(option.getVendor(), functionCall.getFunctionName().toString(), functionCall.getArgs().size(), k)) { 26728 directExpressions.add(expr); 26729 } 26730 if(FunctionUtility.isIndirectRelation(option.getVendor(), functionCall.getFunctionName().toString(), functionCall.getArgs().size(), k)) { 26731 if("DECODE".equalsIgnoreCase(functionCall.getFunctionName().toString()) && option.getVendor() == EDbVendor.dbvoracle) { 26732 if(option.isShowCaseWhenAsDirect()) { 26733 directExpressions.add(expr); 26734 continue; 26735 } 26736 } 26737 indirectExpressions.add(expr); 26738 } 26739 } 26740 } 26741 collectOracleXmlFunctionExpressions(functionCall, directExpressions, indirectExpressions); 26742 collectArrayAggOrderByExpressions(functionCall, directExpressions, indirectExpressions); 26743 if (functionCall.getTrimArgument() != null) { 26744 TTrimArgument args = functionCall.getTrimArgument(); 26745 TExpression expr = args.getStringExpression(); 26746 if (expr != null) { 26747 directExpressions.add(expr); 26748 } 26749 expr = args.getTrimCharacter(); 26750 if (expr != null) { 26751 directExpressions.add(expr); 26752 } 26753 } 26754 26755 if (functionCall.getAgainstExpr() != null) { 26756 directExpressions.add(functionCall.getAgainstExpr()); 26757 } 26758// if (functionCall.getBetweenExpr() != null) { 26759// directExpressions.add(functionCall.getBetweenExpr()); 26760// } 26761 if (functionCall.getExpr1() != null) { 26762 directExpressions.add(functionCall.getExpr1()); 26763 } 26764 if (functionCall.getExpr2() != null) { 26765 directExpressions.add(functionCall.getExpr2()); 26766 } 26767 if (functionCall.getExpr3() != null) { 26768 directExpressions.add(functionCall.getExpr3()); 26769 } 26770 if (functionCall.getParameter() != null) { 26771 directExpressions.add(functionCall.getParameter()); 26772 } 26773 if (functionCall.getWindowDef() != null && functionCall.getWindowDef().getPartitionClause() != null) { 26774 TExpressionList args = functionCall.getWindowDef().getPartitionClause().getExpressionList(); 26775 if (args != null) { 26776 for (int k = 0; k < args.size(); k++) { 26777 TExpression expr = args.getExpression(k); 26778 if (expr != null) { 26779 indirectExpressions.add(expr); 26780 } 26781 } 26782 } 26783 } 26784 if (functionCall.getWindowDef() != null && functionCall.getWindowDef().getOrderBy() != null) { 26785 TOrderByItemList orderByList = functionCall.getWindowDef().getOrderBy().getItems(); 26786 for (int i = 0; i < orderByList.size(); i++) { 26787 TOrderByItem element = orderByList.getOrderByItem(i); 26788 TExpression expression = element.getSortKey(); 26789 indirectExpressions.add(expression); 26790 } 26791 } 26792 if (functionCall.getWithinGroup() != null && functionCall.getWithinGroup().getOrderBy() != null) { 26793 TOrderByItemList orderByList = functionCall.getWithinGroup().getOrderBy().getItems(); 26794 for (int i = 0; i < orderByList.size(); i++) { 26795 TOrderByItem element = orderByList.getOrderByItem(i); 26796 TExpression expression = element.getSortKey(); 26797 indirectExpressions.add(expression); 26798 } 26799 } 26800 if (functionCall.getCallTarget() != null) { 26801 directExpressions.add(functionCall.getCallTarget().getExpr()); 26802 } 26803 if (functionCall.getFieldValues() != null) { 26804 for (int k = 0; k < functionCall.getFieldValues().size(); k++) { 26805 TExpression expr = functionCall.getFieldValues().getResultColumn(k).getExpr(); 26806 directExpressions.add(expr); 26807 } 26808 } 26809 if (functionCall instanceof TJsonObjectFunction) { 26810 TJsonObjectFunction jsonObject = (TJsonObjectFunction)functionCall; 26811 for (int k = 0; k < jsonObject.getKeyValues().size(); k++) { 26812 TExpression expr = jsonObject.getKeyValues().get(k).getValue(); 26813 directExpressions.add(expr); 26814 } 26815 } 26816 if (functionCall.getGroupConcatParam() != null) { 26817 for (int k = 0; k < functionCall.getGroupConcatParam().getExprList().size(); k++) { 26818 TExpression expr = functionCall.getGroupConcatParam().getExprList().getExpression(k); 26819 directExpressions.add(expr); 26820 } 26821 } 26822 } 26823 26824 private void getFunctionExpressions(List<TExpression> directExpressions, List<TExpression> indirectExpressions, 26825 TFunctionCall functionCall, int argumentIndex) { 26826 if (functionCall.getArgs() != null && argumentIndex < functionCall.getArgs().size()) { 26827 TExpression expr = functionCall.getArgs().getExpression(argumentIndex); 26828 if (FunctionUtility.isDirectRelation(option.getVendor(), functionCall.getFunctionName().toString(), functionCall.getArgs().size(), argumentIndex)) { 26829 directExpressions.add(expr); 26830 } 26831 if (FunctionUtility.isIndirectRelation(option.getVendor(), functionCall.getFunctionName().toString(), functionCall.getArgs().size(), argumentIndex)) { 26832 indirectExpressions.add(expr); 26833 } 26834 } 26835 } 26836 26837 private void getFunctionExpressions(List<TExpression> directExpressions, List<TExpression> indirectExpressions, 26838 TCallStatement functionCall, int argumentIndex) { 26839 if (functionCall.getArgs() != null && argumentIndex < functionCall.getArgs().size()) { 26840 TExpression expr = functionCall.getArgs().getExpression(argumentIndex); 26841 if (FunctionUtility.isDirectRelation(option.getVendor(), functionCall.getRoutineName().toString(), 26842 functionCall.getArgs().size(), argumentIndex)) { 26843 directExpressions.add(expr); 26844 } 26845 if (FunctionUtility.isIndirectRelation(option.getVendor(), functionCall.getRoutineName().toString(), 26846 functionCall.getArgs().size(), argumentIndex)) { 26847 indirectExpressions.add(expr); 26848 } 26849 } 26850 } 26851 26852 private void getFunctionExpressions(List<TExpression> directExpressions, List<TExpression> indirectExpressions, 26853 TDb2CallStmt functionCall, int argumentIndex) { 26854 if (functionCall.getParameters() != null && argumentIndex < functionCall.getParameters().size()) { 26855 TExpression expr = functionCall.getParameters().getExpression(argumentIndex); 26856 if (FunctionUtility.isDirectRelation(option.getVendor(), functionCall.getProcedureName().toString(), 26857 functionCall.getParameters().size(), argumentIndex)) { 26858 directExpressions.add(expr); 26859 } 26860 if (FunctionUtility.isIndirectRelation(option.getVendor(), functionCall.getProcedureName().toString(), 26861 functionCall.getParameters().size(), argumentIndex)) { 26862 indirectExpressions.add(expr); 26863 } 26864 } 26865 } 26866 26867 private void getFunctionExpressions(List<TExpression> directExpressions, List<TExpression> indirectExpressions, 26868 TMssqlExecute functionCall, String argumentName, int argumentIndex) { 26869 if (functionCall.getParameters() != null) { 26870 for (int i = 0; i < functionCall.getParameters().size(); i++) { 26871 TExecParameter param = functionCall.getParameters().getExecParameter(i); 26872 if (param.getParameterName() != null) { 26873 if (DlineageUtil.compareColumnIdentifier(param.getParameterName().toString(), argumentName)) { 26874 TExpression expr = param.getParameterValue(); 26875 directExpressions.add(expr); 26876 } 26877 } else if (i == argumentIndex) { 26878 TExpression expr = param.getParameterValue(); 26879 directExpressions.add(expr); 26880 } 26881 } 26882 } 26883 } 26884 26885 private void analyzeJoin(TJoin join, EffectType effectType) { 26886 if (join.getJoinItems() != null) { 26887 for (int j = 0; j < join.getJoinItems().size(); j++) { 26888 TJoinItem joinItem = join.getJoinItems().getJoinItem(j); 26889 TExpression expr = joinItem.getOnCondition(); 26890 if (expr != null) { 26891 analyzeFilterCondition(null, expr, joinItem.getJoinType(), JoinClauseType.on, effectType); 26892 } 26893 } 26894 } 26895 26896 if (join.getJoin() != null) { 26897 analyzeJoin(join.getJoin(), effectType); 26898 } 26899 } 26900 26901 private TSelectSqlStatement getParentSetSelectStmt(TSelectSqlStatement stmt) { 26902 TCustomSqlStatement parent = stmt.getParentStmt(); 26903 if (parent == null) 26904 return null; 26905 if (parent.getStatements() != null) { 26906 for (int i = 0; i < parent.getStatements().size(); i++) { 26907 TCustomSqlStatement temp = parent.getStatements().get(i); 26908 if (temp instanceof TSelectSqlStatement) { 26909 TSelectSqlStatement select = (TSelectSqlStatement) temp; 26910 if (select.getLeftStmt() == stmt || select.getRightStmt() == stmt) 26911 return select; 26912 } 26913 } 26914 } 26915 if (parent instanceof TSelectSqlStatement) { 26916 TSelectSqlStatement select = (TSelectSqlStatement) parent; 26917 if (select.getLeftStmt() == stmt || select.getRightStmt() == stmt) 26918 return select; 26919 } 26920 return null; 26921 } 26922 26923 private void createSelectSetResultColumns(SelectSetResultSet resultSet, TSelectSqlStatement stmt) { 26924 if (stmt.getSetOperatorType() != ESetOperatorType.none) { 26925 createSelectSetResultColumns(resultSet, stmt.getLeftStmt()); 26926 } else { 26927 TResultColumnList columnList = stmt.getResultColumnList(); 26928 ResultSet subqueryResultSet = (ResultSet) modelManager.getModel(columnList); 26929 if(subqueryResultSet!=null && subqueryResultSet.isDetermined()) { 26930 for (int j = 0; j < subqueryResultSet.getColumns().size(); j++) { 26931 ResultColumn tableColumn = subqueryResultSet.getColumns().get(j); 26932 String name = tableColumn.getRefColumnName() != null 26933 ? tableColumn.getRefColumnName() : tableColumn.getName(); 26934 boolean emptyName = SQLUtil.isEmpty(name); 26935 if (emptyName) { 26936 // Empty delimited identifiers have no normalized name. Keep 26937 // their set-column ordinal and recover the lexical spelling. 26938 name = tableColumn.getFullName(); 26939 } 26940 TObjectName columnName = new TObjectName(); 26941 columnName.setString(name == null ? "" : name); 26942 if (emptyName && tableColumn.getColumnObject() != null) { 26943 // Reuse the real span instead of publishing a synthetic 1:1 26944 // coordinate for the recovered spelling. 26945 columnName.setStartTokenDirectly(tableColumn.getColumnObject().getStartToken()); 26946 columnName.setEndTokenDirectly(tableColumn.getColumnObject().getEndToken()); 26947 } 26948 modelFactory.createDeterminedResultColumn(resultSet, columnName); 26949 } 26950 resultSet.setDetermined(true); 26951 return; 26952 } 26953 26954 boolean isDetermined = true; 26955 for (int i = 0; i < columnList.size(); i++) { 26956 TResultColumn column = columnList.getResultColumn(i); 26957 26958 if ("*".equals(column.getColumnNameOnly())) { 26959 TObjectName columnObject = column.getFieldAttr(); 26960 TTable sourceTable = columnObject.getSourceTable(); 26961 if (sourceTable != null) { 26962 Object tableModel = modelManager.getModel(sourceTable); 26963 if (tableModel instanceof Table && ((Table) tableModel).isCreateTable()) { 26964 Table table = (Table) tableModel; 26965 for (int j = 0; j < table.getColumns().size(); j++) { 26966 TableColumn tableColumn = table.getColumns().get(j); 26967 if (column.getExceptColumnList() != null) { 26968 boolean except = false; 26969 for (TObjectName objectName : column.getExceptColumnList()) { 26970 if (getColumnName(objectName.toString()) 26971 .equals(getColumnName(tableColumn.getName()))) { 26972 except = true; 26973 break; 26974 } 26975 } 26976 if (!except && tableColumn.isStruct()) { 26977 List<String> names = SQLUtil 26978 .parseNames(tableColumn.getName()); 26979 for (String name : names) { 26980 for (TObjectName objectName : column 26981 .getExceptColumnList()) { 26982 if (getColumnName(objectName.toString()) 26983 .equals(getColumnName(name))) { 26984 except = true; 26985 break; 26986 } 26987 } 26988 if (except) { 26989 break; 26990 } 26991 } 26992 } 26993 if (except) { 26994 continue; 26995 } 26996 } 26997 TObjectName columnName = new TObjectName(); 26998 columnName.setString(tableColumn.getName()); 26999 ResultColumn resultColumn = modelFactory.createResultColumn( 27000 resultSet, columnName); 27001 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 27002 relation.setEffectType(EffectType.select); 27003 relation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 27004 relation.addSource(new TableColumnRelationshipElement(tableColumn)); 27005 } 27006 continue; 27007 } else if (tableModel instanceof ResultSet 27008 && ((ResultSet) tableModel).isDetermined()) { 27009 ResultSet table = (ResultSet) tableModel; 27010 for (int j = 0; j < table.getColumns().size(); j++) { 27011 ResultColumn tableColumn = table.getColumns().get(j); 27012 if (column.getExceptColumnList() != null) { 27013 boolean except = false; 27014 for (TObjectName objectName : column.getExceptColumnList()) { 27015 if (getColumnName(objectName.toString()) 27016 .equals(getColumnName(tableColumn.getName()))) { 27017 except = true; 27018 break; 27019 } 27020 } 27021 if (!except && tableColumn.isStruct()) { 27022 List<String> names = SQLUtil 27023 .parseNames(tableColumn.getName()); 27024 for (String name : names) { 27025 for (TObjectName objectName : column 27026 .getExceptColumnList()) { 27027 if (getColumnName(objectName.toString()) 27028 .equals(getColumnName(name))) { 27029 except = true; 27030 break; 27031 } 27032 } 27033 if (except) { 27034 break; 27035 } 27036 } 27037 } 27038 if (except) { 27039 continue; 27040 } 27041 } 27042 if (tableColumn.getRefColumnName() != null) { 27043 TObjectName columnName = new TObjectName(); 27044 columnName.setString(tableColumn.getRefColumnName()); 27045 ResultColumn resultColumn = modelFactory.createResultColumn( 27046 resultSet, columnName); 27047 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 27048 relation.setEffectType(EffectType.select); 27049 relation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 27050 relation.addSource(new ResultColumnRelationshipElement(tableColumn)); 27051 } else { 27052 TObjectName columnName = new TObjectName(); 27053 columnName.setString(tableColumn.getName()); 27054 ResultColumn resultColumn = modelFactory.createResultColumn( 27055 resultSet, columnName); 27056 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 27057 relation.setEffectType(EffectType.select); 27058 relation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 27059 relation.addSource(new ResultColumnRelationshipElement(tableColumn)); 27060 } 27061 } 27062 continue; 27063 } 27064 else { 27065 isDetermined = false; 27066 } 27067 } 27068 } 27069 27070 ResultColumn resultColumn = modelFactory.createSelectSetResultColumn(resultSet, column, i); 27071 27072 if (resultColumn.getColumnObject() instanceof TResultColumn) { 27073 TResultColumn columnObject = (TResultColumn) resultColumn.getColumnObject(); 27074 if (columnObject.getFieldAttr() != null) { 27075 if ("*".equals(getColumnName(columnObject.getFieldAttr()))) { 27076 TObjectName fieldAttr = columnObject.getFieldAttr(); 27077 TTable sourceTable = fieldAttr.getSourceTable(); 27078 if (fieldAttr.getTableToken() != null && sourceTable != null) { 27079 TObjectName[] columns = modelManager.getTableColumns(sourceTable); 27080 for (int j = 0; j < columns.length; j++) { 27081 TObjectName columnName = columns[j]; 27082 if (columnName == null) { 27083 continue; 27084 } 27085 if ("*".equals(getColumnName(columnName))) { 27086 continue; 27087 } 27088 resultColumn.bindStarLinkColumn(columnName); 27089 } 27090 27091 if (modelManager.getModel(sourceTable) instanceof Table) { 27092 Table tableModel = (Table) modelManager.getModel(sourceTable); 27093 if (tableModel != null && !tableModel.getColumns().isEmpty()) { 27094 for (int z = 0; z < tableModel.getColumns().size(); z++) { 27095 if ("*".equals( 27096 getColumnName(tableModel.getColumns().get(z).getColumnObject()))) { 27097 continue; 27098 } 27099 resultColumn.bindStarLinkColumn( 27100 tableModel.getColumns().get(z).getColumnObject()); 27101 } 27102 } 27103 } else if (modelManager.getModel(sourceTable) instanceof QueryTable) { 27104 QueryTable tableModel = (QueryTable) modelManager.getModel(sourceTable); 27105 if (tableModel != null && !tableModel.getColumns().isEmpty()) { 27106 for (ResultColumn item : tableModel.getColumns()) { 27107 if (item.hasStarLinkColumn()) { 27108 for (TObjectName starLinkColumn : item.getStarLinkColumnList()) { 27109 if ("*".equals(getColumnName(starLinkColumn))) { 27110 continue; 27111 } 27112 resultColumn.bindStarLinkColumn(starLinkColumn); 27113 } 27114 } else if (item.getColumnObject() instanceof TObjectName) { 27115 TObjectName starLinkColumn = (TObjectName) item.getColumnObject(); 27116 if ("*".equals(getColumnName(starLinkColumn))) { 27117 continue; 27118 } 27119 resultColumn.bindStarLinkColumn(starLinkColumn); 27120 } 27121 } 27122 } 27123 } 27124 27125 } else { 27126 TTableList tables = stmt.getTables(); 27127 for (int k = 0; k < tables.size(); k++) { 27128 TTable tableElement = tables.getTable(k); 27129 TObjectName[] columns = modelManager.getTableColumns(tableElement); 27130 for (int j = 0; j < columns.length; j++) { 27131 TObjectName columnName = columns[j]; 27132 if (columnName == null) { 27133 continue; 27134 } 27135 if ("*".equals(getColumnName(columnName))) { 27136 if (modelManager.getModel(tableElement) instanceof Table) { 27137 Table tableModel = (Table) modelManager.getModel(tableElement); 27138 if (tableModel != null && !tableModel.getColumns().isEmpty()) { 27139 for (int z = 0; z < tableModel.getColumns().size(); z++) { 27140 resultColumn.bindStarLinkColumn( 27141 tableModel.getColumns().get(z).getColumnObject()); 27142 } 27143 } 27144 } else if (modelManager.getModel(tableElement) instanceof QueryTable) { 27145 QueryTable tableModel = (QueryTable) modelManager 27146 .getModel(tableElement); 27147 if (tableModel != null && !tableModel.getColumns().isEmpty()) { 27148 for (ResultColumn item : tableModel.getColumns()) { 27149 if (item.hasStarLinkColumn()) { 27150 for (TObjectName starLinkColumn : item 27151 .getStarLinkColumnList()) { 27152 resultColumn.bindStarLinkColumn(starLinkColumn); 27153 } 27154 } else if (item.getColumnObject() instanceof TObjectName) { 27155 resultColumn.bindStarLinkColumn( 27156 (TObjectName) item.getColumnObject()); 27157 } 27158 } 27159 } 27160 } 27161 continue; 27162 } 27163 resultColumn.bindStarLinkColumn(columnName); 27164 } 27165 } 27166 } 27167 } 27168 } 27169 } 27170 27171 resultSet.setDetermined(isDetermined); 27172 } 27173 } 27174 } 27175 27176 private void analyzeResultColumn(TResultColumn column, EffectType effectType) { 27177 // A SELECT * star column whose source resolves to a determined result set 27178 // is bound to a LinkedHashMap of per-column ResultColumns (see 27179 // ModelFactory.createStarResultColumn), and its data-flow relationships 27180 // are already created inline during star expansion in analyzeSelectStmt(). 27181 // Re-analyzing the raw "*" here is redundant; skip it. (analyzeDataFlowRelation 27182 // keeps the same guard as a defensive backstop.) 27183 if (modelManager.getModel(column) instanceof LinkedHashMap) { 27184 return; 27185 } 27186 TExpression expression = column.getExpr(); 27187 if (expression.getExpressionType() == EExpressionType.sqlserver_proprietary_column_alias_t) { 27188 expression = expression.getRightOperand(); 27189 } 27190 27191 if (expression.getExpressionType() == EExpressionType.array_t) { 27192 if (expression.getExprList() != null) { 27193 for (TExpression expr : expression.getExprList()) { 27194 columnsInExpr visitor = new columnsInExpr(); 27195 expr.inOrderTraverse(visitor); 27196 List<TObjectName> objectNames = visitor.getObjectNames(); 27197 27198 List<TParseTreeNode> functions = visitor.getFunctions(); 27199 27200 if (functions != null && !functions.isEmpty()) { 27201 analyzeFunctionDataFlowRelation(column, functions, effectType); 27202 } 27203 27204 List<TSelectSqlStatement> subquerys = visitor.getSubquerys(); 27205 if (subquerys != null && !subquerys.isEmpty()) { 27206 analyzeSubqueryDataFlowRelation(column, subquerys, effectType); 27207 } 27208 27209 analyzeDataFlowRelation(column, objectNames, column.getExceptColumnList(), effectType, functions); 27210 27211 List<TParseTreeNode> constants = visitor.getConstants(); 27212 Object columnObject = modelManager.getModel(column); 27213 analyzeConstantDataFlowRelation(columnObject, constants, effectType, functions); 27214 27215 analyzeRecordSetRelation(functions, effectType); 27216 // analyzeResultColumnImpact( column, effectType, functions); 27217 } 27218 } 27219 else { 27220 List<TParseTreeNode> constants = new ArrayList<TParseTreeNode>(); 27221 TConstant constant = new TConstant(); 27222 constant.setString(expression.toString()); 27223 constants.add(constant); 27224 Object columnObject = modelManager.getModel(column); 27225 analyzeConstantDataFlowRelation(columnObject, constants, effectType, null); 27226 } 27227 } else { 27228 columnsInExpr visitor = new columnsInExpr(); 27229 expression.inOrderTraverse(visitor); 27230 List<TObjectName> objectNames = visitor.getObjectNames(); 27231 27232 List<TParseTreeNode> functions = visitor.getFunctions(); 27233 27234 if (functions != null && !functions.isEmpty()) { 27235 analyzeFunctionDataFlowRelation(column, functions, effectType); 27236 } 27237 27238 List<TSelectSqlStatement> subquerys = visitor.getSubquerys(); 27239 if (subquerys != null && !subquerys.isEmpty()) { 27240 analyzeSubqueryDataFlowRelation(column, subquerys, effectType); 27241 } 27242 27243 analyzeDataFlowRelation(column, objectNames, column.getExceptColumnList(), effectType, functions); 27244 27245 List<TParseTreeNode> constants = visitor.getConstants(); 27246 Object columnObject = modelManager.getModel(column); 27247 analyzeConstantDataFlowRelation(columnObject, constants, effectType, functions); 27248 27249 analyzeRecordSetRelation(functions, effectType); 27250 // analyzeResultColumnImpact( column, effectType, functions); 27251 } 27252 } 27253 27254 27255 private void analyzeValueColumn(Object object, TResultColumn column, EffectType effectType) { 27256 TExpression expression = column.getExpr(); 27257 if (expression.getExpressionType() == EExpressionType.sqlserver_proprietary_column_alias_t) { 27258 expression = expression.getRightOperand(); 27259 } 27260 27261 if (expression.getExpressionType() == EExpressionType.array_t) { 27262 if (expression.getExprList() != null) { 27263 for (TExpression expr : expression.getExprList()) { 27264 columnsInExpr visitor = new columnsInExpr(); 27265 expr.inOrderTraverse(visitor); 27266 List<TObjectName> objectNames = visitor.getObjectNames(); 27267 analyzeDataFlowRelation(object, objectNames, column.getExceptColumnList(), effectType, null, null); 27268 List<TParseTreeNode> constants = visitor.getConstants(); 27269 analyzeConstantDataFlowRelation(object, constants, effectType, null); 27270 } 27271 } 27272 else { 27273 List<TParseTreeNode> constants = new ArrayList<TParseTreeNode>(); 27274 TConstant constant = new TConstant(); 27275 constant.setString(expression.toString()); 27276 constants.add(constant); 27277 Object columnObject = modelManager.getModel(column); 27278 analyzeConstantDataFlowRelation(object, constants, effectType, null); 27279 } 27280 } else { 27281 columnsInExpr visitor = new columnsInExpr(); 27282 expression.inOrderTraverse(visitor); 27283 List<TObjectName> objectNames = visitor.getObjectNames(); 27284 analyzeDataFlowRelation(object, objectNames, column.getExceptColumnList(), effectType, null, null); 27285 List<TParseTreeNode> constants = visitor.getConstants(); 27286 analyzeConstantDataFlowRelation(object, constants, effectType, null); } 27287 } 27288 27289 private void analyzeTableColumn(TableColumn tableColumn, TFunctionCall functionCall, EffectType effectType) { 27290 // ClickHouse external table functions are modelled as source TABLES 27291 // (modelClickhouseExternalTableFunction); running the generic 27292 // function-argument analysis on top fabricates argument-named columns 27293 // on that table (remote('h', mydb, mytable) invented a column "mydb") 27294 // and re-attaches the generic function node the modelling replaced. 27295 if (option.getVendor() == EDbVendor.dbvclickhouse && functionCall != null 27296 && functionCall.getFunctionName() != null) { 27297 String rawName = functionCall.getFunctionName().toString(); 27298 if (isClickhouseTableFunction(rawName, "remote") 27299 || isClickhouseTableFunction(rawName, "remoteSecure") 27300 || isClickhouseTableFunction(rawName, "file") 27301 || isClickhouseTableFunction(rawName, "s3") 27302 || isClickhouseTableFunction(rawName, "url") 27303 || isClickhouseTableFunction(rawName, "hdfs") 27304 || isClickhouseTableFunction(rawName, "fileCluster") 27305 || isClickhouseTableFunction(rawName, "s3Cluster") 27306 || isClickhouseTableFunction(rawName, "urlCluster") 27307 || isClickhouseTableFunction(rawName, "hdfsCluster") 27308 || isClickhouseTableFunction(rawName, "azureBlobStorage") 27309 || isClickhouseTableFunction(rawName, "azureBlobStorageCluster")) { 27310 return; 27311 } 27312 } 27313 List<TParseTreeNode> functions = new ArrayList<TParseTreeNode>(); 27314 functions.add(functionCall); 27315 27316 if (functions != null && !functions.isEmpty()) { 27317 analyzeFunctionDataFlowRelation(tableColumn, functions, effectType); 27318 } 27319 27320 analyzeRecordSetRelation(functions, effectType); 27321 } 27322 27323 private void analyzeRecordSetRelation(List<TParseTreeNode> functions, EffectType effectType) { 27324 if (functions == null || functions.size() == 0) 27325 return; 27326 27327 List<TFunctionCall> aggregateFunctions = new ArrayList<TFunctionCall>(); 27328 for (TParseTreeNode function : functions) { 27329 if (function instanceof TFunctionCall && isAggregateFunction((TFunctionCall) function)) { 27330 aggregateFunctions.add((TFunctionCall) function); 27331 } 27332 } 27333 27334 if (aggregateFunctions.size() == 0) 27335 return; 27336 27337 for (int i = 0; i < aggregateFunctions.size(); i++) { 27338 TFunctionCall function = aggregateFunctions.get(i); 27339 27340 TCustomSqlStatement stmt = stmtStack.peek(); 27341 if (stmt instanceof TSelectSqlStatement) { 27342 TSelectSqlStatement select = (TSelectSqlStatement) stmt; 27343 if (select.getGroupByClause() != null) { 27344 if (select.getGroupByClause().isAllModifier()) { 27345 // GROUP BY ALL: implicit grouping columns are the 27346 // non-aggregate expressions in the SELECT list. 27347 TResultColumnList resultColumns = select.getResultColumnList(); 27348 if (resultColumns != null) { 27349 for (int j = 0; j < resultColumns.size(); j++) { 27350 TResultColumn column = resultColumns.getResultColumn(j); 27351 TExpression expr = column.getExpr(); 27352 if (expr == null) 27353 continue; 27354 columnsInExpr aggVisitor = new columnsInExpr(); 27355 expr.inOrderTraverse(aggVisitor); 27356 boolean containsAggregate = false; 27357 for (TParseTreeNode funcNode : aggVisitor.getFunctions()) { 27358 if (funcNode instanceof TFunctionCall 27359 && isAggregateFunction((TFunctionCall) funcNode)) { 27360 containsAggregate = true; 27361 break; 27362 } 27363 } 27364 if (!containsAggregate) { 27365 analyzeAggregate(function, expr); 27366 } 27367 } 27368 } 27369 } else { 27370 TGroupByItemList groupByList = select.getGroupByClause().getItems(); 27371 for (int j = 0; j < groupByList.size(); j++) { 27372 TGroupByItem groupBy = groupByList.getGroupByItem(j); 27373 TExpression expr = groupBy.getExpr(); 27374 analyzeAggregate(function, expr); 27375 } 27376 } 27377 27378 if (select.getGroupByClause().getHavingClause() != null) { 27379 analyzeAggregate(function, select.getGroupByClause().getHavingClause()); 27380 } 27381 // if ("COUNT".equalsIgnoreCase(function.getFunctionName().toString())) 27382 { 27383 analyzeAggregate(function, null); 27384 } 27385 } else { 27386 analyzeAggregate(function, null); 27387 } 27388 } 27389 } 27390 } 27391 27392 private void analyzeDataFlowRelation(TParseTreeNode gspObject, List<TObjectName> objectNames, 27393 TObjectNameList exceptColumnList, EffectType effectType, List<TParseTreeNode> functions) { 27394 Object columnObject = modelManager.getModel(gspObject); 27395 analyzeDataFlowRelation(columnObject, objectNames, exceptColumnList, effectType, functions, null); 27396 } 27397 27398 private DataFlowRelationship analyzeDataFlowRelation(Object modelObject, List<TObjectName> objectNames, EffectType effectType, 27399 List<TParseTreeNode> functions) { 27400 return analyzeDataFlowRelation(modelObject, objectNames, null, effectType, functions, null); 27401 } 27402 27403 private DataFlowRelationship analyzeDataFlowRelation(Object modelObject, List<TObjectName> objectNames, EffectType effectType, 27404 List<TParseTreeNode> functions, Process process) { 27405 return analyzeDataFlowRelation(modelObject, objectNames, null, effectType, functions, process); 27406 } 27407 27408 private DataFlowRelationship analyzeDataFlowRelation(Object modelObject, List<TObjectName> objectNames, 27409 TObjectNameList exceptColumnList, EffectType effectType, List<TParseTreeNode> functions, Process process) { 27410 return analyzeDataFlowRelation(modelObject, objectNames, exceptColumnList, effectType, functions, process, null); 27411 } 27412 27413 private DataFlowRelationship analyzeDataFlowRelation(Object modelObject, List<TObjectName> objectNames, 27414 TObjectNameList exceptColumnList, EffectType effectType, List<TParseTreeNode> functions, Process process, Integer valueIndex) { 27415 if (objectNames == null || objectNames.size() == 0) 27416 return null; 27417 27418 // Reject model objects this method cannot turn into a relationship BEFORE 27419 // creating one, since createDataFlowRelation() registers the relation 27420 // globally and an early return afterwards would leak an empty relation. 27421 // 27422 // - LinkedHashMap: a SELECT * star column whose source resolves to a 27423 // determined result set is expanded into per-column ResultColumns held 27424 // in a LinkedHashMap by ModelFactory.createStarResultColumn(), with its 27425 // data-flow relationships created inline during star expansion in 27426 // analyzeSelectStmt(). Re-analyzing the raw "*" here is a no-op. 27427 // - null: modelManager.getModel() found no binding for the gsp object. 27428 if (modelObject == null || modelObject instanceof LinkedHashMap) { 27429 return null; 27430 } 27431 // Lineage is best-effort: an unexpected model type should not abort the 27432 // whole statement's lineage (this used to throw UnsupportedOperationException). 27433 // Log it for diagnosis and skip just this column instead. 27434 if (!(modelObject instanceof ResultColumn) && !(modelObject instanceof TableColumn)) { 27435 logger.warn("analyzeDataFlowRelation: unhandled model type " 27436 + modelObject.getClass().getName() + ", effectType=" + effectType); 27437 return null; 27438 } 27439 27440 boolean isStar = false; 27441 boolean showStar = false; 27442 27443 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 27444 relation.setEffectType(effectType); 27445 relation.setProcess(process); 27446 27447 if (functions != null && !functions.isEmpty()) { 27448 relation.setFunction(getFunctionName(functions.get(0))); 27449 } 27450 27451 int columnIndex = -1; 27452 27453 boolean isOut = false; 27454 27455 if (modelObject instanceof ResultColumn) { 27456 relation.setTarget(new ResultColumnRelationshipElement((ResultColumn) modelObject)); 27457 27458 if ("*".equals(((ResultColumn) modelObject).getName())) { 27459 isStar = true; 27460 showStar = ((ResultColumn) modelObject).isShowStar(); 27461 } 27462 27463 if (((ResultColumn) modelObject).getResultSet() != null) { 27464 columnIndex = ((ResultColumn) modelObject).getResultSet().getColumns().indexOf(modelObject); 27465 } 27466 } else if (modelObject instanceof TableColumn) { 27467 Table table = ((TableColumn) modelObject).getTable(); 27468 if(table.getSubType() == SubType.out && isNotInProcedure(table)){ 27469 isOut = true; 27470 relation.addSource(new TableColumnRelationshipElement((TableColumn) modelObject)); 27471 } 27472 else { 27473 relation.setTarget(new TableColumnRelationshipElement((TableColumn) modelObject)); 27474 } 27475 27476 if ("*".equals(((TableColumn) modelObject).getName())) { 27477 isStar = true; 27478 } 27479 27480 if (((TableColumn) modelObject).getTable() != null) { 27481 columnIndex = ((TableColumn) modelObject).getTable().getColumns().indexOf(modelObject); 27482 } 27483 } 27484 // No trailing else: modelObject is guaranteed to be a ResultColumn or 27485 // TableColumn here (validated and logged above before relation creation). 27486 27487 for (int i = 0; i < objectNames.size(); i++) { 27488 TObjectName columnName = objectNames.get(i); 27489 if (columnName.toString().indexOf(".") == -1 && isConstant(columnName)) { 27490 boolean isConstant = true; 27491 if (columnName.getSourceTable() != null) { 27492 Table tableModel = modelManager.getTableByName( 27493 DlineageUtil.getTableFullName(columnName.getSourceTable().getTableName().toString())); 27494 if (tableModel != null && tableModel.getColumns() != null) { 27495 for (int j = 0; j < tableModel.getColumns().size(); j++) { 27496 if (DlineageUtil.compareColumnIdentifier(getColumnName(columnName), 27497 getColumnName(tableModel.getColumns().get(j).getName()))) { 27498 isConstant = false; 27499 break; 27500 } 27501 } 27502 } 27503 } 27504 27505 if (isConstant) { 27506 if (option.isShowConstantTable()) { 27507 Table constantTable = modelFactory.createConstantsTable(stmtStack.peek()); 27508 TableColumn tableColumn = modelFactory.createTableColumn(constantTable, columnName, false); 27509 if(tableColumn!=null) { 27510 relation.addSource(new TableColumnRelationshipElement(tableColumn)); 27511 } 27512 } 27513 continue; 27514 } 27515 } 27516 if (columnName.getDbObjectType() == EDbObjectType.variable) { 27517 boolean find = false; 27518 List<String> segments = SQLUtil.parseNames(columnName.toString()); 27519 if (segments.size() == 1) { 27520 Table variable = modelManager.getTableByName(DlineageUtil.getTableFullName(columnName.toString())); 27521 if (variable != null && !variable.getColumns().isEmpty()) { 27522 TableColumn columnModel = variable.getColumns().get(0); 27523 TableColumn matchColumn = matchColumn(variable.getColumns(), columnName); 27524 if(matchColumn!=null){ 27525 columnModel = matchColumn; 27526 } else if (variable.getColumns().size() > 1 && variable instanceof Variable) { 27527 // A bare reference to a field-expanded record is the 27528 // WHOLE record (#695): bind its star, never an 27529 // arbitrary first field — citing one field asserts 27530 // the value ignores the others. 27531 columnModel = compositeBindingColumn((Variable) variable); 27532 } 27533 if(isOut){ 27534 relation.setTarget(new TableColumnRelationshipElement(columnModel)); 27535 } 27536 else { 27537 relation.addSource(new TableColumnRelationshipElement(columnModel)); 27538 } 27539 find = true; 27540 } else { 27541 if (columnName.toString().matches("\\$\\d+")) { 27542 TCustomSqlStatement stmt = stmtStack.peek(); 27543 Procedure procedure = modelManager 27544 .getProcedureByName(DlineageUtil.getTableFullName(getProcedureParentName(stmt))); 27545 if(procedure!=null) { 27546 Variable cursorVariable = modelFactory.createVariable(procedure.getArguments().get(Integer.valueOf(columnName.toString().replace("$", "")) - 1).getName()); 27547 if (isOut) { 27548 relation.setTarget(new TableColumnRelationshipElement(cursorVariable.getColumns().get(0))); 27549 } else { 27550 relation.addSource(new TableColumnRelationshipElement(cursorVariable.getColumns().get(0))); 27551 } 27552 } 27553 } else { 27554 Variable cursorVariable = modelFactory.createVariable(columnName); 27555 cursorVariable.setCreateTable(true); 27556 cursorVariable.setSubType(SubType.record); 27557 TableColumn variableProperty = null; 27558 if(cursorVariable.getColumns() == null || cursorVariable.getColumns().isEmpty()) { 27559 variableProperty = modelFactory.createTableColumn(cursorVariable, columnName, 27560 true); 27561 } 27562 else{ 27563 variableProperty = compositeBindingColumn(cursorVariable); 27564 } 27565 if (isOut) { 27566 relation.setTarget(new TableColumnRelationshipElement(variableProperty)); 27567 } else { 27568 relation.addSource(new TableColumnRelationshipElement(variableProperty)); 27569 } 27570 } 27571 find = true; 27572 } 27573 } else if (option.getVendor() == EDbVendor.dbvoracle && columnName.getTableToken()!=null) { 27574 Variable cursorVariable = modelFactory 27575 .createVariable(columnName.getTableToken().toString()); 27576 27577 TObjectName variableColumnName = new TObjectName(); 27578 variableColumnName.setString(segments.get(segments.size() - 1)); 27579 27580 if (cursorVariable.getColumns() != null) { 27581 for (int j = 0; j < cursorVariable.getColumns().size(); j++) { 27582 if (getColumnName(variableColumnName) 27583 .equals(getColumnName(cursorVariable.getColumns().get(j).getColumnObject()))) { 27584 TableColumn columnModel = cursorVariable.getColumns().get(j); 27585 if (isOut) { 27586 relation.setTarget(new TableColumnRelationshipElement(columnModel)); 27587 } else { 27588 relation.addSource(new TableColumnRelationshipElement(columnModel)); 27589 } 27590 find = true; 27591 } 27592 } 27593 } 27594 27595 if (!find) { 27596 TableColumn variableColumn = new TableColumn(cursorVariable, variableColumnName); 27597 cursorVariable.addColumn(variableColumn); 27598 if (isOut) { 27599 relation.setTarget(new TableColumnRelationshipElement(variableColumn)); 27600 } else { 27601 relation.addSource(new TableColumnRelationshipElement(variableColumn)); 27602 } 27603 } 27604 } else { 27605 Table variable = modelManager 27606 .getTableByName(DlineageUtil.getTableFullName(segments.get(segments.size() - 2))); 27607 if (variable != null) { 27608 for (int j = 0; j < variable.getColumns().size(); j++) { 27609 if (getColumnName(columnName) 27610 .equals(getColumnName(variable.getColumns().get(j).getColumnObject()))) { 27611 TableColumn columnModel = variable.getColumns().get(j); 27612 if (isOut) { 27613 relation.setTarget(new TableColumnRelationshipElement(columnModel)); 27614 } else { 27615 relation.addSource(new TableColumnRelationshipElement(columnModel)); 27616 } 27617 find = true; 27618 } 27619 } 27620 } 27621 } 27622 if (!find) { 27623 TCustomSqlStatement stmt = stmtStack.peek(); 27624 if (getProcedureParentName(stmt) != null) { 27625 Procedure procedure = modelManager 27626 .getProcedureByName(DlineageUtil.getTableFullName(getProcedureParentName(stmt))); 27627 if (procedure != null && procedure.getArguments() != null) { 27628 for (Argument argument : procedure.getArguments()) { 27629 if (DlineageUtil.getTableFullName(argument.getName()) 27630 .equals(DlineageUtil.getTableFullName(columnName.toString()))) { 27631 relation.addSource(new ArgumentRelationshipElement(argument)); 27632 } 27633 } 27634 } 27635 } 27636 } 27637 continue; 27638 } 27639 27640 // Handle sequence pseudocolumn syntax (sequence.NEXTVAL or sequence.CURRVAL) 27641 // Used by Oracle, Snowflake, and accepted by other vendors for compatibility 27642 if(("NEXTVAL".equalsIgnoreCase(columnName.getColumnNameOnly()) || "CURRVAL".equalsIgnoreCase(columnName.getColumnNameOnly()))){ 27643 List<String> segments = SQLUtil.parseNames(columnName.toString()); 27644 if (segments.size() > 1) { 27645 segments.remove(segments.size()-1); 27646 Table table = modelFactory.createTableByName(SQLUtil.mergeSegments(segments, 0), true); 27647 table.setSequence(true); 27648 TableColumn seqCursor = modelFactory.createTableColumn(table, columnName, true); 27649 relation.addSource(new TableColumnRelationshipElement(seqCursor)); 27650 continue; 27651 } 27652 } 27653 27654 { 27655 if (columnName.getSourceTable() != null) { 27656 27657 } 27658 else { 27659 Table variable = modelManager.getTableByName(DlineageUtil.getTableFullName(columnName.toString())); 27660 if (variable == null) { 27661 variable = modelManager 27662 .getTableByName(DlineageUtil.getTableFullName(columnName.getTableString())); 27663 if (variable != null && variable.isCursor()) { 27664 TableColumn variableColumn = modelFactory.createInsertTableColumn(variable, columnName); 27665 if (variableColumn != null) { 27666 if(isOut){ 27667 relation.setTarget(new TableColumnRelationshipElement(variableColumn)); 27668 } 27669 else { 27670 relation.addSource(new TableColumnRelationshipElement(variableColumn)); 27671 } 27672 } else { 27673 TableColumn wholeValue = variable instanceof Variable 27674 ? compositeBindingColumn((Variable) variable) 27675 : variable.getColumns().get(0); 27676 if(isOut){ 27677 relation.setTarget(new TableColumnRelationshipElement(wholeValue)); 27678 } 27679 else { 27680 relation.addSource(new TableColumnRelationshipElement(wholeValue)); 27681 } 27682 } 27683 continue; 27684 } 27685 } else if (variable.isVariable() || variable.isCursor()) { 27686 TableColumn columnModel = variable.getColumns().get(0); 27687 if (valueIndex != null) { 27688 if(isOut){ 27689 relation.setTarget(new TableColumnRelationshipElement(columnModel, valueIndex)); 27690 } 27691 else { 27692 relation.addSource(new TableColumnRelationshipElement(columnModel, valueIndex)); 27693 } 27694 } else { 27695 if(isOut){ 27696 relation.setTarget(new TableColumnRelationshipElement(columnModel)); 27697 } 27698 else { 27699 relation.addSource(new TableColumnRelationshipElement(columnModel)); 27700 } 27701 } 27702 continue; 27703 } 27704 } 27705 } 27706 27707 if (columnName.getColumnNameOnly().startsWith("@") 27708 && (option.getVendor() == EDbVendor.dbvmssql || option.getVendor() == EDbVendor.dbvazuresql)) { 27709 continue; 27710 } 27711 27712 if (columnName.getColumnNameOnly().startsWith(":") 27713 && (option.getVendor() == EDbVendor.dbvhana || option.getVendor() == EDbVendor.dbvteradata)) { 27714 Table variable = modelManager 27715 .getTableByName(DlineageUtil.getTableFullName(columnName.getColumnNameOnly().replace(":", ""))); 27716 if (variable != null) { 27717 for (int j = 0; j < variable.getColumns().size(); j++) { 27718 if (getColumnName(columnName).replace(":", "") 27719 .equals(getColumnName(variable.getColumns().get(j).getColumnObject()))) { 27720 TableColumn columnModel = variable.getColumns().get(j); 27721 relation.addSource(new TableColumnRelationshipElement(columnModel)); 27722 } 27723 } 27724 } 27725 continue; 27726 } 27727 27728 boolean linkedFirstTable = false; 27729 27730 TCustomSqlStatement stmt = stmtStack.peek(); 27731 TTableList tableList = stmt.tables; 27732 if ((tableList == null || tableList.size() == 0) && (hiveFromTables != null && hiveFromTables.size() > 0)) { 27733 tableList = hiveFromTables; 27734 } 27735 27736 List<TTable> tables = new ArrayList<TTable>(); 27737 { 27738 TTable table = columnName.getSourceTable(); 27739 27740 // 针对CursorVariable特殊处理 27741 if (columnName.getTableToken() != null) { 27742 27743 Table tableModel = null; 27744 if (table != null && modelManager.getModel(table) instanceof Table) { 27745 tableModel = (Table) modelManager.getModel(table); 27746 } 27747 27748 if (tableModel == null) { 27749 tableModel = modelManager 27750 .getTableByName(DlineageUtil.getTableFullName(columnName.getTableToken().toString())); 27751 } 27752 27753 if (tableModel == null) { 27754 TCustomSqlStatement currentStmt = ModelBindingManager.getGlobalStmtStack().peek(); 27755 String procedureName = DlineageUtil.getProcedureParentName(currentStmt); 27756 String variableString = columnName.getTableToken().toString(); 27757 if (variableString.startsWith(":")) { 27758 variableString = variableString.substring(variableString.indexOf(":") + 1); 27759 } 27760 if (!SQLUtil.isEmpty(procedureName)) { 27761 variableString = procedureName + "." + SQLUtil.getIdentifierNormalTableName(variableString); 27762 } 27763 27764 if (modelManager 27765 .getTableByName(DlineageUtil.getTableFullName(variableString)) instanceof Variable) { 27766 tableModel = modelManager.getTableByName(DlineageUtil.getTableFullName(variableString)); 27767 } 27768 } 27769 27770 if (tableModel != null) { 27771 27772 if (table == null) { 27773 table = tableModel.getTableObject(); 27774 } 27775 27776 if (tableModel.isVariable()) { 27777 if (!isStar && "*".equals(getColumnName(columnName))) { 27778 TObjectName[] columns = modelManager.getTableColumns(table); 27779 for (int j = 0; j < columns.length; j++) { 27780 TObjectName objectName = columns[j]; 27781 if (objectName == null || "*".equals(getColumnName(objectName))) { 27782 continue; 27783 } 27784 TableColumn columnModel = modelFactory.createTableColumn(tableModel, objectName, 27785 false); 27786 relation.addSource(new TableColumnRelationshipElement(columnModel)); 27787 } 27788 } else { 27789 if ("*".equals(getColumnName(columnName)) && !tableModel.getColumns().isEmpty()) { 27790 27791 for (int j = 0; j < tableModel.getColumns().size(); j++) { 27792 TableColumn columnModel = tableModel.getColumns().get(j); 27793 if (exceptColumnList != null) { 27794 boolean flag = false; 27795 for (TObjectName objectName : exceptColumnList) { 27796 if (getColumnName(objectName) 27797 .equals(getColumnName(columnModel.getColumnObject()))) { 27798 flag = true; 27799 break; 27800 } 27801 } 27802 if (flag) { 27803 continue; 27804 } 27805 } 27806 relation.addSource(new TableColumnRelationshipElement(columnModel)); 27807 } 27808 27809 if (isStar && showStar) { 27810 TableColumn columnModel = modelFactory.createTableColumn(tableModel, columnName, 27811 false); 27812 if (columnModel == null) { 27813 if (tableModel.isCreateTable()) { 27814 for (TableColumn tableColumn : tableModel.getColumns()) { 27815 relation.addSource(new TableColumnRelationshipElement(tableColumn)); 27816 relation.setShowStarRelation(true); 27817 } 27818 } 27819 } else { 27820 relation.addSource(new TableColumnRelationshipElement(columnModel)); 27821 relation.setShowStarRelation(true); 27822 } 27823 } 27824 } else { 27825 TableColumn columnModel = modelFactory.createTableColumn(tableModel, columnName, 27826 false); 27827 27828 if(columnModel == null && containStarColumn(tableModel.getColumns())){ 27829 columnModel = getStarColumn(tableModel.getColumns()); 27830 if (columnModel != null && tableModel.getSubType() == SubType.record_type) { 27831 if(!"*".equals(getColumnName(columnName))){ 27832 columnModel.bindStarLinkColumn(columnName); 27833 } 27834 } 27835 } 27836 27837 if (columnModel == null) { 27838 continue; 27839 } 27840 if (columnModel.hasStarLinkColumn()) { 27841 relation.addSource(new TableColumnRelationshipElement(columnModel, 27842 columnModel.getStarLinkColumnNames() 27843 .indexOf(DlineageUtil.getColumnName(columnName)))); 27844 } else { 27845 if (isOut) { 27846 relation.setTarget(new TableColumnRelationshipElement(columnModel)); 27847 } else { 27848 relation.addSource(new TableColumnRelationshipElement(columnModel)); 27849 } 27850 } 27851 if (columnName.getSourceTable() != null 27852 && columnName.getSourceTable().getTableType() == ETableSource.function) { 27853 analyzeTableColumn(columnModel, columnName.getSourceTable().getFuncCall(), 27854 effectType); 27855 } 27856 } 27857 } 27858 continue; 27859 } 27860 } 27861 } 27862 27863 if (table == null) { 27864 table = modelManager.getTable(stmt, columnName); 27865 } 27866 27867 if (table == null) { 27868 if (columnName.getTableToken() != null || !"*".equals(getColumnName(columnName))) { 27869 table = columnName.getSourceTable(); 27870 } 27871 27872 if (table == null && !SQLUtil.isEmpty(columnName.getTableString())) { 27873 table = modelManager.getTableFromColumn(columnName); 27874 } 27875 } 27876 27877 if (table == null) { 27878 if (tableList != null) { 27879 for (int j = 0; j < tableList.size(); j++) { 27880 if (table != null) 27881 break; 27882 27883 TTable tTable = tableList.getTable(j); 27884 if (tTable.getTableType().name().startsWith("open")) { 27885 continue; 27886 } 27887 27888 if (getTableLinkedColumns(tTable) != null && getTableLinkedColumns(tTable).size() > 0) { 27889 for (int z = 0; z < getTableLinkedColumns(tTable).size(); z++) { 27890 TObjectName refer = getTableLinkedColumns(tTable).getObjectName(z); 27891 if ("*".equals(getColumnName(refer))) 27892 continue; 27893 // For BigQuery struct field access, match base column name from FieldPath 27894 String structFullName = getStructFieldFullName(columnName); 27895 if (structFullName != null) { 27896 String baseName = getStructFieldBaseName(columnName); 27897 if (baseName != null && DlineageUtil.getIdentifierNormalColumnName(getColumnName(refer)) 27898 .equals(DlineageUtil.getIdentifierNormalColumnName(baseName))) { 27899 table = tTable; 27900 break; 27901 } 27902 } 27903 if (getColumnName(refer).equals(getColumnName(columnName))) { 27904 table = tTable; 27905 break; 27906 } 27907 } 27908 } 27909 27910 if (tTable.getLinkTable() != null) { 27911 tTable = tTable.getLinkTable(); 27912 for (int z = 0; z < getTableLinkedColumns(tTable).size(); z++) { 27913 TObjectName refer = getTableLinkedColumns(tTable).getObjectName(z); 27914 if ("*".equals(getColumnName(refer))) 27915 continue; 27916 if (getColumnName(refer).equals(getColumnName(columnName))) { 27917 table = tTable; 27918 break; 27919 } 27920 } 27921 } 27922 27923 if (table != null) 27924 break; 27925 27926 if (columnName.getTableToken() != null && (SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotTable, 27927 columnName.getTableToken().getAstext(), tTable.getName()) 27928 || SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotTable, columnName.getTableToken().getAstext(), tTable.getAliasName()))) { 27929 table = tTable; 27930 break; 27931 } 27932 } 27933 } 27934 27935 if (table == null) { 27936 for (int j = 0; j < tableList.size(); j++) { 27937 if (table != null) 27938 break; 27939 27940 TTable tTable = tableList.getTable(j); 27941 Object model = ModelBindingManager.get().getModel(tTable); 27942 if (model instanceof Table) { 27943 Table tableModel = (Table) model; 27944 for (int z = 0; tableModel.getColumns() != null 27945 && z < tableModel.getColumns().size(); z++) { 27946 TableColumn refer = tableModel.getColumns().get(z); 27947 if (getColumnName(refer.getName()).equals(getColumnName(columnName))) { 27948 table = tTable; 27949 break; 27950 } 27951 if (refer.hasStarLinkColumn()) { 27952 for (TObjectName linkColumn : refer.getStarLinkColumnList()) { 27953 if (getColumnName(linkColumn).equals(getColumnName(columnName))) { 27954 table = tTable; 27955 break; 27956 } 27957 } 27958 } 27959 } 27960 } else if (model instanceof QueryTable) { 27961 QueryTable tableModel = (QueryTable) model; 27962 for (int z = 0; tableModel.getColumns() != null 27963 && z < tableModel.getColumns().size(); z++) { 27964 ResultColumn refer = tableModel.getColumns().get(z); 27965 // Try FieldPath-based matching first 27966 String structFullName = getStructFieldFullName(columnName); 27967 if (structFullName != null) { 27968 String baseName = getStructFieldBaseName(columnName); 27969 if (baseName != null && DlineageUtil.getIdentifierNormalColumnName(refer.getName()) 27970 .equals(DlineageUtil.getIdentifierNormalColumnName(baseName))) { 27971 table = tTable; 27972 break; 27973 } 27974 } 27975 List<String> splits = SQLUtil.parseNames(columnName.toString()); 27976 if (splits.size() > 1 && EDbVendor.dbvbigquery == getOption().getVendor()) { 27977 if (DlineageUtil.getIdentifierNormalColumnName(refer.getName()) 27978 .equals(DlineageUtil 27979 .getIdentifierNormalColumnName(getColumnName(splits.get(0))))) { 27980 table = tTable; 27981 break; 27982 } 27983 } 27984 else if (DlineageUtil.getIdentifierNormalColumnName(refer.getName()).equals( 27985 DlineageUtil.getIdentifierNormalColumnName(getColumnName(columnName)))) { 27986 table = tTable; 27987 break; 27988 } 27989 if (refer.hasStarLinkColumn()) { 27990 for (TObjectName linkColumn : refer.getStarLinkColumnList()) { 27991 if (getColumnName(linkColumn).equals(getColumnName(columnName))) { 27992 table = tTable; 27993 break; 27994 } 27995 } 27996 } 27997 } 27998 } 27999 } 28000 } 28001 } 28002 28003 if (columnName.getTableToken() == null && "*".equals(getColumnName(columnName))) { 28004 if (!hasJoin(stmt)) { 28005 tables.add(table); 28006 } else { 28007 for (int j = 0; j < tableList.size(); j++) { 28008 tables.add(tableList.getTable(j)); 28009 } 28010 } 28011 } else if (table != null) { 28012 tables.add(table); 28013 } 28014 28015 // 此处特殊处理,多表关联无法找到 column 所属的 Table, tTable.getLinkedColumns 28016 // 也找不到,退而求其次采用第一个表 28017 28018 if (stmt.getParentStmt() != null && isApplyJoin(stmt.getParentStmt()) 28019 && (tableList != null && tableList.size() > 0)) { 28020 stmt = stmt.getParentStmt(); 28021 TTable applyTable = tableList.getTable(0); 28022 if (modelManager.getModel(table) == null) { 28023 modelFactory.createTable(applyTable); 28024 } 28025 } 28026 28027 if (columnName.toString().indexOf(".")==-1 && isConstant(columnName)) { 28028 boolean isConstant = true; 28029 if (columnName.getSourceTable() != null) { 28030 Table tableModel = modelManager.getTableByName( 28031 DlineageUtil.getTableFullName(columnName.getSourceTable().getTableName().toString())); 28032 if (tableModel != null && tableModel.getColumns() != null) { 28033 for (int j = 0; j < tableModel.getColumns().size(); j++) { 28034 if (DlineageUtil.compareColumnIdentifier(getColumnName(columnName), 28035 getColumnName(tableModel.getColumns().get(j).getName()))) { 28036 isConstant = false; 28037 break; 28038 } 28039 } 28040 } 28041 } 28042 28043 if (isConstant) { 28044 if (option.isShowConstantTable()) { 28045 Table constantTable = modelFactory.createConstantsTable(stmtStack.peek()); 28046 TableColumn tableColumn = modelFactory.createTableColumn(constantTable, columnName, false); 28047 relation.addSource(new TableColumnRelationshipElement(tableColumn)); 28048 } 28049 continue; 28050 } 28051 } 28052 28053 if (tableList != null && tableList.size() != 0 && tables.size() == 0 28054 && !(isBuiltInFunctionName(columnName) && isFromFunction(columnName))) { 28055 if (modelManager.getModel(stmt) instanceof ResultSet) { 28056 ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt); 28057 boolean find = false; 28058 for (ResultColumn resultColumn : resultSetModel.getColumns()) { 28059 if(resultColumn.equals(modelObject)) { 28060 continue; 28061 } 28062 if (!TSQLEnv.isAliasReferenceForbidden.get(option.getVendor())) { 28063 if (getColumnName(columnName).equals(getColumnName(resultColumn.getName()))) { 28064 if (resultColumn.getColumnObject() != null) { 28065 int startToken = resultColumn.getColumnObject().getStartToken().posinlist; 28066 int endToken = resultColumn.getColumnObject().getEndToken().posinlist; 28067 if (columnName.getStartToken().posinlist >= startToken 28068 && columnName.getEndToken().posinlist <= endToken) { 28069 continue; 28070 } 28071 } 28072 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 28073 find = true; 28074 break; 28075 } 28076 } 28077 } 28078 if (find) { 28079 continue; 28080 } 28081 } 28082 28083 // Structured-dataflow outer field resolution: an unresolved 28084 // reference like nodes1.key (or t.nodes1.key) may be a 28085 // struct-field projection produced by a structured generator 28086 // (e.g. Spark explode(from_json(...))) in a visible subquery. 28087 // Link directly to the matching FunctionResultColumn so the 28088 // exact source path (nodes[*].key) flows through. 28089 if (tryAppendStructuredFieldRelation(relation, columnName, tableList)) { 28090 continue; 28091 } 28092 28093 TObjectName pseudoTableName = new TObjectName(); 28094 // Use qualified prefix from column name if available (e.g., sch.pk_constv2 from sch.pk_constv2.c_cdsl) 28095 // Otherwise fall back to default pseudo table name 28096 String qualifiedPrefix = getQualifiedPrefixFromColumn(columnName); 28097 pseudoTableName.setString(qualifiedPrefix != null ? qualifiedPrefix : "pseudo_table_include_orphan_column"); 28098 Table pseudoTable = modelFactory.createTableByName(pseudoTableName); 28099 pseudoTable.setPseudo(true); 28100 TableColumn pseudoTableColumn = modelFactory.createTableColumn(pseudoTable, columnName, true); 28101 28102 // If not linking to first table and column has qualified prefix (3-part name like sch.pkg.col), 28103 // add the pseudo table column as source 28104 if (!isLinkOrphanColumnToFirstTable() && pseudoTableColumn != null && qualifiedPrefix != null) { 28105 if (isOut) { 28106 relation.setTarget(new TableColumnRelationshipElement(pseudoTableColumn)); 28107 } else { 28108 relation.addSource(new TableColumnRelationshipElement(pseudoTableColumn)); 28109 } 28110 } 28111 28112 if (isLinkOrphanColumnToFirstTable()) { 28113 TTable orphanTable = tableList.getTable(0); 28114 tables.add(orphanTable); 28115 Object tableModel = modelManager.getModel(orphanTable); 28116 if (tableModel == null) { 28117 if(orphanTable.getSubquery()!=null) { 28118 QueryTable queryTable = modelFactory.createQueryTable(orphanTable); 28119 TSelectSqlStatement subquery = orphanTable.getSubquery(); 28120 analyzeSelectStmt(subquery); 28121 } 28122 else { 28123 tableModel = modelFactory.createTable(orphanTable); 28124 } 28125 } 28126 if (tableModel instanceof Table) { 28127 TableColumn orphanColum = modelFactory.createTableColumn((Table) tableModel, columnName, false); 28128 if(orphanColum!=null) { 28129 ErrorInfo errorInfo = new ErrorInfo(); 28130 errorInfo.setErrorType(ErrorInfo.LINK_ORPHAN_COLUMN); 28131 errorInfo.setErrorMessage("Link orphan column [" + columnName.toString() 28132 + "] to the first table [" + orphanTable.getFullNameWithAliasString() + "]"); 28133 errorInfo.setStartPosition(new Pair3<Long, Long, String>(columnName.getStartToken().lineNo, 28134 columnName.getStartToken().columnNo, ModelBindingManager.getGlobalHash())); 28135 errorInfo.setEndPosition(new Pair3<Long, Long, String>(columnName.getEndToken().lineNo, 28136 columnName.getEndToken().columnNo + columnName.getEndToken().getAstext().length(), 28137 ModelBindingManager.getGlobalHash())); 28138 errorInfo.fillInfo(this); 28139 errorInfos.add(errorInfo); 28140 } 28141 if (orphanTable.getSubquery() != null) { 28142 TSelectSqlStatement subquery = orphanTable.getSubquery(); 28143 if (subquery.getResultColumnList().toString().endsWith("*") && subquery.getTables().size() == 1) { 28144 TTable subqueryTable = subquery.getTables().getTable(0); 28145 Object sourceTable = modelManager.getModel(subqueryTable); 28146 if(sourceTable instanceof Table) { 28147 modelFactory.createTableColumn((Table) sourceTable, columnName, false); 28148 } 28149 else if(sourceTable instanceof ResultSet) { 28150 modelFactory.createResultColumn((ResultSet) sourceTable, columnName, false); 28151 } 28152 } 28153 } 28154 else if (orphanTable.getCTE()!=null && orphanTable.getCTE().getSubquery() != null) { 28155 TSelectSqlStatement subquery = orphanTable.getCTE().getSubquery(); 28156 if (subquery.getResultColumnList().toString().endsWith("*") && subquery.getTables().size() == 1) { 28157 TTable subqueryTable = subquery.getTables().getTable(0); 28158 Object sourceTable = modelManager.getModel(subqueryTable); 28159 if(sourceTable instanceof Table) { 28160 modelFactory.createTableColumn((Table) sourceTable, columnName, false); 28161 } 28162 else if(sourceTable instanceof ResultSet) { 28163 modelFactory.createResultColumn((ResultSet) sourceTable, columnName, false); 28164 } 28165 } 28166 } 28167 } 28168 28169 linkedFirstTable = true; 28170 } 28171 } 28172 } 28173 28174 for (int k = 0; k < tables.size(); k++) { 28175 TTable table = tables.get(k); 28176 if (table != null) { 28177 Object object = modelManager.getModel(table); 28178 if(object instanceof PivotedTable) { 28179 TPivotClause clause = (TPivotClause)((PivotedTable)object).getGspObject(); 28180 if(clause.getAliasClause()!=null) { 28181 object = modelManager.getModel(clause.getAliasClause()); 28182 } 28183 } 28184 if (object == null && table.getTableType() == ETableSource.objectname) { 28185 if (table.getCTE() != null) { 28186 QueryTable queryTable = modelFactory.createQueryTable(table); 28187 TSelectSqlStatement subquery = table.getCTE().getSubquery(); 28188 analyzeSelectStmt(subquery); 28189 ResultSet resultSetModel = (ResultSet) modelManager.getModel(subquery); 28190 28191 if (resultSetModel != null && resultSetModel != queryTable 28192 && !resultSetModel.getRelationRows().getHoldRelations().isEmpty()) { 28193 ImpactRelationship impactRelation = modelFactory.createImpactRelation(); 28194 impactRelation.setEffectType(EffectType.select); 28195 impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>( 28196 resultSetModel.getRelationRows())); 28197 impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>( 28198 queryTable.getRelationRows())); 28199 } 28200 28201 if (resultSetModel != null && resultSetModel != queryTable) { 28202 for (int j = 0; j < resultSetModel.getColumns().size(); j++) { 28203 ResultColumn sourceColumn = resultSetModel.getColumns().get(j); 28204 ResultColumn targetColumn = modelFactory.createSelectSetResultColumn(queryTable, 28205 sourceColumn); 28206 28207 DataFlowRelationship queryRalation = modelFactory.createDataFlowRelation(); 28208 queryRalation.setEffectType(EffectType.select); 28209 queryRalation.setTarget(new ResultColumnRelationshipElement(targetColumn)); 28210 queryRalation.addSource(new ResultColumnRelationshipElement(sourceColumn)); 28211 } 28212 } 28213 28214 object = queryTable; 28215 28216 } else { 28217 object = modelFactory.createTable(table); 28218 } 28219 } 28220 if (object instanceof Function) { 28221 relation.addSource(new ResultColumnRelationshipElement(((Function)object).getColumns().get(0))); 28222 continue; 28223 } else if (object instanceof ResultSet && !(object instanceof QueryTable)) { 28224 //Object tableModel = modelManager.getModel(columnName.getSourceTable()); 28225 appendResultColumnRelationSource(modelObject, relation, columnIndex, columnName, 28226 (ResultSet)object); 28227 if(table.getTableType() == ETableSource.function && !relation.getSources().isEmpty()) { 28228 relation.getSources().stream().reduce((first, second) -> second).get().addTransform(Transform.FUNCTION, table); 28229 } 28230 continue; 28231 } 28232 else if (object instanceof Table) { 28233 Table tableModel = (Table) modelManager.getModel(table); 28234 if (tableModel != null) { 28235 if (!isStar && "*".equals(getColumnName(columnName))) { 28236 TObjectName[] columns = modelManager.getTableColumns(table); 28237 for (int j = 0; j < columns.length; j++) { 28238 TObjectName objectName = columns[j]; 28239 if (objectName == null || ("*".equals(getColumnName(objectName)) && tableModel.isDetermined())) { 28240 continue; 28241 } 28242 TableColumn columnModel = modelFactory.createTableColumn(tableModel, objectName, 28243 false); 28244 if(columnModel == null) { 28245 continue; 28246 } 28247 relation.addSource(new TableColumnRelationshipElement(columnModel)); 28248 } 28249 } else { 28250 if ("*".equals(getColumnName(columnName)) && !tableModel.getColumns().isEmpty()) { 28251 Map<String, Pair<String, TExpression>> replaceAsIdentifierMap = new HashMap<String, Pair<String, TExpression>>(); 28252 Map<String, TObjectName> replaceColumnMap = new HashMap<String, TObjectName>(); 28253 if(modelObject instanceof ResultColumn && ((ResultColumn)modelObject).getColumnObject() instanceof TResultColumn) { 28254 TResultColumn resultColumn = (TResultColumn )((ResultColumn)modelObject).getColumnObject(); 28255 if(resultColumn.getReplaceExprAsIdentifiers()!=null && resultColumn.getReplaceExprAsIdentifiers().size()>0) { 28256 for(TReplaceExprAsIdentifier replace: resultColumn.getReplaceExprAsIdentifiers()) { 28257 replaceAsIdentifierMap.put(replace.getIdentifier().toString(), new Pair<String, TExpression>(resultColumn.getExpr().getExceptReplaceClause().toString(), replace.getExpr())); 28258 replaceColumnMap.put(replace.getIdentifier().toString(), replace.getIdentifier()); 28259 } 28260 ResultSet resultSet = ((ResultColumn)modelObject).getResultSet(); 28261 if(tableModel.isDetermined()) { 28262 resultSet.getColumns().clear(); 28263 } 28264 } 28265 } 28266 28267 for (int j = 0; j < tableModel.getColumns().size(); j++) { 28268 TableColumn columnModel = tableModel.getColumns().get(j); 28269 if (exceptColumnList != null) { 28270 boolean flag = false; 28271 for (TObjectName objectName : exceptColumnList) { 28272 if (getColumnName(objectName) 28273 .equals(getColumnName(columnModel.getColumnObject()))) { 28274 flag = true; 28275 break; 28276 } 28277 } 28278 if (flag) { 28279 continue; 28280 } 28281 } 28282 28283 if (replaceAsIdentifierMap.containsKey(tableModel.getColumns().get(j).getName())) { 28284 Pair<String, TExpression> expr = replaceAsIdentifierMap.get(tableModel.getColumns().get(j).getName()); 28285 ResultSet resultSet = ((ResultColumn)modelObject).getResultSet(); 28286 ResultColumn resultColumn = modelFactory.createResultColumn(resultSet, replaceColumnMap.get(tableModel.getColumns().get(j).getName())); 28287 Transform transform = new Transform(); 28288 transform.setType(Transform.EXPRESSION); 28289 TObjectName expression = new TObjectName(); 28290 expression.setString(expr.first); 28291 transform.setCode(expression); 28292 resultColumn.setTransform(transform); 28293 analyzeResultColumnExpressionRelation(resultColumn, expr.second); 28294 } else { 28295 if(!replaceAsIdentifierMap.isEmpty()) { 28296 ResultSet resultSet = ((ResultColumn) modelObject).getResultSet(); 28297 ResultColumn resultColumn = modelFactory.createResultColumn(resultSet, 28298 columnModel.getColumnObject()); 28299 DataFlowRelationship relation1 = modelFactory.createDataFlowRelation(); 28300 relation1.setEffectType(effectType); 28301 relation1.setProcess(process); 28302 relation1.setTarget(new ResultColumnRelationshipElement(resultColumn)); 28303 relation1.addSource(new TableColumnRelationshipElement(columnModel)); 28304 } 28305 else { 28306 relation.addSource( 28307 new TableColumnRelationshipElement(columnModel)); 28308 } 28309 } 28310 } 28311 28312 if (isStar && showStar) { 28313 TableColumn columnModel = modelFactory.createTableColumn(tableModel, columnName, 28314 false); 28315 if (columnModel == null) { 28316 if(tableModel.isCreateTable()) { 28317 for (TableColumn tableColumn : tableModel.getColumns()) { 28318 relation.addSource(new TableColumnRelationshipElement(tableColumn)); 28319 relation.setShowStarRelation(true); 28320 } 28321 } 28322 } else { 28323 relation.addSource(new TableColumnRelationshipElement(columnModel)); 28324 relation.setShowStarRelation(true); 28325 } 28326 } 28327 } else { 28328 TableColumn columnModel = modelFactory.createTableColumn(tableModel, columnName, 28329 false); 28330 if(columnModel == null) { 28331 if(tableModel.isCreateTable()) { 28332 boolean flag = false; 28333 // Try FieldPath-based matching first for BigQuery/Redshift struct fields 28334 String structFullName = getStructFieldFullName(columnName); 28335 if (structFullName != null && !flag) { 28336 for (TableColumn tableColumn : tableModel.getColumns()) { 28337 if (tableColumn.isStruct() 28338 && DlineageUtil.getIdentifierNormalColumnName(tableColumn.getName()) 28339 .equals(DlineageUtil.getIdentifierNormalColumnName(structFullName))) { 28340 relation.addSource(new TableColumnRelationshipElement(tableColumn)); 28341 flag = true; 28342 if (modelObject instanceof ResultColumn) { 28343 ((ResultColumn) modelObject).setStruct(true); 28344 } else if (modelObject instanceof TableColumn) { 28345 ((TableColumn) modelObject).setStruct(true); 28346 } 28347 break; 28348 } 28349 } 28350 } 28351 if (ModelBindingManager.getGlobalVendor() == EDbVendor.dbvbigquery || ModelBindingManager.getGlobalVendor() == EDbVendor.dbvredshift) { 28352 for (TableColumn tableColumn : tableModel.getColumns()) { 28353 if(tableColumn.isStruct()) { 28354 List<String> names = SQLUtil.parseNames(tableColumn.getName()); 28355 if (modelObject instanceof TableColumn) { 28356 TableColumn targetColumn = (TableColumn) modelObject; 28357 if (targetColumn.isStruct()) { 28358 List<String> targetNames = SQLUtil 28359 .parseNames(targetColumn.getName()); 28360 if (!getColumnName(targetNames.get(0)) 28361 .equals(getColumnName(names.get(0)))) { 28362 continue; 28363 } 28364 } 28365 } 28366 else if (modelObject instanceof ResultColumn) { 28367 ResultColumn targetColumn = (ResultColumn) modelObject; 28368 if (targetColumn.isStruct()) { 28369 List<String> targetNames = SQLUtil 28370 .parseNames(targetColumn.getName()); 28371 if (!getColumnName(targetNames.get(0)) 28372 .equals(getColumnName(names.get(0)))) { 28373 continue; 28374 } 28375 } 28376 } 28377 for(String name: names) { 28378 if (getColumnName(name) 28379 .equals(getColumnName(modelObject.toString()))) { 28380 relation.addSource( 28381 new TableColumnRelationshipElement(tableColumn)); 28382 flag = true; 28383 if (modelObject instanceof ResultColumn) { 28384 ((ResultColumn)modelObject).setStruct(true); 28385 } 28386 else if (modelObject instanceof TableColumn) { 28387 ((TableColumn)modelObject).setStruct(true); 28388 } 28389 break; 28390 } 28391 } 28392 28393 if (!flag && tableModel.getColumns().size() == 1 && tableModel 28394 .getColumns().get(0).getSourceColumn() != null) { 28395 TableColumn sourceColumn = tableModel 28396 .getColumns().get(0).getSourceColumn(); 28397 Table sourceTable = sourceColumn.getTable(); 28398 TObjectName sourceColumnName = new TObjectName(); 28399 sourceColumnName.setString(sourceColumn.getName() + "." 28400 + columnName.getColumnNameOnly()); 28401 TableColumn sourceTableColumn = modelFactory.createTableColumn(sourceTable, sourceColumnName, true); 28402 relation.addSource( 28403 new TableColumnRelationshipElement(sourceTableColumn)); 28404 flag = true; 28405 break; 28406 } 28407 } 28408 else if (getColumnName(tableColumn.getName()) 28409 .equals(getColumnName(modelObject.toString()))) { 28410 relation.addSource( 28411 new TableColumnRelationshipElement(tableColumn)); 28412 flag = true; 28413 break; 28414 } 28415 } 28416 if (modelObject instanceof TableColumn) { 28417 TableColumn column = (TableColumn) modelObject; 28418 for (TableColumn tableColumn : tableModel.getColumns()) { 28419 if (tableColumn.getColumnIndex() == null) { 28420 continue; 28421 } 28422 if (tableColumn.getName().toLowerCase() 28423 .indexOf(column.getName().toLowerCase()) == -1 28424 && tableColumn.getName().toLowerCase() 28425 .indexOf(column.getColumnObject().toString() 28426 .toLowerCase()) == -1) { 28427 continue; 28428 } 28429 flag = true; 28430 relation.addSource( 28431 new TableColumnRelationshipElement(tableColumn)); 28432 relation.setShowStarRelation(true); 28433 } 28434 if (flag) 28435 break; 28436 28437 } else if (modelObject instanceof ResultColumn) { 28438 ResultColumn column = (ResultColumn) modelObject; 28439 for (TableColumn tableColumn : tableModel.getColumns()) { 28440 if (tableColumn.getColumnIndex() == null) { 28441 continue; 28442 } 28443 if (tableColumn.getName().toLowerCase() 28444 .indexOf(column.getName().toLowerCase()) == -1 28445 && tableColumn.getName().toLowerCase() 28446 .indexOf(column.getColumnObject().toString() 28447 .toLowerCase()) == -1) { 28448 continue; 28449 } 28450 flag = true; 28451 relation.addSource( 28452 new TableColumnRelationshipElement(tableColumn)); 28453 relation.setShowStarRelation(true); 28454 } 28455 if (flag) 28456 break; 28457 28458 } 28459 } 28460 if (!flag 28461 && (isStar 28462 || getColumnName(columnName).equals(getColumnName(tableModel.getName())) 28463 || getColumnName(columnName).equals(getColumnName(tableModel.getAlias())))) { 28464 for (TableColumn tableColumn : tableModel.getColumns()) { 28465 relation.addSource(new TableColumnRelationshipElement(tableColumn)); 28466 relation.setShowStarRelation(true); 28467 } 28468 } 28469 } 28470 } 28471 else { 28472 if (linkedFirstTable || columnModel.getCandidateParents() != null) { 28473 if (columnName.getCandidateTables() != null 28474 && columnName.getCandidateTables().size() > 1) { 28475 List<Object> candidateParents = new ArrayList<Object>(); 28476 for(TTable tableItem: columnName.getCandidateTables()) { 28477 Object model = modelManager.getModel(tableItem); 28478 if(model!=null) { 28479 candidateParents.add(model); 28480 } 28481 } 28482 if (candidateParents.size() > 1) { 28483 columnModel.setCandidateParents(candidateParents); 28484 } 28485 } 28486 } 28487 relation.addSource(new TableColumnRelationshipElement(columnModel)); 28488 relation.setShowStarRelation(true); 28489 if(modelObject instanceof TableColumn) { 28490 TableColumn targetTableColumn = (TableColumn)modelObject; 28491 if(targetTableColumn.getTable().getSubType() == SubType.unnest && targetTableColumn.getTable().getColumns().size() == 1) { 28492 targetTableColumn.setSourceColumn(columnModel); 28493 targetTableColumn.setStruct(true); 28494 } 28495 } 28496 if (columnName.getSourceTable() != null 28497 && columnName.getSourceTable().getTableType() == ETableSource.function) { 28498 analyzeTableColumn(columnModel, columnName.getSourceTable().getFuncCall(), 28499 effectType); 28500 } 28501 } 28502 } 28503 } 28504 } 28505 } else if (modelManager.getModel(table) instanceof QueryTable) { 28506 QueryTable queryTable = (QueryTable) modelManager.getModel(table); 28507 28508 TObjectNameList cteColumns = null; 28509 TSelectSqlStatement subquery = null; 28510 if (queryTable.getTableObject().getCTE() != null) { 28511 subquery = queryTable.getTableObject().getCTE().getSubquery(); 28512 cteColumns = queryTable.getTableObject().getCTE().getColumnList(); 28513 } else if (queryTable.getTableObject().getAliasClause() != null 28514 && queryTable.getTableObject().getAliasClause().getColumns() != null) { 28515 28516 } else if (queryTable.getTableObject().getTableExpr() != null 28517 && queryTable.getTableObject().getTableExpr().getSubQuery() != null) { 28518 subquery = queryTable.getTableObject().getTableExpr().getSubQuery(); 28519 } else { 28520 subquery = queryTable.getTableObject().getSubquery(); 28521 } 28522 28523 if (cteColumns != null) { 28524 for (int j = 0; j < cteColumns.size(); j++) { 28525 modelFactory.createResultColumn(queryTable, cteColumns.getObjectName(j)); 28526 } 28527 } 28528 28529 if (subquery != null && subquery.isCombinedQuery()) { 28530 SelectSetResultSet selectSetResultSetModel = (SelectSetResultSet) modelManager 28531 .getModel(subquery); 28532 28533 if (selectSetResultSetModel != null 28534 && !selectSetResultSetModel.getRelationRows().getHoldRelations().isEmpty()) { 28535 ImpactRelationship impactRelation = modelFactory.createImpactRelation(); 28536 impactRelation.setEffectType(EffectType.select); 28537 impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>( 28538 selectSetResultSetModel.getRelationRows())); 28539 impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>( 28540 queryTable.getRelationRows())); 28541 } 28542 28543 if (selectSetResultSetModel != null) { 28544 if (getColumnName(columnName).equals("*")) { 28545 Map<String, Pair<String, TExpression>> replaceAsIdentifierMap = new HashMap<String, Pair<String, TExpression>>(); 28546 Map<String, TObjectName> replaceColumnMap = new HashMap<String, TObjectName>(); 28547 if(modelObject instanceof ResultColumn && ((ResultColumn)modelObject).getColumnObject() instanceof TResultColumn) { 28548 TResultColumn resultColumn = (TResultColumn )((ResultColumn)modelObject).getColumnObject(); 28549 if(resultColumn.getReplaceExprAsIdentifiers()!=null && resultColumn.getReplaceExprAsIdentifiers().size()>0) { 28550 for(TReplaceExprAsIdentifier replace: resultColumn.getReplaceExprAsIdentifiers()) { 28551 replaceAsIdentifierMap.put(replace.getIdentifier().toString(), new Pair<String, TExpression>(resultColumn.getExpr().getExceptReplaceClause().toString(), replace.getExpr())); 28552 replaceColumnMap.put(replace.getIdentifier().toString(), replace.getIdentifier()); 28553 } 28554 ResultSet resultSet = ((ResultColumn)modelObject).getResultSet(); 28555 if(selectSetResultSetModel.isDetermined()) { 28556 resultSet.getColumns().clear(); 28557 } 28558 } 28559 } 28560 28561 28562 for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) { 28563 ResultColumn sourceColumn = selectSetResultSetModel.getColumns().get(j); 28564 if (cteColumns != null) { 28565 if (j < cteColumns.size()) { 28566 ResultColumn targetColumn = queryTable.getColumns().get(j); 28567 28568 if (exceptColumnList != null) { 28569 boolean flag = false; 28570 for (TObjectName objectName : exceptColumnList) { 28571 if (getColumnName(objectName) 28572 .equals(getColumnName(targetColumn.getName()))) { 28573 flag = true; 28574 break; 28575 } 28576 } 28577 if (flag) { 28578 continue; 28579 } 28580 } 28581 28582 if (replaceAsIdentifierMap.containsKey(targetColumn.getName())) { 28583 Pair<String, TExpression> expr = replaceAsIdentifierMap.get(targetColumn.getName()); 28584 ResultSet resultSet = ((ResultColumn)modelObject).getResultSet(); 28585 ResultColumn resultColumn = modelFactory.createResultColumn(resultSet, replaceColumnMap.get(targetColumn.getName())); 28586 Transform transform = new Transform(); 28587 transform.setType(Transform.EXPRESSION); 28588 TObjectName expression = new TObjectName(); 28589 expression.setString(expr.first); 28590 transform.setCode(expression); 28591 resultColumn.setTransform(transform); 28592 analyzeResultColumnExpressionRelation(resultColumn, expr.second); 28593 } else { 28594 if(!replaceAsIdentifierMap.isEmpty()) { 28595 ResultSet resultSet = ((ResultColumn) modelObject).getResultSet(); 28596 TObjectName resultColumnName = new TObjectName(); 28597 resultColumnName.setString(targetColumn.getName()); 28598 ResultColumn resultColumn = modelFactory.createResultColumn(resultSet, 28599 resultColumnName); 28600 DataFlowRelationship relation1 = modelFactory.createDataFlowRelation(); 28601 relation1.setEffectType(effectType); 28602 relation1.setProcess(process); 28603 relation1.setTarget(new ResultColumnRelationshipElement(resultColumn)); 28604 relation1.addSource(new ResultColumnRelationshipElement(targetColumn)); 28605 } 28606 else { 28607 relation.addSource( 28608 new ResultColumnRelationshipElement(targetColumn)); 28609 } 28610 } 28611 28612 } 28613 } else { 28614 ResultColumn targetColumn = modelFactory 28615 .createSelectSetResultColumn(queryTable, sourceColumn); 28616 28617 DataFlowRelationship combinedQueryRelation = modelFactory 28618 .createDataFlowRelation(); 28619 combinedQueryRelation.setEffectType(effectType); 28620 combinedQueryRelation 28621 .setTarget(new ResultColumnRelationshipElement(targetColumn)); 28622 combinedQueryRelation 28623 .addSource(new ResultColumnRelationshipElement(sourceColumn)); 28624 28625 relation.addSource(new ResultColumnRelationshipElement(targetColumn)); 28626 } 28627 } 28628 break; 28629 } else { 28630 boolean flag = false; 28631 28632 for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) { 28633 ResultColumn sourceColumn = selectSetResultSetModel 28634 .getColumns().get(j); 28635 List<String> splits = SQLUtil.parseNames(columnName.toString()); 28636 if (splits.size() > 1 && EDbVendor.dbvbigquery == getOption().getVendor()) { 28637 if (DlineageUtil.sameColumnSegment(sourceColumn.getName(), splits.get(0)) 28638 || DlineageUtil.sameColumnSegment(sourceColumn.getName(), splits.get(1))) { 28639 ResultColumn targetColumn = modelFactory 28640 .createSelectSetResultColumn(queryTable, sourceColumn); 28641 28642 DataFlowRelationship combinedQueryRelation = modelFactory 28643 .createDataFlowRelation(); 28644 combinedQueryRelation.setEffectType(effectType); 28645 combinedQueryRelation.setTarget( 28646 new ResultColumnRelationshipElement(targetColumn)); 28647 combinedQueryRelation.addSource( 28648 new ResultColumnRelationshipElement(sourceColumn)); 28649 28650 relation.addSource( 28651 new ResultColumnRelationshipElement(targetColumn)); 28652 flag = true; 28653 break; 28654 } 28655 } 28656 else if (DlineageUtil.sameColumnName(sourceColumn.getName(), columnName)) { 28657 ResultColumn targetColumn = modelFactory 28658 .createSelectSetResultColumn(queryTable, sourceColumn); 28659 28660 DataFlowRelationship combinedQueryRelation = modelFactory 28661 .createDataFlowRelation(); 28662 combinedQueryRelation.setEffectType(effectType); 28663 combinedQueryRelation 28664 .setTarget(new ResultColumnRelationshipElement(targetColumn)); 28665 combinedQueryRelation 28666 .addSource(new ResultColumnRelationshipElement(sourceColumn)); 28667 28668 relation.addSource(new ResultColumnRelationshipElement(targetColumn)); 28669 flag = true; 28670 break; 28671 } 28672 else if (sourceColumn instanceof SelectSetResultColumn && ((SelectSetResultColumn)sourceColumn).getAliasSet().size() > 1) { 28673 for (String alias : ((SelectSetResultColumn)sourceColumn).getAliasSet()) { 28674 if (DlineageUtil.sameColumnName(alias, columnName)) { 28675 ResultColumn targetColumn = modelFactory 28676 .createSelectSetResultColumn(queryTable, sourceColumn); 28677 28678 DataFlowRelationship combinedQueryRelation = modelFactory 28679 .createDataFlowRelation(); 28680 combinedQueryRelation.setEffectType(effectType); 28681 combinedQueryRelation.setTarget( 28682 new ResultColumnRelationshipElement(targetColumn)); 28683 combinedQueryRelation.addSource( 28684 new ResultColumnRelationshipElement(sourceColumn)); 28685 28686 relation.addSource( 28687 new ResultColumnRelationshipElement(targetColumn)); 28688 flag = true; 28689 break; 28690 } 28691 } 28692 if (flag) { 28693 break; 28694 } 28695 } 28696 } 28697 28698 if (flag) { 28699 break; 28700 } else if (columnIndex != -1) { 28701 for (int j = 0; j < selectSetResultSetModel.getColumns().size(); j++) { 28702 ResultColumn sourceColumn = selectSetResultSetModel.getColumns().get(j); 28703 if (!sourceColumn.getStarLinkColumns().isEmpty()) { 28704 if (cteColumns != null) { 28705 if (j < cteColumns.size()) { 28706 ResultColumn targetColumn = queryTable.getColumns().get(j); 28707 relation.addSource( 28708 new ResultColumnRelationshipElement(targetColumn)); 28709 } 28710 } 28711 else { 28712 if(sourceColumn.hasStarLinkColumn()) { 28713 for (TObjectName linkColumn : sourceColumn.getStarLinkColumnList()) { 28714 if (getColumnName(linkColumn).equals(getColumnName(columnName))) { 28715 ResultColumn targetColumn = modelFactory 28716 .createSelectSetResultColumn(queryTable, sourceColumn); 28717 28718 DataFlowRelationship combinedQueryRelation = modelFactory 28719 .createDataFlowRelation(); 28720 combinedQueryRelation.setEffectType(effectType); 28721 combinedQueryRelation.setTarget( 28722 new ResultColumnRelationshipElement(targetColumn, linkColumn)); 28723 combinedQueryRelation.addSource( 28724 new ResultColumnRelationshipElement(sourceColumn)); 28725 28726 relation.addSource( 28727 new ResultColumnRelationshipElement(targetColumn, linkColumn)); 28728 flag = true; 28729 break; 28730 } 28731 } 28732 } 28733 28734 if(!flag) { 28735 ResultColumn targetColumn = modelFactory 28736 .createSelectSetResultColumn(queryTable, sourceColumn); 28737 28738 DataFlowRelationship combinedQueryRelation = modelFactory 28739 .createDataFlowRelation(); 28740 combinedQueryRelation.setEffectType(effectType); 28741 combinedQueryRelation.setTarget( 28742 new ResultColumnRelationshipElement(targetColumn)); 28743 combinedQueryRelation.addSource( 28744 new ResultColumnRelationshipElement(sourceColumn)); 28745 28746 relation.addSource( 28747 new ResultColumnRelationshipElement(targetColumn)); 28748 } 28749 } 28750 flag = true; 28751 break; 28752 } 28753 } 28754 } 28755 28756 if (flag) { 28757 break; 28758 } else if (columnIndex < selectSetResultSetModel.getColumns().size() 28759 && columnIndex != -1) { 28760 ResultColumn sourceColumn = selectSetResultSetModel.getColumns() 28761 .get(columnIndex); 28762 if (cteColumns != null) { 28763 boolean flag1 = false; 28764 for (ResultColumn targetColumn : queryTable.getColumns()) { 28765 if (DlineageUtil.sameColumnName(targetColumn.getName(), columnName)) { 28766 relation.addSource( 28767 new ResultColumnRelationshipElement(targetColumn)); 28768 flag1 = true; 28769 break; 28770 } 28771 } 28772 if (!flag1 && columnIndex < cteColumns.size()){ 28773 ResultColumn targetColumn = queryTable.getColumns() 28774 .get(columnIndex); 28775 relation.addSource( 28776 new ResultColumnRelationshipElement(targetColumn)); 28777 } 28778 } else { 28779 ResultColumn targetColumn = modelFactory 28780 .createSelectSetResultColumn(queryTable, sourceColumn); 28781 28782 DataFlowRelationship combinedQueryRelation = modelFactory 28783 .createDataFlowRelation(); 28784 combinedQueryRelation.setEffectType(effectType); 28785 combinedQueryRelation 28786 .setTarget(new ResultColumnRelationshipElement(targetColumn)); 28787 combinedQueryRelation 28788 .addSource(new ResultColumnRelationshipElement(sourceColumn)); 28789 28790 relation.addSource(new ResultColumnRelationshipElement(targetColumn)); 28791 } 28792 flag = true; 28793 break; 28794 } 28795 28796 if (flag) { 28797 break; 28798 } 28799 } 28800 } else if (cteColumns != null) { 28801 if (getColumnName(columnName).equals("*")) { 28802 Map<String, Pair<String, TExpression>> replaceAsIdentifierMap = new HashMap<String, Pair<String, TExpression>>(); 28803 Map<String, TObjectName> replaceColumnMap = new HashMap<String, TObjectName>(); 28804 if(modelObject instanceof ResultColumn && ((ResultColumn)modelObject).getColumnObject() instanceof TResultColumn) { 28805 TResultColumn resultColumn = (TResultColumn )((ResultColumn)modelObject).getColumnObject(); 28806 if(resultColumn.getReplaceExprAsIdentifiers()!=null && resultColumn.getReplaceExprAsIdentifiers().size()>0) { 28807 for(TReplaceExprAsIdentifier replace: resultColumn.getReplaceExprAsIdentifiers()) { 28808 replaceAsIdentifierMap.put(replace.getIdentifier().toString(), new Pair<String, TExpression>(resultColumn.getExpr().getExceptReplaceClause().toString(), replace.getExpr())); 28809 replaceColumnMap.put(replace.getIdentifier().toString(), replace.getIdentifier()); 28810 } 28811 ResultSet resultSet = ((ResultColumn)modelObject).getResultSet(); 28812 28813 if (columnName.getSourceColumn() != null) { 28814 Object model = modelManager.getModel(columnName.getSourceColumn()); 28815 if (model instanceof ResultColumn && ((ResultColumn)model).getResultSet().isDetermined()) { 28816 resultSet.getColumns().clear(); 28817 } 28818 } else if (columnName.getSourceTable() != null) { 28819 Object tableModel = modelManager.getModel(columnName.getSourceTable()); 28820 if (tableModel instanceof Table && ((Table)tableModel).isDetermined()) { 28821 resultSet.getColumns().clear(); 28822 } 28823 } 28824 } 28825 } 28826 28827 for (int j = 0; j < cteColumns.size(); j++) { 28828 ResultColumn targetColumn = queryTable.getColumns().get(j); 28829 28830 if (exceptColumnList != null) { 28831 boolean flag = false; 28832 for (TObjectName objectName : exceptColumnList) { 28833 if (getColumnName(objectName) 28834 .equals(getColumnName(targetColumn.getName()))) { 28835 flag = true; 28836 break; 28837 } 28838 } 28839 if (flag) { 28840 continue; 28841 } 28842 } 28843 28844 if (replaceAsIdentifierMap.containsKey(targetColumn.getName())) { 28845 Pair<String, TExpression> expr = replaceAsIdentifierMap.get(targetColumn.getName()); 28846 ResultSet resultSet = ((ResultColumn)modelObject).getResultSet(); 28847 ResultColumn resultColumn = modelFactory.createResultColumn(resultSet, replaceColumnMap.get(targetColumn.getName())); 28848 Transform transform = new Transform(); 28849 transform.setType(Transform.EXPRESSION); 28850 TObjectName expression = new TObjectName(); 28851 expression.setString(expr.first); 28852 transform.setCode(expression); 28853 resultColumn.setTransform(transform); 28854 analyzeResultColumnExpressionRelation(resultColumn, expr.second); 28855 } else { 28856 if(!replaceAsIdentifierMap.isEmpty()) { 28857 ResultSet resultSet = ((ResultColumn) modelObject).getResultSet(); 28858 TObjectName resultColumnName = new TObjectName(); 28859 resultColumnName.setString(targetColumn.getName()); 28860 ResultColumn resultColumn = modelFactory.createResultColumn(resultSet, 28861 resultColumnName); 28862 DataFlowRelationship relation1 = modelFactory.createDataFlowRelation(); 28863 relation1.setEffectType(effectType); 28864 relation1.setProcess(process); 28865 relation1.setTarget(new ResultColumnRelationshipElement(resultColumn)); 28866 relation1.addSource(new ResultColumnRelationshipElement(targetColumn)); 28867 } 28868 else { 28869 relation.addSource( 28870 new ResultColumnRelationshipElement(targetColumn)); 28871 } 28872 } 28873 } 28874 break; 28875 } else { 28876 boolean flag = false; 28877 28878 for (int j = 0; j < cteColumns.size(); j++) { 28879 TObjectName sourceColumn = cteColumns.getObjectName(j); 28880 28881 if (DlineageUtil.sameColumnName(sourceColumn, columnName)) { 28882 ResultColumn targetColumn = queryTable.getColumns().get(j); 28883 28884 relation.addSource(new ResultColumnRelationshipElement(targetColumn)); 28885 flag = true; 28886 break; 28887 } 28888 } 28889 28890 if (flag) { 28891 break; 28892 } 28893 } 28894 } 28895 28896 if (columnName.getSourceColumn() != null) { 28897 Object model = modelManager.getModel(columnName.getSourceColumn()); 28898 if (model instanceof ResultColumn) { 28899 ResultColumn resultColumn = (ResultColumn) model; 28900 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 28901 } 28902 } else if (columnName.getSourceTable() != null) { 28903 Object tableModel = modelManager.getModel(columnName.getSourceTable()); 28904 if (tableModel instanceof Table) { 28905 Object model = modelManager 28906 .getModel(new Pair<Table, TObjectName>((Table) tableModel, columnName)); 28907 if (model instanceof TableColumn) { 28908 relation.addSource(new TableColumnRelationshipElement((TableColumn) model)); 28909 } 28910 } 28911 } 28912 } else { 28913 List<ResultColumn> columns = queryTable.getColumns(); 28914 if (getColumnName(columnName).equals("*")) { 28915 Map<String, Pair<String, TExpression>> replaceAsIdentifierMap = new HashMap<String, Pair<String, TExpression>>(); 28916 Map<String, TObjectName> replaceColumnMap = new HashMap<String, TObjectName>(); 28917 if(modelObject instanceof ResultColumn && ((ResultColumn)modelObject).getColumnObject() instanceof TResultColumn) { 28918 TResultColumn resultColumn = (TResultColumn )((ResultColumn)modelObject).getColumnObject(); 28919 if(resultColumn.getReplaceExprAsIdentifiers()!=null && resultColumn.getReplaceExprAsIdentifiers().size()>0) { 28920 for(TReplaceExprAsIdentifier replace: resultColumn.getReplaceExprAsIdentifiers()) { 28921 replaceAsIdentifierMap.put(replace.getIdentifier().toString(), new Pair<String, TExpression>(resultColumn.getExpr().getExceptReplaceClause().toString(), replace.getExpr())); 28922 replaceColumnMap.put(replace.getIdentifier().toString(), replace.getIdentifier()); 28923 } 28924 ResultSet resultSet = ((ResultColumn)modelObject).getResultSet(); 28925 if(queryTable.isDetermined()) { 28926 resultSet.getColumns().clear(); 28927 } 28928 } 28929 } 28930 28931 int index = 0; 28932 for (int j = 0; j < queryTable.getColumns().size(); j++) { 28933 ResultColumn targetColumn = queryTable.getColumns().get(j); 28934 if (exceptColumnList != null) { 28935 boolean flag = false; 28936 for (TObjectName objectName : exceptColumnList) { 28937 if (getColumnName(objectName) 28938 .equals(getColumnName(targetColumn.getName()))) { 28939 flag = true; 28940 break; 28941 } 28942 } 28943 if (flag) { 28944 continue; 28945 } 28946 } 28947 28948 if (replaceAsIdentifierMap.containsKey(targetColumn.getName())) { 28949 Pair<String, TExpression> expr = replaceAsIdentifierMap.get(targetColumn.getName()); 28950 ResultSet resultSet = ((ResultColumn)modelObject).getResultSet(); 28951 ResultColumn resultColumn = modelFactory.createResultColumn(resultSet, replaceColumnMap.get(targetColumn.getName())); 28952 Transform transform = new Transform(); 28953 transform.setType(Transform.EXPRESSION); 28954 TObjectName expression = new TObjectName(); 28955 expression.setString(expr.first); 28956 transform.setCode(expression); 28957 resultColumn.setTransform(transform); 28958 analyzeResultColumnExpressionRelation(resultColumn, expr.second); 28959 } else { 28960 if(!replaceAsIdentifierMap.isEmpty()) { 28961 ResultSet resultSet = ((ResultColumn) modelObject).getResultSet(); 28962 TObjectName resultColumnName = new TObjectName(); 28963 resultColumnName.setString(targetColumn.getName()); 28964 ResultColumn resultColumn = modelFactory.createResultColumn(resultSet, 28965 resultColumnName); 28966 DataFlowRelationship relation1 = modelFactory.createDataFlowRelation(); 28967 relation1.setEffectType(effectType); 28968 relation1.setProcess(process); 28969 relation1.setTarget(new ResultColumnRelationshipElement(resultColumn)); 28970 relation1.addSource(new ResultColumnRelationshipElement(targetColumn)); 28971 } 28972 else { 28973 relation.addSource( 28974 new ResultColumnRelationshipElement(targetColumn)); 28975 } 28976 } 28977 index++; 28978 } 28979 } else { 28980 if (table.getCTE() != null) { 28981 28982 if (modelObject instanceof TableColumn) { 28983 Table modelTable = ((TableColumn) modelObject).getTable(); 28984 if (modelTable.getSubType() == SubType.unnest) { 28985 boolean find = false; 28986 for (k = 0; k < columns.size(); k++) { 28987 ResultColumn column = columns.get(k); 28988 if (column.isStruct()) { 28989 List<String> names = SQLUtil.parseNames(column.getName()); 28990 for (String name : names) { 28991 if (getColumnName(name).equals(getColumnName(columnName))) { 28992 DataFlowRelationship unnestRelation = modelFactory.createDataFlowRelation(); 28993 unnestRelation.setEffectType(effectType); 28994 unnestRelation.setProcess(process); 28995 unnestRelation.addSource(new ResultColumnRelationshipElement( 28996 column, columnName)); 28997 TObjectName unnestTableColumnName = new TObjectName(); 28998 unnestTableColumnName.setString(names.get(names.size()-1)); 28999 TableColumn unnestTableColumn = modelFactory.createTableColumn(modelTable, unnestTableColumnName, true); 29000 unnestRelation.setTarget(new TableColumnRelationshipElement(unnestTableColumn)); 29001 find = true; 29002 } 29003 } 29004 List<String> names1 = SQLUtil.parseNames(column.getName()); 29005 if (names.size() == 1 && names1.size() >= 1) { 29006 for (String name : names1) { 29007 if (getColumnName(name) 29008 .equals(getColumnName(column.getName()))) { 29009 DataFlowRelationship unnestRelation = modelFactory.createDataFlowRelation(); 29010 unnestRelation.setEffectType(effectType); 29011 unnestRelation.setProcess(process); 29012 unnestRelation.addSource(new ResultColumnRelationshipElement( 29013 column, columnName)); 29014 TObjectName unnestTableColumnName = new TObjectName(); 29015 unnestTableColumnName.setString(names1.get(names.size()-1)); 29016 TableColumn unnestTableColumn = modelFactory.createTableColumn(modelTable, unnestTableColumnName, true); 29017 unnestRelation.setTarget(new TableColumnRelationshipElement(unnestTableColumn)); 29018 find = true; 29019 } 29020 } 29021 } 29022 } 29023 } 29024 if (find) { 29025 modelTable.getColumns().remove(modelObject); 29026 break; 29027 } 29028 } 29029 } 29030 29031 ColumnNameIndex nameIdx = getColumnNameIndex(columns); 29032 if (nameIdx.byNormalizedName != null) { 29033 // Fast path: simple column list (no star/struct). A direct index lookup 29034 // reproduces the linear scan below for such lists: first-match-wins, 29035 // skip when the match IS modelObject, and stop at the first match. 29036 ResultColumn match = nameIdx.byNormalizedName.get(getColumnName(columnName)); 29037 if (match != null && !match.equals(modelObject)) { 29038 relation.addSource( 29039 new ResultColumnRelationshipElement(match, columnName)); 29040 } 29041 } else { 29042 String targetColName = getColumnName(columnName); // loop-invariant, hoisted 29043 for (k = 0; k < columns.size(); k++) { 29044 ResultColumn column = columns.get(k); 29045 if ("*".equals(column.getName())) { 29046 if (!containsStarColumn(column, columnName)) { 29047 column.bindStarLinkColumn(columnName); 29048 } 29049 relation.addSource(new ResultColumnRelationshipElement(column, columnName)); 29050 } else if (DlineageUtil.compareColumnIdentifier(targetColName, 29051 DlineageUtil.getIdentifierNormalColumnName(column.getName()))) { 29052 if (!column.equals(modelObject)) { 29053 relation.addSource( 29054 new ResultColumnRelationshipElement(column, columnName)); 29055 } 29056 break; 29057 } else if(column.isStruct()) { 29058 List<String> names = SQLUtil.parseNames(column.getName()); 29059 for(String name: names) { 29060 if (getColumnName(name) 29061 .equals(getColumnName(columnName))) { 29062 relation.addSource( 29063 new ResultColumnRelationshipElement(column, columnName)); 29064 } 29065 } 29066 List<String> names1 = SQLUtil.parseNames(column.getName()); 29067 if (names.size() == 1 && names1.size() >= 1) { 29068 for(String name: names1) { 29069 if (getColumnName(name) 29070 .equals(getColumnName(column.getName()))) { 29071 relation.addSource( 29072 new ResultColumnRelationshipElement(column, columnName)); 29073 } 29074 } 29075 } 29076 } 29077 } 29078 } 29079 } else if (table.getAliasClause() != null 29080 && table.getAliasClause().getColumns() != null) { 29081 for (k = 0; k < columns.size(); k++) { 29082 ResultColumn column = columns.get(k); 29083 List<String> splits = SQLUtil.parseNames(columnName.toString()); 29084 if ("*".equals(column.getName())) { 29085 if (!containsStarColumn(column, columnName)) { 29086 column.bindStarLinkColumn(columnName); 29087 } 29088 relation.addSource(new ResultColumnRelationshipElement(column, columnName)); 29089 } else if (splits.size() > 1 && EDbVendor.dbvbigquery == getOption().getVendor()) { 29090 if (DlineageUtil.compareColumnIdentifier(getColumnName(splits.get(0)), 29091 DlineageUtil.getIdentifierNormalColumnName(column.getName()))) { 29092 if (!column.equals(modelObject)) { 29093 relation.addSource(new ResultColumnRelationshipElement(column, 29094 columnName)); 29095 } 29096 break; 29097 } 29098 } else if (DlineageUtil.compareColumnIdentifier(getColumnName(columnName), 29099 DlineageUtil.getIdentifierNormalColumnName(column.getName()))) { 29100 if (!column.equals(modelObject)) { 29101 relation.addSource( 29102 new ResultColumnRelationshipElement(column, columnName)); 29103 } 29104 break; 29105 } 29106 } 29107 } else if (table.getSubquery() != null || (table.getTableExpr() != null 29108 && table.getTableExpr().getSubQuery() != null)) { 29109 TSelectSqlStatement select = table.getSubquery(); 29110 if (select == null) { 29111 select = table.getTableExpr().getSubQuery(); 29112 } 29113 if (columnName.getSourceTable() != null) { 29114 Object tableModel = modelManager.getModel(columnName.getSourceTable()); 29115 appendResultColumnRelationSource(modelObject, relation, columnIndex, columnName, 29116 tableModel); 29117 } else if (columnName.getObjectToken() != null 29118 && !SQLUtil.isEmpty(table.getAliasName())) { 29119 if (DlineageUtil.compareTableIdentifier(columnName.getObjectToken().toString(), 29120 table.getAliasName())) { 29121 Object tableModel = modelManager.getModel(table); 29122 appendResultColumnRelationSource(modelObject, relation, columnIndex, 29123 columnName, tableModel); 29124 } 29125 } else if(columns!=null) { 29126 for (k = 0; k < columns.size(); k++) { 29127 ResultColumn column = columns.get(k); 29128 List<String> splits = SQLUtil.parseNames(columnName.toString()); 29129 if ("*".equals(column.getName())) { 29130 if (!containsStarColumn(column, columnName)) { 29131 column.bindStarLinkColumn(columnName); 29132 } 29133 relation.addSource(new ResultColumnRelationshipElement(column, columnName)); 29134 } else if (splits.size() > 1 && EDbVendor.dbvbigquery == getOption().getVendor()) { 29135 if (DlineageUtil.compareColumnIdentifier(getColumnName(splits.get(0)), 29136 DlineageUtil.getIdentifierNormalColumnName(column.getName()))) { 29137 if (!column.equals(modelObject)) { 29138 relation.addSource(new ResultColumnRelationshipElement(column, 29139 columnName)); 29140 } 29141 break; 29142 } 29143 } else if (DlineageUtil.compareColumnIdentifier(getColumnName(columnName), 29144 DlineageUtil.getIdentifierNormalColumnName(column.getName()))) { 29145 if (!column.equals(modelObject)) { 29146 relation.addSource( 29147 new ResultColumnRelationshipElement(column, columnName)); 29148 } 29149 break; 29150 } 29151 } 29152 } 29153 } else if (table.getOutputMerge() != null) { 29154 if (columnName.getSourceColumn() != null) { 29155 Object model = modelManager.getModel(columnName.getSourceColumn()); 29156 if (model instanceof ResultColumn) { 29157 ResultColumn resultColumn = (ResultColumn) model; 29158 if ("*".equals(resultColumn.getName()) 29159 && !containsStarColumn(resultColumn, columnName)) { 29160 resultColumn.bindStarLinkColumn(columnName); 29161 } 29162 relation.addSource( 29163 new ResultColumnRelationshipElement(resultColumn, columnName)); 29164 } 29165 } else if (columnName.getSourceTable() != null) { 29166 Object tableModel = modelManager.getModel(columnName.getSourceTable()); 29167 appendResultColumnRelationSource(modelObject, relation, columnIndex, columnName, 29168 tableModel); 29169 } else if (columnName.getObjectToken() != null 29170 && !SQLUtil.isEmpty(table.getAliasName())) { 29171 if (DlineageUtil.compareTableIdentifier(columnName.getObjectToken().toString(), 29172 table.getAliasName())) { 29173 Object tableModel = modelManager.getModel(table); 29174 appendResultColumnRelationSource(modelObject, relation, columnIndex, 29175 columnName, tableModel); 29176 } 29177 } 29178 } 29179 } 29180 } 29181 } 29182 } 29183 } 29184 if (relation.getSources().size() == 0 && isKeyword(columnName)) { 29185 Table constantTable = modelFactory.createConstantsTable(stmtStack.peek()); 29186 TableColumn constantColumn = modelFactory.createTableColumn(constantTable, columnName, true); 29187 relation.addSource(new ConstantRelationshipElement(constantColumn)); 29188 } 29189 29190 if (relation.getSources().size() > 0) { 29191 for (RelationshipElement<?> sourceItem: relation.getSources()) { 29192 Object source = sourceItem.getElement(); 29193 ImpactRelationship impactRelation = null; 29194 if (source instanceof ResultColumn 29195 && !((ResultColumn) source).getResultSet().getRelationRows().getHoldRelations().isEmpty()) { 29196 impactRelation = modelFactory.createImpactRelation(); 29197 impactRelation.addSource(new RelationRowsRelationshipElement<ResultSetRelationRows>( 29198 ((ResultColumn) source).getResultSet().getRelationRows())); 29199 impactRelation.setEffectType(effectType); 29200 } else if (source instanceof TableColumn 29201 && !((TableColumn) source).getTable().getRelationRows().getHoldRelations().isEmpty()) { 29202 impactRelation = modelFactory.createImpactRelation(); 29203 impactRelation.addSource(new RelationRowsRelationshipElement<TableRelationRows>( 29204 ((TableColumn) source).getTable().getRelationRows())); 29205 impactRelation.setEffectType(effectType);; 29206 } 29207 29208 if (impactRelation == null) { 29209 continue; 29210 } 29211 29212 if (relation.getTarget() != null) { 29213 Object target = relation.getTarget().getElement(); 29214 if (target instanceof ResultColumn) { 29215 impactRelation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>( 29216 ((ResultColumn) target).getResultSet().getRelationRows())); 29217 } else if (target instanceof TableColumn) { 29218 impactRelation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>( 29219 ((TableColumn) target).getTable().getRelationRows())); 29220 } 29221 } 29222 29223 if (impactRelation.getSources() != null && !impactRelation.getSources().isEmpty() 29224 && impactRelation.getTarget() != null 29225 && impactRelation.getSources().iterator().next().getElement() == impactRelation.getTarget().getElement()) { 29226 modelManager.removeRelation(impactRelation); 29227 } 29228 } 29229 } 29230 } 29231 return relation; 29232 } 29233 29234 private boolean isNotInProcedure(Table table) { 29235 TStoredProcedureSqlStatement stmt = getProcedureParent(stmtStack.peek()); 29236 Procedure procedure = this.modelFactory.createProcedure(stmt); 29237 if(procedure!=null && procedure.getName().equals(table.getParent())){ 29238 return false; 29239 } 29240 return true; 29241 } 29242 29243 private boolean hasJoin(TCustomSqlStatement stmt) { 29244 if (stmt.getJoins() == null || stmt.getJoins().size() == 0) 29245 return false; 29246 if (stmt.getJoins().size() > 1) { 29247 return true; 29248 } 29249 TJoinItemList joinItems = stmt.getJoins().getJoin(0).getJoinItems(); 29250 if (joinItems == null || joinItems.size() == 0) { 29251 return false; 29252 } 29253 return true; 29254 } 29255 29256 private boolean isInQuery(TSelectSqlStatement query, TResultColumn column) { 29257 if(query == null) 29258 return false; 29259 TResultColumnList columns = query.getResultColumnList(); 29260 if (columns != null) { 29261 for (int i = 0; i < columns.size(); i++) { 29262 if (columns.getResultColumn(i).equals(column)) { 29263 return true; 29264 } 29265 } 29266 } 29267 return false; 29268 } 29269 29270 private void appendResultColumnRelationSource(Object modelObject, DataFlowRelationship relation, int columnIndex, 29271 TObjectName columnName, Object tableModel) { 29272 if (tableModel instanceof Table) { 29273 Object model = modelManager.getModel(new Pair<Table, TObjectName>((Table) tableModel, columnName)); 29274 if (model instanceof TableColumn) { 29275 relation.addSource(new TableColumnRelationshipElement((TableColumn) model)); 29276 } 29277 } else if (tableModel instanceof ResultSet) { 29278 List<ResultColumn> queryColumns = ((ResultSet) tableModel).getColumns(); 29279 boolean flag = false; 29280 for (int l = 0; l < queryColumns.size(); l++) { 29281 ResultColumn column = queryColumns.get(l); 29282 if (DlineageUtil.compareColumnIdentifier(getColumnName(columnName), 29283 DlineageUtil.getIdentifierNormalColumnName(column.getName()))) { 29284 if (!column.equals(modelObject)) { 29285 relation.addSource(new ResultColumnRelationshipElement(column, columnName)); 29286 flag = true; 29287 } 29288 break; 29289 } 29290 } 29291 if (!flag) { 29292 for (int l = 0; l < queryColumns.size(); l++) { 29293 ResultColumn column = queryColumns.get(l); 29294 if ("*".equals(column.getName())) { 29295 if (!containsStarColumn(column, columnName)) { 29296 column.bindStarLinkColumn(columnName); 29297 } 29298 relation.addSource(new ResultColumnRelationshipElement(column, columnName)); 29299 flag = true; 29300 break; 29301 } 29302 } 29303 } 29304 if (!flag && columnIndex < queryColumns.size() && columnIndex != -1) { 29305 relation.addSource(new ResultColumnRelationshipElement(queryColumns.get(columnIndex), columnName)); 29306 } 29307 } 29308 } 29309 29310 private boolean isApplyJoin(TCustomSqlStatement stmt) { 29311 if (stmt.getJoins() == null || stmt.getJoins().size() == 0) 29312 return false; 29313 TJoinItemList joinItems = stmt.getJoins().getJoin(0).getJoinItems(); 29314 if (joinItems == null || joinItems.size() == 0) { 29315 return false; 29316 } 29317 if (joinItems.getJoinItem(0).getJoinType() == EJoinType.crossapply 29318 || joinItems.getJoinItem(0).getJoinType() == EJoinType.outerapply) 29319 return true; 29320 return false; 29321 } 29322 29323 private boolean containsStarColumn(ResultColumn resultColumn, TObjectName columnName) { 29324 String targetColumnName = getColumnName(columnName); 29325 if (resultColumn.hasStarLinkColumn()) { 29326 return resultColumn.getStarLinkColumns().containsKey(targetColumnName); 29327 } 29328 return false; 29329 } 29330 29331 private void analyzeAggregate(TFunctionCall function, TExpression expr) { 29332 TCustomSqlStatement stmt = stmtStack.peek(); 29333 ResultSet resultSet = (ResultSet) modelManager.getModel(stmt.getResultColumnList()); 29334 if (resultSet == null) { 29335 return; 29336 } 29337 29338 if (expr != null) { 29339 columnsInExpr visitor = new columnsInExpr(); 29340 expr.inOrderTraverse(visitor); 29341 List<TObjectName> objectNames = visitor.getObjectNames(); 29342 for (int j = 0; j < objectNames.size(); j++) { 29343 TObjectName columnName = objectNames.get(j); 29344 29345 if (columnName.getDbObjectType() == EDbObjectType.variable) { 29346 continue; 29347 } 29348 29349 if (columnName.getColumnNameOnly().startsWith("@") 29350 && (option.getVendor() == EDbVendor.dbvmssql || option.getVendor() == EDbVendor.dbvazuresql)) { 29351 continue; 29352 } 29353 29354 if (columnName.getColumnNameOnly().startsWith(":") 29355 && (option.getVendor() == EDbVendor.dbvhana || option.getVendor() == EDbVendor.dbvteradata)) { 29356 continue; 29357 } 29358 29359 Object targetModel0 = modelManager.getModel(function.getFunctionName()); 29360 if (targetModel0 == null && function.getFunctionType() == EFunctionType.array_agg_t && modelManager.getModel(function) instanceof Function) { 29361 Function functionModel = (Function) modelManager.getModel(function); 29362 if (functionModel.getColumns().size() == 1) { 29363 targetModel0 = ((Function) functionModel).getColumns().get(0); 29364 } 29365 } 29366 if (!(targetModel0 instanceof ResultColumn)) { 29367 continue; 29368 } 29369 ResultColumn targetResultColumn0 = (ResultColumn) targetModel0; 29370 AbstractRelationship relation = modelFactory.createRecordSetRelation(); 29371 relation.setEffectType(EffectType.function); 29372 relation.setFunction(function.getFunctionName().toString()); 29373 relation.setTarget(new ResultColumnRelationshipElement(targetResultColumn0)); 29374 TTable table = modelManager.getTable(stmt, columnName); 29375 if (table != null) { 29376 if (modelManager.getModel(table) instanceof Table) { 29377 Table tableModel = (Table) modelManager.getModel(table); 29378 if (tableModel != null) { 29379 TableColumn columnModel = modelFactory.createTableColumn(tableModel, columnName, false); 29380 if (columnModel != null) { 29381 relation.addSource( 29382 new TableColumnRelationshipElement(columnModel, columnName.getLocation(), 29383 columnName)); 29384 } 29385 } 29386 } else if (modelManager.getModel(table) instanceof QueryTable) { 29387 Object model = modelManager.getModel(columnName.getSourceColumn()); 29388 if (model instanceof ResultColumn) { 29389 ResultColumn resultColumn = (ResultColumn) model; 29390 if (resultColumn != null) { 29391 relation.addSource( 29392 new ResultColumnRelationshipElement(resultColumn, columnName.getLocation(), columnName)); 29393 } 29394 } 29395 } 29396 } 29397 } 29398 29399 List<TParseTreeNode> functions = visitor.getFunctions(); 29400 for (int j = 0; j < functions.size(); j++) { 29401 TParseTreeNode functionObj = functions.get(j); 29402 Object functionModel = modelManager.getModel(functionObj); 29403 if (functionModel == null) { 29404 functionModel = createFunction(functionObj); 29405 } 29406 if (functionModel instanceof Function) { 29407 Object targetModel1 = modelManager.getModel(function.getFunctionName()); 29408 if (targetModel1 == null && function.getFunctionType() == EFunctionType.array_agg_t && modelManager.getModel(function) instanceof Function) { 29409 Function functionModel1 = (Function) modelManager.getModel(function); 29410 if (functionModel1.getColumns().size() == 1) { 29411 targetModel1 = ((Function) functionModel1).getColumns().get(0); 29412 } 29413 } 29414 if (!(targetModel1 instanceof ResultColumn)) { 29415 continue; 29416 } 29417 ResultColumn targetRC_func = (ResultColumn) targetModel1; 29418 AbstractRelationship relation; 29419 if ("COUNT".equalsIgnoreCase(function.getFunctionName().toString())) { 29420 // relation = modelFactory.createDataFlowRelation(); 29421 relation = modelFactory.createRecordSetRelation(); 29422 } else { 29423 relation = modelFactory.createRecordSetRelation(); 29424 } 29425 relation.setEffectType(EffectType.function); 29426 relation.setFunction(function.getFunctionName().toString()); 29427 relation.setTarget(new ResultColumnRelationshipElement(targetRC_func)); 29428 29429 if (functionObj instanceof TFunctionCall) { 29430 ResultColumn resultColumn = (ResultColumn) modelManager 29431 .getModel(((TFunctionCall) functionObj).getFunctionName()); 29432 if (resultColumn != null) { 29433 ResultColumnRelationshipElement element = new ResultColumnRelationshipElement(resultColumn, 29434 ((TFunctionCall) functionObj).getFunctionName().getLocation()); 29435 relation.addSource(element); 29436 } 29437 } 29438 if (functionObj instanceof TCaseExpression) { 29439 ResultColumn resultColumn = (ResultColumn) modelManager 29440 .getModel(((TCaseExpression) functionObj).getWhenClauseItemList()); 29441 if (resultColumn != null) { 29442 ResultColumnRelationshipElement element = new ResultColumnRelationshipElement(resultColumn); 29443 relation.addSource(element); 29444 } 29445 } 29446 } else if (functionModel instanceof Table) { 29447 Object targetModel2 = modelManager.getModel(function.getFunctionName()); 29448 if (targetModel2 == null && function.getFunctionType() == EFunctionType.array_agg_t && modelManager.getModel(function) instanceof Function) { 29449 Function functionModel1 = (Function) modelManager.getModel(function); 29450 if (functionModel1.getColumns().size() == 1) { 29451 targetModel2 = ((Function) functionModel1).getColumns().get(0); 29452 } 29453 } 29454 if (!(targetModel2 instanceof ResultColumn)) { 29455 continue; 29456 } 29457 ResultColumn targetRC_table = (ResultColumn) targetModel2; 29458 TableColumn tableColumn = modelFactory.createTableColumn((Table) functionModel, 29459 function.getFunctionName(), false); 29460 AbstractRelationship relation; 29461 if ("COUNT".equalsIgnoreCase(function.getFunctionName().toString())) { 29462 // relation = modelFactory.createDataFlowRelation(); 29463 relation = modelFactory.createRecordSetRelation(); 29464 } else { 29465 relation = modelFactory.createRecordSetRelation(); 29466 } 29467 relation.setEffectType(EffectType.function); 29468 relation.setFunction(function.getFunctionName().toString()); 29469 relation.setTarget(new ResultColumnRelationshipElement(targetRC_table)); 29470 TableColumnRelationshipElement element = new TableColumnRelationshipElement(tableColumn); 29471 relation.addSource(element); 29472 } 29473 } 29474 } 29475 29476 if (expr == null || "COUNT".equalsIgnoreCase(function.getFunctionName().toString())) { 29477 if ("COUNT".equalsIgnoreCase(function.getFunctionName().toString()) 29478 // https://e.gitee.com/gudusoft/issues/list?issue=I4L5EO 29479 // 对于非 count() 函数,当有group by clause时, RelationRows 不参与indirect dataflow, 29480 // 让位与group by clause中的column. 29481 || ((TSelectSqlStatement) stmt).getGroupByClause() == null) { 29482 TTableList tables = stmt.getTables(); 29483 if (tables != null) { 29484 for (int i = 0; i < tables.size(); i++) { 29485 TTable table = tables.getTable(i); 29486 if (modelManager.getModel(table) == null && table.getSubquery() == null) { 29487 modelFactory.createTable(table); 29488 } 29489 if (modelManager.getModel(table) instanceof Table) { 29490 Object targetModel3 = modelManager.getModel(function.getFunctionName()); 29491 if (targetModel3 == null && function.getFunctionType() == EFunctionType.array_agg_t && modelManager.getModel(function) instanceof Function) { 29492 Function functionModel = (Function) modelManager.getModel(function); 29493 if (functionModel.getColumns().size() == 1) { 29494 targetModel3 = functionModel.getColumns().get(0); 29495 } 29496 } 29497 if (!(targetModel3 instanceof ResultColumn)) { 29498 continue; 29499 } 29500 ResultColumn targetRC_tbl = (ResultColumn) targetModel3; 29501 Table tableModel = (Table) modelManager.getModel(table); 29502 AbstractRelationship relation; 29503 if ("COUNT".equalsIgnoreCase(function.getFunctionName().toString())) { 29504 // relation = modelFactory.createDataFlowRelation(); 29505 relation = modelFactory.createRecordSetRelation(); 29506 } else { 29507 relation = modelFactory.createRecordSetRelation(); 29508 } 29509 relation.setEffectType(EffectType.function); 29510 relation.setFunction(function.getFunctionName().toString()); 29511 relation.setTarget(new ResultColumnRelationshipElement(targetRC_tbl)); 29512 29513 if (relation instanceof DataFlowRelationship && option.isShowCountTableColumn()) { 29514 List<TExpression> expressions = new ArrayList<TExpression>(); 29515 getFunctionExpressions(expressions, new ArrayList<TExpression>(), function); 29516 for (int j = 0; j < expressions.size(); j++) { 29517 columnsInExpr visitor = new columnsInExpr(); 29518 expressions.get(j).inOrderTraverse(visitor); 29519 List<TObjectName> objectNames = visitor.getObjectNames(); 29520 if (objectNames != null) { 29521 for (TObjectName columnName : objectNames) { 29522 TTable tempTable = modelManager.getTable(stmt, columnName); 29523 if (table.equals(tempTable)) { 29524 TableColumn tableColumn = modelFactory.createTableColumn(tableModel, 29525 columnName, false); 29526 TableColumnRelationshipElement element = new TableColumnRelationshipElement( 29527 tableColumn); 29528 relation.addSource(element); 29529 } 29530 } 29531 } 29532 } 29533 } 29534 if (relation.getSources().size() == 0) { 29535 RelationRowsRelationshipElement element = new RelationRowsRelationshipElement<TableRelationRows>( 29536 tableModel.getRelationRows()); 29537 relation.addSource(element); 29538 } 29539 } else if (modelManager.getModel(table) instanceof QueryTable) { 29540 Object targetModel4 = modelManager.getModel(function.getFunctionName()); 29541 if (targetModel4 == null && function.getFunctionType() == EFunctionType.array_agg_t && modelManager.getModel(function) instanceof Function) { 29542 Function functionModel = (Function) modelManager.getModel(function); 29543 targetModel4 = functionModel.getColumns().get(0); 29544 } 29545 if (!(targetModel4 instanceof ResultColumn)) { 29546 continue; 29547 } 29548 ResultColumn targetRC_qt = (ResultColumn) targetModel4; 29549 QueryTable tableModel = (QueryTable) modelManager.getModel(table); 29550 AbstractRelationship relation; 29551 if ("COUNT".equalsIgnoreCase(function.getFunctionName().toString())) { 29552 // relation = modelFactory.createDataFlowRelation(); 29553 relation = modelFactory.createRecordSetRelation(); 29554 } else { 29555 relation = modelFactory.createRecordSetRelation(); 29556 } 29557 relation.setEffectType(EffectType.function); 29558 relation.setFunction(function.getFunctionName().toString()); 29559 relation.setTarget(new ResultColumnRelationshipElement(targetRC_qt)); 29560 RelationRowsRelationshipElement element = new RelationRowsRelationshipElement<ResultSetRelationRows>( 29561 tableModel.getRelationRows()); 29562 relation.addSource(element); 29563 } 29564 } 29565 } 29566 } 29567 29568 if (stmt.getWhereClause() == null || stmt.getWhereClause().getCondition() == null) { 29569 return; 29570 } 29571 29572 columnsInExpr visitor = new columnsInExpr(); 29573 stmt.getWhereClause().getCondition().inOrderTraverse(visitor); 29574 List<TObjectName> objectNames = visitor.getObjectNames(); 29575 for (int j = 0; j < objectNames.size(); j++) { 29576 TObjectName columnName = objectNames.get(j); 29577 if (columnName.getDbObjectType() == EDbObjectType.variable) { 29578 Variable tableModel; 29579 if (columnName.toString().indexOf(".") != -1) { 29580 List<String> splits = SQLUtil.parseNames(columnName.toString()); 29581 tableModel = modelFactory.createVariable(splits.get(splits.size() - 2)); 29582 } else { 29583 tableModel = modelFactory.createVariable(columnName); 29584 } 29585 tableModel.setCreateTable(true); 29586 tableModel.setSubType(SubType.record); 29587 TObjectName variableProperties = new TObjectName(); 29588 variableProperties.setString("*"); 29589 modelFactory.createTableColumn(tableModel, variableProperties, true); 29590 } 29591 29592// if (columnName.getColumnNameOnly().startsWith("@") 29593// && (option.getVendor() == EDbVendor.dbvmssql || option.getVendor() == EDbVendor.dbvazuresql)) { 29594// continue; 29595// } 29596// 29597// if (columnName.getColumnNameOnly().startsWith(":") && (option.getVendor() == EDbVendor.dbvhana || option.getVendor() == EDbVendor.dbvteradata)) { 29598// continue; 29599// } 29600 29601 Object targetModel5 = modelManager.getModel(function.getFunctionName()); 29602 if (targetModel5 == null && function.getFunctionType() == EFunctionType.array_agg_t && modelManager.getModel(function) instanceof Function) { 29603 Function functionModel = (Function) modelManager.getModel(function); 29604 if (functionModel.getColumns().size() == 1) { 29605 targetModel5 = functionModel.getColumns().get(0); 29606 } 29607 } 29608 if (!(targetModel5 instanceof ResultColumn)) { 29609 continue; 29610 } 29611 AbstractRelationship relation = modelFactory.createRecordSetRelation(); 29612 relation.setEffectType(EffectType.function); 29613 relation.setFunction(function.getFunctionName().toString()); 29614 relation.setTarget(new ResultColumnRelationshipElement((ResultColumn) targetModel5)); 29615 29616 TTable table = modelManager.getTable(stmt, columnName); 29617 if (table != null) { 29618 if (modelManager.getModel(table) instanceof Table) { 29619 Table tableModel = (Table) modelManager.getModel(table); 29620 if (tableModel != null) { 29621 TableColumn columnModel = modelFactory.createTableColumn(tableModel, columnName, false); 29622 if(columnModel == null) { 29623 continue; 29624 } 29625 relation.addSource( 29626 new TableColumnRelationshipElement(columnModel, columnName.getLocation(), 29627 columnName)); 29628 } 29629 } else if (modelManager.getModel(table) instanceof QueryTable) { 29630 Object model = modelManager.getModel(columnName.getSourceColumn()); 29631 if (model instanceof ResultColumn) { 29632 ResultColumn resultColumn = (ResultColumn) model; 29633 if (resultColumn != null) { 29634 relation.addSource( 29635 new ResultColumnRelationshipElement(resultColumn, columnName.getLocation(), columnName)); 29636 } 29637 } 29638 } 29639 } 29640 } 29641 } 29642 } 29643 29644 private void analyzeFilterCondition(Object modelObject, TExpression expr, EJoinType joinType, 29645 JoinClauseType joinClauseType, EffectType effectType) { 29646 if (expr == null) { 29647 return; 29648 } 29649 29650 TCustomSqlStatement stmt = stmtStack.peek(); 29651 29652 columnsInExpr visitor = new columnsInExpr(); 29653 expr.inOrderTraverse(visitor); 29654 29655 List<TObjectName> objectNames = visitor.getObjectNames(); 29656 List<TParseTreeNode> functions = visitor.getFunctions(); 29657 List<TResultColumn> resultColumns = visitor.getResultColumns(); 29658 List<TParseTreeNode> constants = visitor.getConstants(); 29659 29660 ImpactRelationship relation = modelFactory.createImpactRelation(); 29661 relation.setEffectType(effectType); 29662 relation.setJoinClauseType(joinClauseType); 29663 if (modelObject instanceof ResultColumn) { 29664 relation.setTarget(new ResultColumnRelationshipElement((ResultColumn) modelObject)); 29665 } else if (modelObject instanceof Table) { 29666 Table targetTable = (Table) modelObject; 29667 relation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>( 29668 targetTable.getRelationRows())); 29669 } else if (modelObject instanceof ResultSet) { 29670 ResultSet targetResultSet = (ResultSet) modelObject; 29671 relation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>( 29672 targetResultSet.getRelationRows())); 29673 } else { 29674 ResultSet resultSet = (ResultSet) modelManager.getModel(stmt.getResultColumnList()); 29675 if (resultSet == null && stmt instanceof TUpdateSqlStatement) { 29676 resultSet = (ResultSet) modelManager.getModel(stmt); 29677 } 29678 if (resultSet == null && stmt instanceof TMergeSqlStatement) { 29679 TSelectSqlStatement subquery = ((TMergeSqlStatement) stmt).getUsingTable().getSubquery(); 29680 if (subquery != null) { 29681 resultSet = (ResultSet) modelManager.getModel(((TMergeSqlStatement) stmt).getUsingTable()); 29682 } 29683 else { 29684 resultSet = modelFactory.createQueryTable(((TMergeSqlStatement) stmt).getUsingTable()); 29685 } 29686 } 29687 if (resultSet != null) { 29688 relation.setTarget( 29689 new RelationRowsRelationshipElement<ResultSetRelationRows>(resultSet.getRelationRows())); 29690 } 29691 if (stmt instanceof TDeleteSqlStatement) { 29692 Object deleteTarget = modelManager.getModel(((TDeleteSqlStatement) stmt).getTargetTable()); 29693 if (deleteTarget instanceof Table) { 29694 Table table = (Table) deleteTarget; 29695 relation.setTarget(new RelationRowsRelationshipElement<TableRelationRows>(table.getRelationRows())); 29696 } else if (deleteTarget instanceof ResultSet) { 29697 ResultSet resultSetTarget = (ResultSet) deleteTarget; 29698 relation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>( 29699 resultSetTarget.getRelationRows())); 29700 } 29701 } 29702 } 29703 if (relation.getTarget() != null) { 29704 29705 if (constants != null && constants.size() > 0) { 29706 if (option.isShowConstantTable()) { 29707 Table constantTable = modelFactory.createConstantsTable(stmtStack.peek()); 29708 for (int i = 0; i < constants.size(); i++) { 29709 TParseTreeNode constant = constants.get(i); 29710 if (constant instanceof TConstant) { 29711 TableColumn constantColumn = modelFactory.createTableColumn(constantTable, 29712 (TConstant) constant); 29713 relation.addSource(new ConstantRelationshipElement(constantColumn)); 29714 } else if (constant instanceof TObjectName) { 29715 TableColumn constantColumn = modelFactory.createTableColumn(constantTable, 29716 (TObjectName) constant, false); 29717 if(constantColumn == null) { 29718 continue; 29719 } 29720 relation.addSource(new ConstantRelationshipElement(constantColumn)); 29721 } 29722 } 29723 } 29724 } 29725 29726 for (int j = 0; j < objectNames.size(); j++) { 29727 TObjectName columnName = objectNames.get(j); 29728 if (columnName.getDbObjectType() == EDbObjectType.variable) { 29729 Variable variable = modelFactory.createVariable(columnName); 29730 variable.setSubType(SubType.record); 29731 if (variable.getColumns().isEmpty()) { 29732 TObjectName variableProperties = new TObjectName(); 29733 variableProperties.setString("*"); 29734 modelFactory.createTableColumn(variable, variableProperties, true); 29735 } 29736 relation.addSource(new TableColumnRelationshipElement(variable.getColumns().get(0), 29737 columnName.getLocation(), columnName)); 29738 continue; 29739 } 29740 29741 if (columnName.getColumnNameOnly().startsWith("@") 29742 && (option.getVendor() == EDbVendor.dbvmssql || option.getVendor() == EDbVendor.dbvazuresql)) { 29743 continue; 29744 } 29745 29746 if (columnName.getColumnNameOnly().startsWith(":") 29747 && (option.getVendor() == EDbVendor.dbvhana || option.getVendor() == EDbVendor.dbvteradata)) { 29748 continue; 29749 } 29750 29751 TTable table = modelManager.getTable(stmt, columnName); 29752 29753 if (table == null) { 29754 table = columnName.getSourceTable(); 29755 } 29756 29757 if (table == null && stmt.tables != null) { 29758 for (int k = 0; k < stmt.tables.size(); k++) { 29759 if (table != null) 29760 break; 29761 29762 TTable tTable = stmt.tables.getTable(k); 29763 if (tTable.getTableType().name().startsWith("open")) { 29764 continue; 29765 } else if (getTableLinkedColumns(tTable) != null && getTableLinkedColumns(tTable).size() > 0) { 29766 for (int z = 0; z < getTableLinkedColumns(tTable).size(); z++) { 29767 TObjectName refer = getTableLinkedColumns(tTable).getObjectName(z); 29768 if ("*".equals(getColumnName(refer))) 29769 continue; 29770 if (getColumnName(refer).equals(getColumnName(columnName))) { 29771 table = tTable; 29772 break; 29773 } 29774 } 29775 } else if (columnName.getTableToken() != null 29776 && (SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotTable, columnName.getTableToken().getAstext(), tTable.getName()) 29777 || SQLUtil.sameName(option.getVendor(), ESQLDataObjectType.dotTable, columnName.getTableToken().getAstext(), tTable.getAliasName()))) { 29778 table = tTable; 29779 break; 29780 } 29781 } 29782 29783 if (table == null) { 29784 for (int k = 0; k < stmt.tables.size(); k++) { 29785 if (table != null) 29786 break; 29787 29788 TTable tTable = stmt.tables.getTable(k); 29789 Object model = ModelBindingManager.get().getModel(tTable); 29790 if (model instanceof Table) { 29791 Table tableModel = (Table) model; 29792 for (int z = 0; tableModel.getColumns() != null 29793 && z < tableModel.getColumns().size(); z++) { 29794 TableColumn refer = tableModel.getColumns().get(z); 29795 if (getColumnName(refer.getName()).equals(getColumnName(columnName))) { 29796 table = tTable; 29797 break; 29798 } 29799 if (refer.hasStarLinkColumn()) { 29800 for (TObjectName linkColumn : refer.getStarLinkColumnList()) { 29801 if (getColumnName(linkColumn).equals(getColumnName(columnName))) { 29802 table = tTable; 29803 break; 29804 } 29805 } 29806 } 29807 } 29808 } else if (model instanceof QueryTable) { 29809 QueryTable tableModel = (QueryTable) model; 29810 for (int z = 0; tableModel.getColumns() != null 29811 && z < tableModel.getColumns().size(); z++) { 29812 ResultColumn refer = tableModel.getColumns().get(z); 29813 if (DlineageUtil.getIdentifierNormalColumnName(refer.getName()).equals( 29814 DlineageUtil.getIdentifierNormalColumnName(getColumnName(columnName)))) { 29815 table = tTable; 29816 break; 29817 } 29818 if (refer.hasStarLinkColumn()) { 29819 for (TObjectName linkColumn : refer.getStarLinkColumnList()) { 29820 if (getColumnName(linkColumn).equals(getColumnName(columnName))) { 29821 table = tTable; 29822 break; 29823 } 29824 } 29825 } 29826 } 29827 } 29828 } 29829 } 29830 } 29831 29832 if (table == null && stmt.tables != null && stmt.tables.size() != 0 29833 && !(isBuiltInFunctionName(columnName) && isFromFunction(columnName))) { 29834 29835 if (modelManager.getModel(stmt) instanceof ResultSet) { 29836 ResultSet resultSetModel = (ResultSet) modelManager.getModel(stmt); 29837 boolean find = false; 29838 for (ResultColumn resultColumn : resultSetModel.getColumns()) { 29839 if(resultColumn.equals(modelObject)) { 29840 continue; 29841 } 29842 if (!TSQLEnv.isAliasReferenceForbidden.get(option.getVendor())) { 29843 if (getColumnName(columnName).equals(getColumnName(resultColumn.getName()))) { 29844 if (resultColumn.getColumnObject() != null) { 29845 int startToken = resultColumn.getColumnObject().getStartToken().posinlist; 29846 int endToken = resultColumn.getColumnObject().getEndToken().posinlist; 29847 if (columnName.getStartToken().posinlist >= startToken 29848 && columnName.getEndToken().posinlist <= endToken) { 29849 continue; 29850 } 29851 } 29852 relation.addSource(new ResultColumnRelationshipElement(resultColumn)); 29853 find = true; 29854 break; 29855 } 29856 } 29857 } 29858 if (find) { 29859 continue; 29860 } 29861 } 29862 29863 TObjectName pseudoTableName = new TObjectName(); 29864 // Use qualified prefix from column name if available (e.g., sch.pk_constv2 from sch.pk_constv2.c_cdsl) 29865 // Otherwise fall back to default pseudo table name 29866 String qualifiedPrefix = getQualifiedPrefixFromColumn(columnName); 29867 pseudoTableName.setString(qualifiedPrefix != null ? qualifiedPrefix : "pseudo_table_include_orphan_column"); 29868 Table pseudoTable = modelFactory.createTableByName(pseudoTableName); 29869 pseudoTable.setPseudo(true); 29870 TableColumn pseudoTableColumn = modelFactory.createTableColumn(pseudoTable, columnName, true); 29871 29872 // If not linking to first table and column has qualified prefix (3-part name like sch.pkg.col), 29873 // add the pseudo table column as source 29874 if (!isLinkOrphanColumnToFirstTable() && pseudoTableColumn != null && qualifiedPrefix != null) { 29875 relation.addSource(new TableColumnRelationshipElement(pseudoTableColumn)); 29876 } 29877 29878 if (isLinkOrphanColumnToFirstTable()) { 29879 TTable orphanTable = stmt.tables.getTable(0); 29880 table = stmt.tables.getTable(0); 29881 Object tableModel = modelManager.getModel(table); 29882 if (tableModel == null) { 29883 tableModel = modelFactory.createTable(orphanTable); 29884 } 29885 if (tableModel instanceof Table) { 29886 modelFactory.createTableColumn((Table) tableModel, columnName, false); 29887 ErrorInfo errorInfo = new ErrorInfo(); 29888 errorInfo.setErrorType(ErrorInfo.LINK_ORPHAN_COLUMN); 29889 errorInfo.setErrorMessage("Link orphan column [" + columnName.toString() 29890 + "] to the first table [" + orphanTable.getFullNameWithAliasString() + "]"); 29891 errorInfo.setStartPosition(new Pair3<Long, Long, String>(columnName.getStartToken().lineNo, 29892 columnName.getStartToken().columnNo, ModelBindingManager.getGlobalHash())); 29893 errorInfo.setEndPosition(new Pair3<Long, Long, String>(columnName.getEndToken().lineNo, 29894 columnName.getEndToken().columnNo + columnName.getEndToken().getAstext().length(), 29895 ModelBindingManager.getGlobalHash())); 29896 errorInfo.fillInfo(this); 29897 errorInfos.add(errorInfo); 29898 } 29899 } 29900 } 29901 29902 if (table != null) { 29903 if (modelManager.getModel(table) instanceof Table) { 29904 Table tableModel = (Table) modelManager.getModel(table); 29905 if (tableModel != null) { 29906 TableColumn columnModel = modelFactory.createTableColumn(tableModel, columnName, false); 29907 if(columnModel!=null) { 29908 // Carry the operand's own reference site: for created 29909 // (e.g. #temp) tables the column model is shared with 29910 // the definition site, whose coordinate is not where 29911 // this ON/WHERE operand sits. 29912 TableColumnRelationshipElement element = new TableColumnRelationshipElement(columnModel, 29913 columnName.getLocation(), columnName); 29914 relation.addSource(element); 29915 } 29916 } 29917 } else if (modelManager.getModel(table) instanceof QueryTable) { 29918 QueryTable tableModel = (QueryTable)modelManager.getModel(table); 29919 if (table.getSubquery() != null && table.getSubquery().isCombinedQuery()) { 29920 TSelectSqlStatement subquery = table.getSubquery(); 29921 List<ResultSet> resultSets = new ArrayList<ResultSet>(); 29922 if (!subquery.getLeftStmt().isCombinedQuery()) { 29923 ResultSet sourceResultSet = (ResultSet) modelManager 29924 .getModel(subquery.getLeftStmt().getResultColumnList()); 29925 resultSets.add(sourceResultSet); 29926 } else { 29927 ResultSet sourceResultSet = (ResultSet) modelManager.getModel(subquery.getLeftStmt()); 29928 resultSets.add(sourceResultSet); 29929 } 29930 29931 if (!subquery.getRightStmt().isCombinedQuery()) { 29932 ResultSet sourceResultSet = (ResultSet) modelManager 29933 .getModel(subquery.getRightStmt().getResultColumnList()); 29934 resultSets.add(sourceResultSet); 29935 } else { 29936 ResultSet sourceResultSet = (ResultSet) modelManager.getModel(subquery.getRightStmt()); 29937 resultSets.add(sourceResultSet); 29938 } 29939 29940 for (ResultSet sourceResultSet : resultSets) { 29941 if (sourceResultSet != null && columnName.getSourceColumn() != null) { 29942 for (int k = 0; k < sourceResultSet.getColumns().size(); k++) { 29943 if (getColumnName(sourceResultSet.getColumns().get(k).getName()).equals( 29944 getColumnName(columnName.getSourceColumn().getColumnNameOnly()))) { 29945 Set<TObjectName> starLinkColumnSet = sourceResultSet.getColumns().get(k) 29946 .getStarLinkColumns().get(getColumnName(columnName)); 29947 if (starLinkColumnSet != null && !starLinkColumnSet.isEmpty()) { 29948 ResultColumn column = modelFactory.createResultColumn(sourceResultSet, 29949 starLinkColumnSet.iterator().next(), true); 29950 ResultColumnRelationshipElement setopElement = new ResultColumnRelationshipElement(column); 29951 setopElement.setReferencePosition(columnName); 29952 relation.addSource(setopElement); 29953 } else { 29954 ResultColumnRelationshipElement setopElement = new ResultColumnRelationshipElement( 29955 sourceResultSet.getColumns().get(k)); 29956 setopElement.setReferencePosition(columnName); 29957 relation.addSource(setopElement); 29958 } 29959 } 29960 } 29961 } 29962 } 29963 } else { 29964 Object model = modelManager.getModel(columnName.getSourceColumn()); 29965 if (model instanceof ResultColumn) { 29966 ResultColumn resultColumn = (ResultColumn) model; 29967 if (resultColumn != null) { 29968 if (resultColumn.hasStarLinkColumn()) { 29969 Set<TObjectName> starLinkColumnSet = resultColumn.getStarLinkColumns() 29970 .get(getColumnName(columnName)); 29971 if (starLinkColumnSet != null && !starLinkColumnSet.isEmpty()) { 29972 ResultColumn column = modelFactory.createResultColumn( 29973 resultColumn.getResultSet(), starLinkColumnSet.iterator().next(), true); 29974 ResultColumnRelationshipElement starElement = new ResultColumnRelationshipElement(column); 29975 starElement.setReferencePosition(columnName); 29976 relation.addSource(starElement); 29977 } else { 29978 resultColumn.bindStarLinkColumn(columnName); 29979 ResultColumn column = modelFactory 29980 .createResultColumn(resultColumn.getResultSet(), columnName, true); 29981 ResultColumnRelationshipElement starElement = new ResultColumnRelationshipElement(column); 29982 starElement.setReferencePosition(columnName); 29983 relation.addSource(starElement); 29984 } 29985 } else { 29986 // Carry the operand's own reference site: the 29987 // derived table's / CTE's result column model 29988 // carries its select-list definition span, not 29989 // where this ON/WHERE operand sits. 29990 ResultColumnRelationshipElement element = new ResultColumnRelationshipElement( 29991 resultColumn, columnName.getLocation(), columnName); 29992 relation.addSource(element); 29993 } 29994 } 29995 } 29996 else{ 29997 boolean find = false; 29998 for (int i = 0; i < tableModel.getColumns().size(); i++) { 29999 ResultColumn resultColumn = tableModel.getColumns().get(i); 30000 if (DlineageUtil.getIdentifierNormalColumnName(resultColumn.getName()).equals( 30001 DlineageUtil.getIdentifierNormalColumnName(getColumnName(columnName)))) { 30002 // An explicit CTE column list resolves here; 30003 // the model span is the column-list entry, not 30004 // the operand. 30005 ResultColumnRelationshipElement element = new ResultColumnRelationshipElement( 30006 resultColumn, columnName.getLocation(), columnName); 30007 relation.addSource(element); 30008 find = true; 30009 break; 30010 } 30011 else if (resultColumn.getName().endsWith("*")) { 30012 resultColumn.bindStarLinkColumn(columnName); 30013 } 30014 } 30015 if(!find){ 30016 ResultColumn resultColumn = new ResultColumn(tableModel, columnName); 30017 ResultColumnRelationshipElement element = new ResultColumnRelationshipElement( 30018 resultColumn, columnName.getLocation(), columnName); 30019 relation.addSource(element); 30020 } 30021 } 30022 } 30023 } 30024 } 30025 } 30026 30027 for (int j = 0; j < functions.size(); j++) { 30028 TParseTreeNode functionObj = functions.get(j); 30029 Object functionModel = modelManager.getModel(functionObj); 30030 if (functionModel == null) { 30031 functionModel = createFunction(functionObj); 30032 } 30033 if (functionModel instanceof Function) { 30034 if (functionObj instanceof TFunctionCall) { 30035 ResultColumn resultColumn = (ResultColumn) modelManager 30036 .getModel(((TFunctionCall) functionObj).getFunctionName()); 30037 if (resultColumn != null) { 30038 ResultColumnRelationshipElement element = new ResultColumnRelationshipElement(resultColumn, 30039 ((TFunctionCall) functionObj).getFunctionName().getLocation()); 30040 relation.addSource(element); 30041 } 30042 } 30043 if (functionObj instanceof TCaseExpression) { 30044 ResultColumn resultColumn = (ResultColumn) modelManager 30045 .getModel(((TCaseExpression) functionObj).getWhenClauseItemList()); 30046 if (resultColumn != null) { 30047 ResultColumnRelationshipElement element = new ResultColumnRelationshipElement(resultColumn); 30048 relation.addSource(element); 30049 } 30050 } 30051 } else if (functionModel instanceof Table && ((TFunctionCall) functionObj).getFunctionName()!=null) { 30052 TableColumn tableColumn = modelFactory.createTableColumn((Table) functionModel, 30053 ((TFunctionCall) functionObj).getFunctionName(), false); 30054 if (tableColumn != null) { 30055 TableColumnRelationshipElement element = new TableColumnRelationshipElement(tableColumn); 30056 relation.addSource(element); 30057 } 30058 } 30059 } 30060 30061 for (int j = 0; j < resultColumns.size(); j++) { 30062 TResultColumn resultColumn = resultColumns.get(j); 30063 if (modelManager.getModel(resultColumn) instanceof ResultColumn) { 30064 ResultColumn resultColumnModel = (ResultColumn) modelManager.getModel(resultColumn); 30065 relation.addSource(new ResultColumnRelationshipElement(resultColumnModel, ESqlClause.selectList)); 30066 } 30067 } 30068 } 30069 30070 if (isShowJoin() && joinClauseType != null) { 30071 joinInExpr joinVisitor = new joinInExpr(joinType, joinClauseType, effectType); 30072 expr.inOrderTraverse(joinVisitor); 30073 } 30074 } 30075 30076 public void dispose() { 30077 authoritativeEvidenceCollector.reset(); 30078 cteSelectRelationKeys.clear(); 30079 accessedSubqueries.clear(); 30080 accessedStatements.clear(); 30081 stmtStack.clear(); 30082 viewDDLMap.clear(); 30083 procedureDDLMap.clear(); 30084 structObjectMap.clear(); 30085 appendResultSets.clear(); 30086 appendStarColumns.clear(); 30087 appendTableStarColumns.clear(); 30088 columnNameIndexCache.clear(); 30089 resultSetColumnLookupCache.clear(); 30090 tableColumnLookupCache.clear(); 30091 modelManager.DISPLAY_ID.clear(); 30092 modelManager.DISPLAY_NAME.clear(); 30093 tableIds.clear(); 30094 ModelBindingManager.remove(); 30095 } 30096 30097 class joinTreatColumnsInExpr implements IExpressionVisitor { 30098 30099 private List<TObjectName> objectNames = new ArrayList<TObjectName>(); 30100 30101 private TTable table; 30102 30103 public joinTreatColumnsInExpr(TTable table) { 30104 this.table = table; 30105 } 30106 30107 public List<TObjectName> getObjectNames() { 30108 return objectNames; 30109 } 30110 30111 boolean is_compare_condition(EExpressionType t) { 30112 return t == EExpressionType.simple_comparison_t; 30113 } 30114 30115 @Override 30116 public boolean exprVisit(TParseTreeNode pNode, boolean isLeafNode) { 30117 TExpression expr = (TExpression) pNode; 30118 if (is_compare_condition(expr.getExpressionType())) { 30119 TExpression leftExpr = expr.getLeftOperand(); 30120 columnsInExpr leftVisitor = new columnsInExpr(); 30121 leftExpr.inOrderTraverse(leftVisitor); 30122 List<TObjectName> leftObjectNames = leftVisitor.getObjectNames(); 30123 30124 TExpression rightExpr = expr.getRightOperand(); 30125 columnsInExpr rightVisitor = new columnsInExpr(); 30126 rightExpr.inOrderTraverse(rightVisitor); 30127 List<TObjectName> rightObjectNames = rightVisitor.getObjectNames(); 30128 30129 if (!leftObjectNames.isEmpty() && !rightObjectNames.isEmpty()) { 30130 for (TObjectName column : leftObjectNames) { 30131 if (column.getSourceTable() != null && column.getSourceTable().equals(table)) { 30132 objectNames.add(column); 30133 return false; 30134 } 30135 } 30136 for (TObjectName column : rightObjectNames) { 30137 if (column.getSourceTable() != null && column.getSourceTable().equals(table)) { 30138 objectNames.add(column); 30139 return false; 30140 } 30141 } 30142 } 30143 return false; 30144 } 30145 return true; 30146 } 30147 } 30148 30149 class columnsInExpr implements IExpressionVisitor { 30150 30151 private List<TParseTreeNode> constants = new ArrayList<TParseTreeNode>(); 30152 private List<TObjectName> objectNames = new ArrayList<TObjectName>(); 30153 private List<TParseTreeNode> functions = new ArrayList<TParseTreeNode>(); 30154 private List<TResultColumn> resultColumns = new ArrayList<TResultColumn>(); 30155 private List<TSelectSqlStatement> subquerys = new ArrayList<TSelectSqlStatement>(); 30156 private boolean skipFunction = false; 30157 // ClickHouse expression-CTE aliases already substituted during this 30158 // traversal — guards against cyclic alias definitions. 30159 private Set<TCTE> expandedExpressionCtes = new HashSet<TCTE>(); 30160 // Alias-chain expansion is depth-bounded hybrid: direct recursion 30161 // preserves depth-first substitution order (source order of the 30162 // collected columns) up to CTE_EXPANSION_MAX_DEPTH; deeper chains 30163 // queue their definitions on an explicit deque drained by the 30164 // outermost frame — a 2000-alias chain must not overflow the -Xss2m 30165 // stack (same class as the repo's UNION/expression traversal rule). 30166 private static final int CTE_EXPANSION_MAX_DEPTH = 64; 30167 private Deque<TExpression> pendingCteExpressions = new ArrayDeque<TExpression>(); 30168 private int cteExpansionDepth = 0; 30169 30170 /** 30171 * Collects an expression operand as a column reference — unless it is a 30172 * ClickHouse expression-CTE alias (WITH <expr> AS ident), in which case 30173 * the CTE's definition is traversed instead so its real column 30174 * dependencies flow to the consumer. Every branch of exprVisit that 30175 * adds an operand's TObjectName directly must go through this method, 30176 * or alias references in that expression shape leak out as columns. 30177 */ 30178 private void addColumnOrExpandCteAlias(TObjectName object) { 30179 if (object.getExpressionCteRef() == null 30180 || object.getSourceTable() != null 30181 || object.getExpressionCteRef().getExpression() == null) { 30182 objectNames.add(object); 30183 return; 30184 } 30185 if (!expandedExpressionCtes.add(object.getExpressionCteRef())) { 30186 return; // cyclic or already substituted 30187 } 30188 TExpression definition = object.getExpressionCteRef().getExpression(); 30189 if (cteExpansionDepth >= CTE_EXPANSION_MAX_DEPTH) { 30190 pendingCteExpressions.add(definition); 30191 return; // drained below when the outermost frame unwinds 30192 } 30193 cteExpansionDepth++; 30194 try { 30195 definition.inOrderTraverse(this); 30196 } finally { 30197 cteExpansionDepth--; 30198 } 30199 if (cteExpansionDepth == 0) { 30200 while (!pendingCteExpressions.isEmpty()) { 30201 TExpression pending = pendingCteExpressions.poll(); 30202 cteExpansionDepth++; 30203 try { 30204 pending.inOrderTraverse(this); 30205 } finally { 30206 cteExpansionDepth--; 30207 } 30208 } 30209 } 30210 } 30211 30212 public void setSkipFunction(boolean skipFunction) { 30213 this.skipFunction = skipFunction; 30214 } 30215 30216 public List<TParseTreeNode> getFunctions() { 30217 return functions; 30218 } 30219 30220 public List<TSelectSqlStatement> getSubquerys() { 30221 return subquerys; 30222 } 30223 30224 public List<TParseTreeNode> getConstants() { 30225 return constants; 30226 } 30227 30228 public List<TObjectName> getObjectNames() { 30229 return objectNames; 30230 } 30231 30232 public List<TResultColumn> getResultColumns() { 30233 return resultColumns; 30234 } 30235 30236 @Override 30237 public boolean exprVisit(TParseTreeNode pNode, boolean isLeafNode) { 30238 TExpression lcexpr = (TExpression) pNode; 30239 // Handle named argument expressions (e.g., "INPUT => value" in Snowflake FLATTEN) 30240 // The left operand is the parameter name, NOT a column reference. 30241 // Only traverse the right operand (the value). 30242 if (lcexpr.getExpressionType() == EExpressionType.assignment_t) { 30243 // Skip left operand (parameter name) - only traverse right operand (value) 30244 if (lcexpr.getRightOperand() != null) { 30245 lcexpr.getRightOperand().inOrderTraverse(this); 30246 } 30247 return false; // Don't continue default traversal 30248 } 30249 if (lcexpr.getExpressionType() == EExpressionType.simple_constant_t) { 30250 if (lcexpr.getConstantOperand() != null) { 30251 if(lcexpr.getConstantOperand().getInt64_expression()!=null 30252 && lcexpr.getConstantOperand().getInt64_expression().getExpressionType() == EExpressionType.function_t) { 30253 lcexpr.getConstantOperand().getInt64_expression().inOrderTraverse(this); 30254 } 30255 else { 30256 constants.add(lcexpr.getConstantOperand()); 30257 } 30258 } 30259 } else if (lcexpr.getExpressionType() == EExpressionType.array_t) { 30260 if(lcexpr.getObjectOperand()!=null) { 30261 TObjectName object = lcexpr.getObjectOperand(); 30262 addColumnOrExpandCteAlias(object); 30263 } else if (lcexpr.getExprList() != null) { 30264 for (int j = 0; j < lcexpr.getExprList().size(); j++) { 30265 TExpression expr = lcexpr.getExprList().getExpression(j); 30266 if (expr != null) 30267 expr.inOrderTraverse(this); 30268 } 30269 } 30270 } else if (lcexpr.getExpressionType() == EExpressionType.simple_object_name_t) { 30271 if (lcexpr.getObjectOperand() != null && !(isBuiltInFunctionName(lcexpr.getObjectOperand()) 30272 && isFromFunction(lcexpr.getObjectOperand()))) { 30273 TObjectName object = lcexpr.getObjectOperand(); 30274 // Skip named argument parameter names (e.g., INPUT in "INPUT => value") 30275 // These are function parameter names, NOT column references 30276 if (object.getObjectType() == TObjectName.ttobjNamedArgParameter) { 30277 // Skip - this is a named argument parameter name 30278 } else if (object.getDbObjectType() == EDbObjectType.column 30279 || object.getDbObjectType() == EDbObjectType.column_alias 30280 || object.getDbObjectType() == EDbObjectType.alias 30281 || object.getDbObjectType() == EDbObjectType.unknown 30282 || object.getDbObjectType() == EDbObjectType.variable 30283 || isTableScopedPseudoColumn(object)) { 30284 addColumnOrExpandCteAlias(object); 30285 } else if (object.getDbObjectType() == EDbObjectType.notAColumn 30286 || object.getDbObjectType() == EDbObjectType.date_time_part ) { 30287 constants.add(object); 30288 } 30289 } 30290 } else if (lcexpr.getExpressionType() == EExpressionType.between_t) { 30291 if (lcexpr.getBetweenOperand() != null && lcexpr.getBetweenOperand().getObjectOperand() != null) { 30292 TObjectName object = lcexpr.getBetweenOperand().getObjectOperand(); 30293 if (object.getDbObjectType() == EDbObjectType.column 30294 || object.getDbObjectType() == EDbObjectType.column_alias 30295 || object.getDbObjectType() == EDbObjectType.alias 30296 || object.getDbObjectType() == EDbObjectType.unknown 30297 || object.getDbObjectType() == EDbObjectType.variable) { 30298 addColumnOrExpandCteAlias(object); 30299 } 30300 } 30301 } else if (lcexpr.getExpressionType() == EExpressionType.object_access_t) { 30302 if (lcexpr.getObjectAccess() != null) { 30303 TObjectNameList objects = lcexpr.getObjectAccess().getAttributes(); 30304 TFunctionCall function = lcexpr.getObjectAccess().getObjectExpr().getFunctionCall(); 30305 if (objects != null && function != null) { 30306 for (TObjectName object : objects) { 30307 TGSqlParser sqlparser = createSqlParser(option.getVendor()); 30308 sqlparser.sqltext = "select " + function.getFunctionName().toString() + "." 30309 + object.getColumnNameOnly() + " from " + function.getFunctionName().toString(); 30310 if (sqlparser.parse() == 0) { 30311 TObjectName objectName = sqlparser.sqlstatements.get(0).getResultColumnList() 30312 .getResultColumn(0).getFieldAttr(); 30313 objectNames.add(objectName); 30314 } 30315 } 30316 } 30317 } 30318 } else if (lcexpr.getExpressionType() == EExpressionType.function_t || lcexpr.getExpressionType() == EExpressionType.fieldselection_t) { 30319 TFunctionCall func = lcexpr.getFunctionCall(); 30320 if (func == null) { 30321 return true; 30322 } 30323 if (skipFunction) { 30324 if (func.getArgs() != null) { 30325 for (int k = 0; k < func.getArgs().size(); k++) { 30326 TExpression expr = func.getArgs().getExpression(k); 30327 if (expr != null) 30328 expr.inOrderTraverse(this); 30329 } 30330 } 30331 30332 if (func.getTrimArgument() != null) { 30333 TTrimArgument args = func.getTrimArgument(); 30334 TExpression expr = args.getStringExpression(); 30335 if (expr != null) { 30336 expr.inOrderTraverse(this); 30337 } 30338 expr = args.getTrimCharacter(); 30339 if (expr != null) { 30340 expr.inOrderTraverse(this); 30341 } 30342 } 30343 30344 if (func.getAgainstExpr() != null) { 30345 func.getAgainstExpr().inOrderTraverse(this); 30346 } 30347// if (func.getBetweenExpr() != null) { 30348// func.getBetweenExpr().inOrderTraverse(this); 30349// } 30350 if (func.getExpr1() != null) { 30351 func.getExpr1().inOrderTraverse(this); 30352 } 30353 if (func.getExpr2() != null) { 30354 func.getExpr2().inOrderTraverse(this); 30355 } 30356 if (func.getExpr3() != null) { 30357 func.getExpr3().inOrderTraverse(this); 30358 } 30359 if (func.getParameter() != null) { 30360 func.getParameter().inOrderTraverse(this); 30361 } 30362 } else { 30363 functions.add(func); 30364 } 30365 30366 } else if (lcexpr.getExpressionType() == EExpressionType.case_t) { 30367 TCaseExpression expr = lcexpr.getCaseExpression(); 30368 if (skipFunction) { 30369 TExpression defaultExpr = expr.getElse_expr(); 30370 if (defaultExpr != null) { 30371 defaultExpr.inOrderTraverse(this); 30372 } 30373 TWhenClauseItemList list = expr.getWhenClauseItemList(); 30374 for (int i = 0; i < list.size(); i++) { 30375 TWhenClauseItem element = (TWhenClauseItem) list.getElement(i); 30376 (((TWhenClauseItem) element).getReturn_expr()).inOrderTraverse(this); 30377 30378 } 30379 } else { 30380 functions.add(expr); 30381 } 30382 } else if (lcexpr.getSubQuery() != null) { 30383 TSelectSqlStatement select = lcexpr.getSubQuery(); 30384 analyzeSelectStmt(select); 30385 subquerys.add(select); 30386 if (select.getResultColumnList() != null && select.getResultColumnList().size() > 0) { 30387 for (TResultColumn column : select.getResultColumnList()) { 30388 resultColumns.add(column); 30389 } 30390 } 30391 } 30392 return true; 30393 } 30394 } 30395 30396 class joinInExpr implements IExpressionVisitor { 30397 30398 private EJoinType joinType; 30399 private JoinClauseType joinClauseType; 30400 private EffectType effectType; 30401 30402 public joinInExpr(EJoinType joinType, JoinClauseType joinClauseType, EffectType effectType) { 30403 this.joinType = joinType; 30404 this.joinClauseType = joinClauseType; 30405 this.effectType = effectType; 30406 } 30407 30408 boolean is_compare_condition(EExpressionType t) { 30409 return ((t == EExpressionType.simple_comparison_t) || (t == EExpressionType.group_comparison_t) 30410 || (t == EExpressionType.in_t) || (t == EExpressionType.pattern_matching_t) 30411 || (t == EExpressionType.left_join_t) || (t == EExpressionType.right_join_t)); 30412 } 30413 30414 @Override 30415 public boolean exprVisit(TParseTreeNode pNode, boolean isLeafNode) { 30416 TExpression expr = (TExpression) pNode; 30417 if (is_compare_condition(expr.getExpressionType())) { 30418 TExpression leftExpr = expr.getLeftOperand(); 30419 columnsInExpr leftVisitor = new columnsInExpr(); 30420 leftExpr.inOrderTraverse(leftVisitor); 30421 List<TObjectName> leftObjectNames = leftVisitor.getObjectNames(); 30422 List<TParseTreeNode> leftObjects = leftVisitor.getFunctions(); 30423 leftObjects.addAll(leftObjectNames); 30424 30425 TExpression rightExpr = expr.getRightOperand(); 30426 columnsInExpr rightVisitor = new columnsInExpr(); 30427 rightExpr.inOrderTraverse(rightVisitor); 30428 List<TObjectName> rightObjectNames = rightVisitor.getObjectNames(); 30429 List<TParseTreeNode> rightObjects = rightVisitor.getFunctions(); 30430 rightObjects.addAll(rightObjectNames); 30431 30432 if (!leftObjects.isEmpty() && !rightObjects.isEmpty()) { 30433 TCustomSqlStatement stmt = stmtStack.peek(); 30434 30435 for (int i = 0; i < leftObjects.size(); i++) { 30436 TParseTreeNode leftObject = leftObjects.get(i); 30437 TTable leftTable = null; 30438 TFunctionCall leftFunction = null; 30439 TObjectName leftObjectName = null; 30440 if (leftObject instanceof TObjectName) { 30441 leftObjectName = (TObjectName)leftObject; 30442 30443 if (leftObjectName.getDbObjectType() == EDbObjectType.variable) { 30444 continue; 30445 } 30446 30447 if (leftObjectName.getColumnNameOnly().startsWith("@") 30448 && (option.getVendor() == EDbVendor.dbvmssql 30449 || option.getVendor() == EDbVendor.dbvazuresql)) { 30450 continue; 30451 } 30452 30453 if (leftObjectName.getColumnNameOnly().startsWith(":") 30454 && (option.getVendor() == EDbVendor.dbvhana 30455 || option.getVendor() == EDbVendor.dbvteradata)) { 30456 continue; 30457 } 30458 30459 leftTable = modelManager.getTable(stmt, leftObjectName); 30460 30461 if (leftTable == null) { 30462 leftTable = leftObjectName.getSourceTable(); 30463 } 30464 30465 if (leftTable == null) { 30466 leftTable = modelManager.guessTable(stmt, leftObjectName); 30467 } 30468 } 30469 else if(leftObject instanceof TFunctionCall){ 30470 leftFunction = (TFunctionCall)leftObject; 30471 } 30472 30473 if (leftTable != null || leftFunction != null) { 30474 for (int j = 0; j < rightObjects.size(); j++) { 30475 JoinRelationship joinRelation = modelFactory.createJoinRelation(); 30476 joinRelation.setEffectType(effectType); 30477 if (joinType != null) { 30478 joinRelation.setJoinType(joinType); 30479 } else { 30480 if (expr.getLeftOperand().isOracleOuterJoin()) { 30481 joinRelation.setJoinType(right); 30482 } else if (expr.getRightOperand().isOracleOuterJoin()) { 30483 joinRelation.setJoinType(EJoinType.left); 30484 } else if (expr.getExpressionType() == EExpressionType.left_join_t) { 30485 joinRelation.setJoinType(EJoinType.left); 30486 } else if (expr.getExpressionType() == EExpressionType.right_join_t) { 30487 joinRelation.setJoinType(right); 30488 } else { 30489 joinRelation.setJoinType(EJoinType.inner); 30490 } 30491 } 30492 30493 joinRelation.setJoinClauseType(joinClauseType); 30494 joinRelation.setJoinCondition(expr.toString()); 30495 30496 30497 if (leftTable != null) { 30498 if (modelManager.getModel(leftTable) instanceof Table) { 30499 Table tableModel = (Table) modelManager.getModel(leftTable); 30500 if (tableModel != null) { 30501 TableColumn columnModel = modelFactory.createTableColumn(tableModel, 30502 leftObjectName, false); 30503 if (columnModel != null) { 30504 joinRelation.addSource(new TableColumnRelationshipElement(columnModel)); 30505 } 30506 } 30507 } else if (modelManager.getModel(leftTable) instanceof QueryTable) { 30508 QueryTable table = (QueryTable) modelManager.getModel(leftTable); 30509 TSelectSqlStatement subquery = table.getTableObject().getSubquery(); 30510 if (subquery != null && subquery.isCombinedQuery()) { 30511 ResultColumn resultColumn = matchResultColumn(table.getColumns(), 30512 leftObjectName); 30513 if (resultColumn != null) { 30514 joinRelation 30515 .addSource(new ResultColumnRelationshipElement(resultColumn)); 30516 } 30517 } else if (leftObjectName.getSourceColumn() != null) { 30518 Object model = modelManager.getModel(leftObjectName); 30519 if (model == null) { 30520 model = modelFactory.createResultColumn(table, leftObjectName); 30521 } 30522 if (model instanceof ResultColumn) { 30523 ResultColumn resultColumn = (ResultColumn) model; 30524 if (resultColumn != null) { 30525 joinRelation.addSource( 30526 new ResultColumnRelationshipElement(resultColumn)); 30527 } 30528 } else if (model instanceof LinkedHashMap) { 30529 String columnName = getColumnNameOnly(leftObjectName.toString()); 30530 LinkedHashMap<String, ResultColumn> resultColumns = (LinkedHashMap<String, ResultColumn>) model; 30531 if (resultColumns.containsKey(columnName)) { 30532 ResultColumn resultColumn = resultColumns.get(columnName); 30533 joinRelation.addSource( 30534 new ResultColumnRelationshipElement(resultColumn)); 30535 } 30536 } 30537 } else { 30538 ResultColumn resultColumn = matchResultColumn(table.getColumns(), 30539 leftObjectName); 30540 if (resultColumn != null) { 30541 joinRelation 30542 .addSource(new ResultColumnRelationshipElement(resultColumn)); 30543 } 30544 } 30545 } 30546 } 30547 else if(leftFunction!=null) { 30548 Object functionObj = createFunction(leftFunction); 30549 if(functionObj instanceof Function) { 30550 Function function = (Function)functionObj; 30551 joinRelation.addSource(new ResultColumnRelationshipElement(function.getColumns().get(0))); 30552 } 30553 } 30554 30555 TParseTreeNode rightObject = rightObjects.get(j); 30556 if(rightObject instanceof TObjectName) { 30557 TObjectName rightObjectName = (TObjectName)rightObject; 30558 30559 if (rightObjectName.getDbObjectType() == EDbObjectType.variable) { 30560 continue; 30561 } 30562 30563 if (rightObjectName.getColumnNameOnly().startsWith("@") 30564 && (option.getVendor() == EDbVendor.dbvmssql 30565 || option.getVendor() == EDbVendor.dbvazuresql)) { 30566 continue; 30567 } 30568 30569 if (rightObjectName.getColumnNameOnly().startsWith(":") 30570 && (option.getVendor() == EDbVendor.dbvhana 30571 || option.getVendor() == EDbVendor.dbvteradata)) { 30572 continue; 30573 } 30574 30575 TTable rightTable = modelManager.getTable(stmt, rightObjectName); 30576 if (rightTable == null) { 30577 rightTable = rightObjectName.getSourceTable(); 30578 } 30579 30580 if (rightTable == null) { 30581 rightTable = modelManager.guessTable(stmt, rightObjectName); 30582 } 30583 30584 if (modelManager.getModel(rightTable) instanceof Table) { 30585 Table tableModel = (Table) modelManager.getModel(rightTable); 30586 if (tableModel != null) { 30587 TableColumn columnModel = modelFactory.createTableColumn(tableModel, 30588 rightObjectName, false); 30589 if(columnModel != null) { 30590 joinRelation.setTarget(new TableColumnRelationshipElement(columnModel)); 30591 } 30592 } 30593 } else if (modelManager.getModel(rightTable) instanceof QueryTable) { 30594 QueryTable table = (QueryTable) modelManager.getModel(rightTable); 30595 TSelectSqlStatement subquery = table.getTableObject().getSubquery(); 30596 if (subquery != null && subquery.isCombinedQuery()) { 30597 ResultColumn resultColumn = matchResultColumn(table.getColumns(), 30598 rightObjectName); 30599 if (resultColumn != null) { 30600 joinRelation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 30601 } 30602 } else if (rightObjectName.getSourceColumn() != null) { 30603 Object model = modelManager.getModel(rightObjectName); 30604 if (model == null) { 30605 model = modelManager 30606 .getModel(rightObjectName.getSourceColumn()); 30607 } 30608 if (model instanceof ResultColumn) { 30609 joinRelation.setTarget(new ResultColumnRelationshipElement((ResultColumn)model)); 30610 } 30611 else if (model instanceof LinkedHashMap) { 30612 String columnName = getColumnNameOnly(rightObjectName.toString()); 30613 LinkedHashMap<String, ResultColumn> resultColumns = (LinkedHashMap<String, ResultColumn>)model; 30614 if (resultColumns.containsKey(columnName)) { 30615 ResultColumn resultColumn = resultColumns.get(columnName); 30616 joinRelation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 30617 } 30618 } 30619 } else { 30620 ResultColumn resultColumn = matchResultColumn(table.getColumns(), 30621 rightObjectName); 30622 if (resultColumn != null) { 30623 joinRelation.setTarget(new ResultColumnRelationshipElement(resultColumn)); 30624 } 30625 } 30626 } 30627 } 30628 else if(rightObject instanceof TFunctionCall) { 30629 Object functionObj = createFunction(rightObject); 30630 if(functionObj instanceof Function) { 30631 Function function = (Function)functionObj; 30632 joinRelation.setTarget(new ResultColumnRelationshipElement(function.getColumns().get(0))); 30633 } 30634 } 30635 } 30636 } 30637 } 30638 } 30639 } 30640 return true; 30641 } 30642 } 30643 30644 @Deprecated 30645 public static Dataflow getSqlflowJSONModel(dataflow dataflow) { 30646 EDbVendor vendor = ModelBindingManager.getGlobalVendor(); 30647 if(vendor == null) { 30648 throw new IllegalArgumentException("getSqlflowJSONModel(dataflow dataflow) is deprecated, please call method getSqlflowJSONModel(dataflow dataflow, EDbVendor vendor)."); 30649 } 30650 return getSqlflowJSONModel(vendor, dataflow, false); 30651 } 30652 30653 public static Dataflow getSqlflowJSONModel(dataflow dataflow, EDbVendor vendor) { 30654 return getSqlflowJSONModel(vendor, dataflow, false); 30655 } 30656 30657 public static Dataflow getSqlflowJSONModel(EDbVendor vendor, dataflow dataflow, boolean normalizeIdentifier) { 30658 Dataflow model = new Dataflow(); 30659 30660 if (dataflow.getErrors() != null && !dataflow.getErrors().isEmpty()) { 30661 List<Error> errorList = new ArrayList<Error>(); 30662 for (error error : dataflow.getErrors()) { 30663 Error err = new Error(); 30664 err.setErrorMessage(error.getErrorMessage()); 30665 err.setErrorType(error.getErrorType()); 30666 err.setCoordinates(Coordinate.parse(error.getCoordinate())); 30667 err.setFile(err.getFile()); 30668 err.setOriginCoordinates(Coordinate.parse(error.getOriginCoordinate())); 30669 errorList.add(err); 30670 } 30671 model.setErrors(errorList.toArray(new Error[0])); 30672 } 30673 30674 Sqlflow sqlflow = MetadataUtil.convertDataflowToMetadata(vendor, dataflow); 30675 sqlflow.setErrorMessages(null); 30676 model.setDbobjs(sqlflow); 30677 model.setOrientation(dataflow.getOrientation()); 30678 30679 30680 List<gudusoft.gsqlparser.dlineage.dataflow.model.json.Process> processes = new ArrayList<gudusoft.gsqlparser.dlineage.dataflow.model.json.Process>(); 30681 if(dataflow.getProcesses()!=null){ 30682 for(process process: dataflow.getProcesses()){ 30683 gudusoft.gsqlparser.dlineage.dataflow.model.json.Process processModel = new gudusoft.gsqlparser.dlineage.dataflow.model.json.Process(); 30684 processModel.setId(process.getId()); 30685 processModel.setName(process.getName()); 30686 processModel.setProcedureId(process.getProcedureId()); 30687 processModel.setProcedureName(process.getProcedureName()); 30688 processModel.setType(process.getType()); 30689 processModel.setCoordinate(process.getCoordinate()); 30690 processModel.setDatabase(process.getDatabase()); 30691 processModel.setSchema(process.getSchema()); 30692 processModel.setServer(process.getServer()); 30693 processModel.setQueryHashId(process.getQueryHashId()); 30694 if (process.getTransforms() != null && !process.getTransforms().isEmpty()) { 30695 List<gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform> transforms = new ArrayList<gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform>(); 30696 for (transform transform : process.getTransforms()) { 30697 gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform item = new gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform(); 30698 item.setCode(transform.getCode()); 30699 item.setType(transform.getType()); 30700 item.setCoordinate(transform.getCoordinate(true)); 30701 transforms.add(item); 30702 } 30703 processModel.setTransforms(transforms 30704 .toArray(new gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform[0])); 30705 } 30706 processes.add(processModel); 30707 } 30708 } 30709 model.setProcesses(processes.toArray(new gudusoft.gsqlparser.dlineage.dataflow.model.json.Process[0])); 30710 30711 Map<String, String> tableIdToProcedureId = new HashMap<>(); 30712 if (dataflow.getTables() != null) { 30713 for (table t : dataflow.getTables()) { 30714 if (t.getProcedureId() != null) { 30715 tableIdToProcedureId.put(t.getId(), t.getProcedureId()); 30716 } 30717 } 30718 } 30719 30720 List<gudusoft.gsqlparser.dlineage.dataflow.model.json.Relationship> relations = new ArrayList<gudusoft.gsqlparser.dlineage.dataflow.model.json.Relationship>(); 30721 if (dataflow.getRelationships() != null) { 30722 for (relationship relation : dataflow.getRelationships()) { 30723 gudusoft.gsqlparser.dlineage.dataflow.model.json.Relationship relationModel; 30724 if (relation.getType().equals("join")) { 30725 gudusoft.gsqlparser.dlineage.dataflow.model.json.JoinRelationship joinRelationModel = new gudusoft.gsqlparser.dlineage.dataflow.model.json.JoinRelationship(); 30726 joinRelationModel.setCondition(relation.getCondition()); 30727 joinRelationModel.setJoinType(relation.getJoinType()); 30728 joinRelationModel.setClause(relation.getClause()); 30729 relationModel = joinRelationModel; 30730 } else { 30731 relationModel = new gudusoft.gsqlparser.dlineage.dataflow.model.json.Relationship(); 30732 } 30733 30734 relationModel.setId(relation.getId()); 30735 relationModel.setProcessId(relation.getProcessId()); 30736 relationModel.setProcessType(relation.getProcessType()); 30737 relationModel.setType(relation.getType()); 30738 relationModel.setEffectType(relation.getEffectType()); 30739 relationModel.setPartition(relation.getPartition()); 30740 relationModel.setFunction(relation.getFunction()); 30741 relationModel.setProcedureId(relation.getProcedureId()); 30742 relationModel.setSqlHash(relation.getSqlHash()); 30743 relationModel.setCondition(relation.getCondition()); 30744 relationModel.setSqlComment(relation.getSqlComment()); 30745 relationModel.setTimestampMax(relation.getTimestampMax()); 30746 relationModel.setTimestampMin(relation.getTimestampMin()); 30747 if (Boolean.TRUE.equals(relation.getBuiltIn())) { 30748 relationModel.setBuiltIn(relation.getBuiltIn()); 30749 } 30750 relationModel.setCallStmt(relation.getCallStmt()); 30751 relationModel.setCallCoordinate(relation.getCallCoordinate()); 30752 30753 if (relation.getTarget() != null && relation.getSources() != null && !relation.getSources().isEmpty()) { 30754 { 30755 gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement targetModel = new gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement(); 30756 targetColumn target = relation.getTarget(); 30757 if (normalizeIdentifier) { 30758 targetModel.setColumn(SQLUtil.getIdentifierNormalColumnName(vendor, target.getColumn())); 30759 targetModel.setParentName( 30760 SQLUtil.getIdentifierNormalTableName(vendor, target.getParent_name())); 30761 targetModel.setTargetName( 30762 SQLUtil.getIdentifierNormalColumnName(vendor, target.getTarget_name())); 30763 } else { 30764 targetModel.setColumn(target.getColumn()); 30765 targetModel.setParentName(target.getParent_name()); 30766 targetModel.setTargetName(target.getTarget_name()); 30767 } 30768 targetModel.setId(target.getId()); 30769 targetModel.setTargetId(target.getTarget_id()); 30770 targetModel.setParentId(target.getParent_id()); 30771 targetModel.setCoordinates(Coordinate.parse(target.getCoordinate())); 30772 targetModel.setFunction(target.getFunction()); 30773 targetModel.setType(target.getType()); 30774 String targetProcId = tableIdToProcedureId.get(target.getParent_id()); 30775 if (targetProcId != null) { 30776 targetModel.setProcedureId(targetProcId); 30777 } 30778 relationModel.setTarget(targetModel); 30779 } 30780 30781 List<gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement> sourceModels = new ArrayList<gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement>(); 30782 for (sourceColumn source : relation.getSources()) { 30783 gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement sourceModel = new gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement(); 30784 if (normalizeIdentifier) { 30785 sourceModel.setColumn(SQLUtil.getIdentifierNormalColumnName(vendor, source.getColumn())); 30786 sourceModel.setParentName( 30787 SQLUtil.getIdentifierNormalTableName(vendor, source.getParent_name())); 30788 sourceModel.setSourceName( 30789 SQLUtil.getIdentifierNormalColumnName(vendor, source.getSource_name())); 30790 } else { 30791 sourceModel.setColumn(source.getColumn()); 30792 sourceModel.setParentName(source.getParent_name()); 30793 sourceModel.setSourceName(source.getSource_name()); 30794 } 30795 sourceModel.setColumnType(source.getColumn_type()); 30796 sourceModel.setId(source.getId()); 30797 sourceModel.setParentId(source.getParent_id()); 30798 sourceModel.setSourceId(source.getSource_id()); 30799 sourceModel.setCoordinates(Coordinate.parse(source.getCoordinate())); 30800 sourceModel.setClauseType(source.getClauseType()); 30801 sourceModel.setType(source.getType()); 30802 String sourceProcId = tableIdToProcedureId.get(source.getParent_id()); 30803 if (sourceProcId != null) { 30804 sourceModel.setProcedureId(sourceProcId); 30805 } 30806 sourceModels.add(sourceModel); 30807 if (source.getTransforms() != null && !source.getTransforms().isEmpty()) { 30808 List<gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform> transforms = new ArrayList<gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform>(); 30809 for (transform transform : source.getTransforms()) { 30810 gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform item = new gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform(); 30811 item.setCode(transform.getCode()); 30812 item.setType(transform.getType()); 30813 item.setCoordinate(transform.getCoordinate(true)); 30814 transforms.add(item); 30815 } 30816 sourceModel.setTransforms(transforms 30817 .toArray(new gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform[0])); 30818 } 30819 30820 if (source.getCandidateParents() != null && !source.getCandidateParents().isEmpty()) { 30821 List<gudusoft.gsqlparser.dlineage.dataflow.model.json.CandidateTable> candidateParents = new ArrayList<gudusoft.gsqlparser.dlineage.dataflow.model.json.CandidateTable>(); 30822 for (candidateTable candidateTable : source.getCandidateParents()) { 30823 gudusoft.gsqlparser.dlineage.dataflow.model.json.CandidateTable item = new gudusoft.gsqlparser.dlineage.dataflow.model.json.CandidateTable(); 30824 item.setId(candidateTable.getId()); 30825 if (normalizeIdentifier) { 30826 item.setName( 30827 SQLUtil.getIdentifierNormalTableName(vendor, candidateTable.getName())); 30828 } else { 30829 item.setName(candidateTable.getName()); 30830 } 30831 candidateParents.add(item); 30832 } 30833 sourceModel.setCandidateParents(candidateParents 30834 .toArray(new gudusoft.gsqlparser.dlineage.dataflow.model.json.CandidateTable[0])); 30835 } 30836 } 30837 relationModel.setSources(sourceModels 30838 .toArray(new gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement[0])); 30839 relations.add(relationModel); 30840 } else if (relation.getCaller() != null && relation.getCallees() != null 30841 && !relation.getCallees().isEmpty()) { 30842 { 30843 gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement targetModel = new gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement(); 30844 targetColumn target = relation.getCaller(); 30845 if (normalizeIdentifier) { 30846 targetModel.setName(SQLUtil.getIdentifierNormalColumnName(vendor, target.getName())); 30847 } else { 30848 targetModel.setName(target.getName()); 30849 } 30850 targetModel.setId(target.getId()); 30851 targetModel.setCoordinates(Coordinate.parse(target.getCoordinate())); 30852 targetModel.setType(target.getType()); 30853 relationModel.setCaller(targetModel); 30854 } 30855 30856 List<gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement> sourceModels = new ArrayList<gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement>(); 30857 for (sourceColumn source : relation.getCallees()) { 30858 gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement sourceModel = new gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement(); 30859 if (normalizeIdentifier) { 30860 sourceModel.setName(SQLUtil.getIdentifierNormalColumnName(vendor, source.getName())); 30861 } else { 30862 sourceModel.setName(source.getName()); 30863 } 30864 sourceModel.setId(source.getId()); 30865 sourceModel.setCoordinates(Coordinate.parse(source.getCoordinate())); 30866 sourceModel.setType(source.getType()); 30867 sourceModels.add(sourceModel); 30868 } 30869 relationModel.setCallees(sourceModels 30870 .toArray(new gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement[0])); 30871 relations.add(relationModel); 30872 } 30873 } 30874 } 30875 model.setRelationships(relations.toArray(new gudusoft.gsqlparser.dlineage.dataflow.model.json.Relationship[0])); 30876 return model; 30877 } 30878 30879 public static String getVersion() { 30880 // Return the authoritative product version instead of a stale hardcoded 30881 // constant so version-reporting stays in sync with each release (MantisBT 30882 // / QS-009). TBaseType.versionid is the single source of truth (also used 30883 // by the JAR manifest and release notes). 30884 return TBaseType.versionid; 30885 } 30886 30887 public static String getReleaseDate() { 30888 return TBaseType.releaseDate; 30889 } 30890 30891 public static void main(String[] args) { 30892 if (args.length < 1) { 30893 System.out.println( 30894 "Usage: java DataFlowAnalyzer [/f <path_to_sql_file>] [/d <path_to_directory_includes_sql_files>] [/s [/text]] [/json] [/traceView] [/t <database type>] [/o <output file path>][/version]"); 30895 System.out.println("/f: Option, specify the sql file path to analyze fdd relation."); 30896 System.out.println("/d: Option, specify the sql directory path to analyze fdd relation."); 30897 System.out.println("/j: Option, analyze the join relation."); 30898 System.out.println("/s: Option, simple output, ignore the intermediate results."); 30899 System.out.println("/i: Option, ignore all result sets."); 30900 System.out.println("/traceView: Option, analyze the source tables of views."); 30901 System.out.println("/text: Option, print the plain text format output."); 30902 System.out.println("/json: Option, print the json format output."); 30903 System.out.println( 30904 "/t: Option, set the database type. Support oracle, mysql, mssql, db2, netezza, teradata, informix, sybase, postgresql, hive, greenplum and redshift, the default type is oracle"); 30905 System.out.println("/o: Option, write the output stream to the specified file."); 30906 System.out.println("/log: Option, generate a dataflow.log file to log information."); 30907 return; 30908 } 30909 30910 File sqlFiles = null; 30911 30912 List<String> argList = Arrays.asList(args); 30913 30914 if (argList.indexOf("/version") != -1) { 30915 System.out.println("Version: " + DataFlowAnalyzer.getVersion()); 30916 System.out.println("Release Date: " + DataFlowAnalyzer.getReleaseDate()); 30917 return; 30918 } 30919 30920 if (argList.indexOf("/f") != -1 && argList.size() > argList.indexOf("/f") + 1) { 30921 sqlFiles = new File(args[argList.indexOf("/f") + 1]); 30922 if (!sqlFiles.exists() || !sqlFiles.isFile()) { 30923 System.out.println(sqlFiles + " is not a valid file."); 30924 return; 30925 } 30926 } else if (argList.indexOf("/d") != -1 && argList.size() > argList.indexOf("/d") + 1) { 30927 sqlFiles = new File(args[argList.indexOf("/d") + 1]); 30928 if (!sqlFiles.exists() || !sqlFiles.isDirectory()) { 30929 System.out.println(sqlFiles + " is not a valid directory."); 30930 return; 30931 } 30932 } else { 30933 System.out.println("Please specify a sql file path or directory path to analyze dlineage."); 30934 return; 30935 } 30936 30937 EDbVendor vendor = EDbVendor.dbvoracle; 30938 30939 int index = argList.indexOf("/t"); 30940 30941 if (index != -1 && args.length > index + 1) { 30942 vendor = TGSqlParser.getDBVendorByName(args[index + 1]); 30943 } 30944 30945 String outputFile = null; 30946 30947 index = argList.indexOf("/o"); 30948 30949 if (index != -1 && args.length > index + 1) { 30950 outputFile = args[index + 1]; 30951 } 30952 30953 FileOutputStream writer = null; 30954 if (outputFile != null) { 30955 try { 30956 writer = new FileOutputStream(outputFile); 30957 System.setOut(new PrintStream(writer)); 30958 } catch (FileNotFoundException e) { 30959 logger.error("output file is not found.", e); 30960 } 30961 } 30962 30963 boolean simple = argList.indexOf("/s") != -1; 30964 boolean ignoreResultSets = argList.indexOf("/i") != -1; 30965 boolean showJoin = argList.indexOf("/j") != -1; 30966 boolean textFormat = false; 30967 boolean jsonFormat = false; 30968 if (simple) { 30969 textFormat = argList.indexOf("/text") != -1; 30970 } 30971 30972 boolean traceView = argList.indexOf("/traceView") != -1; 30973 if (traceView) { 30974 simple = true; 30975 } 30976 30977 jsonFormat = argList.indexOf("/json") != -1; 30978 30979 DataFlowAnalyzer dlineage = new DataFlowAnalyzer(sqlFiles, vendor, simple); 30980 30981 dlineage.setShowJoin(showJoin); 30982 dlineage.setIgnoreRecordSet(ignoreResultSets); 30983 // dlineage.setShowImplicitSchema(true); 30984 30985 if (simple && !jsonFormat) { 30986 dlineage.setTextFormat(textFormat); 30987 } 30988 30989 String result = dlineage.generateDataFlow(); 30990 30991// dataflow dataflow = ProcessUtility.generateTableLevelLineage(dlineage, dlineage.getDataFlow()); 30992// System.out.println(result); 30993 30994 if (jsonFormat) { 30995 // Map jsonResult = new LinkedHashMap(); 30996 Dataflow model = getSqlflowJSONModel(vendor, dlineage.getDataFlow(), true); 30997 // jsonResult.put("data", BeanUtils.bean2Map(model)); 30998 result = JSON.toJSONString(model); 30999 } else if (traceView) { 31000 result = dlineage.traceView(); 31001 } 31002 31003 if (result != null) { 31004 System.out.println(result); 31005 31006 if (writer != null && result.length() < 1024 * 1024) { 31007 System.err.println(result); 31008 } 31009 } 31010 31011 try { 31012 if (writer != null) { 31013 writer.close(); 31014 } 31015 } catch (IOException e) { 31016 logger.error("close writer failed.", e); 31017 } 31018 31019 boolean log = argList.indexOf("/log") != -1; 31020 31021 PrintStream systemSteam = System.err; 31022 ByteArrayOutputStream sw = new ByteArrayOutputStream(); 31023 PrintStream pw = new PrintStream(sw); 31024 System.setErr(pw); 31025 31026 31027 List<ErrorInfo> errors = dlineage.getErrorMessages(); 31028 if (!errors.isEmpty()) { 31029 System.err.println("Error log:\n"); 31030 for (int i = 0; i < errors.size(); i++) { 31031 System.err.println(errors.get(i).getErrorMessage()); 31032 } 31033 } 31034 31035 if (sw != null) { 31036 String errorMessage = sw.toString().trim(); 31037 if (errorMessage.length() > 0) { 31038 if (log) { 31039 try { 31040 pw = new PrintStream(new File(".", "dataflow.log")); 31041 pw.print(errorMessage); 31042 } catch (FileNotFoundException e) { 31043 logger.error("error log file is not found.", e); 31044 } 31045 } 31046 31047 System.setErr(systemSteam); 31048 System.err.println(errorMessage); 31049 } 31050 } 31051 } 31052 31053 public List<ErrorInfo> getErrorMessages() { 31054 return errorInfos; 31055 } 31056 31057 /** 31058 * Every dynamic-SQL execution site (T-SQL {@code EXEC(...)} / {@code sp_executesql}) the analyzer 31059 * encountered, with whether it was statically resolved. Diagnostic only; not consumed by lineage. 31060 * Read-only snapshot; see {@link DynamicSqlSite}. 31061 */ 31062 public List<DynamicSqlSite> getDynamicSqlSites() { 31063 return java.util.Collections.unmodifiableList(new ArrayList<DynamicSqlSite>(dynamicSqlSites)); 31064 } 31065 31066 /** 31067 * Resolve dynamic-SQL object names by abstractly evaluating a stored procedure 31068 * with a set of parameter bindings. 31069 * 31070 * <p>Given a T-SQL {@code CREATE PROC} parse tree and the values its parameters 31071 * actually had at a call site (e.g. {@code @SourceDB='QSP'}, 31072 * {@code @STG_AEG='STG_AEG'}), this evaluates the procedure's dynamic-SQL string 31073 * building (assignment + concatenation + name-building functions), materializes 31074 * the concrete SQL each {@code EXEC} / {@code sp_executesql} site runs, analyzes 31075 * it with ordinary dlineage, and returns the lineage edges tagged with dynamic 31076 * provenance ({@code origin=DYNAMIC_RESOLVED}, source proc, stable dynamic-site 31077 * id, binding hash). Sites that cannot be reduced are reported as honest 31078 * {@code UNRESOLVED} diagnostics — never guessed. 31079 * 31080 * <p>This is abstract string evaluation, NOT execution: no table reads, no 31081 * cursors, no data-dependent control flow. It is purely additive — it builds 31082 * throwaway analyzers for the materialized strings and does not change the 31083 * default dlineage output of this analyzer. 31084 * 31085 * @param procAst the {@code CREATE PROC} parse tree (currently T-SQL) 31086 * @param vendor the SQL dialect ({@code dbvmssql}) 31087 * @param sqlEnv catalog env for the resolved analysis (may be null) 31088 * @param currentDatabase the proc's database (resolves unqualified names) 31089 * @param defaultSchema the proc's default schema (e.g. {@code dbo}) 31090 * @param bindings parameter name → concrete value 31091 * @param options resource bounds (may be null for defaults) 31092 */ 31093 public gudusoft.gsqlparser.dlineage.dynamicsql.DynamicLineageResult resolveDynamicSqlLineage( 31094 TCustomSqlStatement procAst, EDbVendor vendor, gudusoft.gsqlparser.sqlenv.TSQLEnv sqlEnv, 31095 String currentDatabase, String defaultSchema, 31096 Map<String, gudusoft.gsqlparser.dlineage.dynamicsql.SqlValue> bindings, 31097 gudusoft.gsqlparser.dlineage.dynamicsql.DynamicLineageOptions options) { 31098 return gudusoft.gsqlparser.dlineage.dynamicsql.DynamicSqlLineageResolver.resolve( 31099 procAst, vendor, sqlEnv, currentDatabase, defaultSchema, bindings, options); 31100 } 31101 31102 public String traceView() { 31103 StringBuilder buffer = new StringBuilder(); 31104 dataflow dataflow = this.getDataFlow(); 31105 Map<table, Set<table>> traceViewMap = new LinkedHashMap<table, Set<table>>(); 31106 if (dataflow != null && dataflow.getViews() != null) { 31107 List<relationship> relations = dataflow.getRelationships(); 31108 Map<String, table> viewMap = new HashMap<String, table>(); 31109 Map<String, table> tableMap = new HashMap<String, table>(); 31110 for (table view : dataflow.getViews()) { 31111 viewMap.put(view.getId(), view); 31112 tableMap.put(view.getId(), view); 31113 } 31114 for (table table : dataflow.getTables()) { 31115 tableMap.put(table.getId(), table); 31116 } 31117 for (relationship relation : relations) { 31118 if (!RelationshipType.fdd.name().equals(relation.getType())) { 31119 continue; 31120 } 31121 String parentId = relation.getTarget().getParent_id(); 31122 if (viewMap.containsKey(parentId)) { 31123 if (!traceViewMap.containsKey(viewMap.get(parentId))) { 31124 traceViewMap.put(viewMap.get(parentId), new LinkedHashSet<table>()); 31125 } 31126 31127 for (sourceColumn sourceColumn : relation.getSources()) { 31128 traceViewMap.get(viewMap.get(parentId)).add(tableMap.get(sourceColumn.getParent_id())); 31129 } 31130 } 31131 } 31132 31133 Map<table, Set<table>> viewTableMap = new LinkedHashMap<table, Set<table>>(); 31134 for (table view : traceViewMap.keySet()) { 31135 Set<table> tables = new LinkedHashSet<table>(); 31136 traverseViewSourceTables(tables, view, traceViewMap); 31137 viewTableMap.put(view, tables); 31138 } 31139 31140 for (table view : viewTableMap.keySet()) { 31141 buffer.append(view.getFullName()); 31142 for (table table : viewTableMap.get(view)) { 31143 buffer.append(",").append(table.getFullName()); 31144 } 31145 buffer.append(System.getProperty("line.separator")); 31146 } 31147 } 31148 return buffer.toString().trim(); 31149 } 31150 31151 private void traverseViewSourceTables(Set<table> tables, table view, Map<table, Set<table>> traceViewMap) { 31152 Set<table> sourceTables = traceViewMap.get(view); 31153 for (table sourceTable : sourceTables) { 31154 if (sourceTable.isTable()) { 31155 tables.add(sourceTable); 31156 } else if (sourceTable.isView()) { 31157 traverseViewSourceTables(tables, sourceTable, traceViewMap); 31158 } 31159 } 31160 } 31161 31162 protected List<SqlInfo> convertSQL(EDbVendor vendor, String json) { 31163 List<SqlInfo> sqlInfos = new ArrayList<SqlInfo>(); 31164 List sqlContents = (List) JSON.parseObject(json); 31165 for (int j = 0; j < sqlContents.size(); j++) { 31166 Map sqlContent = (Map) sqlContents.get(j); 31167 String sql = (String) sqlContent.get("sql"); 31168 String fileName = (String) sqlContent.get("fileName"); 31169 String filePath = (String) sqlContent.get("filePath"); 31170 if (sql != null && sql.trim().startsWith("{")) { 31171 if (sql.indexOf("createdBy") != -1) { 31172 String shardBaseDir = filePath != null ? new File(filePath).getParent() : null; 31173 // Route on the authoritative `format` first. A sqlflow-sharded 31174 // manifest carries the product brand (SQLdep/grabit) in createdBy 31175 // but has no single-file `queries` array — reading it through the 31176 // brand path silently dropped every source record. 31177 if (MetadataReader.isSqlflowSharded(sql)) { 31178 if (MetadataReader.isSupportedSqlflowSharded(sql)) { 31179 appendShardedSourceSqlInfos(sql, shardBaseDir, fileName, sqlInfos); 31180 } 31181 // Unsupported version, or no base dir to locate shards: 31182 // nothing is added — never misrouted, never crashes. 31183 } else if (sql.toLowerCase().indexOf("sqldep") != -1 31184 || sql.toLowerCase().indexOf("grabit") != -1) { 31185 Map queryObject = (Map) JSON.parseObject(sql); 31186 List querys = (List) queryObject.get("queries"); 31187 if (querys != null) { 31188 for (int i = 0; i < querys.size(); i++) { 31189 Map object = (Map) querys.get(i); 31190 SqlInfo info = new SqlInfo(); 31191 info.setSql(JSON.toJSONString(object)); 31192 info.setFileName(fileName); 31193 info.setFilePath(filePath); 31194 info.setOriginIndex(i); 31195 sqlInfos.add(info); 31196 } 31197 queryObject.remove("queries"); 31198 SqlInfo info = new SqlInfo(); 31199 info.setSql(JSON.toJSONString(queryObject)); 31200 info.setFileName(fileName); 31201 info.setFilePath(filePath); 31202 info.setOriginIndex(querys.size()); 31203 sqlInfos.add(info); 31204 } else { 31205 SqlInfo info = new SqlInfo(); 31206 info.setSql(JSON.toJSONString(queryObject)); 31207 info.setFileName(fileName); 31208 info.setFilePath(filePath); 31209 info.setOriginIndex(0); 31210 sqlInfos.add(info); 31211 } 31212 } else if (sql.toLowerCase().indexOf("sqlflow") != -1) { 31213 Map sqlflow = (Map) JSON.parseObject(sql); 31214 List<Map> servers = (List<Map>) sqlflow.get("servers"); 31215 if (servers != null) { 31216 for (Map queryObject : servers) { 31217 String name = (String) queryObject.get("name"); 31218 String dbVendor = (String) queryObject.get("dbVendor"); 31219 List querys = (List) queryObject.get("queries"); 31220 if (querys != null) { 31221 for (int i = 0; i < querys.size(); i++) { 31222 Map object = (Map) querys.get(i); 31223 SqlInfo info = new SqlInfo(); 31224 info.setSql(JSON.toJSONString(object)); 31225 info.setFileName(fileName); 31226 info.setFilePath(filePath); 31227 info.setOriginIndex(i); 31228 info.setDbVendor(dbVendor); 31229 info.setServer(name); 31230 sqlInfos.add(info); 31231 } 31232 queryObject.remove("queries"); 31233 Map serverObject = new IndexedLinkedHashMap(); 31234 serverObject.put("createdBy", sqlflow.get("createdBy")); 31235 serverObject.put("servers", Arrays.asList(queryObject)); 31236 SqlInfo info = new SqlInfo(); 31237 info.setSql(JSON.toJSONString(serverObject)); 31238 info.setFileName(fileName); 31239 info.setFilePath(filePath); 31240 info.setOriginIndex(querys.size()); 31241 info.setDbVendor(dbVendor); 31242 info.setServer(filePath); 31243 sqlInfos.add(info); 31244 } else { 31245 SqlInfo info = new SqlInfo(); 31246 info.setSql(JSON.toJSONString(queryObject)); 31247 info.setFileName(fileName); 31248 info.setFilePath(filePath); 31249 info.setOriginIndex(0); 31250 sqlInfos.add(info); 31251 } 31252 } 31253 } 31254 31255 List<Map> errorMessages = (List<Map>) sqlflow.get("errorMessages"); 31256 if(errorMessages!=null && !errorMessages.isEmpty()) { 31257 for(Map error: errorMessages){ 31258 ErrorInfo errorInfo = new ErrorInfo(); 31259 errorInfo.setErrorType(ErrorInfo.METADATA_ERROR); 31260 errorInfo.setErrorMessage((String)error.get("errorMessage")); 31261 errorInfo.setFileName(fileName); 31262 errorInfo.setFilePath(filePath); 31263 errorInfo.setStartPosition(new Pair3<Long, Long, String>(-1L, -1L, 31264 ModelBindingManager.getGlobalHash())); 31265 errorInfo.setEndPosition(new Pair3<Long, Long, String>(-1L, -1L, 31266 ModelBindingManager.getGlobalHash())); 31267 errorInfo.setOriginStartPosition(new Pair<Long, Long>(-1L, -1L)); 31268 errorInfo.setOriginEndPosition(new Pair<Long, Long>(-1L, -1L)); 31269 metadataErrors.add(errorInfo); 31270 } 31271 } 31272 } 31273 } 31274 } else if (sql != null) { 31275 SqlInfo info = new SqlInfo(); 31276 info.setSql(sql); 31277 info.setFileName(fileName); 31278 info.setFilePath(filePath); 31279 info.setOriginIndex(0); 31280 sqlInfos.add(info); 31281 } else if (filePath != null) { 31282 SqlInfo info = new SqlInfo(); 31283 info.setFileName(fileName); 31284 info.setFilePath(filePath); 31285 info.setOriginIndex(0); 31286 sqlInfos.add(info); 31287 } 31288 } 31289 return sqlInfos; 31290 } 31291 31292 public void setTextFormat(boolean textFormat) { 31293 option.setTextFormat(textFormat); 31294 } 31295 31296 public boolean isBuiltInFunctionName(TObjectName object) { 31297 if (object == null || object.getGsqlparser() == null) 31298 return false; 31299 try { 31300 EDbVendor vendor = object.getGsqlparser().getDbVendor(); 31301 if (vendor == EDbVendor.dbvteradata) { 31302 boolean result = TERADATA_BUILTIN_FUNCTIONS.contains(object.toString().toUpperCase()); 31303 if (result) { 31304 return true; 31305 } 31306 } 31307 31308 List<String> versions = functionChecker.getAvailableDbVersions(vendor); 31309 if (versions != null && versions.size() > 0) { 31310 for (int i = 0; i < versions.size(); i++) { 31311 boolean result = functionChecker.isBuiltInFunction(object.toString(), 31312 object.getGsqlparser().getDbVendor(), versions.get(i)); 31313 if (result) { 31314 return result; 31315 } 31316 } 31317 31318 // boolean result = 31319 // TERADATA_BUILTIN_FUNCTIONS.contains(object.toString()); 31320 // if (result) { 31321 // return true; 31322 // } 31323 } 31324 } catch (Exception e) { 31325 } 31326 31327 return false; 31328 } 31329 31330 public boolean isBuiltInFunctionName(String functionName) { 31331 if (functionName == null) 31332 return false; 31333 try { 31334 EDbVendor vendor = getOption().getVendor(); 31335 if (vendor == EDbVendor.dbvteradata) { 31336 boolean result = TERADATA_BUILTIN_FUNCTIONS.contains(functionName.toUpperCase()); 31337 if (result) { 31338 return true; 31339 } 31340 } 31341 31342 List<String> versions = functionChecker.getAvailableDbVersions(vendor); 31343 if (versions != null && versions.size() > 0) { 31344 for (int i = 0; i < versions.size(); i++) { 31345 // no caller-side folding: isBuiltInFunction folds with 31346 // Locale.ROOT itself; a default-locale toUpperCase here 31347 // breaks the lookup on Turkish JVMs 31348 boolean result = functionChecker.isBuiltInFunction(functionName, 31349 vendor, versions.get(i)); 31350 if (result) { 31351 return result; 31352 } 31353 } 31354 31355 // boolean result = 31356 // TERADATA_BUILTIN_FUNCTIONS.contains(object.toString()); 31357 // if (result) { 31358 // return true; 31359 // } 31360 } 31361 } catch (Exception e) { 31362 } 31363 31364 return false; 31365 } 31366 31367 public boolean isKeyword(TObjectName object) { 31368 if (object == null || object.getGsqlparser() == null) 31369 return false; 31370 try { 31371 EDbVendor vendor = object.getGsqlparser().getDbVendor(); 31372 31373 List<String> versions = keywordChecker.getAvailableDbVersions(vendor); 31374 if (versions != null && versions.size() > 0) { 31375 for (int i = 0; i < versions.size(); i++) { 31376 List<String> segments = SQLUtil.parseNames(object.toString()); 31377 boolean result = keywordChecker.isKeyword(segments.get(segments.size() - 1), 31378 object.getGsqlparser().getDbVendor(), versions.get(i), true); 31379 if (result) { 31380 return result; 31381 } 31382 } 31383 } 31384 } catch (Exception e) { 31385 } 31386 31387 return false; 31388 } 31389 31390 public boolean isKeyword(String objectName) { 31391 if (objectName == null) 31392 return false; 31393 try { 31394 EDbVendor vendor = getOption().getVendor(); 31395 31396 List<String> versions = keywordChecker.getAvailableDbVersions(vendor); 31397 if (versions != null && versions.size() > 0) { 31398 for (int i = 0; i < versions.size(); i++) { 31399 List<String> segments = SQLUtil.parseNames(objectName); 31400 boolean result = keywordChecker.isKeyword(segments.get(segments.size() - 1), 31401 vendor, versions.get(i), false); 31402 if (result) { 31403 return result; 31404 } 31405 } 31406 } 31407 } catch (Exception e) { 31408 } 31409 31410 return false; 31411 } 31412 31413 public boolean isAggregateFunction(TFunctionCall func) { 31414 if (func == null) 31415 return false; 31416 return Arrays 31417 .asList(new String[] { "AVG", "COUNT", "MAX", "MIN", "SUM", "COLLECT", "CORR", "COVAR_POP", 31418 "COVAR_SAMP", "CUME_DIST", "DENSE_RANK", "FIRST", "GROUP_ID", "GROUPING", "GROUPING_ID", "LAST", 31419 "LISTAGG", "MEDIAN", "PERCENT_RANK", "PERCENTILE_CONT", "PERCENTILE_DISC", "RANK", 31420 "STATS_BINOMIAL_TEST", "STATS_CROSSTAB", "STATS_F_TEST", "STATS_KS_TEST", "STATS_MODE", 31421 "STATS_MW_TEST", "STATS_ONE_WAY_ANOVA", "STATS_WSR_TEST", "STDDEV", "STDDEV_POP", "STDDEV_SAMP", 31422 "SYS_XMLAGG", "VAR_ POP", "VAR_ SAMP", "VARI ANCE", "XMLAGG", "ARRAY_AGG" }) 31423 .contains(func.getFunctionName().toString().toUpperCase()); 31424 } 31425 31426 public boolean isConstant(TObjectName object) { 31427 if (object == null || object.getGsqlparser() == null) 31428 return false; 31429 List<String> constants = Arrays.asList(new String[] { "NEXTVAL", "CURRVAL", "SYSDATE", "CENTURY", "YEAR", 31430 "MONTH", "DAY", "HOUR", "MINUTE", "SECOND" }); 31431 List<String> segments = SQLUtil.parseNames(object.toString()); 31432 // sequence.NEXTVAL or sequence.CURRVAL is a sequence reference, not a constant 31433 // This syntax is used by Oracle, Snowflake, and accepted by other vendors for compatibility 31434 String columnNameOnly = object.getColumnNameOnly(); 31435 if (segments.size() > 1 && ("NEXTVAL".equalsIgnoreCase(columnNameOnly) || "CURRVAL".equalsIgnoreCase(columnNameOnly))) { 31436 return false; 31437 } 31438 boolean result = constants.indexOf(segments.get(segments.size() - 1).toUpperCase()) != -1; 31439 if (result) { 31440 return result; 31441 } 31442 if (isKeyword(object)) { 31443 return true; 31444 } 31445 return false; 31446 } 31447 31448 private Pair3<Long, Long, Integer> convertCoordinate(Pair3<Long, Long, String> position) { 31449// if (ModelBindingManager.getGlobalOption()!=null && ModelBindingManager.getGlobalOption().isIgnoreCoordinate()) { 31450// return new Pair3<>(-1L, -1L, -1); 31451// } 31452 return new Pair3<Long, Long, Integer>(position.first, position.second, 31453 ModelBindingManager.getGlobalSqlInfo().getIndexOf(position.third)); 31454 } 31455 31456 /** 31457 * Analyze MDX SELECT statement to generate data lineage. 31458 * Maps MDX cube as source table, measures/dimensions as columns, 31459 * and creates dataflow relationships to the result set. 31460 */ 31461 private void analyzeMdxSelectStmt(gudusoft.gsqlparser.stmt.mdx.TMdxSelect stmt) { 31462 // Get cube name from FROM clause 31463 gudusoft.gsqlparser.nodes.mdx.TMdxIdentifierNode cube = stmt.getCube(); 31464 if (cube == null) { 31465 return; 31466 } 31467 31468 String cubeName = getMdxIdentifierName(cube); 31469 Table cubeTable = modelFactory.createTableByName(cubeName, false); 31470 31471 // Collect all MDX identifier references from axes and WHERE clause 31472 List<String> measureNames = new ArrayList<String>(); 31473 List<String> dimensionNames = new ArrayList<String>(); 31474 31475 // Process axes (COLUMNS, ROWS, etc.) 31476 if (stmt.getAxes() != null) { 31477 for (int i = 0; i < stmt.getAxes().size(); i++) { 31478 gudusoft.gsqlparser.nodes.mdx.TMdxAxisNode axis = stmt.getAxes().getElement(i); 31479 if (axis.getExpNode() != null) { 31480 collectMdxReferences(axis.getExpNode(), measureNames, dimensionNames); 31481 } 31482 } 31483 } 31484 31485 // Process WHERE clause (slicer dimension) 31486 if (stmt.getWhere() != null && stmt.getWhere().getFilter() != null) { 31487 collectMdxReferences(stmt.getWhere().getFilter(), measureNames, dimensionNames); 31488 } 31489 31490 // Process WITH MEMBER definitions 31491 if (stmt.getWiths() != null) { 31492 for (int i = 0; i < stmt.getWiths().size(); i++) { 31493 gudusoft.gsqlparser.nodes.mdx.TMdxWithNode withNode = stmt.getWiths().getElement(i); 31494 if (withNode.getNameNode() != null) { 31495 String withName = getMdxIdentifierName(withNode.getNameNode()); 31496 // Calculated members are treated as derived measures 31497 if (withName.toLowerCase().startsWith("[measures].") 31498 || withName.toLowerCase().startsWith("measures.")) { 31499 measureNames.add(withName); 31500 } 31501 } 31502 // Also collect references used in the WITH expression 31503 if (withNode.getExprNode() != null) { 31504 collectMdxReferences(withNode.getExprNode(), measureNames, dimensionNames); 31505 } 31506 } 31507 } 31508 31509 // Create columns on the cube table for all referenced measures and dimensions 31510 Set<String> addedColumns = new LinkedHashSet<String>(); 31511 for (String measure : measureNames) { 31512 if (addedColumns.add(measure)) { 31513 modelFactory.createTableColumn(cubeTable, measure); 31514 } 31515 } 31516 for (String dimension : dimensionNames) { 31517 if (addedColumns.add(dimension)) { 31518 modelFactory.createTableColumn(cubeTable, dimension); 31519 } 31520 } 31521 31522 // Create result set with all referenced columns 31523 ResultSet resultSet = modelFactory.createResultSet(stmt, false); 31524 if (resultSet != null) { 31525 for (String colName : addedColumns) { 31526 TableColumn sourceCol = findTableColumnByName(cubeTable, colName); 31527 if (sourceCol != null) { 31528 DataFlowRelationship relation = modelFactory.createDataFlowRelation(); 31529 relation.setEffectType(EffectType.select); 31530 relation.addSource(new TableColumnRelationshipElement(sourceCol)); 31531 relation.setTarget(new RelationRowsRelationshipElement<ResultSetRelationRows>( 31532 resultSet.getRelationRows())); 31533 } 31534 } 31535 } 31536 } 31537 31538 private TableColumn findTableColumnByName(Table table, String name) { 31539 for (TableColumn col : table.getColumns()) { 31540 if (col.getName().equals(name)) { 31541 return col; 31542 } 31543 } 31544 return null; 31545 } 31546 31547 /** 31548 * Extract the display name from an MDX identifier node. 31549 * E.g., [Measures].[Departures NEAT] -> [Measures].[Departures NEAT] 31550 */ 31551 private String getMdxIdentifierName(gudusoft.gsqlparser.nodes.mdx.TMdxIdentifierNode idNode) { 31552 StringBuilder sb = new StringBuilder(); 31553 for (int i = 0; i < idNode.getSegmentList().size(); i++) { 31554 if (i > 0) sb.append("."); 31555 gudusoft.gsqlparser.nodes.mdx.IMdxIdentifierSegment seg = idNode.getSegmentList().getElement(i); 31556 if (seg.getQuoting() == gudusoft.gsqlparser.nodes.mdx.EMdxQuoting.QUOTED) { 31557 sb.append("[").append(seg.getName()).append("]"); 31558 } else { 31559 sb.append(seg.getName()); 31560 } 31561 } 31562 return sb.toString(); 31563 } 31564 31565 /** 31566 * Recursively collect measure and dimension references from MDX expression tree. 31567 * Uses iterative DFS to avoid StackOverflow on deeply nested expressions. 31568 */ 31569 private void collectMdxReferences(gudusoft.gsqlparser.nodes.mdx.TMdxExpNode expr, 31570 List<String> measures, List<String> dimensions) { 31571 Deque<gudusoft.gsqlparser.nodes.mdx.TMdxExpNode> stack = new ArrayDeque<gudusoft.gsqlparser.nodes.mdx.TMdxExpNode>(); 31572 stack.push(expr); 31573 31574 while (!stack.isEmpty()) { 31575 gudusoft.gsqlparser.nodes.mdx.TMdxExpNode current = stack.pop(); 31576 if (current == null) continue; 31577 31578 if (current instanceof gudusoft.gsqlparser.nodes.mdx.TMdxIdentifierNode) { 31579 gudusoft.gsqlparser.nodes.mdx.TMdxIdentifierNode idNode = 31580 (gudusoft.gsqlparser.nodes.mdx.TMdxIdentifierNode) current; 31581 String name = getMdxIdentifierName(idNode); 31582 if (name.toLowerCase().startsWith("[measures].") 31583 || name.toLowerCase().startsWith("measures.")) { 31584 measures.add(name); 31585 } else if (idNode.getSegmentList().size() > 1) { 31586 // Multi-segment identifiers that aren't measures are dimensions 31587 dimensions.add(name); 31588 } 31589 } else if (current instanceof gudusoft.gsqlparser.nodes.mdx.TMdxSetNode) { 31590 gudusoft.gsqlparser.nodes.mdx.TMdxSetNode setNode = 31591 (gudusoft.gsqlparser.nodes.mdx.TMdxSetNode) current; 31592 if (setNode.getTupleList() != null) { 31593 for (int i = 0; i < setNode.getTupleList().size(); i++) { 31594 stack.push(setNode.getTupleList().getElement(i)); 31595 } 31596 } 31597 } else if (current instanceof gudusoft.gsqlparser.nodes.mdx.TMdxTupleNode) { 31598 gudusoft.gsqlparser.nodes.mdx.TMdxTupleNode tupleNode = 31599 (gudusoft.gsqlparser.nodes.mdx.TMdxTupleNode) current; 31600 if (tupleNode.getExprList() != null) { 31601 for (int i = 0; i < tupleNode.getExprList().size(); i++) { 31602 stack.push(tupleNode.getExprList().getElement(i)); 31603 } 31604 } 31605 } else if (current instanceof gudusoft.gsqlparser.nodes.mdx.TMdxBinOpNode) { 31606 gudusoft.gsqlparser.nodes.mdx.TMdxBinOpNode binOp = 31607 (gudusoft.gsqlparser.nodes.mdx.TMdxBinOpNode) current; 31608 if (binOp.getRightExprNode() != null) stack.push(binOp.getRightExprNode()); 31609 if (binOp.getLeftExprNode() != null) stack.push(binOp.getLeftExprNode()); 31610 } else if (current instanceof gudusoft.gsqlparser.nodes.mdx.TMdxFunctionNode) { 31611 gudusoft.gsqlparser.nodes.mdx.TMdxFunctionNode funcNode = 31612 (gudusoft.gsqlparser.nodes.mdx.TMdxFunctionNode) current; 31613 if (funcNode.getArguments() != null) { 31614 for (int i = 0; i < funcNode.getArguments().size(); i++) { 31615 stack.push(funcNode.getArguments().getElement(i)); 31616 } 31617 } 31618 } 31619 } 31620 } 31621 31622 /** 31623 * Analyze a Power Query M-language document for data lineage. 31624 * 31625 * Delegates to TPowerQueryAnalyzer to extract navigation chains and 31626 * NativeQuery embedded SQL, then feeds the results back through the 31627 * standard SQL analysis pipeline so the dataflow model contains 31628 * regular table/column lineage. 31629 */ 31630 private void analyzePowerQueryDocumentStmt(TPowerQueryDocumentStmt pqStmt) { 31631 TPowerQueryAnalyzer pqAnalyzer = new TPowerQueryAnalyzer(pqStmt); 31632 31633 if (option.getPowerQueryInnerVendor() != null) { 31634 pqAnalyzer.withExplicitInnerVendor(option.getPowerQueryInnerVendor()); 31635 } 31636 31637 PowerQueryLineageResult result = pqAnalyzer.analyze(); 31638 31639 if (result.isEmpty()) { 31640 for (String w : result.getWarnings()) { 31641 logger.warn("Power Query: " + w); 31642 } 31643 return; 31644 } 31645 31646 for (PowerQueryLineageResult.NativeQueryRef nq : result.getNativeQueryReferences()) { 31647 if (nq.innerParser != null && nq.innerParseReturnCode == 0) { 31648 for (int i = 0; i < nq.innerParser.sqlstatements.size(); i++) { 31649 analyzeCustomSqlStmt(nq.innerParser.sqlstatements.get(i)); 31650 } 31651 } 31652 } 31653 31654 for (PowerQueryLineageResult.NavigationRef nav : result.getNavigationReferences()) { 31655 if (nav.syntheticSelect != null && nav.resolvedVendor != null) { 31656 TGSqlParser synParser = createSqlParser(nav.resolvedVendor); 31657 synParser.sqltext = nav.syntheticSelect; 31658 int rc = synParser.parse(); 31659 if (rc == 0) { 31660 for (int i = 0; i < synParser.sqlstatements.size(); i++) { 31661 analyzeCustomSqlStmt(synParser.sqlstatements.get(i)); 31662 } 31663 } 31664 } 31665 } 31666 31667 for (String w : result.getWarnings()) { 31668 logger.warn("Power Query: " + w); 31669 } 31670 } 31671 31672 31673 /** 31674 * True for an Oracle pseudocolumn that is scoped to one table, i.e. ROWID. 31675 * 31676 * <p>These report {@link EDbObjectType#notAColumn} so that nothing resolves 31677 * them against a catalog that will never contain them, but they still carry 31678 * a real source table: {@code SELECT a.ROWID AS rn FROM t a} genuinely does 31679 * derive {@code rn} from a row of {@code t}. Letting the notAColumn branch 31680 * claim them would route them to constants and delete that edge 31681 * (Mantis #4675).</p> 31682 */ 31683 private boolean isTableScopedPseudoColumn(TObjectName object) { 31684 if (object == null) return false; 31685 if (object.getDbObjectType() != EDbObjectType.notAColumn) return false; 31686 if (object.getSourceTable() == null) return false; 31687 return gudusoft.gsqlparser.util.OraclePseudoColumnUtil 31688 .isTableScopedPseudoColumn(object.getColumnNameOnly()); 31689 } 31690 31691}