001package gudusoft.gsqlparser.dlineage.dynamicsql; 002 003import gudusoft.gsqlparser.EDbVendor; 004import gudusoft.gsqlparser.TCustomSqlStatement; 005import gudusoft.gsqlparser.TSourceToken; 006import gudusoft.gsqlparser.TStatementList; 007import gudusoft.gsqlparser.dlineage.DataFlowAnalyzer; 008import gudusoft.gsqlparser.dlineage.dataflow.model.Option; 009import gudusoft.gsqlparser.dlineage.dataflow.model.xml.dataflow; 010import gudusoft.gsqlparser.dlineage.dataflow.model.xml.relationship; 011import gudusoft.gsqlparser.dlineage.dataflow.model.xml.sourceColumn; 012import gudusoft.gsqlparser.dlineage.dataflow.model.xml.table; 013import gudusoft.gsqlparser.dlineage.dataflow.model.xml.targetColumn; 014import gudusoft.gsqlparser.nodes.TParameterDeclaration; 015import gudusoft.gsqlparser.nodes.TParameterDeclarationList; 016import gudusoft.gsqlparser.sqlenv.TSQLEnv; 017import gudusoft.gsqlparser.stmt.mssql.TMssqlCreateProcedure; 018 019import java.io.UnsupportedEncodingException; 020import java.security.MessageDigest; 021import java.security.NoSuchAlgorithmException; 022import java.util.ArrayList; 023import java.util.HashMap; 024import java.util.List; 025import java.util.Map; 026import java.util.TreeMap; 027 028/** 029 * Orchestrates dynamic-SQL lineage resolution: evaluate the procedure's 030 * dynamic-SQL string building under the given bindings 031 * ({@link TDynamicSqlStringEvaluator}), then for each materialized EXEC site run 032 * ordinary dlineage on the concrete string and tag the resulting edges with 033 * dynamic provenance. 034 * 035 * <p>This is purely additive — it constructs fresh, throwaway 036 * {@link DataFlowAnalyzer} instances for the materialized strings and never 037 * touches the default analysis path, so default dlineage output is unchanged. 038 */ 039public final class DynamicSqlLineageResolver { 040 041 private DynamicSqlLineageResolver() { 042 } 043 044 public static DynamicLineageResult resolve(TCustomSqlStatement procAst, EDbVendor vendor, TSQLEnv sqlEnv, 045 String currentDatabase, String defaultSchema, Map<String, SqlValue> bindings, 046 DynamicLineageOptions options) { 047 048 if (options == null) { 049 options = DynamicLineageOptions.createDefault(); 050 } 051 if (bindings != null && bindings.size() > options.getMaxBindings()) { 052 throw new IllegalArgumentException( 053 "binding count " + bindings.size() + " exceeds maxBindings " + options.getMaxBindings()); 054 } 055 056 TStatementList body = extractBody(procAst); 057 List<TParameterDeclaration> params = extractParams(procAst); 058 String sourceProc = extractProcName(procAst); 059 String bindingHash = hashBindings(bindings); 060 061 DynamicLineageResult result = new DynamicLineageResult(sourceProc, bindingHash); 062 if (body == null) { 063 return result; // not a recognized procedure shape — nothing to evaluate 064 } 065 066 TDynamicSqlStringEvaluator evaluator = new TDynamicSqlStringEvaluator(vendor, options); 067 evaluator.seedBindings(bindings); 068 evaluator.seedParameterDefaults(params); 069 070 List<TDynamicSqlStringEvaluator.RawSite> rawSites = evaluator.evaluate(body); 071 for (TDynamicSqlStringEvaluator.RawSite raw : rawSites) { 072 result.addSite(buildSite(raw, vendor, sqlEnv, currentDatabase, defaultSchema, sourceProc, 073 bindingHash, options)); 074 } 075 // Never let a truncated walk look complete — emit an honest diagnostic. 076 if (evaluator.isSitesTruncated()) { 077 result.addSite(new DynamicSiteResult(DynamicSiteResult.Kind.EXEC_STRING, 078 DynamicSiteResult.Status.UNRESOLVED, null, null, sourceProc, 079 dynamicSiteId(sourceProc, "truncated"), bindingHash, 080 "site/time limit reached (maxSites=" + options.getMaxSites() 081 + ", timeout=" + options.getTimeoutMillis() + "ms); further sites dropped", 082 0L, 0L, new ArrayList<DynamicLineageEdge>())); 083 } 084 return result; 085 } 086 087 /* --------------------------------------------------------- site building */ 088 089 private static DynamicSiteResult buildSite(TDynamicSqlStringEvaluator.RawSite raw, EDbVendor vendor, 090 TSQLEnv sqlEnv, String currentDatabase, String defaultSchema, String sourceProc, String bindingHash, 091 DynamicLineageOptions options) { 092 093 long[] pos = coordinate(raw.node); 094 String dynamicSite = dynamicSiteId(sourceProc, raw.ordinalPath); 095 SqlStringValue sql = raw.sql; 096 // The EXEC site is wrapped in an IF whose predicate we cannot decide from 097 // the bindings (e.g. IF @Debug = 0 EXEC(@sql) with @Debug unbound). 098 boolean executionGated = !raw.controlFlowCertain; 099 100 // Honest unresolved reasons — never guess. 101 // Case (a): the dynamic string itself is not usable — NULL, opaque, or 102 // assigned under an undecidable branch and collapsed to UNKNOWN. Its 103 // *value* is untrustworthy, so suppress regardless of the gate. 104 if (sql == null || !sql.usable()) { 105 String reason = sql != null && sql.reason != null ? sql.reason 106 : executionGated ? "site guarded by an undecidable conditional" 107 : "dynamic SQL did not reduce to a concrete string"; 108 return unresolved(raw, dynamicSite, sourceProc, bindingHash, pos, reason); 109 } 110 // Case (b): the EXEC site is execution-gated, but the string itself is usable 111 // (CONCRETE or PARTIAL) — only the *did-it-run* fact is uncertain, so we 112 // degrade-don't-fail rather than dropping the lineage. We fall through to 113 // analyzeResolvedSql either way: a fully-concrete gated site emits all its 114 // edges (flagged CONDITIONAL below), and a PARTIAL gated site emits exactly 115 // the placeholder-free subset a non-gated PARTIAL site would, suppressing only 116 // the placeholder edges. The downstream status/reason block surfaces both the 117 // placeholder suppression and the execution gate. (Case (a) — an unusable 118 // string, NULL/UNKNOWN — was already returned UNRESOLVED above.) 119 String resolvedSql = sql.text; 120 if (resolvedSql.length() > options.getMaxResolvedSqlLength()) { 121 return unresolved(raw, dynamicSite, sourceProc, bindingHash, pos, 122 "resolved SQL length " + resolvedSql.length() + " exceeds maxResolvedSqlLength"); 123 } 124 125 String concreteHash = sha1Hex(resolvedSql); 126 List<DynamicLineageEdge> edges; 127 boolean parseError; 128 boolean partialParseError; 129 try { 130 Analysis a = analyzeResolvedSql(resolvedSql, vendor, sqlEnv, currentDatabase, defaultSchema, 131 sourceProc, dynamicSite, bindingHash, options); 132 edges = a.edges; 133 // No lineage + recorded errors = parse failure (generateDataFlow() 134 // does not throw on bad SQL). Errors WITH some lineage = a mixed batch 135 // where one statement failed; keep the good edges but flag it honestly. 136 parseError = a.hadErrors && a.edges.isEmpty(); 137 partialParseError = a.hadErrors && !a.edges.isEmpty(); 138 } catch (RuntimeException ex) { 139 edges = new ArrayList<DynamicLineageEdge>(); 140 parseError = true; 141 partialParseError = false; 142 } 143 144 boolean partialPlaceholders = sql.state == SqlStringValue.State.PARTIAL; 145 146 DynamicSiteResult.Status status; 147 if (parseError) { 148 status = DynamicSiteResult.Status.PARSE_ERROR; 149 } else if (executionGated) { 150 // The undecidable gate dominates the status: whether the string is fully 151 // concrete or PARTIAL, every emitted edge is reached only if the guarded 152 // branch runs. CONDITIONAL keeps that "might not execute" caveat visible — 153 // the strictly-stronger signal to preserve — and the reason below adds the 154 // placeholder-suppression note when the string is also PARTIAL. 155 status = DynamicSiteResult.Status.CONDITIONAL; 156 } else if (partialPlaceholders) { 157 status = DynamicSiteResult.Status.PARTIAL; 158 } else { 159 status = DynamicSiteResult.Status.RESOLVED; 160 } 161 162 // Compose the reason from every applicable fact rather than picking one — a 163 // single site can be partial-parse AND placeholder-suppressed AND gated at 164 // once, and a consumer needs all three to gauge how far to trust the edges. 165 List<String> reasons = new ArrayList<String>(); 166 if (parseError) { 167 reasons.add("materialized SQL failed to parse"); 168 } else if (partialParseError) { 169 reasons.add("some statements in the materialized SQL failed to parse; emitted edges are from the parseable statements"); 170 } 171 if (partialPlaceholders) { 172 reasons.add("materialized SQL still contains unbound placeholder(s); placeholder edges suppressed"); 173 } 174 if (executionGated) { 175 boolean anyEdges = !edges.isEmpty(); 176 reasons.add("site reached through an undecidable conditional" 177 + (anyEdges ? " — emitted edges assume the guarded statement executed" 178 : " — no resolvable edges were produced")); 179 } 180 String reason = reasons.isEmpty() ? null : String.join("; ", reasons); 181 182 return new DynamicSiteResult(raw.kind, status, resolvedSql, concreteHash, sourceProc, dynamicSite, 183 bindingHash, reason, pos[0], pos[1], edges); 184 } 185 186 private static DynamicSiteResult unresolved(TDynamicSqlStringEvaluator.RawSite raw, String dynamicSite, 187 String sourceProc, String bindingHash, long[] pos, String reason) { 188 String text = raw.sql != null && raw.sql.usable() ? raw.sql.text : null; 189 return new DynamicSiteResult(raw.kind, DynamicSiteResult.Status.UNRESOLVED, text, 190 text != null ? sha1Hex(text) : null, sourceProc, dynamicSite, bindingHash, reason, 191 pos[0], pos[1], new ArrayList<DynamicLineageEdge>()); 192 } 193 194 /** 195 * Shared "analyze a materialized dynamic-SQL string" routine: run ordinary 196 * dlineage with the proc's database/schema context and lift each column edge 197 * into a provenance-tagged {@link DynamicLineageEdge}. 198 */ 199 /** Edges from analyzing one materialized string, plus whether dlineage recorded parse/analysis errors. */ 200 private static final class Analysis { 201 final List<DynamicLineageEdge> edges; 202 final boolean hadErrors; 203 204 Analysis(List<DynamicLineageEdge> edges, boolean hadErrors) { 205 this.edges = edges; 206 this.hadErrors = hadErrors; 207 } 208 } 209 210 private static Analysis analyzeResolvedSql(String resolvedSql, EDbVendor vendor, 211 TSQLEnv sqlEnv, String currentDatabase, String defaultSchema, String sourceProc, String dynamicSite, 212 String bindingHash, DynamicLineageOptions options) { 213 214 Option opt = new Option(); 215 opt.setVendor(vendor); 216 opt.setSimpleOutput(false); 217 opt.setOutput(false); 218 opt.setShowImplicitSchema(options.isShowImplicitSchema()); 219 if (currentDatabase != null && !currentDatabase.isEmpty()) { 220 opt.setDefaultDatabase(currentDatabase); 221 } 222 if (defaultSchema != null && !defaultSchema.isEmpty()) { 223 opt.setDefaultSchema(defaultSchema); 224 } 225 226 DataFlowAnalyzer nested = new DataFlowAnalyzer(resolvedSql, opt); 227 if (sqlEnv != null) { 228 nested.setSqlEnv(sqlEnv); 229 } 230 nested.generateDataFlow(); 231 dataflow df = nested.getDataFlow(); 232 233 List<DynamicLineageEdge> edges = new ArrayList<DynamicLineageEdge>(); 234 if (df == null) { 235 return new Analysis(edges, true); 236 } 237 boolean hadErrors = df.getErrors() != null && !df.getErrors().isEmpty(); 238 if (df.getRelationships() == null) { 239 return new Analysis(edges, hadErrors); 240 } 241 242 // Index the real tables/views by id so we can rebuild fully-qualified 243 // names from the structured model and fill in the proc's database/schema 244 // for names the dynamic SQL left unqualified. 245 Map<String, table> realTables = new HashMap<String, table>(); 246 indexTables(realTables, df.getTables()); 247 indexTables(realTables, df.getViews()); 248 249 boolean showImplicit = options.isShowImplicitSchema(); 250 for (relationship rel : df.getRelationships()) { 251 if (rel == null || rel.getTarget() == null || rel.getSources() == null) { 252 continue; 253 } 254 String target = fullName(rel.getTarget(), realTables, currentDatabase, defaultSchema, showImplicit); 255 // An edge touching an unresolved placeholder (a name still carrying a 256 // @variable from an unbound binding) must NOT be emitted as resolved 257 // lineage — it would be a guess. The site stays PARTIAL and its 258 // resolvedSql still shows the placeholder; only the fully-resolved 259 // edges flow out. 260 if (target == null || referencesPlaceholder(target)) { 261 continue; 262 } 263 for (sourceColumn src : rel.getSources()) { 264 if (isSystem(src)) { 265 continue; 266 } 267 String source = fullName(src, realTables, currentDatabase, defaultSchema, showImplicit); 268 if (source == null || referencesPlaceholder(source)) { 269 continue; 270 } 271 edges.add(new DynamicLineageEdge(source, target, rel.getEffectType(), rel.getType(), 272 sourceProc, dynamicSite, bindingHash)); 273 } 274 } 275 return new Analysis(edges, hadErrors); 276 } 277 278 /** A resolved object name never legitimately contains '@'; if it does, it still holds an unbound placeholder. */ 279 private static boolean referencesPlaceholder(String name) { 280 return name != null && name.indexOf('@') >= 0; 281 } 282 283 /* ------------------------------------------------------------- helpers */ 284 285 private static void indexTables(Map<String, table> index, List<table> tables) { 286 if (tables == null) { 287 return; 288 } 289 for (table t : tables) { 290 if (t != null && t.getId() != null) { 291 index.put(t.getId(), t); 292 } 293 } 294 } 295 296 private static String fullName(targetColumn c, Map<String, table> realTables, String currentDatabase, 297 String defaultSchema, boolean showImplicit) { 298 if (c == null || "system".equals(c.getSource())) { 299 return null; 300 } 301 return joinName(c.getParent_id(), c.getParent_name(), c.getColumn(), realTables, currentDatabase, 302 defaultSchema, showImplicit); 303 } 304 305 private static String fullName(sourceColumn c, Map<String, table> realTables, String currentDatabase, 306 String defaultSchema, boolean showImplicit) { 307 return joinName(c.getParent_id(), c.getParent_name(), c.getColumn(), realTables, currentDatabase, 308 defaultSchema, showImplicit); 309 } 310 311 private static boolean isSystem(sourceColumn c) { 312 return c == null || "system".equals(c.getSource()) || "RelationRows".equals(c.getColumn()); 313 } 314 315 private static String joinName(String parentId, String parentName, String column, 316 Map<String, table> realTables, String currentDatabase, String defaultSchema, boolean showImplicit) { 317 if (column == null) { 318 return null; 319 } 320 return qualifiedParent(parentId, parentName, realTables, currentDatabase, defaultSchema, showImplicit) 321 + "." + stripBrackets(column); 322 } 323 324 /** 325 * Build the qualified parent name from the real-table model. dlineage may 326 * return the name already qualified (db.schema.table), partly qualified 327 * (schema.table), bare (table), or with an empty schema slot (db..table). We 328 * normalize per-segment brackets positionally — last = table, then schema, 329 * then database — and, only when {@code showImplicit} is on, fill a missing 330 * schema/database from the proc's {@code defaultSchema}/{@code currentDatabase} 331 * so an unqualified {@code RECON_JE_02_QSP_…} binds to 332 * {@code JOURNALENGINE.dbo.RECON_JE_02_QSP_…}. Intermediate result sets 333 * (no model table) keep their display name unchanged. 334 */ 335 private static String qualifiedParent(String parentId, String parentName, Map<String, table> realTables, 336 String currentDatabase, String defaultSchema, boolean showImplicit) { 337 table t = parentId == null ? null : realTables.get(parentId); 338 if (t == null) { 339 return stripBrackets(parentName); // result set / function / intermediate node 340 } 341 String full = t.getFullName() != null ? t.getFullName() : t.getName(); 342 if (full == null || full.isEmpty()) { 343 full = parentName; 344 } 345 if (full == null) { 346 return stripBrackets(parentName); 347 } 348 349 // Keep empty segments so positions are preserved (e.g. db..table). 350 String[] raw = full.split("\\.", -1); 351 for (int i = 0; i < raw.length; i++) { 352 raw[i] = stripBrackets(raw[i].trim()); 353 } 354 int n = raw.length; 355 String table = n >= 1 ? raw[n - 1] : stripBrackets(parentName); 356 String schema = n >= 2 ? raw[n - 2] : ""; 357 String db = n >= 3 ? raw[n - 3] : ""; 358 // Anything left of the database segment (e.g. linked-server) is kept as a prefix. 359 StringBuilder prefix = new StringBuilder(); 360 for (int i = 0; i < n - 3; i++) { 361 if (!raw[i].isEmpty()) { 362 prefix.append(raw[i]).append('.'); 363 } 364 } 365 if (showImplicit) { 366 if (schema.isEmpty()) { 367 schema = stripBrackets(defaultSchema); 368 } 369 if (db.isEmpty()) { 370 db = stripBrackets(currentDatabase); 371 } 372 } 373 StringBuilder sb = new StringBuilder(prefix.toString()); 374 if (db != null && !db.isEmpty()) { 375 sb.append(db).append('.'); 376 } 377 if (schema != null && !schema.isEmpty()) { 378 sb.append(schema).append('.'); 379 } 380 sb.append(table == null ? "" : table); 381 return sb.length() == 0 ? stripBrackets(parentName) : sb.toString(); 382 } 383 384 private static TStatementList extractBody(TCustomSqlStatement procAst) { 385 if (procAst instanceof TMssqlCreateProcedure) { 386 return ((TMssqlCreateProcedure) procAst).getBodyStatements(); 387 } 388 return null; 389 } 390 391 private static List<TParameterDeclaration> extractParams(TCustomSqlStatement procAst) { 392 List<TParameterDeclaration> out = new ArrayList<TParameterDeclaration>(); 393 if (procAst instanceof TMssqlCreateProcedure) { 394 TParameterDeclarationList list = ((TMssqlCreateProcedure) procAst).getParameterDeclarations(); 395 if (list != null) { 396 for (int i = 0; i < list.size(); i++) { 397 out.add(list.getParameterDeclarationItem(i)); 398 } 399 } 400 } 401 return out; 402 } 403 404 private static String extractProcName(TCustomSqlStatement procAst) { 405 if (procAst instanceof TMssqlCreateProcedure) { 406 TMssqlCreateProcedure cp = (TMssqlCreateProcedure) procAst; 407 if (cp.getProcedureName() != null) { 408 return cp.getProcedureName().toString(); 409 } 410 } 411 return procAst == null ? "" : procAst.getClass().getSimpleName(); 412 } 413 414 private static long[] coordinate(TCustomSqlStatement node) { 415 try { 416 TSourceToken t = node.getStartToken(); 417 if (t != null) { 418 return new long[] { t.lineNo, t.columnNo }; 419 } 420 } catch (Throwable ignore) { 421 // fall through 422 } 423 return new long[] { 0L, 0L }; 424 } 425 426 /** Stable id robust to reformatting: proc identity + structural ordinal path, hashed. */ 427 private static String dynamicSiteId(String sourceProc, String ordinalPath) { 428 return "dyn:" + sha1Hex(stripBrackets(sourceProc) + "#" + ordinalPath).substring(0, 12); 429 } 430 431 private static String hashBindings(Map<String, SqlValue> bindings) { 432 if (bindings == null || bindings.isEmpty()) { 433 return sha1Hex(""); 434 } 435 TreeMap<String, String> sorted = new TreeMap<String, String>(); 436 for (Map.Entry<String, SqlValue> e : bindings.entrySet()) { 437 if (e.getKey() == null) { 438 continue; 439 } 440 sorted.put(stripBrackets(e.getKey().trim()).toLowerCase(), 441 e.getValue() == null ? "NULL" : e.getValue().toString()); 442 } 443 StringBuilder sb = new StringBuilder(); 444 for (Map.Entry<String, String> e : sorted.entrySet()) { 445 sb.append(e.getKey()).append('=').append(e.getValue()).append(';'); 446 } 447 return sha1Hex(sb.toString()); 448 } 449 450 private static String stripBrackets(String s) { 451 if (s == null) { 452 return ""; 453 } 454 s = s.trim(); 455 if (s.length() >= 2) { 456 char a = s.charAt(0); 457 char z = s.charAt(s.length() - 1); 458 if ((a == '[' && z == ']') || (a == '"' && z == '"') || (a == '`' && z == '`')) { 459 return s.substring(1, s.length() - 1); 460 } 461 } 462 return s; 463 } 464 465 private static String sha1Hex(String input) { 466 try { 467 MessageDigest md = MessageDigest.getInstance("SHA-1"); 468 byte[] digest = md.digest(input.getBytes("UTF-8")); 469 StringBuilder sb = new StringBuilder(digest.length * 2); 470 for (byte b : digest) { 471 sb.append(Character.forDigit((b >> 4) & 0xF, 16)); 472 sb.append(Character.forDigit(b & 0xF, 16)); 473 } 474 return sb.toString(); 475 } catch (NoSuchAlgorithmException | UnsupportedEncodingException e) { 476 return Integer.toHexString(input.hashCode()); 477 } 478 } 479}