001package gudusoft.gsqlparser.dlineage.dynamicsql;
002
003import gudusoft.gsqlparser.EDbVendor;
004import gudusoft.gsqlparser.TCustomSqlStatement;
005import gudusoft.gsqlparser.TSourceToken;
006import gudusoft.gsqlparser.TStatementList;
007import gudusoft.gsqlparser.dlineage.DataFlowAnalyzer;
008import gudusoft.gsqlparser.dlineage.dataflow.model.Option;
009import gudusoft.gsqlparser.dlineage.dataflow.model.xml.dataflow;
010import gudusoft.gsqlparser.dlineage.dataflow.model.xml.relationship;
011import gudusoft.gsqlparser.dlineage.dataflow.model.xml.sourceColumn;
012import gudusoft.gsqlparser.dlineage.dataflow.model.xml.table;
013import gudusoft.gsqlparser.dlineage.dataflow.model.xml.targetColumn;
014import gudusoft.gsqlparser.nodes.TParameterDeclaration;
015import gudusoft.gsqlparser.nodes.TParameterDeclarationList;
016import gudusoft.gsqlparser.sqlenv.TSQLEnv;
017import gudusoft.gsqlparser.stmt.mssql.TMssqlCreateProcedure;
018
019import java.io.UnsupportedEncodingException;
020import java.security.MessageDigest;
021import java.security.NoSuchAlgorithmException;
022import java.util.ArrayList;
023import java.util.HashMap;
024import java.util.List;
025import java.util.Map;
026import java.util.TreeMap;
027
028/**
029 * Orchestrates dynamic-SQL lineage resolution: evaluate the procedure's
030 * dynamic-SQL string building under the given bindings
031 * ({@link TDynamicSqlStringEvaluator}), then for each materialized EXEC site run
032 * ordinary dlineage on the concrete string and tag the resulting edges with
033 * dynamic provenance.
034 *
035 * <p>This is purely additive — it constructs fresh, throwaway
036 * {@link DataFlowAnalyzer} instances for the materialized strings and never
037 * touches the default analysis path, so default dlineage output is unchanged.
038 */
039public final class DynamicSqlLineageResolver {
040
041    private DynamicSqlLineageResolver() {
042    }
043
044    public static DynamicLineageResult resolve(TCustomSqlStatement procAst, EDbVendor vendor, TSQLEnv sqlEnv,
045            String currentDatabase, String defaultSchema, Map<String, SqlValue> bindings,
046            DynamicLineageOptions options) {
047
048        if (options == null) {
049            options = DynamicLineageOptions.createDefault();
050        }
051        if (bindings != null && bindings.size() > options.getMaxBindings()) {
052            throw new IllegalArgumentException(
053                    "binding count " + bindings.size() + " exceeds maxBindings " + options.getMaxBindings());
054        }
055
056        TStatementList body = extractBody(procAst);
057        List<TParameterDeclaration> params = extractParams(procAst);
058        String sourceProc = extractProcName(procAst);
059        String bindingHash = hashBindings(bindings);
060
061        DynamicLineageResult result = new DynamicLineageResult(sourceProc, bindingHash);
062        if (body == null) {
063            return result; // not a recognized procedure shape — nothing to evaluate
064        }
065
066        TDynamicSqlStringEvaluator evaluator = new TDynamicSqlStringEvaluator(vendor, options);
067        evaluator.seedBindings(bindings);
068        evaluator.seedParameterDefaults(params);
069
070        List<TDynamicSqlStringEvaluator.RawSite> rawSites = evaluator.evaluate(body);
071        for (TDynamicSqlStringEvaluator.RawSite raw : rawSites) {
072            result.addSite(buildSite(raw, vendor, sqlEnv, currentDatabase, defaultSchema, sourceProc,
073                    bindingHash, options));
074        }
075        // Never let a truncated walk look complete — emit an honest diagnostic.
076        if (evaluator.isSitesTruncated()) {
077            result.addSite(new DynamicSiteResult(DynamicSiteResult.Kind.EXEC_STRING,
078                    DynamicSiteResult.Status.UNRESOLVED, null, null, sourceProc,
079                    dynamicSiteId(sourceProc, "truncated"), bindingHash,
080                    "site/time limit reached (maxSites=" + options.getMaxSites()
081                            + ", timeout=" + options.getTimeoutMillis() + "ms); further sites dropped",
082                    0L, 0L, new ArrayList<DynamicLineageEdge>()));
083        }
084        return result;
085    }
086
087    /* --------------------------------------------------------- site building */
088
089    private static DynamicSiteResult buildSite(TDynamicSqlStringEvaluator.RawSite raw, EDbVendor vendor,
090            TSQLEnv sqlEnv, String currentDatabase, String defaultSchema, String sourceProc, String bindingHash,
091            DynamicLineageOptions options) {
092
093        long[] pos = coordinate(raw.node);
094        String dynamicSite = dynamicSiteId(sourceProc, raw.ordinalPath);
095        SqlStringValue sql = raw.sql;
096        // The EXEC site is wrapped in an IF whose predicate we cannot decide from
097        // the bindings (e.g. IF @Debug = 0 EXEC(@sql) with @Debug unbound).
098        boolean executionGated = !raw.controlFlowCertain;
099
100        // Honest unresolved reasons — never guess.
101        // Case (a): the dynamic string itself is not usable — NULL, opaque, or
102        // assigned under an undecidable branch and collapsed to UNKNOWN. Its
103        // *value* is untrustworthy, so suppress regardless of the gate.
104        if (sql == null || !sql.usable()) {
105            String reason = sql != null && sql.reason != null ? sql.reason
106                    : executionGated ? "site guarded by an undecidable conditional"
107                            : "dynamic SQL did not reduce to a concrete string";
108            return unresolved(raw, dynamicSite, sourceProc, bindingHash, pos, reason);
109        }
110        // Case (b): the EXEC site is execution-gated, but the string itself is usable
111        // (CONCRETE or PARTIAL) — only the *did-it-run* fact is uncertain, so we
112        // degrade-don't-fail rather than dropping the lineage. We fall through to
113        // analyzeResolvedSql either way: a fully-concrete gated site emits all its
114        // edges (flagged CONDITIONAL below), and a PARTIAL gated site emits exactly
115        // the placeholder-free subset a non-gated PARTIAL site would, suppressing only
116        // the placeholder edges. The downstream status/reason block surfaces both the
117        // placeholder suppression and the execution gate. (Case (a) — an unusable
118        // string, NULL/UNKNOWN — was already returned UNRESOLVED above.)
119        String resolvedSql = sql.text;
120        if (resolvedSql.length() > options.getMaxResolvedSqlLength()) {
121            return unresolved(raw, dynamicSite, sourceProc, bindingHash, pos,
122                    "resolved SQL length " + resolvedSql.length() + " exceeds maxResolvedSqlLength");
123        }
124
125        String concreteHash = sha1Hex(resolvedSql);
126        List<DynamicLineageEdge> edges;
127        boolean parseError;
128        boolean partialParseError;
129        try {
130            Analysis a = analyzeResolvedSql(resolvedSql, vendor, sqlEnv, currentDatabase, defaultSchema,
131                    sourceProc, dynamicSite, bindingHash, options);
132            edges = a.edges;
133            // No lineage + recorded errors = parse failure (generateDataFlow()
134            // does not throw on bad SQL). Errors WITH some lineage = a mixed batch
135            // where one statement failed; keep the good edges but flag it honestly.
136            parseError = a.hadErrors && a.edges.isEmpty();
137            partialParseError = a.hadErrors && !a.edges.isEmpty();
138        } catch (RuntimeException ex) {
139            edges = new ArrayList<DynamicLineageEdge>();
140            parseError = true;
141            partialParseError = false;
142        }
143
144        boolean partialPlaceholders = sql.state == SqlStringValue.State.PARTIAL;
145
146        DynamicSiteResult.Status status;
147        if (parseError) {
148            status = DynamicSiteResult.Status.PARSE_ERROR;
149        } else if (executionGated) {
150            // The undecidable gate dominates the status: whether the string is fully
151            // concrete or PARTIAL, every emitted edge is reached only if the guarded
152            // branch runs. CONDITIONAL keeps that "might not execute" caveat visible —
153            // the strictly-stronger signal to preserve — and the reason below adds the
154            // placeholder-suppression note when the string is also PARTIAL.
155            status = DynamicSiteResult.Status.CONDITIONAL;
156        } else if (partialPlaceholders) {
157            status = DynamicSiteResult.Status.PARTIAL;
158        } else {
159            status = DynamicSiteResult.Status.RESOLVED;
160        }
161
162        // Compose the reason from every applicable fact rather than picking one — a
163        // single site can be partial-parse AND placeholder-suppressed AND gated at
164        // once, and a consumer needs all three to gauge how far to trust the edges.
165        List<String> reasons = new ArrayList<String>();
166        if (parseError) {
167            reasons.add("materialized SQL failed to parse");
168        } else if (partialParseError) {
169            reasons.add("some statements in the materialized SQL failed to parse; emitted edges are from the parseable statements");
170        }
171        if (partialPlaceholders) {
172            reasons.add("materialized SQL still contains unbound placeholder(s); placeholder edges suppressed");
173        }
174        if (executionGated) {
175            boolean anyEdges = !edges.isEmpty();
176            reasons.add("site reached through an undecidable conditional"
177                    + (anyEdges ? " — emitted edges assume the guarded statement executed"
178                            : " — no resolvable edges were produced"));
179        }
180        String reason = reasons.isEmpty() ? null : String.join("; ", reasons);
181
182        return new DynamicSiteResult(raw.kind, status, resolvedSql, concreteHash, sourceProc, dynamicSite,
183                bindingHash, reason, pos[0], pos[1], edges);
184    }
185
186    private static DynamicSiteResult unresolved(TDynamicSqlStringEvaluator.RawSite raw, String dynamicSite,
187            String sourceProc, String bindingHash, long[] pos, String reason) {
188        String text = raw.sql != null && raw.sql.usable() ? raw.sql.text : null;
189        return new DynamicSiteResult(raw.kind, DynamicSiteResult.Status.UNRESOLVED, text,
190                text != null ? sha1Hex(text) : null, sourceProc, dynamicSite, bindingHash, reason,
191                pos[0], pos[1], new ArrayList<DynamicLineageEdge>());
192    }
193
194    /**
195     * Shared "analyze a materialized dynamic-SQL string" routine: run ordinary
196     * dlineage with the proc's database/schema context and lift each column edge
197     * into a provenance-tagged {@link DynamicLineageEdge}.
198     */
199    /** Edges from analyzing one materialized string, plus whether dlineage recorded parse/analysis errors. */
200    private static final class Analysis {
201        final List<DynamicLineageEdge> edges;
202        final boolean hadErrors;
203
204        Analysis(List<DynamicLineageEdge> edges, boolean hadErrors) {
205            this.edges = edges;
206            this.hadErrors = hadErrors;
207        }
208    }
209
210    private static Analysis analyzeResolvedSql(String resolvedSql, EDbVendor vendor,
211            TSQLEnv sqlEnv, String currentDatabase, String defaultSchema, String sourceProc, String dynamicSite,
212            String bindingHash, DynamicLineageOptions options) {
213
214        Option opt = new Option();
215        opt.setVendor(vendor);
216        opt.setSimpleOutput(false);
217        opt.setOutput(false);
218        opt.setShowImplicitSchema(options.isShowImplicitSchema());
219        if (currentDatabase != null && !currentDatabase.isEmpty()) {
220            opt.setDefaultDatabase(currentDatabase);
221        }
222        if (defaultSchema != null && !defaultSchema.isEmpty()) {
223            opt.setDefaultSchema(defaultSchema);
224        }
225
226        DataFlowAnalyzer nested = new DataFlowAnalyzer(resolvedSql, opt);
227        if (sqlEnv != null) {
228            nested.setSqlEnv(sqlEnv);
229        }
230        nested.generateDataFlow();
231        dataflow df = nested.getDataFlow();
232
233        List<DynamicLineageEdge> edges = new ArrayList<DynamicLineageEdge>();
234        if (df == null) {
235            return new Analysis(edges, true);
236        }
237        boolean hadErrors = df.getErrors() != null && !df.getErrors().isEmpty();
238        if (df.getRelationships() == null) {
239            return new Analysis(edges, hadErrors);
240        }
241
242        // Index the real tables/views by id so we can rebuild fully-qualified
243        // names from the structured model and fill in the proc's database/schema
244        // for names the dynamic SQL left unqualified.
245        Map<String, table> realTables = new HashMap<String, table>();
246        indexTables(realTables, df.getTables());
247        indexTables(realTables, df.getViews());
248
249        boolean showImplicit = options.isShowImplicitSchema();
250        for (relationship rel : df.getRelationships()) {
251            if (rel == null || rel.getTarget() == null || rel.getSources() == null) {
252                continue;
253            }
254            String target = fullName(rel.getTarget(), realTables, currentDatabase, defaultSchema, showImplicit);
255            // An edge touching an unresolved placeholder (a name still carrying a
256            // @variable from an unbound binding) must NOT be emitted as resolved
257            // lineage — it would be a guess. The site stays PARTIAL and its
258            // resolvedSql still shows the placeholder; only the fully-resolved
259            // edges flow out.
260            if (target == null || referencesPlaceholder(target)) {
261                continue;
262            }
263            for (sourceColumn src : rel.getSources()) {
264                if (isSystem(src)) {
265                    continue;
266                }
267                String source = fullName(src, realTables, currentDatabase, defaultSchema, showImplicit);
268                if (source == null || referencesPlaceholder(source)) {
269                    continue;
270                }
271                edges.add(new DynamicLineageEdge(source, target, rel.getEffectType(), rel.getType(),
272                        sourceProc, dynamicSite, bindingHash));
273            }
274        }
275        return new Analysis(edges, hadErrors);
276    }
277
278    /** A resolved object name never legitimately contains '@'; if it does, it still holds an unbound placeholder. */
279    private static boolean referencesPlaceholder(String name) {
280        return name != null && name.indexOf('@') >= 0;
281    }
282
283    /* ------------------------------------------------------------- helpers */
284
285    private static void indexTables(Map<String, table> index, List<table> tables) {
286        if (tables == null) {
287            return;
288        }
289        for (table t : tables) {
290            if (t != null && t.getId() != null) {
291                index.put(t.getId(), t);
292            }
293        }
294    }
295
296    private static String fullName(targetColumn c, Map<String, table> realTables, String currentDatabase,
297            String defaultSchema, boolean showImplicit) {
298        if (c == null || "system".equals(c.getSource())) {
299            return null;
300        }
301        return joinName(c.getParent_id(), c.getParent_name(), c.getColumn(), realTables, currentDatabase,
302                defaultSchema, showImplicit);
303    }
304
305    private static String fullName(sourceColumn c, Map<String, table> realTables, String currentDatabase,
306            String defaultSchema, boolean showImplicit) {
307        return joinName(c.getParent_id(), c.getParent_name(), c.getColumn(), realTables, currentDatabase,
308                defaultSchema, showImplicit);
309    }
310
311    private static boolean isSystem(sourceColumn c) {
312        return c == null || "system".equals(c.getSource()) || "RelationRows".equals(c.getColumn());
313    }
314
315    private static String joinName(String parentId, String parentName, String column,
316            Map<String, table> realTables, String currentDatabase, String defaultSchema, boolean showImplicit) {
317        if (column == null) {
318            return null;
319        }
320        return qualifiedParent(parentId, parentName, realTables, currentDatabase, defaultSchema, showImplicit)
321                + "." + stripBrackets(column);
322    }
323
324    /**
325     * Build the qualified parent name from the real-table model. dlineage may
326     * return the name already qualified (db.schema.table), partly qualified
327     * (schema.table), bare (table), or with an empty schema slot (db..table). We
328     * normalize per-segment brackets positionally — last = table, then schema,
329     * then database — and, only when {@code showImplicit} is on, fill a missing
330     * schema/database from the proc's {@code defaultSchema}/{@code currentDatabase}
331     * so an unqualified {@code RECON_JE_02_QSP_…} binds to
332     * {@code JOURNALENGINE.dbo.RECON_JE_02_QSP_…}. Intermediate result sets
333     * (no model table) keep their display name unchanged.
334     */
335    private static String qualifiedParent(String parentId, String parentName, Map<String, table> realTables,
336            String currentDatabase, String defaultSchema, boolean showImplicit) {
337        table t = parentId == null ? null : realTables.get(parentId);
338        if (t == null) {
339            return stripBrackets(parentName); // result set / function / intermediate node
340        }
341        String full = t.getFullName() != null ? t.getFullName() : t.getName();
342        if (full == null || full.isEmpty()) {
343            full = parentName;
344        }
345        if (full == null) {
346            return stripBrackets(parentName);
347        }
348
349        // Keep empty segments so positions are preserved (e.g. db..table).
350        String[] raw = full.split("\\.", -1);
351        for (int i = 0; i < raw.length; i++) {
352            raw[i] = stripBrackets(raw[i].trim());
353        }
354        int n = raw.length;
355        String table = n >= 1 ? raw[n - 1] : stripBrackets(parentName);
356        String schema = n >= 2 ? raw[n - 2] : "";
357        String db = n >= 3 ? raw[n - 3] : "";
358        // Anything left of the database segment (e.g. linked-server) is kept as a prefix.
359        StringBuilder prefix = new StringBuilder();
360        for (int i = 0; i < n - 3; i++) {
361            if (!raw[i].isEmpty()) {
362                prefix.append(raw[i]).append('.');
363            }
364        }
365        if (showImplicit) {
366            if (schema.isEmpty()) {
367                schema = stripBrackets(defaultSchema);
368            }
369            if (db.isEmpty()) {
370                db = stripBrackets(currentDatabase);
371            }
372        }
373        StringBuilder sb = new StringBuilder(prefix.toString());
374        if (db != null && !db.isEmpty()) {
375            sb.append(db).append('.');
376        }
377        if (schema != null && !schema.isEmpty()) {
378            sb.append(schema).append('.');
379        }
380        sb.append(table == null ? "" : table);
381        return sb.length() == 0 ? stripBrackets(parentName) : sb.toString();
382    }
383
384    private static TStatementList extractBody(TCustomSqlStatement procAst) {
385        if (procAst instanceof TMssqlCreateProcedure) {
386            return ((TMssqlCreateProcedure) procAst).getBodyStatements();
387        }
388        return null;
389    }
390
391    private static List<TParameterDeclaration> extractParams(TCustomSqlStatement procAst) {
392        List<TParameterDeclaration> out = new ArrayList<TParameterDeclaration>();
393        if (procAst instanceof TMssqlCreateProcedure) {
394            TParameterDeclarationList list = ((TMssqlCreateProcedure) procAst).getParameterDeclarations();
395            if (list != null) {
396                for (int i = 0; i < list.size(); i++) {
397                    out.add(list.getParameterDeclarationItem(i));
398                }
399            }
400        }
401        return out;
402    }
403
404    private static String extractProcName(TCustomSqlStatement procAst) {
405        if (procAst instanceof TMssqlCreateProcedure) {
406            TMssqlCreateProcedure cp = (TMssqlCreateProcedure) procAst;
407            if (cp.getProcedureName() != null) {
408                return cp.getProcedureName().toString();
409            }
410        }
411        return procAst == null ? "" : procAst.getClass().getSimpleName();
412    }
413
414    private static long[] coordinate(TCustomSqlStatement node) {
415        try {
416            TSourceToken t = node.getStartToken();
417            if (t != null) {
418                return new long[] { t.lineNo, t.columnNo };
419            }
420        } catch (Throwable ignore) {
421            // fall through
422        }
423        return new long[] { 0L, 0L };
424    }
425
426    /** Stable id robust to reformatting: proc identity + structural ordinal path, hashed. */
427    private static String dynamicSiteId(String sourceProc, String ordinalPath) {
428        return "dyn:" + sha1Hex(stripBrackets(sourceProc) + "#" + ordinalPath).substring(0, 12);
429    }
430
431    private static String hashBindings(Map<String, SqlValue> bindings) {
432        if (bindings == null || bindings.isEmpty()) {
433            return sha1Hex("");
434        }
435        TreeMap<String, String> sorted = new TreeMap<String, String>();
436        for (Map.Entry<String, SqlValue> e : bindings.entrySet()) {
437            if (e.getKey() == null) {
438                continue;
439            }
440            sorted.put(stripBrackets(e.getKey().trim()).toLowerCase(),
441                    e.getValue() == null ? "NULL" : e.getValue().toString());
442        }
443        StringBuilder sb = new StringBuilder();
444        for (Map.Entry<String, String> e : sorted.entrySet()) {
445            sb.append(e.getKey()).append('=').append(e.getValue()).append(';');
446        }
447        return sha1Hex(sb.toString());
448    }
449
450    private static String stripBrackets(String s) {
451        if (s == null) {
452            return "";
453        }
454        s = s.trim();
455        if (s.length() >= 2) {
456            char a = s.charAt(0);
457            char z = s.charAt(s.length() - 1);
458            if ((a == '[' && z == ']') || (a == '"' && z == '"') || (a == '`' && z == '`')) {
459                return s.substring(1, s.length() - 1);
460            }
461        }
462        return s;
463    }
464
465    private static String sha1Hex(String input) {
466        try {
467            MessageDigest md = MessageDigest.getInstance("SHA-1");
468            byte[] digest = md.digest(input.getBytes("UTF-8"));
469            StringBuilder sb = new StringBuilder(digest.length * 2);
470            for (byte b : digest) {
471                sb.append(Character.forDigit((b >> 4) & 0xF, 16));
472                sb.append(Character.forDigit(b & 0xF, 16));
473            }
474            return sb.toString();
475        } catch (NoSuchAlgorithmException | UnsupportedEncodingException e) {
476            return Integer.toHexString(input.hashCode());
477        }
478    }
479}