001package gudusoft.gsqlparser.lineage.v2.validation;
002
003import gudusoft.gsqlparser.lineage.v2.Finding;
004import gudusoft.gsqlparser.lineage.v2.SqllrResources;
005import gudusoft.gsqlparser.lineage.v2.identity.DocumentIds;
006
007import java.nio.ByteBuffer;
008import java.nio.charset.CodingErrorAction;
009import java.nio.charset.StandardCharsets;
010import java.util.ArrayList;
011import java.util.Arrays;
012import java.util.Collections;
013import java.util.HashSet;
014import java.util.LinkedHashMap;
015import java.util.List;
016import java.util.Locale;
017import java.util.Map;
018import java.util.Set;
019
020import static gudusoft.gsqlparser.lineage.v2.validation.DocumentIndex.items;
021
022/**
023 * Validation stage {@code IDENTITY} (specification Section 14.2) as the reference runner implements it
024 * ({@code conformance/run.py}, {@code stage_identity}): every analysis context names a published dependency
025 * snapshot ({@code VERSION_UNKNOWN_DEPENDENCY_SNAPSHOT}); every id recomputes ({@link DocumentIds}); full names
026 * follow the dialect's qualification profile ({@code ID_QUALIFICATION_MISMATCH}); and declared canonical
027 * segment keys are what the identifier-rule data gives ({@code ID_SEGMENT_KEY_MISMATCH}).
028 *
029 * <p>Segment keys are checked where the reference checks them: ASCII text under every fold, and any text under a
030 * collation profile. Other non-ASCII text needs the Unicode case tables of the snapshot's Unicode version and is
031 * listed as unchecked, as the reference lists it.</p>
032 *
033 * @since 4.2.17
034 */
035public final class IdentityStage {
036
037    private static final List<String> RELATION_NAME_KINDS = Arrays.asList("RELATION", "TABLE", "VIEW",
038            "MATERIALIZED_VIEW", "EXTERNAL_TABLE", "SYNONYM");
039    private static final List<String> NAME_COMPUTED_STATUSES = Arrays.asList("PORTABLE", "UNAVAILABLE_COLLATION",
040            "UNAVAILABLE_MIXED_PROFILE");
041
042    private IdentityStage() {
043    }
044
045    /** The findings of the stage; {@code notes} receives what was not checked. */
046    public static List<Finding> run(Map<?, ?> doc, List<String> notes) {
047        List<Map<?, ?>> contexts = items(doc.get("analysisContexts"));
048        List<Map<?, ?>> snapshots = new ArrayList<Map<?, ?>>();
049        for (int i = 0; i < contexts.size(); i++) {
050            Object snapshotId = contexts.get(i).get("dependencySnapshotId");
051            Map<?, ?> snapshot = findSnapshot(snapshotId);
052            if (snapshot == null) {
053                List<Finding> out = new ArrayList<Finding>();
054                out.add(new Finding("VERSION_UNKNOWN_DEPENDENCY_SNAPSHOT", "$.analysisContexts[" + i
055                        + "].dependencySnapshotId", "no published snapshot has id " + snapshotId));
056                return out;
057            }
058            snapshots.add(snapshot);
059        }
060        DocumentIds.Report report = DocumentIds.check(doc);
061        for (String note : report.getNotes()) {
062            notes.add("check-ids: " + note);
063        }
064        List<Finding> findings = new ArrayList<Finding>(report.getFindings());
065        Set<List<Object>> domains = new HashSet<List<Object>>();
066        for (Map<?, ?> context : contexts) {
067            Object dialect = context.get("dialect");
068            domains.add(Arrays.asList(context.get("dependencySnapshotId"),
069                    dialect instanceof Map<?, ?> ? ((Map<?, ?>) dialect).get("id") : null));
070        }
071        if (domains.size() != 1) {
072            notes.add("several identity domains; canonical segment keys not checked");
073            return findings;
074        }
075        Object dialect = contexts.get(0).get("dialect");
076        SegmentKeys keys = SegmentKeys.of(snapshots.get(0), dialect instanceof Map<?, ?> ? ((Map<?, ?>) dialect).get("id")
077                : null);
078        if (keys == null) {
079            notes.add("no identifier rules for the dialect; canonical segment keys not checked");
080            return findings;
081        }
082        findings.addAll(qualificationFindings(doc, keys.profile));
083        for (Map<?, ?> endpoint : items(doc.get("endpoints"))) {
084            Object name = endpoint.get("name");
085            List<Map<?, ?>> segments = items(name instanceof Map<?, ?> ? ((Map<?, ?>) name).get("segments") : null);
086            for (int i = 0; i < segments.size(); i++) {
087                Map<?, ?> segment = segments.get(i);
088                if (!"NAMED".equals(segment.get("kind"))) {
089                    continue;
090                }
091                String objectType = keys.objectType(segment.get("role"), endpoint.get("kind"));
092                if (objectType == null) {
093                    continue;
094                }
095                List<Object> expected = keys.key(segment, objectType);
096                String where = "endpoints ref " + endpoint.get("ref") + " segment " + i;
097                if (expected == null) {
098                    notes.add(where + ": canonical key not checked");
099                    continue;
100                }
101                List<Object> declared = segment.containsKey("canonicalKey")
102                        ? Arrays.<Object>asList("KEY", segment.get("canonicalKey"))
103                        : segment.containsKey("keyUnavailable")
104                        ? Arrays.<Object>asList("UNAVAILABLE", segment.get("keyUnavailable")) : null;
105                if (declared != null && !declared.equals(expected)) {
106                    findings.add(new Finding("ID_SEGMENT_KEY_MISMATCH", where, "declared " + declared + ", rules give "
107                            + expected));
108                }
109            }
110        }
111        return findings;
112    }
113
114    /** The shipped snapshot with this id, or null. */
115    static Map<?, ?> findSnapshot(Object snapshotId) {
116        for (String path : DocumentIds.shippedSnapshots()) {
117            Map<?, ?> snapshot = (Map<?, ?>) SqllrResources.json(path);
118            Object id = snapshot.get("snapshotId");
119            if (id == null ? snapshotId == null : id.equals(snapshotId)) {
120                return snapshot;
121            }
122        }
123        return null;
124    }
125
126    /**
127     * Sections 6.3 and 9.1: the full name of an endpoint whose identity is computed from its name carries the
128     * segment roles of the dialect's qualification profile, in order.
129     */
130    static List<Finding> qualificationFindings(Map<?, ?> doc, Map<?, ?> profile) {
131        List<Finding> findings = new ArrayList<Finding>();
132        for (Map<?, ?> endpoint : items(doc.get("endpoints"))) {
133            Object kind = endpoint.get("kind");
134            Object identity = endpoint.get("identity");
135            Object status = identity instanceof Map<?, ?> ? ((Map<?, ?>) identity).get("status") : null;
136            if (!NAME_COMPUTED_STATUSES.contains(status)) {
137                continue;
138            }
139            String nameKind = RELATION_NAME_KINDS.contains(kind) ? "relationName" : "ROUTINE".equals(kind) ? "routineName" : null;
140            if (nameKind == null || !profile.containsKey(nameKind)) {
141                continue;
142            }
143            Object name = endpoint.get("name");
144            List<Object> roles = new ArrayList<Object>();
145            for (Map<?, ?> segment : items(name instanceof Map<?, ?> ? ((Map<?, ?>) name).get("segments") : null)) {
146                roles.add(segment.get("role"));
147            }
148            int position = 0;
149            String problem = null;
150            for (Map<?, ?> entry : items(((Map<?, ?>) profile.get(nameKind)).get("identityRoles"))) {
151                if (position < roles.size() && entry.get("role") != null && entry.get("role").equals(roles.get(position))) {
152                    position++;
153                } else if (!"OPTIONAL".equals(entry.get("presence"))) {
154                    problem = "role " + entry.get("role") + " (" + entry.get("presence") + ") is missing";
155                    break;
156                }
157            }
158            if (problem == null && position < roles.size()) {
159                problem = "role " + roles.get(position) + " is not in the profile at this position";
160            }
161            if (problem != null) {
162                findings.add(new Finding("ID_QUALIFICATION_MISMATCH", "endpoints ref " + endpoint.get("ref"),
163                        status + " " + kind + ", roles " + roles + ": " + problem));
164            }
165        }
166        return findings;
167    }
168
169    /**
170     * {@code CanonicalSegmentKeyV1} ({@code identifier-rules/FORMAT.md}, Section 9) where it can be decided
171     * without the Unicode case tables, as the reference runner's {@code SegmentKeys}.
172     */
173    static final class SegmentKeys {
174        final Map<?, ?> rules;
175        final Map<String, Object> flags = new LinkedHashMap<String, Object>();
176        final Map<?, ?> collations;
177        final Map<?, ?> profile;
178
179        private SegmentKeys(Map<?, ?> rules, Map<?, ?> entry, Map<?, ?> profile) {
180            this.rules = rules;
181            Object declared = rules.get("flags");
182            if (declared instanceof Map<?, ?>) {
183                for (Map.Entry<?, ?> flag : ((Map<?, ?>) declared).entrySet()) {
184                    flags.put((String) flag.getKey(), ((Map<?, ?>) flag.getValue()).get("default"));
185                }
186            }
187            if (entry.get("effectiveFlags") instanceof Map<?, ?>) {
188                for (Map.Entry<?, ?> flag : ((Map<?, ?>) entry.get("effectiveFlags")).entrySet()) {
189                    flags.put((String) flag.getKey(), flag.getValue());
190                }
191            }
192            this.collations = entry.get("collations") instanceof Map<?, ?> ? (Map<?, ?>) entry.get("collations")
193                    : Collections.emptyMap();
194            this.profile = profile;
195        }
196
197        static SegmentKeys of(Map<?, ?> snapshot, Object dialect) {
198            Map<?, ?> entry = null;
199            for (Map<?, ?> candidate : items(snapshot.get("identifierRules"))) {
200                if (candidate.get("dialect") != null && candidate.get("dialect").equals(dialect)) {
201                    entry = candidate;
202                    break;
203                }
204            }
205            if (entry == null) {
206                return null;
207            }
208            Map<?, ?> rules = (Map<?, ?>) SqllrResources.json((String) entry.get("path"));
209            Map<?, ?> profile = Collections.emptyMap();
210            for (Map<?, ?> d : items(((Map<?, ?>) SqllrResources.qualificationProfiles()).get("dialects"))) {
211                if (dialect != null && dialect.equals(d.get("dialect"))) {
212                    profile = d;
213                    break;
214                }
215            }
216            return new SegmentKeys(rules, entry, profile);
217        }
218
219        Map<?, ?> cell(String objectType) {
220            Map<?, ?> types = (Map<?, ?>) rules.get("objectTypes");
221            Map<?, ?> entry = (Map<?, ?>) types.get(objectType);
222            if (entry == null) {
223                return null;
224            }
225            if (entry.containsKey("sameRulesAs")) {
226                objectType = (String) entry.get("sameRulesAs");
227                entry = (Map<?, ?>) types.get(objectType);
228            }
229            Object label = entry.get("cell");
230            for (Map<?, ?> variant : items(rules.get("variants"))) {
231                boolean applies = true;
232                for (Map.Entry<?, ?> condition : ((Map<?, ?>) variant.get("when")).entrySet()) {
233                    Object value = flags.get(condition.getKey());
234                    applies &= value == null ? condition.getValue() == null : value.equals(condition.getValue());
235                }
236                Map<?, ?> variantTypes = (Map<?, ?>) variant.get("objectTypes");
237                if (applies && variantTypes.containsKey(objectType)) {
238                    Object override = ((Map<?, ?>) variantTypes.get(objectType)).get("cell");
239                    if (override != null) {
240                        label = override;
241                    }
242                }
243            }
244            return (Map<?, ?>) ((Map<?, ?>) rules.get("cells")).get(label);
245        }
246
247        String objectType(Object role, Object endpointKind) {
248            if ("AUTHORITY".equals(role) || "LOCATION".equals(role)) {
249                return null;
250            }
251            if ("FIELD".equals(role)) {
252                return "COLUMN";
253            }
254            if ("REMOTE_LINK".equals(role)) {
255                return (String) profile.get("remoteLinkObjectType");
256            }
257            if ("OBJECT".equals(role)) {
258                if (Arrays.asList("RELATION", "TABLE", "VIEW", "MATERIALIZED_VIEW", "EXTERNAL_TABLE", "SYNONYM",
259                        "TEMP_TABLE", "TABLE_VARIABLE", "CTE").contains(endpointKind)) {
260                    return "TABLE";
261                }
262                return "SEQUENCE".equals(endpointKind) ? "SEQUENCE" : "PARAMETER".equals(endpointKind) ? "PARAMETER" : null;
263            }
264            return role instanceof String ? (String) role : null; // CATALOG, SCHEMA, PACKAGE
265        }
266
267        String truncate(String key) {
268            Object limit = rules.get("truncation");
269            if (!(limit instanceof Map<?, ?>)) {
270                return key;
271            }
272            int max = ((Number) ((Map<?, ?>) limit).get("maxLength")).intValue();
273            byte[] bytes = key.getBytes(StandardCharsets.UTF_8);
274            if (bytes.length <= max) {
275                return key;
276            }
277            try {
278                return StandardCharsets.UTF_8.newDecoder().onMalformedInput(CodingErrorAction.IGNORE)
279                        .onUnmappableCharacter(CodingErrorAction.IGNORE)
280                        .decode(ByteBuffer.wrap(bytes, 0, max)).toString();
281            } catch (java.nio.charset.CharacterCodingException e) {
282                throw new IllegalStateException(e);
283            }
284        }
285
286        private static boolean isAscii(String text) {
287            for (int i = 0; i < text.length(); i++) {
288                if (text.charAt(i) >= 0x80) {
289                    return false;
290                }
291            }
292            return true;
293        }
294
295        private static String fold(String fold, String text) {
296            if ("UPPER".equals(fold) || "UPPER_SIMPLE".equals(fold)) {
297                return text.toUpperCase(Locale.ROOT); // ASCII text only reaches here
298            }
299            if ("LOWER_SIMPLE".equals(fold) || "LOWER_ASCII".equals(fold)) {
300                return text.toLowerCase(Locale.ROOT);
301            }
302            return null;
303        }
304
305        private List<Object> collationKey(Map<?, ?> cell, String text) {
306            Object name = collations.get(cell.get("collationScope"));
307            Object namesValue = rules.get("collations");
308            Object label = namesValue instanceof Map<?, ?> ? ((Map<?, ?>) namesValue).get(name) : null;
309            if (label == null) {
310                return Arrays.<Object>asList("UNAVAILABLE", "COLLATION");
311            }
312            Map<?, ?> profile = (Map<?, ?>) ((Map<?, ?>) rules.get("collationProfiles")).get(label);
313            List<long[]> ranges = new ArrayList<long[]>();
314            for (Object range : (List<?>) profile.get("repertoire")) {
315                List<?> pair = (List<?>) range;
316                ranges.add(new long[] {Long.parseLong((String) pair.get(0), 16), Long.parseLong((String) pair.get(1), 16)});
317            }
318            int last = -1;
319            for (int i = 0; i < text.length(); ) {
320                int cp = text.codePointAt(i);
321                boolean inside = false;
322                for (long[] range : ranges) {
323                    inside |= range[0] <= cp && cp <= range[1];
324                }
325                if (!inside) {
326                    return Arrays.<Object>asList("UNAVAILABLE", "COLLATION");
327                }
328                last = cp;
329                i += Character.charCount(cp);
330            }
331            Object excluded = profile.get("excludedLast");
332            if (last >= 0 && excluded instanceof List<?>
333                    && ((List<?>) excluded).contains(String.format(Locale.ROOT, "%04X", last))) {
334                return Arrays.<Object>asList("UNAVAILABLE", "COLLATION");
335            }
336            boolean simpleFold = "SIMPLE_FOLD".equals(profile.get("key"));
337            if (simpleFold && !isAscii(text)) {
338                return null;
339            }
340            return Arrays.<Object>asList("KEY", truncate(simpleFold ? text.toLowerCase(Locale.ROOT) : text));
341        }
342
343        List<Object> key(Map<?, ?> segment, String objectType) {
344            Map<?, ?> cell = cell(objectType);
345            Object textValue = segment.get("text");
346            if (cell == null || !(textValue instanceof String)) {
347                return null;
348            }
349            String text = (String) textValue;
350            Map<?, ?> unquoted = (Map<?, ?>) cell.get("unquoted");
351            Map<?, ?> quoted = (Map<?, ?>) cell.get("quoted");
352            if ("COLLATION".equals(unquoted.get("compare"))) {
353                return collationKey(cell, text);
354            }
355            if (!isAscii(text)) {
356                return null;
357            }
358            if ("SQL_LEXICAL".equals(segment.get("inputForm"))) {
359                Map<?, ?> branch = "QUOTED".equals(segment.get("quoteState")) ? quoted : unquoted;
360                String k = "NONE".equals(branch.get("fold")) ? text : fold((String) branch.get("fold"), text);
361                if (k == null) {
362                    throw new IllegalStateException("unknown fold " + branch.get("fold"));
363                }
364                if ("INSENSITIVE".equals(branch.get("compare")) && "NONE".equals(branch.get("fold"))) {
365                    k = k.toLowerCase(Locale.ROOT); // SIMPLE_FOLD on ASCII
366                }
367                return Arrays.<Object>asList("KEY", truncate(k));
368            }
369            if ("CATALOG_STORED".equals(segment.get("inputForm"))) {
370                if ("SENSITIVE".equals(quoted.get("compare"))) {
371                    return Arrays.<Object>asList("KEY", truncate(text));
372                }
373                String f = !"NONE".equals(unquoted.get("fold")) ? (String) unquoted.get("fold") : (String) quoted.get("fold");
374                String folded = fold(f, text);
375                return Arrays.<Object>asList("KEY", truncate(folded != null ? folded : text.toLowerCase(Locale.ROOT)));
376            }
377            return null;
378        }
379    }
380}