001package gudusoft.gsqlparser.lineage.v2.validation;
002
003import gudusoft.gsqlparser.lineage.v2.CoreRegistry;
004import gudusoft.gsqlparser.lineage.v2.Finding;
005import gudusoft.gsqlparser.lineage.v2.SqllrException;
006import gudusoft.gsqlparser.lineage.v2.json.CanonicalJson;
007
008import java.util.ArrayDeque;
009import java.util.ArrayList;
010import java.util.Arrays;
011import java.util.Collections;
012import java.util.Comparator;
013import java.util.Deque;
014import java.util.List;
015import java.util.Map;
016import java.util.regex.Matcher;
017import java.util.regex.Pattern;
018
019/**
020 * Validation stage {@code SCHEMA} (specification Section 14.2), as the reference runner implements it
021 * ({@code conformance/run.py}, {@code stage_schema}):
022 * <ol>
023 *   <li>the strict reader of {@code canonical-json-v1} ({@code SCHEMA_INVALID_JSON},
024 *       {@code SCHEMA_DUPLICATE_KEY}, {@code SCHEMA_NON_INTEGER_NUMBER});</li>
025 *   <li>the version gates: a {@code specVersion} of another major.minor is
026 *       {@code VERSION_UNSUPPORTED_SPEC_VERSION}; a {@code header.registryVersion} at or above the supported
027 *       range is {@code VERSION_REGISTRY_TOO_NEW};</li>
028 *   <li>the JSON Schema: a value outside an enum that names a registry group, and that the group does not
029 *       list, is {@code SCHEMA_UNKNOWN_CODE}; any other schema error is {@code SCHEMA_VIOLATION};</li>
030 *   <li>the array registry in strict mode: an unregistered array is {@code SCHEMA_VIOLATION}; a SET holding
031 *       equal elements or a SEQUENCE with broken ordinals ({@code SEMANTIC_ARRAY_KIND_VIOLATION}) is deferred
032 *       to stage {@code SEMANTIC};</li>
033 *   <li>canonical order: a SET array out of its registered order is {@code SCHEMA_NOT_CANONICAL}.</li>
034 * </ol>
035 * The stage reports the findings of the first step that has any.
036 *
037 * @since 4.2.17
038 */
039public final class SchemaStage {
040
041    /** The supported specification major.minor. */
042    private static final int[] SUPPORTED_SPEC = {0, 1};
043    /** The supported registry range: at least the first, below the second. */
044    private static final String[] SUPPORTED_REGISTRY = {"0.1.0-draft", "0.2.0"};
045
046    private static final Pattern SEMVER = Pattern.compile(
047            "^(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)(?:-([0-9A-Za-z.-]+))?$", Pattern.UNIX_LINES);
048    private static final Pattern REGISTRY_GROUP = Pattern.compile("Registry group '([A-Za-z]+)'");
049
050    /** The outcome of the stage. */
051    public static final class Result {
052        private final Object document;
053        private final List<Finding> findings;
054        private final List<Finding> deferred;
055
056        Result(Object document, List<Finding> findings, List<Finding> deferred) {
057            this.document = document;
058            this.findings = Collections.unmodifiableList(findings);
059            this.deferred = Collections.unmodifiableList(deferred);
060        }
061
062        /** The parsed document, or null when the bytes are not JSON the reader accepts. */
063        public Object getDocument() {
064            return document;
065        }
066
067        /** The findings of the stage; empty when the document passes it. */
068        public List<Finding> getFindings() {
069            return findings;
070        }
071
072        /** Findings this stage detected that belong to stage {@code SEMANTIC}. */
073        public List<Finding> getDeferred() {
074            return deferred;
075        }
076    }
077
078    private final SchemaValidator validator;
079    private final CoreRegistry registry;
080
081    public SchemaStage(SchemaValidator validator, CoreRegistry registry) {
082        this.validator = validator;
083        this.registry = registry;
084    }
085
086    /** The stage with the shipped schema and registry. */
087    public static SchemaStage standard() {
088        return new SchemaStage(SchemaValidator.lineageDocument(), CoreRegistry.standard());
089    }
090
091    public Result run(byte[] data) {
092        List<Finding> deferred = new ArrayList<Finding>();
093        Object doc;
094        try {
095            doc = CanonicalJson.parse(data);
096        } catch (SqllrException e) {
097            return new Result(null, list(new Finding(e.getCode(), "$", e.getDetail())), deferred);
098        }
099        if (!(doc instanceof Map<?, ?>)) {
100            return new Result(doc, list(new Finding("SCHEMA_VIOLATION", "$", "the document is not a JSON object")), deferred);
101        }
102        Map<?, ?> map = (Map<?, ?>) doc;
103        List<Finding> findings = new ArrayList<Finding>();
104        Object specVersion = map.get("specVersion");
105        if (specVersion instanceof String) {
106            List<Object> key = semverKey((String) specVersion);
107            if (key != null && (!key.get(0).equals(java.math.BigInteger.valueOf(SUPPORTED_SPEC[0]))
108                    || !key.get(1).equals(java.math.BigInteger.valueOf(SUPPORTED_SPEC[1])))) {
109                findings.add(new Finding("VERSION_UNSUPPORTED_SPEC_VERSION", "$.specVersion", (String) specVersion));
110            }
111        }
112        Object header = map.get("header");
113        Object registryVersion = header instanceof Map<?, ?> ? ((Map<?, ?>) header).get("registryVersion") : null;
114        if (registryVersion instanceof String) {
115            List<Object> key = semverKey((String) registryVersion);
116            if (key != null && SEMVER_ORDER.compare(key, semverKey(SUPPORTED_REGISTRY[1])) >= 0) {
117                findings.add(new Finding("VERSION_REGISTRY_TOO_NEW", "$.header.registryVersion", (String) registryVersion));
118            }
119        }
120        if (!findings.isEmpty()) {
121            return new Result(doc, findings, deferred);
122        }
123        List<SchemaValidator.SchemaError> errors = validator.errors(doc);
124        if (!errors.isEmpty()) {
125            return new Result(doc, list(classify(errors)), deferred);
126        }
127        Object canonical;
128        try {
129            canonical = CanonicalJson.canonicalizeDocument(doc, true);
130        } catch (SqllrException e) {
131            if (CanonicalJson.SEMANTIC_ARRAY_KIND_VIOLATION.equals(e.getCode())) {
132                deferred.add(new Finding(e.getCode(), "$", e.getDetail()));
133                return new Result(doc, findings, deferred);
134            }
135            return new Result(doc, list(new Finding(e.getCode(), "$", e.getDetail())), deferred);
136        }
137        String where = firstDifference(doc, canonical);
138        if (where != null) {
139            findings.add(new Finding("SCHEMA_NOT_CANONICAL", where, "a SET array is not in its registered canonical order"));
140        }
141        return new Result(doc, findings, deferred);
142    }
143
144    private static List<Finding> list(Finding finding) {
145        List<Finding> out = new ArrayList<Finding>();
146        out.add(finding);
147        return out;
148    }
149
150    /**
151     * The finding the stage reports for schema errors: {@code SCHEMA_UNKNOWN_CODE} when an enum naming a
152     * registry group rejects a string the group lacks (anywhere in the error tree), else {@code SCHEMA_VIOLATION}.
153     */
154    public Finding classify(List<SchemaValidator.SchemaError> errors) {
155        List<Finding> unknown = new ArrayList<Finding>();
156        Deque<SchemaValidator.SchemaError> stack = new ArrayDeque<SchemaValidator.SchemaError>(errors);
157        while (!stack.isEmpty()) {
158            SchemaValidator.SchemaError error = stack.pop();
159            stack.addAll(error.getContext());
160            if (!"enum".equals(error.getKeyword()) || !(error.getInstance() instanceof String)) {
161                continue;
162            }
163            Object description = error.getSchema().get("description");
164            Matcher matcher = description instanceof String ? REGISTRY_GROUP.matcher((String) description) : null;
165            if (matcher != null && matcher.find() && !registry.group(matcher.group(1)).contains(error.getInstance())) {
166                unknown.add(new Finding("SCHEMA_UNKNOWN_CODE", location(error.getPath()),
167                        error.getInstance() + " is not a code of registry group " + matcher.group(1)));
168            }
169        }
170        if (!unknown.isEmpty()) {
171            Collections.sort(unknown, new Comparator<Finding>() {
172                @Override
173                public int compare(Finding a, Finding b) {
174                    int c = a.getLocation().compareTo(b.getLocation());
175                    return c != 0 ? c : a.getDetail().compareTo(b.getDetail());
176                }
177            });
178            return unknown.get(0);
179        }
180        SchemaValidator.SchemaError first = errors.get(0);
181        for (SchemaValidator.SchemaError error : errors) {
182            if (pathOrder(error.getPath(), first.getPath()) < 0) {
183                first = error;
184            }
185        }
186        return new Finding("SCHEMA_VIOLATION", location(first.getPath()), first.toString());
187    }
188
189    private static int pathOrder(List<Object> a, List<Object> b) {
190        for (int i = 0; i < Math.min(a.size(), b.size()); i++) {
191            int c = String.valueOf(a.get(i)).compareTo(String.valueOf(b.get(i)));
192            if (c != 0) {
193                return c;
194            }
195        }
196        return a.size() - b.size();
197    }
198
199    private static String location(List<Object> path) {
200        StringBuilder out = new StringBuilder("$");
201        for (Object step : path) {
202            out.append(step instanceof String ? "['" + step + "']" : "[" + step + "]");
203        }
204        return out.toString();
205    }
206
207    /** Semantic Versioning 2.0.0 precedence as a comparable list, or null when the text is not a version. */
208    static List<Object> semverKey(String text) {
209        Matcher m = SEMVER.matcher(text);
210        if (!m.lookingAt()) {
211            return null;
212        }
213        List<Object> key = new ArrayList<Object>();
214        for (int i = 1; i <= 3; i++) {
215            key.add(new java.math.BigInteger(m.group(i)));
216        }
217        String pre = m.group(4);
218        List<Object> tail = new ArrayList<Object>();
219        if (pre == null) {
220            tail.add(java.math.BigInteger.ONE);
221        } else {
222            tail.add(java.math.BigInteger.ZERO);
223            for (String part : pre.split("\\.", -1)) {
224                if (!part.isEmpty() && part.matches("[0-9]+")) {
225                    tail.add(Arrays.<Object>asList(java.math.BigInteger.ZERO, new java.math.BigInteger(part), ""));
226                } else {
227                    tail.add(Arrays.<Object>asList(java.math.BigInteger.ONE, java.math.BigInteger.ZERO, part));
228                }
229            }
230        }
231        key.add(tail);
232        return key;
233    }
234
235    /** Lexicographic order of the keys above, as Python compares tuples. */
236    static final Comparator<Object> SEMVER_ORDER = new Comparator<Object>() {
237        @Override
238        @SuppressWarnings({"unchecked", "rawtypes"})
239        public int compare(Object a, Object b) {
240            if (a instanceof List && b instanceof List) {
241                List<?> x = (List<?>) a;
242                List<?> y = (List<?>) b;
243                for (int i = 0; i < Math.min(x.size(), y.size()); i++) {
244                    int c = compare(x.get(i), y.get(i));
245                    if (c != 0) {
246                        return c;
247                    }
248                }
249                return x.size() - y.size();
250            }
251            return ((Comparable) a).compareTo(b);
252        }
253    };
254
255    /**
256     * Where {@code doc} first differs from its canonical form, or null ({@code run.py}, {@code first_difference}):
257     * members in the document's order, the first differing element of an array, and the array itself when the
258     * element holds no deeper difference. Members whose canonical bytes are equal hold no difference and are
259     * skipped, so the walk follows differing values only.
260     */
261    static String firstDifference(Object doc, Object canonical) {
262        return difference(doc, canonical, "$");
263    }
264
265    private static boolean sameBytes(Object a, Object b) {
266        return java.util.Arrays.equals(CanonicalJson.jcsBytes(a), CanonicalJson.jcsBytes(b));
267    }
268
269    private static String difference(Object a, Object b, String path) {
270        if (a instanceof Map<?, ?> && b instanceof Map<?, ?>) {
271            for (Map.Entry<?, ?> entry : ((Map<?, ?>) a).entrySet()) {
272                Object other = ((Map<?, ?>) b).get(entry.getKey());
273                if (sameBytes(entry.getValue(), other)) {
274                    continue;
275                }
276                String found = difference(entry.getValue(), other, path + "." + entry.getKey());
277                if (found != null) {
278                    return found;
279                }
280            }
281            return null;
282        }
283        if (a instanceof List<?> && b instanceof List<?>) {
284            List<?> x = (List<?>) a;
285            List<?> y = (List<?>) b;
286            if (x.size() != y.size()) {
287                return path;
288            }
289            for (int i = 0; i < x.size(); i++) {
290                if (!sameBytes(x.get(i), y.get(i))) {
291                    String here = path + "[" + i + "]";
292                    String inner = difference(x.get(i), y.get(i), here);
293                    return inner != null && !inner.equals(here) ? inner : path;
294                }
295            }
296            return null;
297        }
298        return sameBytes(a, b) ? null : path;
299    }
300}