001package gudusoft.gsqlparser.lineage.v2.validation; 002 003import gudusoft.gsqlparser.lineage.v2.CoreRegistry; 004import gudusoft.gsqlparser.lineage.v2.Finding; 005import gudusoft.gsqlparser.lineage.v2.SqllrException; 006import gudusoft.gsqlparser.lineage.v2.json.CanonicalJson; 007 008import java.util.ArrayDeque; 009import java.util.ArrayList; 010import java.util.Arrays; 011import java.util.Collections; 012import java.util.Comparator; 013import java.util.Deque; 014import java.util.List; 015import java.util.Map; 016import java.util.regex.Matcher; 017import java.util.regex.Pattern; 018 019/** 020 * Validation stage {@code SCHEMA} (specification Section 14.2), as the reference runner implements it 021 * ({@code conformance/run.py}, {@code stage_schema}): 022 * <ol> 023 * <li>the strict reader of {@code canonical-json-v1} ({@code SCHEMA_INVALID_JSON}, 024 * {@code SCHEMA_DUPLICATE_KEY}, {@code SCHEMA_NON_INTEGER_NUMBER});</li> 025 * <li>the version gates: a {@code specVersion} of another major.minor is 026 * {@code VERSION_UNSUPPORTED_SPEC_VERSION}; a {@code header.registryVersion} at or above the supported 027 * range is {@code VERSION_REGISTRY_TOO_NEW};</li> 028 * <li>the JSON Schema: a value outside an enum that names a registry group, and that the group does not 029 * list, is {@code SCHEMA_UNKNOWN_CODE}; any other schema error is {@code SCHEMA_VIOLATION};</li> 030 * <li>the array registry in strict mode: an unregistered array is {@code SCHEMA_VIOLATION}; a SET holding 031 * equal elements or a SEQUENCE with broken ordinals ({@code SEMANTIC_ARRAY_KIND_VIOLATION}) is deferred 032 * to stage {@code SEMANTIC};</li> 033 * <li>canonical order: a SET array out of its registered order is {@code SCHEMA_NOT_CANONICAL}.</li> 034 * </ol> 035 * The stage reports the findings of the first step that has any. 036 * 037 * @since 4.2.17 038 */ 039public final class SchemaStage { 040 041 /** The supported specification major.minor. */ 042 private static final int[] SUPPORTED_SPEC = {0, 1}; 043 /** The supported registry range: at least the first, below the second. */ 044 private static final String[] SUPPORTED_REGISTRY = {"0.1.0-draft", "0.2.0"}; 045 046 private static final Pattern SEMVER = Pattern.compile( 047 "^(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)(?:-([0-9A-Za-z.-]+))?$", Pattern.UNIX_LINES); 048 private static final Pattern REGISTRY_GROUP = Pattern.compile("Registry group '([A-Za-z]+)'"); 049 050 /** The outcome of the stage. */ 051 public static final class Result { 052 private final Object document; 053 private final List<Finding> findings; 054 private final List<Finding> deferred; 055 056 Result(Object document, List<Finding> findings, List<Finding> deferred) { 057 this.document = document; 058 this.findings = Collections.unmodifiableList(findings); 059 this.deferred = Collections.unmodifiableList(deferred); 060 } 061 062 /** The parsed document, or null when the bytes are not JSON the reader accepts. */ 063 public Object getDocument() { 064 return document; 065 } 066 067 /** The findings of the stage; empty when the document passes it. */ 068 public List<Finding> getFindings() { 069 return findings; 070 } 071 072 /** Findings this stage detected that belong to stage {@code SEMANTIC}. */ 073 public List<Finding> getDeferred() { 074 return deferred; 075 } 076 } 077 078 private final SchemaValidator validator; 079 private final CoreRegistry registry; 080 081 public SchemaStage(SchemaValidator validator, CoreRegistry registry) { 082 this.validator = validator; 083 this.registry = registry; 084 } 085 086 /** The stage with the shipped schema and registry. */ 087 public static SchemaStage standard() { 088 return new SchemaStage(SchemaValidator.lineageDocument(), CoreRegistry.standard()); 089 } 090 091 public Result run(byte[] data) { 092 List<Finding> deferred = new ArrayList<Finding>(); 093 Object doc; 094 try { 095 doc = CanonicalJson.parse(data); 096 } catch (SqllrException e) { 097 return new Result(null, list(new Finding(e.getCode(), "$", e.getDetail())), deferred); 098 } 099 if (!(doc instanceof Map<?, ?>)) { 100 return new Result(doc, list(new Finding("SCHEMA_VIOLATION", "$", "the document is not a JSON object")), deferred); 101 } 102 Map<?, ?> map = (Map<?, ?>) doc; 103 List<Finding> findings = new ArrayList<Finding>(); 104 Object specVersion = map.get("specVersion"); 105 if (specVersion instanceof String) { 106 List<Object> key = semverKey((String) specVersion); 107 if (key != null && (!key.get(0).equals(java.math.BigInteger.valueOf(SUPPORTED_SPEC[0])) 108 || !key.get(1).equals(java.math.BigInteger.valueOf(SUPPORTED_SPEC[1])))) { 109 findings.add(new Finding("VERSION_UNSUPPORTED_SPEC_VERSION", "$.specVersion", (String) specVersion)); 110 } 111 } 112 Object header = map.get("header"); 113 Object registryVersion = header instanceof Map<?, ?> ? ((Map<?, ?>) header).get("registryVersion") : null; 114 if (registryVersion instanceof String) { 115 List<Object> key = semverKey((String) registryVersion); 116 if (key != null && SEMVER_ORDER.compare(key, semverKey(SUPPORTED_REGISTRY[1])) >= 0) { 117 findings.add(new Finding("VERSION_REGISTRY_TOO_NEW", "$.header.registryVersion", (String) registryVersion)); 118 } 119 } 120 if (!findings.isEmpty()) { 121 return new Result(doc, findings, deferred); 122 } 123 List<SchemaValidator.SchemaError> errors = validator.errors(doc); 124 if (!errors.isEmpty()) { 125 return new Result(doc, list(classify(errors)), deferred); 126 } 127 Object canonical; 128 try { 129 canonical = CanonicalJson.canonicalizeDocument(doc, true); 130 } catch (SqllrException e) { 131 if (CanonicalJson.SEMANTIC_ARRAY_KIND_VIOLATION.equals(e.getCode())) { 132 deferred.add(new Finding(e.getCode(), "$", e.getDetail())); 133 return new Result(doc, findings, deferred); 134 } 135 return new Result(doc, list(new Finding(e.getCode(), "$", e.getDetail())), deferred); 136 } 137 String where = firstDifference(doc, canonical); 138 if (where != null) { 139 findings.add(new Finding("SCHEMA_NOT_CANONICAL", where, "a SET array is not in its registered canonical order")); 140 } 141 return new Result(doc, findings, deferred); 142 } 143 144 private static List<Finding> list(Finding finding) { 145 List<Finding> out = new ArrayList<Finding>(); 146 out.add(finding); 147 return out; 148 } 149 150 /** 151 * The finding the stage reports for schema errors: {@code SCHEMA_UNKNOWN_CODE} when an enum naming a 152 * registry group rejects a string the group lacks (anywhere in the error tree), else {@code SCHEMA_VIOLATION}. 153 */ 154 public Finding classify(List<SchemaValidator.SchemaError> errors) { 155 List<Finding> unknown = new ArrayList<Finding>(); 156 Deque<SchemaValidator.SchemaError> stack = new ArrayDeque<SchemaValidator.SchemaError>(errors); 157 while (!stack.isEmpty()) { 158 SchemaValidator.SchemaError error = stack.pop(); 159 stack.addAll(error.getContext()); 160 if (!"enum".equals(error.getKeyword()) || !(error.getInstance() instanceof String)) { 161 continue; 162 } 163 Object description = error.getSchema().get("description"); 164 Matcher matcher = description instanceof String ? REGISTRY_GROUP.matcher((String) description) : null; 165 if (matcher != null && matcher.find() && !registry.group(matcher.group(1)).contains(error.getInstance())) { 166 unknown.add(new Finding("SCHEMA_UNKNOWN_CODE", location(error.getPath()), 167 error.getInstance() + " is not a code of registry group " + matcher.group(1))); 168 } 169 } 170 if (!unknown.isEmpty()) { 171 Collections.sort(unknown, new Comparator<Finding>() { 172 @Override 173 public int compare(Finding a, Finding b) { 174 int c = a.getLocation().compareTo(b.getLocation()); 175 return c != 0 ? c : a.getDetail().compareTo(b.getDetail()); 176 } 177 }); 178 return unknown.get(0); 179 } 180 SchemaValidator.SchemaError first = errors.get(0); 181 for (SchemaValidator.SchemaError error : errors) { 182 if (pathOrder(error.getPath(), first.getPath()) < 0) { 183 first = error; 184 } 185 } 186 return new Finding("SCHEMA_VIOLATION", location(first.getPath()), first.toString()); 187 } 188 189 private static int pathOrder(List<Object> a, List<Object> b) { 190 for (int i = 0; i < Math.min(a.size(), b.size()); i++) { 191 int c = String.valueOf(a.get(i)).compareTo(String.valueOf(b.get(i))); 192 if (c != 0) { 193 return c; 194 } 195 } 196 return a.size() - b.size(); 197 } 198 199 private static String location(List<Object> path) { 200 StringBuilder out = new StringBuilder("$"); 201 for (Object step : path) { 202 out.append(step instanceof String ? "['" + step + "']" : "[" + step + "]"); 203 } 204 return out.toString(); 205 } 206 207 /** Semantic Versioning 2.0.0 precedence as a comparable list, or null when the text is not a version. */ 208 static List<Object> semverKey(String text) { 209 Matcher m = SEMVER.matcher(text); 210 if (!m.lookingAt()) { 211 return null; 212 } 213 List<Object> key = new ArrayList<Object>(); 214 for (int i = 1; i <= 3; i++) { 215 key.add(new java.math.BigInteger(m.group(i))); 216 } 217 String pre = m.group(4); 218 List<Object> tail = new ArrayList<Object>(); 219 if (pre == null) { 220 tail.add(java.math.BigInteger.ONE); 221 } else { 222 tail.add(java.math.BigInteger.ZERO); 223 for (String part : pre.split("\\.", -1)) { 224 if (!part.isEmpty() && part.matches("[0-9]+")) { 225 tail.add(Arrays.<Object>asList(java.math.BigInteger.ZERO, new java.math.BigInteger(part), "")); 226 } else { 227 tail.add(Arrays.<Object>asList(java.math.BigInteger.ONE, java.math.BigInteger.ZERO, part)); 228 } 229 } 230 } 231 key.add(tail); 232 return key; 233 } 234 235 /** Lexicographic order of the keys above, as Python compares tuples. */ 236 static final Comparator<Object> SEMVER_ORDER = new Comparator<Object>() { 237 @Override 238 @SuppressWarnings({"unchecked", "rawtypes"}) 239 public int compare(Object a, Object b) { 240 if (a instanceof List && b instanceof List) { 241 List<?> x = (List<?>) a; 242 List<?> y = (List<?>) b; 243 for (int i = 0; i < Math.min(x.size(), y.size()); i++) { 244 int c = compare(x.get(i), y.get(i)); 245 if (c != 0) { 246 return c; 247 } 248 } 249 return x.size() - y.size(); 250 } 251 return ((Comparable) a).compareTo(b); 252 } 253 }; 254 255 /** 256 * Where {@code doc} first differs from its canonical form, or null ({@code run.py}, {@code first_difference}): 257 * members in the document's order, the first differing element of an array, and the array itself when the 258 * element holds no deeper difference. Members whose canonical bytes are equal hold no difference and are 259 * skipped, so the walk follows differing values only. 260 */ 261 static String firstDifference(Object doc, Object canonical) { 262 return difference(doc, canonical, "$"); 263 } 264 265 private static boolean sameBytes(Object a, Object b) { 266 return java.util.Arrays.equals(CanonicalJson.jcsBytes(a), CanonicalJson.jcsBytes(b)); 267 } 268 269 private static String difference(Object a, Object b, String path) { 270 if (a instanceof Map<?, ?> && b instanceof Map<?, ?>) { 271 for (Map.Entry<?, ?> entry : ((Map<?, ?>) a).entrySet()) { 272 Object other = ((Map<?, ?>) b).get(entry.getKey()); 273 if (sameBytes(entry.getValue(), other)) { 274 continue; 275 } 276 String found = difference(entry.getValue(), other, path + "." + entry.getKey()); 277 if (found != null) { 278 return found; 279 } 280 } 281 return null; 282 } 283 if (a instanceof List<?> && b instanceof List<?>) { 284 List<?> x = (List<?>) a; 285 List<?> y = (List<?>) b; 286 if (x.size() != y.size()) { 287 return path; 288 } 289 for (int i = 0; i < x.size(); i++) { 290 if (!sameBytes(x.get(i), y.get(i))) { 291 String here = path + "[" + i + "]"; 292 String inner = difference(x.get(i), y.get(i), here); 293 return inner != null && !inner.equals(here) ? inner : path; 294 } 295 } 296 return null; 297 } 298 return sameBytes(a, b) ? null : path; 299 } 300}