001package gudusoft.gsqlparser.lineage.v2.json; 002 003import gudusoft.gsqlparser.lineage.v2.SqllrResources; 004 005import java.util.ArrayList; 006import java.util.Arrays; 007import java.util.Collections; 008import java.util.HashSet; 009import java.util.LinkedHashMap; 010import java.util.LinkedHashSet; 011import java.util.List; 012import java.util.Map; 013import java.util.Set; 014import java.util.SortedMap; 015import java.util.SortedSet; 016import java.util.TreeMap; 017import java.util.TreeSet; 018 019/** 020 * The array registry of {@code canonical-json-v1}, Section 6: the class ({@code SET}, {@code SEQUENCE}, 021 * {@code DISPLAY_ORDER}) and the sort key of every array position of the lineage document, read from the 022 * {@code x-sqllr-order} and {@code x-sqllr-sortKey} annotations of the schema, plus the positions of open 023 * content (Section 5, rule 5), where every array is a {@code SEQUENCE}. 024 * 025 * <p>The positions form a tree: {@link #root()} is the document, {@link Position#member(String)} a member 026 * and {@link Position#element()} any element of an array. A position outside the tree is unregistered.</p> 027 * 028 * <p>The derivation is the reference's {@code schema_array_registry}; the open-content positions are its 029 * {@code OPEN_CONTENT}, which the schema does not mark.</p> 030 * 031 * @since 4.2.17 032 */ 033public final class ArrayRegistry { 034 035 /** The registry group {@code arrayClasses}. */ 036 public enum Order { 037 SET, SEQUENCE, DISPLAY_ORDER 038 } 039 040 /** The sort key of a {@code SET} (Section 6.2). */ 041 public static final class SortKey { 042 /** What the key of an element is. */ 043 public enum Kind { 044 /** The element itself. */ 045 SELF, 046 /** The element's {@code ref} member. */ 047 REF, 048 /** Nothing beyond the canonical bytes of the element. */ 049 BYTES, 050 /** A tuple of members of the element. */ 051 MEMBERS 052 } 053 054 private final Kind kind; 055 private final List<String> members; 056 057 private SortKey(Kind kind, List<String> members) { 058 this.kind = kind; 059 this.members = members; 060 } 061 062 public Kind getKind() { 063 return kind; 064 } 065 066 /** The member names of a {@link Kind#MEMBERS} key; empty otherwise. */ 067 public List<String> getMembers() { 068 return members; 069 } 070 071 static SortKey of(Object annotation, String where) { 072 if ("self".equals(annotation)) { 073 return new SortKey(Kind.SELF, Collections.<String>emptyList()); 074 } 075 if ("ref".equals(annotation)) { 076 return new SortKey(Kind.REF, Collections.<String>emptyList()); 077 } 078 if ("bytes".equals(annotation)) { 079 return new SortKey(Kind.BYTES, Collections.<String>emptyList()); 080 } 081 if (annotation instanceof List<?> && !((List<?>) annotation).isEmpty()) { 082 List<String> names = new ArrayList<String>(); 083 for (Object name : (List<?>) annotation) { 084 if (!(name instanceof String)) { 085 throw new IllegalStateException("x-sqllr-sortKey of " + where + " is not a list of names"); 086 } 087 names.add((String) name); 088 } 089 return new SortKey(Kind.MEMBERS, Collections.unmodifiableList(names)); 090 } 091 throw new IllegalStateException("unknown x-sqllr-sortKey " + annotation + " at " + where); 092 } 093 094 @Override 095 public boolean equals(Object other) { 096 return other instanceof SortKey && ((SortKey) other).kind == kind && ((SortKey) other).members.equals(members); 097 } 098 099 @Override 100 public int hashCode() { 101 return kind.hashCode() * 31 + members.hashCode(); 102 } 103 104 /** {@code self}, {@code ref}, {@code bytes}, or {@code (a, b)}. */ 105 @Override 106 public String toString() { 107 switch (kind) { 108 case SELF: return "self"; 109 case REF: return "ref"; 110 case BYTES: return "bytes"; 111 default: 112 StringBuilder out = new StringBuilder("("); 113 for (int i = 0; i < members.size(); i++) { 114 out.append(i == 0 ? "" : ", ").append(members.get(i)); 115 } 116 return out.append(')').toString(); 117 } 118 } 119 } 120 121 /** The registration of one array position. */ 122 public static final class Registration { 123 /** Every array in open content. */ 124 public static final Registration OPEN_SEQUENCE = new Registration(Order.SEQUENCE, null); 125 126 private final Order order; 127 private final SortKey sortKey; 128 129 Registration(Order order, SortKey sortKey) { 130 this.order = order; 131 this.sortKey = sortKey; 132 } 133 134 public Order getOrder() { 135 return order; 136 } 137 138 /** The sort key of a {@code SET}; null for the other classes. */ 139 public SortKey getSortKey() { 140 return sortKey; 141 } 142 143 @Override 144 public boolean equals(Object other) { 145 if (!(other instanceof Registration)) { 146 return false; 147 } 148 Registration that = (Registration) other; 149 return order == that.order && (sortKey == null ? that.sortKey == null : sortKey.equals(that.sortKey)); 150 } 151 152 @Override 153 public int hashCode() { 154 return order.hashCode() * 31 + (sortKey == null ? 0 : sortKey.hashCode()); 155 } 156 157 /** {@code SET ref}, {@code SEQUENCE}. */ 158 @Override 159 public String toString() { 160 return sortKey == null ? order.name() : order.name() + " " + sortKey; 161 } 162 } 163 164 /** A position of the document: the root, a member of a position, or any element of an array position. */ 165 public static final class Position { 166 private final Map<String, Position> members = new LinkedHashMap<String, Position>(); 167 private Position element; 168 private Registration registration; 169 private boolean openContent; 170 171 private Position() { 172 } 173 174 /** The position of member {@code name}, or null when nothing is registered at or below it. */ 175 public Position member(String name) { 176 return members.get(name); 177 } 178 179 /** The position of any element of this array, or null when nothing is registered at or below it. */ 180 public Position element() { 181 return element; 182 } 183 184 /** The registration of the array at this position, or null. */ 185 public Registration registration() { 186 return registration; 187 } 188 189 /** True when the value at this position, and everything inside it, is open content. */ 190 public boolean isOpenContent() { 191 return openContent; 192 } 193 194 private Position child(String step) { 195 if (ELEMENT.equals(step)) { 196 if (element == null) { 197 element = new Position(); 198 } 199 return element; 200 } 201 Position child = members.get(step); 202 if (child == null) { 203 child = new Position(); 204 members.put(step, child); 205 } 206 return child; 207 } 208 } 209 210 /** The step for "any element" in the path constants of this class. */ 211 private static final String ELEMENT = "[]"; 212 213 /** The module collections whose element schemas are not defined yet (canonical-json-v1, Section 6.3). */ 214 public static final List<String> MODULE_ARRAYS = Collections.unmodifiableList(Arrays.asList( 215 "dynamicSqlSites", "evidence", "expressionSpans", "naryFacts", "refinements", "routineCalls", 216 "structuralSignatures", "typedSteps", "wildcardSelections")); 217 218 /** Open content other than {@code extensions} and the elements of {@link #MODULE_ARRAYS}. */ 219 private static final String[][] OPEN_CONTENT = { 220 {"merge"}, 221 {"reproducibilityManifest"}, 222 {"sourceUnits", ELEMENT, "originMap", ELEMENT}, 223 {"endpoints", ELEMENT, "pathSegments", ELEMENT}, 224 {"factCandidates", ELEMENT, "source", "candidateSet", "details"}, 225 {"factCandidates", ELEMENT, "target", "candidateSet", "details"}, 226 {"explainGraph", "candidateSets", ELEMENT, "details"}, 227 {"candidateSets", ELEMENT, "details"}, 228 }; 229 230 private final Position root; 231 private final SortedMap<String, Registration> registrations; 232 private final SortedSet<String> openContent; 233 234 private ArrayRegistry(Position root, SortedMap<String, Registration> registrations, SortedSet<String> openContent) { 235 this.root = root; 236 this.registrations = Collections.unmodifiableSortedMap(registrations); 237 this.openContent = Collections.unmodifiableSortedSet(openContent); 238 } 239 240 /** The position of the document root. */ 241 public Position root() { 242 return root; 243 } 244 245 /** Every registered array position, as a display path ({@code $.endpoints[].name.segments}). */ 246 public SortedMap<String, Registration> registrations() { 247 return registrations; 248 } 249 250 /** Every open-content position other than {@code extensions} members, as a display path. */ 251 public SortedSet<String> openContent() { 252 return openContent; 253 } 254 255 /** The registry of the schema shipped with this library. */ 256 public static ArrayRegistry standard() { 257 return Standard.INSTANCE; 258 } 259 260 private static final class Standard { 261 static final ArrayRegistry INSTANCE = fromSchema(SqllrResources.lineageDocumentSchema()); 262 } 263 264 /** 265 * Derives the registry from a lineage document schema (a parsed JSON value). 266 * 267 * @throws IllegalStateException when an array position has no class, two classes, or an unknown sort key 268 */ 269 public static ArrayRegistry fromSchema(Object schema) { 270 Map<?, ?> top = (Map<?, ?>) schema; 271 Map<?, ?> defs = (Map<?, ?>) top.get("$defs"); 272 Set<List<String>> open = new HashSet<List<String>>(); 273 for (String[] path : OPEN_CONTENT) { 274 open.add(Arrays.asList(path)); 275 } 276 for (String name : MODULE_ARRAYS) { 277 open.add(Arrays.asList(name, ELEMENT)); 278 } 279 Map<List<String>, Set<Registration>> found = new LinkedHashMap<List<String>, Set<Registration>>(); 280 new SchemaWalker(defs, open, found).walk(schema, new ArrayList<String>(), 0); 281 282 Position root = new Position(); 283 SortedMap<String, Registration> registrations = new TreeMap<String, Registration>(); 284 for (Map.Entry<List<String>, Set<Registration>> entry : found.entrySet()) { 285 String where = displayPath(entry.getKey()); 286 if (entry.getValue().size() != 1) { 287 throw new IllegalStateException("the schema registers " + where + " more than once: " + entry.getValue()); 288 } 289 Registration registration = entry.getValue().iterator().next(); 290 Position position = root; 291 for (String step : entry.getKey()) { 292 position = position.child(step); 293 } 294 position.registration = registration; 295 registrations.put(where, registration); 296 } 297 SortedSet<String> openPaths = new TreeSet<String>(); 298 for (List<String> path : open) { 299 Position position = root; 300 for (String step : path) { 301 position = position.child(step); 302 } 303 position.openContent = true; 304 openPaths.add(displayPath(path)); 305 } 306 return new ArrayRegistry(root, registrations, openPaths); 307 } 308 309 /** {@code $}, then {@code .name} per member and {@code []} per element, as the reference prints paths. */ 310 static String displayPath(List<String> path) { 311 StringBuilder out = new StringBuilder("$"); 312 for (String step : path) { 313 out.append(ELEMENT.equals(step) ? ELEMENT : "." + step); 314 } 315 return out.toString(); 316 } 317 318 /** The reference's {@code schema_array_registry}. */ 319 private static final class SchemaWalker { 320 private static final int MAX_DEPTH = 64; 321 private static final Set<String> REF_ONLY = new HashSet<String>(Arrays.asList("$ref", "description")); 322 323 private final Map<?, ?> defs; 324 private final Set<List<String>> open; 325 private final Map<List<String>, Set<Registration>> found; 326 327 SchemaWalker(Map<?, ?> defs, Set<List<String>> open, Map<List<String>, Set<Registration>> found) { 328 this.defs = defs; 329 this.open = open; 330 this.found = found; 331 } 332 333 private Object resolve(Object ref) { 334 String text = (String) ref; 335 if (!text.startsWith("#/$defs/") || !defs.containsKey(text.substring("#/$defs/".length()))) { 336 throw new IllegalStateException("unresolvable $ref " + text); 337 } 338 return defs.get(text.substring("#/$defs/".length())); 339 } 340 341 private Object deref(Object node) { 342 int guard = 0; 343 while (node instanceof Map<?, ?> && ((Map<?, ?>) node).containsKey("$ref") 344 && REF_ONLY.containsAll(((Map<?, ?>) node).keySet())) { 345 node = resolve(((Map<?, ?>) node).get("$ref")); 346 if (++guard > MAX_DEPTH) { 347 throw new IllegalStateException("$ref cycle"); 348 } 349 } 350 return node; 351 } 352 353 private void membersOf(Object node, List<Map.Entry<String, Object>> out, int depth) { 354 if (depth > MAX_DEPTH) { 355 throw new IllegalStateException("schema nesting too deep"); 356 } 357 node = deref(node); 358 if (!(node instanceof Map<?, ?>)) { 359 return; 360 } 361 Map<?, ?> map = (Map<?, ?>) node; 362 if (map.containsKey("$ref")) { 363 membersOf(resolve(map.get("$ref")), out, depth + 1); 364 } 365 Object properties = map.get("properties"); 366 if (properties instanceof Map<?, ?>) { 367 for (Map.Entry<?, ?> entry : ((Map<?, ?>) properties).entrySet()) { 368 out.add(new java.util.AbstractMap.SimpleImmutableEntry<String, Object>( 369 (String) entry.getKey(), entry.getValue())); 370 } 371 } 372 for (String key : new String[] {"allOf", "anyOf", "oneOf"}) { 373 Object subs = map.get(key); 374 if (subs instanceof List<?>) { 375 for (Object sub : (List<?>) subs) { 376 membersOf(sub, out, depth + 1); 377 } 378 } 379 } 380 for (String key : new String[] {"then", "else"}) { 381 if (map.containsKey(key)) { 382 membersOf(map.get(key), out, depth + 1); 383 } 384 } 385 } 386 387 void walk(Object node, List<String> path, int depth) { 388 if (depth > MAX_DEPTH) { 389 throw new IllegalStateException("schema nesting too deep at " + displayPath(path)); 390 } 391 node = deref(node); 392 if (!(node instanceof Map<?, ?>)) { 393 return; 394 } 395 Map<?, ?> map = (Map<?, ?>) node; 396 if ("array".equals(map.get("type")) || map.containsKey("x-sqllr-order")) { 397 String where = displayPath(path); 398 Object order = map.get("x-sqllr-order"); 399 if (!(order instanceof String)) { 400 throw new IllegalStateException("the array at " + where + " has no x-sqllr-order"); 401 } 402 Order kind = Order.valueOf((String) order); 403 Object key = map.get("x-sqllr-sortKey"); 404 SortKey sortKey = key == null ? null : SortKey.of(key, where); 405 if (kind == Order.SET && sortKey == null) { 406 throw new IllegalStateException("the SET at " + where + " has no x-sqllr-sortKey"); 407 } 408 List<String> at = new ArrayList<String>(path); 409 Set<Registration> registrations = found.get(at); 410 if (registrations == null) { 411 registrations = new LinkedHashSet<Registration>(); 412 found.put(at, registrations); 413 } 414 registrations.add(new Registration(kind, sortKey)); 415 if (map.containsKey("items")) { 416 path.add(ELEMENT); 417 walk(map.get("items"), path, depth + 1); 418 path.remove(path.size() - 1); 419 } 420 return; 421 } 422 if (open.contains(path)) { 423 return; 424 } 425 List<Map.Entry<String, Object>> members = new ArrayList<Map.Entry<String, Object>>(); 426 membersOf(map, members, 0); 427 for (Map.Entry<String, Object> member : members) { 428 if (!"extensions".equals(member.getKey())) { 429 path.add(member.getKey()); 430 walk(member.getValue(), path, depth + 1); 431 path.remove(path.size() - 1); 432 } 433 } 434 } 435 } 436}