001package gudusoft.gsqlparser.demos.prettyscore; 002 003import gudusoft.gsqlparser.EDbVendor; 004import gudusoft.gsqlparser.pp.score.Dimension; 005import gudusoft.gsqlparser.pp.score.Finding; 006import gudusoft.gsqlparser.pp.score.PrettyScore; 007import gudusoft.gsqlparser.pp.score.ScoreProfile; 008import gudusoft.gsqlparser.pp.score.ScoreReport; 009import gudusoft.gsqlparser.pp.score.json.JsonIn; 010import gudusoft.gsqlparser.pp.score.json.JsonOut; 011 012import java.io.File; 013import java.io.IOException; 014import java.io.PrintStream; 015import java.nio.charset.Charset; 016import java.nio.file.Files; 017import java.util.ArrayList; 018import java.util.Arrays; 019import java.util.LinkedHashMap; 020import java.util.List; 021import java.util.Locale; 022import java.util.Map; 023 024/** 025 * PrettyScore command line. 026 * 027 * <pre> 028 * # single file, full explanation 029 * PrettyScoreMain --vendor mssql --profile isf-default --text formatted.sql 030 * # with the original (fidelity gate) and the formatter options used 031 * PrettyScoreMain --vendor oracle --original raw.sql --options fmt.json formatted.sql 032 * # several outputs of one input, side by side 033 * PrettyScoreMain --vendor mssql --compare isf.sql pp.sql pp2.sql 034 * # corpus run, baseline, regression check 035 * PrettyScoreMain --vendor mysql --corpus gsp_benchmark/corpus/mysql.manifest --sqldir gsp_java_core/gsp_sqlfiles \ 036 * --engine pp,pp2 --degrade 20 --baseline out/prettyscore-mysql.json 037 * PrettyScoreMain --vendor mysql --corpus ... --diff-baseline out/prettyscore-mysql.json 038 * # closed loop: gold anchoring, backlog, finding-level diff, blind acceptance sample 039 * PrettyScoreMain --vendor mssql --gold gold/mssql/ --min-gold-ratio 0.95 040 * PrettyScoreMain --vendor mssql --corpus ... --engine pp2 --backlog out/backlog-mssql.md 041 * PrettyScoreMain --diff-run out/run-a.json out/run-b.json 042 * PrettyScoreMain --acceptance-sample out/run-a.json out/run-b.json --n 20 --out accept/ 043 * </pre> 044 * 045 * Exit codes: 0 ok; 1 usage / IO error; 2 fidelity gate failed (with 046 * {@code --original}); 3 baseline regression; 4 gold anchoring failed. 047 */ 048public final class PrettyScoreMain { 049 050 private PrettyScoreMain() { 051 } 052 053 public static void main(String[] args) { 054 int rc; 055 try { 056 rc = run(args, System.out, System.err); 057 } catch (Exception e) { 058 System.err.println("prettyscore: " + e.getMessage()); 059 rc = 1; 060 } 061 System.exit(rc); 062 } 063 064 /** Parsed command line. */ 065 static final class Args { 066 String vendor; 067 String profile = ScoreProfile.ISF_DEFAULT; 068 String original; 069 String options; 070 String explain = "full"; 071 boolean text; 072 String out; 073 List<String> files = new ArrayList<String>(); 074 List<String> compare; 075 String corpus; 076 String sqldir = "."; 077 List<String> engines = Arrays.asList("pp", "pp2"); 078 int degrade = 20; 079 long seed = 20260902L; 080 String baseline; 081 String diffBaseline; 082 double maxDrop = 0.01; 083 String gold; 084 double minGoldRatio = 0.95; 085 String backlog; 086 List<String> diffRun; 087 List<String> acceptance; 088 int n = 20; 089 boolean keepText = true; 090 boolean ast = true; 091 } 092 093 static Args parse(String[] argv) { 094 Args a = new Args(); 095 for (int i = 0; i < argv.length; i++) { 096 String s = argv[i]; 097 if (!s.startsWith("--")) { a.files.add(s); continue; } 098 String name = s.substring(2); 099 if (name.equals("text")) { a.text = true; continue; } 100 if (name.equals("no-degrade")) { a.degrade = 0; continue; } 101 if (name.equals("no-text")) { a.keepText = false; continue; } 102 if (name.equals("no-ast")) { a.ast = false; continue; } 103 if (name.equals("help")) { a.files.clear(); a.vendor = "?"; return a; } 104 if (name.equals("compare")) { a.compare = rest(argv, i + 1); break; } 105 if (name.equals("diff-run")) { a.diffRun = rest(argv, i + 1); i += 2; continue; } 106 if (name.equals("acceptance-sample")) { a.acceptance = Arrays.asList(argv[i + 1], argv[i + 2]); i += 2; continue; } 107 if (i + 1 >= argv.length) throw new IllegalArgumentException("missing value for --" + name); 108 String v = argv[++i]; 109 if (name.equals("vendor")) a.vendor = v; 110 else if (name.equals("profile")) a.profile = v; 111 else if (name.equals("original")) a.original = v; 112 else if (name.equals("options")) a.options = v; 113 else if (name.equals("explain")) a.explain = v; 114 else if (name.equals("out")) a.out = v; 115 else if (name.equals("corpus")) a.corpus = v; 116 else if (name.equals("sqldir")) a.sqldir = v; 117 else if (name.equals("engine")) a.engines = Arrays.asList(v.split(",")); 118 else if (name.equals("degrade")) a.degrade = Integer.parseInt(v); 119 else if (name.equals("seed")) a.seed = Long.parseLong(v); 120 else if (name.equals("baseline")) a.baseline = v; 121 else if (name.equals("diff-baseline")) a.diffBaseline = v; 122 else if (name.equals("max-drop")) a.maxDrop = Double.parseDouble(v); 123 else if (name.equals("gold")) a.gold = v; 124 else if (name.equals("min-gold-ratio")) a.minGoldRatio = Double.parseDouble(v); 125 else if (name.equals("backlog")) a.backlog = v; 126 else if (name.equals("n")) a.n = Integer.parseInt(v); 127 else throw new IllegalArgumentException("unknown option --" + name); 128 } 129 return a; 130 } 131 132 private static List<String> rest(String[] argv, int from) { 133 List<String> l = new ArrayList<String>(); 134 for (int i = from; i < argv.length && !argv[i].startsWith("--"); i++) l.add(argv[i]); 135 return l; 136 } 137 138 static EDbVendor vendor(String s) { 139 if (s == null) return EDbVendor.dbvoracle; 140 EDbVendor v = EDbVendor.fromAlias(s); 141 if (v == null) { 142 try { 143 v = EDbVendor.valueOf(s.startsWith("dbv") ? s : "dbv" + s.toLowerCase(Locale.ROOT)); 144 } catch (IllegalArgumentException e) { 145 throw new IllegalArgumentException("unknown vendor: " + s); 146 } 147 } 148 return v; 149 } 150 151 public static int run(String[] argv, PrintStream out, PrintStream err) throws IOException { 152 Args a; 153 try { 154 a = parse(argv); 155 } catch (RuntimeException e) { 156 err.println("prettyscore: " + e.getMessage()); 157 usage(err); 158 return 1; 159 } 160 if ("?".equals(a.vendor)) { usage(out); return 0; } 161 if (a.diffRun != null) { 162 if (a.diffRun.size() < 2) { err.println("--diff-run needs two run files"); return 1; } 163 RunTools.Diff d = RunTools.diffRuns(RunTools.loadRun(new File(a.diffRun.get(0))), RunTools.loadRun(new File(a.diffRun.get(1)))); 164 out.print(d.text); 165 return 0; 166 } 167 if (a.acceptance != null) { 168 int n = RunTools.acceptanceSample(RunTools.loadRun(new File(a.acceptance.get(0))), RunTools.loadRun(new File(a.acceptance.get(1))), 169 a.n, a.seed, new File(a.out == null ? "accept" : a.out)); 170 out.println("wrote " + n + " anonymised A/B pairs to " + (a.out == null ? "accept" : a.out) + " (key.csv maps sides to runs)"); 171 return 0; 172 } 173 if (a.vendor == null) { err.println("prettyscore: --vendor is required"); usage(err); return 1; } 174 EDbVendor vendor = vendor(a.vendor); 175 ScoreProfile profile = ScoreProfile.load(a.profile); 176 if (a.gold != null) return gold(a, vendor, profile, out, err); 177 if (a.corpus != null) return corpus(a, vendor, profile, out, err); 178 if (a.compare != null) return compare(a, vendor, profile, out); 179 if (a.files.isEmpty()) { usage(err); return 1; } 180 return single(a, vendor, profile, out, err); 181 } 182 183 private static int single(Args a, EDbVendor vendor, ScoreProfile profile, PrintStream out, PrintStream err) throws IOException { 184 String sql = CorpusRunner.readText(new File(a.files.get(0))); 185 String original = a.original == null ? null : CorpusRunner.readText(new File(a.original)); 186 Map<String, Object> options = a.options == null ? null : loadOptions(new File(a.options)); 187 ScoreReport r = new PrettyScore().setAstEnabled(a.ast).score(sql, vendor, profile, original, options); 188 String text = a.text ? r.toText() : r.toJson("summary".equals(a.explain), true); 189 if (a.out != null) Files.write(new File(a.out).toPath(), text.getBytes(Charset.forName("UTF-8"))); 190 else out.print(text); 191 return r.isGateEvaluated() && !r.isGatePassed() ? 2 : 0; 192 } 193 194 /** Options JSON accepts GFmtOpt field names and the ISF page names. */ 195 static Map<String, Object> loadOptions(File f) throws IOException { 196 Map<String, Object> raw = JsonIn.parseObject(CorpusRunner.readText(f)); 197 Map<String, Object> m = new LinkedHashMap<String, Object>(); 198 for (Map.Entry<String, Object> e : raw.entrySet()) { 199 String k = e.getKey(); 200 String lk = k.toLowerCase(Locale.ROOT); 201 if (lk.equals("keywordcs")) k = "caseKeywords"; 202 else if (lk.equals("andorunderwhere")) k = "andOrUnderWhere"; 203 else if (lk.equals("liststyle")) k = "selectColumnlistStyle"; 204 else if (lk.equals("lnbrwithcomma")) k = "selectColumnlistComma"; 205 else if (lk.equals("indent") || lk.equals("indentlen")) k = "indentLen"; 206 m.put(k, e.getValue()); 207 } 208 return m; 209 } 210 211 private static int compare(Args a, EDbVendor vendor, ScoreProfile profile, PrintStream out) throws IOException { 212 List<ScoreReport> reports = new ArrayList<ScoreReport>(); 213 String original = a.original == null ? null : CorpusRunner.readText(new File(a.original)); 214 boolean gateFailed = false; 215 for (String f : a.compare) { 216 ScoreReport r = new PrettyScore().setAstEnabled(a.ast).score(CorpusRunner.readText(new File(f)), vendor, profile, original, null); 217 if (r.isGateEvaluated() && !r.isGatePassed()) gateFailed = true; 218 reports.add(r); 219 } 220 StringBuilder sb = new StringBuilder(); 221 sb.append(String.format(Locale.ROOT, "%-12s", "dimension")); 222 for (String f : a.compare) sb.append(String.format(Locale.ROOT, " %14s", new File(f).getName())); 223 sb.append('\n'); 224 sb.append(String.format(Locale.ROOT, "%-12s", "score")); 225 for (ScoreReport r : reports) sb.append(String.format(Locale.ROOT, " %14.1f", r.getScore())); 226 sb.append('\n'); 227 sb.append(String.format(Locale.ROOT, "%-12s", "ratio")); 228 for (ScoreReport r : reports) sb.append(String.format(Locale.ROOT, " %14.3f", r.getRatio())); 229 sb.append('\n'); 230 for (int d = 0; d < reports.get(0).getDimensions().size(); d++) { 231 sb.append(String.format(Locale.ROOT, "%-12s", reports.get(0).getDimensions().get(d).getId())); 232 for (ScoreReport r : reports) { 233 Dimension dim = r.getDimensions().get(d); 234 sb.append(String.format(Locale.ROOT, " %14.1f", dim.getScore())); 235 } 236 sb.append('\n'); 237 } 238 sb.append("decisive findings (rules present in some outputs but not others):\n"); 239 Map<String, boolean[]> rules = new LinkedHashMap<String, boolean[]>(); 240 for (int i = 0; i < reports.size(); i++) { 241 for (Finding f : reports.get(i).getFindings()) { 242 if (f.isShadowed() || f.getImpact() <= 0) continue; 243 boolean[] p = rules.get(f.getRule()); 244 if (p == null) { p = new boolean[reports.size()]; rules.put(f.getRule(), p); } 245 p[i] = true; 246 } 247 } 248 for (Map.Entry<String, boolean[]> e : rules.entrySet()) { 249 boolean all = true, none = true; 250 for (boolean b : e.getValue()) { all &= b; none &= !b; } 251 if (all || none) continue; 252 sb.append(String.format(Locale.ROOT, " %-24s", e.getKey())); 253 for (boolean b : e.getValue()) sb.append(b ? " x " : " - "); 254 sb.append('\n'); 255 } 256 out.print(sb); 257 return gateFailed ? 2 : 0; 258 } 259 260 private static int corpus(Args a, EDbVendor vendor, ScoreProfile profile, PrintStream out, PrintStream err) throws IOException { 261 File manifest = new File(a.corpus); 262 File sqldir = new File(a.sqldir); 263 List<File> files = CorpusRunner.readManifest(manifest, sqldir); 264 CorpusRunner runner = new CorpusRunner(vendor, profile, a.keepText, a.ast); 265 Map<String, Object> run = runner.run(files, a.engines, a.degrade, a.seed, sqldir); 266 if (a.baseline != null) { 267 File bf = new File(a.baseline); 268 if (bf.getParentFile() != null) bf.getParentFile().mkdirs(); 269 Files.write(bf.toPath(), JsonOut.toJson(run, true).getBytes(Charset.forName("UTF-8"))); 270 out.println("wrote " + bf); 271 } 272 StringBuilder sb = new StringBuilder(); 273 RunTools.summaryLine(sb, vendor.name() + "/" + profile.getName(), run); 274 out.print(sb); 275 if (a.backlog != null) { 276 String md = RunTools.backlog(run, a.engines.size() == 1 ? a.engines.get(0) : null); 277 File bf = new File(a.backlog); 278 if (bf.getParentFile() != null) bf.getParentFile().mkdirs(); 279 Files.write(bf.toPath(), md.getBytes(Charset.forName("UTF-8"))); 280 out.println("wrote " + bf); 281 } 282 if (a.out != null && a.baseline == null) { 283 Files.write(new File(a.out).toPath(), JsonOut.toJson(run, true).getBytes(Charset.forName("UTF-8"))); 284 out.println("wrote " + a.out); 285 } 286 if (a.diffBaseline != null) { 287 Map<String, Object> base = RunTools.loadRun(new File(a.diffBaseline)); 288 List<String> problems = RunTools.checkBaseline(base, run, a.maxDrop); 289 RunTools.Diff d = RunTools.diffRuns(base, run); 290 out.print(d.text); 291 if (!problems.isEmpty()) { 292 for (String p : problems) err.println("REGRESSION: " + p); 293 return 3; 294 } 295 out.println("baseline check passed"); 296 } 297 return 0; 298 } 299 300 /** 301 * Gold anchoring: every {@code *.gold.sql} in the directory is scored (with 302 * the matching {@code *.sql} as original when present); each must reach 303 * {@code --min-gold-ratio} both as ratio and as score / 100, otherwise the 304 * SCORER's rules are wrong. 305 */ 306 private static int gold(Args a, EDbVendor vendor, ScoreProfile profile, PrintStream out, PrintStream err) throws IOException { 307 File dir = new File(a.gold); 308 File[] golds = dir.listFiles(); 309 if (golds == null) { err.println("not a directory: " + dir); return 1; } 310 Arrays.sort(golds); 311 int failed = 0, total = 0; 312 PrettyScore scorer = new PrettyScore().setAstEnabled(a.ast); 313 for (File g : golds) { 314 if (!g.getName().endsWith(".gold.sql")) continue; 315 total++; 316 File orig = new File(dir, g.getName().substring(0, g.getName().length() - ".gold.sql".length()) + ".sql"); 317 String original = orig.isFile() ? CorpusRunner.readText(orig) : null; 318 ScoreReport r = scorer.score(CorpusRunner.readText(g), vendor, profile, original, null); 319 // ratio is 1.0 whenever nothing is unfixable, so gold is anchored on the score itself (min x 100) as well 320 boolean ok = r.getRatio() >= a.minGoldRatio && r.getScore() >= a.minGoldRatio * 100.0 && r.isGatePassed(); 321 if (!ok) failed++; 322 out.println(String.format(Locale.ROOT, "%s %-40s score %5.1f attainable %5.1f ratio %.3f gate %s", 323 ok ? "PASS" : "FAIL", g.getName(), r.getScore(), r.getAttainable(), r.getRatio(), r.isGatePassed() ? "ok" : "FAILED")); 324 if (!ok) { 325 for (Finding f : r.getFindings()) { 326 if (f.isShadowed() || f.getImpact() <= 0) continue; 327 out.println(String.format(Locale.ROOT, " %-24s L%-4d %6.1f %s", f.getRule(), f.getLine(), -f.getImpact(), f.getEvidence())); 328 } 329 } 330 } 331 out.println(String.format(Locale.ROOT, "gold: %d/%d at ratio >= %.2f", total - failed, total, a.minGoldRatio)); 332 if (total == 0) { err.println("prettyscore: no *.gold.sql files in " + dir); return 4; } 333 return failed == 0 ? 0 : 4; 334 } 335 336 static void usage(PrintStream p) { 337 p.println("usage: PrettyScoreMain --vendor <v> [--profile isf-default|river|compact|file.json] [--original raw.sql]"); 338 p.println(" [--options fmt.json] [--explain summary|full] [--text] [--no-ast] [--out report.json] formatted.sql"); 339 p.println(" PrettyScoreMain --vendor <v> [--original raw.sql] --compare a.sql b.sql ..."); 340 p.println(" PrettyScoreMain --vendor <v> --corpus <manifest> --sqldir <root> [--engine pp,pp2] [--degrade N|--no-degrade]"); 341 p.println(" [--seed S] [--baseline out.json] [--diff-baseline base.json [--max-drop 0.01]] [--backlog out.md] [--no-text]"); 342 p.println(" PrettyScoreMain --vendor <v> --gold <dir> [--min-gold-ratio 0.95]"); 343 p.println(" PrettyScoreMain --diff-run a.json b.json"); 344 p.println(" PrettyScoreMain --acceptance-sample a.json b.json [--n 20] [--out dir] [--seed S]"); 345 p.println("exit: 0 ok, 1 usage/io, 2 gate failed, 3 baseline regression, 4 gold anchoring failed"); 346 } 347}