Source code

001package gudusoft.gsqlparser.demos.prettyscore;
002
003import gudusoft.gsqlparser.EDbVendor;
004import gudusoft.gsqlparser.TGSqlParser;
005import gudusoft.gsqlparser.pp.para.GFmtOptFactory;
006import gudusoft.gsqlparser.pp.score.Dimension;
007import gudusoft.gsqlparser.pp.score.Finding;
008import gudusoft.gsqlparser.pp.score.PrettyScore;
009import gudusoft.gsqlparser.pp.score.ScoreProfile;
010import gudusoft.gsqlparser.pp.score.ScoreReport;
011import gudusoft.gsqlparser.pp.stmtformatter.FormatterFactory;
012
013import java.io.BufferedReader;
014import java.io.File;
015import java.io.FileInputStream;
016import java.io.IOException;
017import java.io.InputStreamReader;
018import java.nio.charset.Charset;
019import java.nio.file.Files;
020import java.util.ArrayList;
021import java.util.LinkedHashMap;
022import java.util.List;
023import java.util.Locale;
024import java.util.Map;
025
026/**
027 * Runs pp and/or pp2 over a benchmark manifest, scores every output with
028 * the fidelity gate on, optionally adds seeded syntax-degraded samples (pp2
029 * only), and produces the run record the CLI writes as JSON.
030 *
031 * <p>Run record layout (a published contract for {@code --diff-run} /
032 * {@code --diff-baseline} / {@code --backlog}):
033 * <pre>
034 * { "version": "prettyscore-run/0.1", "vendor", "profile", "engines": [...],
035 *   "entries": [ {file, engine, source: clean|degraded, degrade, parse_ok, gate_passed,
036 *                 score, attainable, ratio, dimensions: {id: score}, findings: [...], formatted} ],
037 *   "summary": { "<engine>": {files, scored, mean_score, mean_ratio, min_ratio, gate_failures, parse_failures},
038 *                "degraded": {samples, pp2_recovery_ratio, pp2_gate_failures} } }
039 * </pre>
040 */
041public final class CorpusRunner {
042
043    public static final String RUN_VERSION = "prettyscore-run/0.1";
044
045    private final EDbVendor vendor;
046    private final ScoreProfile profile;
047    private final PrettyScore scorer;
048    private final boolean keepText;
049
050    public CorpusRunner(EDbVendor vendor, ScoreProfile profile, boolean keepText) {
051        this(vendor, profile, keepText, true);
052    }
053
054    public CorpusRunner(EDbVendor vendor, ScoreProfile profile, boolean keepText, boolean astEnabled) {
055        this.vendor = vendor;
056        this.profile = profile;
057        this.keepText = keepText;
058        this.scorer = new PrettyScore().setAstEnabled(astEnabled);
059    }
060
061    /** Read a manifest: non-empty, non-comment lines are paths relative to {@code sqldir}. */
062    public static List<File> readManifest(File manifest, File sqldir) throws IOException {
063        List<File> out = new ArrayList<File>();
064        BufferedReader r = new BufferedReader(new InputStreamReader(new FileInputStream(manifest), Charset.forName("UTF-8")));
065        try {
066            String line;
067            while ((line = r.readLine()) != null) {
068                String t = line.trim();
069                if (t.isEmpty() || t.startsWith("#")) continue;
070                out.add(new File(sqldir, t));
071            }
072        } finally {
073            r.close();
074        }
075        return out;
076    }
077
078    public static String readText(File f) throws IOException {
079        return new String(Files.readAllBytes(f.toPath()), Charset.forName("UTF-8"));
080    }
081
082    /**
083     * @param files       the corpus files
084     * @param engines     "pp" and/or "pp2"
085     * @param degradeCount number of syntax-degraded samples to derive (0 = none); scored with pp2 only
086     * @param seed        degrader seed
087     */
088    public Map<String, Object> run(List<File> files, List<String> engines, int degradeCount, long seed, File relativeTo) {
089        Map<String, Object> run = new LinkedHashMap<String, Object>();
090        run.put("version", RUN_VERSION);
091        run.put("vendor", vendor.name());
092        run.put("profile", profile.getName());
093        run.put("calibration", profile.getCalibration());
094        run.put("engines", new ArrayList<String>(engines));
095        run.put("degraded_samples", degradeCount);
096        run.put("seed", seed);
097        run.put("ast", scorer.isAstEnabled());
098        List<Object> entries = new ArrayList<Object>();
099        List<String> texts = new ArrayList<String>();
100        List<String> names = new ArrayList<String>();
101        for (File f : files) {
102            String name = relativeTo == null ? f.getPath() : relativize(relativeTo, f);
103            String sql;
104            try {
105                sql = readText(f);
106            } catch (IOException e) {
107                entries.add(errorEntry(name, "-", "clean", null, "read failed: " + e.getMessage()));
108                continue;
109            }
110            texts.add(sql);
111            names.add(name);
112            for (String engine : engines) {
113                entries.add(scoreOne(name, engine, "clean", null, sql));
114            }
115        }
116        for (int i = 0; i < degradeCount && !texts.isEmpty(); i++) {
117            int k = i % texts.size();
118            SyntaxDegrader.Sample s = SyntaxDegrader.degrade(texts.get(k), vendor, seed, i);
119            if (s.stillParses) {
120                entries.add(errorEntry(names.get(k) + "#degraded" + i, "pp2", "degraded-skipped", s.kind.name().toLowerCase(Locale.ROOT) + ": " + s.description,
121                    "degradation still parses; excluded from the recovery corpus"));
122                continue;
123            }
124            entries.add(scoreOne(names.get(k) + "#degraded" + i, "pp2", "degraded", s.kind.name().toLowerCase(Locale.ROOT) + ": " + s.description, s.sql));
125        }
126        run.put("entries", entries);
127        run.put("summary", summarize(entries, engines));
128        return run;
129    }
130
131    static String relativize(File base, File f) {
132        String b = base.getAbsolutePath().replace('\\', '/');
133        String p = f.getAbsolutePath().replace('\\', '/');
134        if (p.startsWith(b)) {
135            p = p.substring(b.length());
136            if (p.startsWith("/")) p = p.substring(1);
137        }
138        return p;
139    }
140
141    /** Format {@code sql} with {@code engine} and score it against the original. */
142    public Map<String, Object> scoreOne(String name, String engine, String source, String degrade, String sql) {
143        String formatted;
144        boolean parseOk = true;
145        try {
146            TGSqlParser parser = new TGSqlParser(vendor);
147            parser.sqltext = sql;
148            if ("pp".equals(engine)) {
149                int rc = parser.parse();
150                if (rc != 0) {
151                    Map<String, Object> e = errorEntry(name, engine, source, degrade, "pp needs a clean parse: " + firstLine(parser.getErrormessage()));
152                    e.put("parse_ok", false);
153                    return e;
154                }
155                formatted = FormatterFactory.pp(parser, GFmtOptFactory.newInstance());
156            } else if ("pp2".equals(engine)) {
157                formatted = FormatterFactory.pp2(parser, GFmtOptFactory.newInstance());
158            } else if ("none".equals(engine)) {
159                formatted = sql;
160            } else {
161                return errorEntry(name, engine, source, degrade, "unknown engine " + engine);
162            }
163        } catch (Throwable t) {
164            return errorEntry(name, engine, source, degrade, "formatter threw " + t.getClass().getSimpleName() + ": " + firstLine(t.getMessage()));
165        }
166        try {
167            ScoreReport r = scorer.score(formatted, vendor, profile, sql, null);
168            return entry(name, engine, source, degrade, r, formatted);
169        } catch (Throwable t) {
170            return errorEntry(name, engine, source, degrade, "scorer threw " + t.getClass().getSimpleName() + ": " + firstLine(t.getMessage()));
171        }
172    }
173
174    private Map<String, Object> entry(String name, String engine, String source, String degrade, ScoreReport r, String formatted) {
175        Map<String, Object> e = new LinkedHashMap<String, Object>();
176        e.put("file", name);
177        e.put("engine", engine);
178        e.put("source", source);
179        e.put("degrade", degrade);
180        e.put("parse_evaluated", r.getParse().isEvaluated());
181        e.put("parse_ok", r.getParse().isEvaluated() && r.getParse().isOk());
182        e.put("gate_passed", r.isGatePassed());
183        e.put("score", r.getScore());
184        e.put("attainable", r.getAttainable());
185        e.put("ratio", r.getRatio());
186        Map<String, Object> dims = new LinkedHashMap<String, Object>();
187        Map<String, Double> weights = new LinkedHashMap<String, Double>();
188        for (Dimension d : r.getDimensions()) {
189            dims.put(d.getId(), d.getScore());
190            weights.put(d.getId(), d.getWeight());
191        }
192        e.put("dimensions", dims);
193        List<Object> fs = new ArrayList<Object>();
194        for (Finding f : r.getFindings()) {
195            if (f.isShadowed() || f.getImpact() <= 0) continue;
196            Map<String, Object> fm = new LinkedHashMap<String, Object>();
197            fm.put("rule", f.getRule());
198            fm.put("fingerprint", f.getFingerprint());
199            fm.put("line", f.getLine());
200            fm.put("col", f.getCol());
201            fm.put("severity", f.getSeverity().wireName());
202            fm.put("fixable", f.isFixable());
203            fm.put("impact", f.getImpact());
204            Double w = weights.get(f.getDimension());
205            fm.put("gain", f.getImpact() * (w == null ? 0 : w));
206            String ev = f.getEvidence();
207            fm.put("evidence", keepText || ev.length() <= 80 ? ev : ev.substring(0, 77) + "...");
208            fs.add(fm);
209        }
210        e.put("findings", fs);
211        if (keepText) {
212            // full runs keep the per-file metrics and the formatted text (needed by --acceptance-sample);
213            // baseline files (--no-text) stay small enough to live in git
214            e.put("metrics", r.getMetrics());
215            e.put("formatted", formatted);
216        }
217        return e;
218    }
219
220    private static Map<String, Object> errorEntry(String name, String engine, String source, String degrade, String error) {
221        Map<String, Object> e = new LinkedHashMap<String, Object>();
222        e.put("file", name);
223        e.put("engine", engine);
224        e.put("source", source);
225        e.put("degrade", degrade);
226        e.put("error", error);
227        return e;
228    }
229
230    private static String firstLine(String s) {
231        if (s == null) return "";
232        int i = s.indexOf('\n');
233        return (i < 0 ? s : s.substring(0, i)).trim();
234    }
235
236    @SuppressWarnings("unchecked")
237    public static Map<String, Object> summarize(List<Object> entries, List<String> engines) {
238        Map<String, Object> summary = new LinkedHashMap<String, Object>();
239        for (String engine : engines) {
240            int files = 0, scored = 0, gateFail = 0, parseFail = 0;
241            double sumScore = 0, sumRatio = 0, minRatio = 1.0;
242            for (Object o : entries) {
243                Map<String, Object> e = (Map<String, Object>) o;
244                if (!engine.equals(e.get("engine")) || !"clean".equals(e.get("source"))) continue;
245                files++;
246                if (e.containsKey("error")) { parseFail++; continue; }
247                scored++;
248                double score = ((Number) e.get("score")).doubleValue();
249                double ratio = ((Number) e.get("ratio")).doubleValue();
250                sumScore += score;
251                sumRatio += ratio;
252                if (ratio < minRatio) minRatio = ratio;
253                if (!Boolean.TRUE.equals(e.get("gate_passed"))) gateFail++;
254            }
255            Map<String, Object> s = new LinkedHashMap<String, Object>();
256            s.put("files", files);
257            s.put("scored", scored);
258            s.put("mean_score", scored == 0 ? 0 : sumScore / scored);
259            s.put("mean_ratio", scored == 0 ? 0 : sumRatio / scored);
260            s.put("min_ratio", scored == 0 ? 0 : minRatio);
261            s.put("gate_failures", gateFail);
262            s.put("parse_failures", parseFail);
263            summary.put(engine, s);
264        }
265        int samples = 0, scored = 0, gateFail = 0, errors = 0;
266        double sumRatio = 0;
267        for (Object o : entries) {
268            Map<String, Object> e = (Map<String, Object>) o;
269            if (!"degraded".equals(e.get("source"))) continue;
270            samples++;
271            if (e.containsKey("error")) { errors++; continue; }
272            scored++;
273            sumRatio += ((Number) e.get("ratio")).doubleValue();
274            if (!Boolean.TRUE.equals(e.get("gate_passed"))) gateFail++;
275        }
276        if (samples > 0) {
277            Map<String, Object> d = new LinkedHashMap<String, Object>();
278            d.put("samples", samples);
279            d.put("scored", scored);
280            d.put("errors", errors);
281            d.put("pp2_recovery_ratio", scored == 0 ? 0 : sumRatio / scored);
282            d.put("pp2_gate_failures", gateFail);
283            summary.put("degraded", d);
284        }
285        return summary;
286    }
287}