001package gudusoft.gsqlparser.pp2.token;
002
003import gudusoft.gsqlparser.EDbVendor;
004import gudusoft.gsqlparser.ETokenType;
005import gudusoft.gsqlparser.TGSqlParser;
006import gudusoft.gsqlparser.TSourceToken;
007import gudusoft.gsqlparser.TSourceTokenList;
008import gudusoft.gsqlparser.pp2.Pp2FormatOptions;
009
010import java.util.ArrayList;
011import java.util.List;
012
013/**
014 * Compares two SQL inputs for token-level equivalence modulo formatting.
015 *
016 * <p>The contract — verified by 12 hand-built cases in
017 * {@code TokenEquivalenceTest} — is that two SQL strings are equivalent
018 * when:
019 *
020 * <ul>
021 *   <li>Their <i>solid + comment</i> token sequences (in order) are equal
022 *       character-by-character.</li>
023 *   <li>Whitespace differences (spaces, tabs, linebreaks, indent) are
024 *       ignored.</li>
025 *   <li>Comment position may shift between the two streams — the comments
026 *       must appear in the same relative order, but their surrounding
027 *       whitespace does not matter.</li>
028 *   <li>Case differences on keywords and unquoted identifiers are tolerated
029 *       when {@code caseInsensitive=true}. Quoted identifiers, string
030 *       literals, and operator/punctuation tokens are always compared
031 *       byte-exact because case changes there alter semantics.</li>
032 * </ul>
033 *
034 * <p>This helper is the safety net for several downstream slices:
035 * <ul>
036 *   <li><b>S13</b> ({@code GuardedAstDelegate}) — verifies that
037 *       {@code FormatterFactory.pp()}'s output preserves every solid
038 *       input token.</li>
039 *   <li><b>S32</b> golden tests — sanity-checks goldens against their
040 *       inputs.</li>
041 *   <li><b>S33</b> overlay annotator — exercises the equivalence helper
042 *       on real corpus inputs.</li>
043 *   <li><b>S34</b> content-preservation — the strongest property test on
044 *       the 50-SQL corpus.</li>
045 *   <li><b>S35</b> per-vendor smoke — confirms every vendor preserves
046 *       content through the pp2 pipeline.</li>
047 * </ul>
048 *
049 * <p>The helper deliberately operates on raw SQL strings (not
050 * {@link Pp2TokenStream} instances) because the comparison is between an
051 * arbitrary input and an arbitrary output — there is no single pp2 stream
052 * shared between the two. The internal tokenization uses the GSP lexer
053 * directly with the same overlap-skip discipline as
054 * {@link Pp2TokenStreamBuilder}.
055 *
056 * <p>Plan reference: §7.3/S10, §7.4/S10, Q8.
057 */
058public final class TokenEquivalence {
059
060    private TokenEquivalence() {
061        // utility class
062    }
063
064    /**
065     * Case-insensitive comparison using the default Oracle dialect. The
066     * "case insensitive" mode tolerates case differences on keywords and
067     * unquoted identifiers; literals and quoted identifiers are always
068     * byte-exact.
069     */
070    public static boolean equalsModuloFormatting(String left, String right,
071                                                 Pp2FormatOptions opts) {
072        return equalsModuloFormatting(left, right, opts,
073            EDbVendor.dbvoracle, true);
074    }
075
076    /**
077     * Full-control comparison.
078     *
079     * @param left  first SQL string; must not be null
080     * @param right second SQL string; must not be null
081     * @param opts  pp2 options (currently unused, reserved for future
082     *              comment-tolerance settings); may be null
083     * @param vendor dialect to tokenize with; must not be null
084     * @param caseInsensitive when {@code true}, keyword and unquoted
085     *                        identifier comparisons are case-insensitive;
086     *                        when {@code false}, all comparisons are
087     *                        byte-exact on text
088     * @return {@code true} iff the two strings have the same token
089     *         sequence under the rules described in the class Javadoc
090     * @throws NullPointerException if {@code left}, {@code right}, or
091     *         {@code vendor} is null
092     */
093    public static boolean equalsModuloFormatting(String left, String right,
094                                                 Pp2FormatOptions opts,
095                                                 EDbVendor vendor,
096                                                 boolean caseInsensitive) {
097        if (left == null) throw new NullPointerException("left");
098        if (right == null) throw new NullPointerException("right");
099        if (vendor == null) throw new NullPointerException("vendor");
100        SplitTokens a = tokenizeSplit(left, vendor);
101        SplitTokens b = tokenizeSplit(right, vendor);
102        // Solid (non-comment) tokens must match in order.
103        if (!compareSequences(a.solids, b.solids, caseInsensitive)) {
104            return false;
105        }
106        // Comments must appear in the same relative order, and each
107        // comment's text must be byte-exact. Position may shift relative
108        // to the solid tokens (plan §16 Q8).
109        return compareSequences(a.comments, b.comments, caseInsensitive);
110    }
111
112    /**
113     * The solid-token half of {@link #equalsModuloFormatting(String, String,
114     * Pp2FormatOptions, EDbVendor, boolean)}: every non-comment token of
115     * {@code left} appears in {@code right} once, in order, with the same
116     * type and (case-insensitively for keywords and unquoted identifiers
117     * when {@code caseInsensitive}) the same text. Exposed so that callers
118     * reporting the two invariants separately (the format worker) apply the
119     * same policy as the combined check.
120     */
121    public static boolean solidTokensEquivalent(String left, String right,
122                                                EDbVendor vendor,
123                                                boolean caseInsensitive) {
124        if (left == null) throw new NullPointerException("left");
125        if (right == null) throw new NullPointerException("right");
126        if (vendor == null) throw new NullPointerException("vendor");
127        return compareSequences(tokenizeSplit(left, vendor).solids,
128            tokenizeSplit(right, vendor).solids, caseInsensitive);
129    }
130
131    /**
132     * The comment half of {@link #equalsModuloFormatting(String, String,
133     * Pp2FormatOptions, EDbVendor, boolean)}: the comments of {@code left}
134     * appear in {@code right} in the same relative order with byte-exact
135     * text; their position relative to the solid tokens may differ.
136     */
137    public static boolean commentsPreserved(String left, String right,
138                                            EDbVendor vendor) {
139        if (left == null) throw new NullPointerException("left");
140        if (right == null) throw new NullPointerException("right");
141        if (vendor == null) throw new NullPointerException("vendor");
142        return compareSequences(tokenizeSplit(left, vendor).comments,
143            tokenizeSplit(right, vendor).comments, false);
144    }
145
146    private static boolean compareSequences(List<TokenInfo> a, List<TokenInfo> b,
147                                            boolean caseInsensitive) {
148        if (a.size() != b.size()) return false;
149        for (int i = 0; i < a.size(); i++) {
150            if (!tokensEquivalent(a.get(i), b.get(i), caseInsensitive)) {
151                return false;
152            }
153        }
154        return true;
155    }
156
157    /** Lightweight value carrier for comparable tokens. */
158    private static final class TokenInfo {
159        final ETokenType type;
160        final String text;
161        TokenInfo(ETokenType type, String text) {
162            this.type = type;
163            this.text = text;
164        }
165    }
166
167    /** Tokenization result split into solid tokens and comments. */
168    private static final class SplitTokens {
169        final List<TokenInfo> solids;
170        final List<TokenInfo> comments;
171        SplitTokens(List<TokenInfo> solids, List<TokenInfo> comments) {
172            this.solids = solids;
173            this.comments = comments;
174        }
175    }
176
177    private static SplitTokens tokenizeSplit(String sql, EDbVendor vendor) {
178        TGSqlParser parser = new TGSqlParser(vendor);
179        parser.sqltext = sql;
180        parser.tokenizeSqltext();
181        TSourceTokenList list = parser.getSourcetokenlist();
182        List<TokenInfo> solids = new ArrayList<TokenInfo>(list.size());
183        List<TokenInfo> comments = new ArrayList<TokenInfo>();
184        long lastEmittedEnd = -1L;
185        for (int i = 0; i < list.size(); i++) {
186            TSourceToken t = list.get(i);
187            if (t == null) continue;
188            String text = t.toString();
189            if (text == null || text.isEmpty()) continue;
190            // Skip phantom shadowed tokens (GSP "${name}" overlap quirk —
191            // see TokenCoverage Javadoc and slice S9 resume doc).
192            if (TokenCoverage.isFullyShadowed(t, lastEmittedEnd)) {
193                continue;
194            }
195            ETokenType type = t.tokentype;
196            // Skip whitespace entirely.
197            if (Pp2TokenStreamBuilder.isFoldable(type)) {
198                continue;
199            }
200            TokenInfo info = new TokenInfo(type, text);
201            if (isCommentType(type)) {
202                comments.add(info);
203            } else {
204                solids.add(info);
205            }
206            lastEmittedEnd = TokenCoverage.endOffset(t);
207        }
208        return new SplitTokens(solids, comments);
209    }
210
211    private static boolean isCommentType(ETokenType type) {
212        if (type == null) return false;
213        switch (type) {
214            case ttsimplecomment:
215            case ttbracketedcomment:
216            case ttCPPComment:
217                return true;
218            default:
219                return false;
220        }
221    }
222
223    private static boolean tokensEquivalent(TokenInfo a, TokenInfo b,
224                                            boolean caseInsensitive) {
225        if (a.type != b.type) {
226            // The Oracle lexer classifies a word at column 0 as a SQL*Plus
227            // command (EXIT, SET ...) and the same word, once indented by the
228            // formatter, as the PL/SQL keyword it really is: same text, only
229            // the lexer's guess changed (sqllog corpus, oracle/020).
230            boolean sqlplusVsKeyword = (a.type == ETokenType.ttsqlpluscmd && b.type == ETokenType.ttkeyword)
231                || (a.type == ETokenType.ttkeyword && b.type == ETokenType.ttsqlpluscmd);
232            return sqlplusVsKeyword && a.text.equalsIgnoreCase(b.text);
233        }
234        if (caseInsensitive && typeAllowsCaseChange(a.type)) {
235            return a.text.equalsIgnoreCase(b.text);
236        }
237        return a.text.equals(b.text);
238    }
239
240    /**
241     * Token types whose case may legitimately change between input and
242     * output: keywords and unquoted identifiers. Everything else
243     * (literals, quoted identifiers, comments, punctuation, operators)
244     * is compared byte-exact even in case-insensitive mode.
245     */
246    private static boolean typeAllowsCaseChange(ETokenType type) {
247        if (type == null) return false;
248        switch (type) {
249            case ttkeyword:
250            case ttnonreservedkeyword:
251            case ttidentifier:
252                return true;
253            default:
254                return false;
255        }
256    }
257}