001package gudusoft.gsqlparser.dlineage.util; 002 003import gudusoft.gsqlparser.EDbVendor; 004import gudusoft.gsqlparser.ETokenType; 005import gudusoft.gsqlparser.TGSqlParser; 006import gudusoft.gsqlparser.TSourceToken; 007import gudusoft.gsqlparser.TSourceTokenList; 008import gudusoft.gsqlparser.sqlenv.CanonKey; 009import gudusoft.gsqlparser.sqlenv.ESQLDataObjectType; 010import gudusoft.gsqlparser.sqlenv.IdentifierCodec; 011import gudusoft.gsqlparser.sqlenv.IdentifierInputForm; 012import gudusoft.gsqlparser.sqlenv.IdentifierProfile; 013import gudusoft.gsqlparser.sqlenv.IdentifierService; 014import gudusoft.gsqlparser.util.SQLUtil; 015 016import java.util.ArrayList; 017import java.util.Collection; 018import java.util.Collections; 019import java.util.EnumSet; 020import java.util.HashMap; 021import java.util.HashSet; 022import java.util.LinkedHashMap; 023import java.util.LinkedHashSet; 024import java.util.List; 025import java.util.Locale; 026import java.util.Map; 027import java.util.Set; 028 029/** 030 * Keeps two different names from sharing one lineage normal name. 031 * 032 * <p>Lineage keys its models by {@link SQLUtil#normalizeIdentifier}, and the legacy fold behind it 033 * can give two different names one string: PostgreSQL {@code "Amount"} and {@code amount} both 034 * become {@code AMOUNT}, ClickHouse {@code userId} and {@code userid} both become {@code USERID}. 035 * Lineage then treats the two as one column, and every reference to either picks up both as its 036 * source. The same string is also displayed in places, so it cannot change for every name.</p> 037 * 038 * <p>So before the first statement is analyzed, {@link #scan} reads every identifier token of the 039 * analysis's SQL and finds the legacy strings that more than one identity shares. Only a name whose 040 * legacy string is contested changes ({@link #normalName}): it keeps the legacy string when that 041 * string, read back as SQL, denotes it ({@code amount} keeps {@code AMOUNT} on PostgreSQL), and 042 * otherwise gets a spelling that denotes it ({@code "Amount"}). Lineage compares these strings as 043 * SQL names, so each one must name its own identity; the first name to arrive keeping the legacy 044 * string would not, because {@code AMOUNT} read back as SQL is {@code amount} whoever holds it. 045 * Column names are contested without case, because lineage matches column normal names 046 * case-insensitively even where the legacy fold keeps case (Oracle, whose quoted {@code "Amount"} 047 * gives {@code Amount} next to a bare {@code amount}'s {@code AMOUNT}). Names are told apart under 048 * the analysis's own rules, so a configured profile's stricter ones count.</p> 049 * 050 * <p>The result is a pure function of the contested set, so all spellings of a legacy string change 051 * together. A name the scan did not see (dynamic SQL, a name only metadata holds) follows the same 052 * rule: it changes if its string is contested, and otherwise keeps the merge the legacy fold always 053 * made. No string is ever split for some of its names and merged for the rest.</p> 054 */ 055public final class NormalNameCollisions { 056 057 /** Tokens that can spell a name: identifiers, delimited identifiers, and keywords used as names. */ 058 private static final Set<ETokenType> NAME_TOKENS = Collections.unmodifiableSet(EnumSet.of( 059 ETokenType.ttidentifier, ETokenType.ttkeyword, ETokenType.ttnonreservedkeyword, 060 ETokenType.ttdqstring, ETokenType.ttdbstring, ETokenType.ttbrstring)); 061 062 /** Characters that only delimit a name; dropping them never separates two spellings of one name. */ 063 private static final String DELIMITERS = "\"`[]{}'"; 064 065 private NormalNameCollisions() { 066 } 067 068 /** 069 * The legacy normal names that two different identities share in {@code sqlTexts}, keyed for 070 * {@link #normalName}. Empty when no two names collide, which is the common case. 071 * 072 * @param vendor the analysis's vendor; names are compared under its default identifier rules 073 * (see {@link #scan(IdentifierService, Collection)} for a configured profile) 074 * @param sqlTexts every unit of SQL the analysis will read 075 * @return the contested keys 076 */ 077 public static Set<String> scan(EDbVendor vendor, Collection<String> sqlTexts) { 078 return scan(IdentifierService.defaultServiceFor(vendor), sqlTexts); 079 } 080 081 /** 082 * {@link #scan(EDbVendor, Collection)} under the analysis's own identifier rules, which a 083 * configured profile may make stricter than the vendor's (SQL Server under a case-sensitive 084 * collation tells {@code Amount} from {@code AMOUNT}). 085 */ 086 public static Set<String> scan(IdentifierService service, Collection<String> sqlTexts) { 087 EDbVendor vendor = service.getProfile().getVendor(); 088 List<TSourceTokenList> tokens = new ArrayList<TSourceTokenList>(); 089 for (String text : sqlTexts) { 090 if (text == null || text.isEmpty()) { 091 continue; 092 } 093 TGSqlParser lexer = new TGSqlParser(vendor); 094 lexer.sqltext = text; 095 lexer.tokenizeSqltext(); 096 tokens.add(lexer.sourcetokenlist); 097 } 098 return scanTokens(service, tokens); 099 } 100 101 /** 102 * {@link #scan} over tokens already read, such as a parser's token list after it parsed the 103 * analysis's only unit, which spares lexing the text a second time. 104 */ 105 public static Set<String> scanTokens(IdentifierService service, Collection<TSourceTokenList> tokenLists) { 106 EDbVendor vendor = service.getProfile().getVendor(); 107 // Names can only share a legacy string when they are the same text apart from case and 108 // delimiters, so only such groups are checked in full. 109 Map<String, Set<String>> byLooseText = new HashMap<String, Set<String>>(); 110 for (TSourceTokenList tokens : tokenLists) { 111 for (int i = 0; tokens != null && i < tokens.size(); i++) { 112 TSourceToken token = tokens.get(i); 113 if (!NAME_TOKENS.contains(token.tokentype)) { 114 continue; 115 } 116 String spelling = token.toString(); 117 String loose = looseText(spelling); 118 Set<String> spellings = byLooseText.get(loose); 119 if (spellings == null) { 120 spellings = new LinkedHashSet<String>(); 121 byLooseText.put(loose, spellings); 122 } 123 spellings.add(spelling); 124 } 125 } 126 127 Map<String, ESQLDataObjectType> nameClasses = nameClasses(service.getProfile()); 128 Map<String, CanonKey> owners = new HashMap<String, CanonKey>(); 129 Set<String> contested = new HashSet<String>(); 130 for (Set<String> spellings : byLooseText.values()) { 131 if (spellings.size() < 2) { 132 continue; 133 } 134 for (String spelling : spellings) { 135 for (Map.Entry<String, ESQLDataObjectType> nameClass : nameClasses.entrySet()) { 136 CanonKey identity = identity(service, nameClass.getValue(), spelling); 137 String legacy = identity == null ? null 138 : SQLUtil.legacyNormalSegment(vendor, nameClass.getValue(), spelling); 139 if (legacy == null) { 140 continue; 141 } 142 String key = key(nameClass.getKey(), nameClass.getValue(), legacy); 143 CanonKey owner = owners.get(key); 144 if (owner == null) { 145 owners.put(key, identity); 146 } else if (!owner.equals(identity)) { 147 contested.add(key); 148 } 149 } 150 } 151 } 152 return contested; 153 } 154 155 /** 156 * The legacy strings of {@code contested}, whatever their class and upper-cased: a cheap first 157 * test, which {@link #normalName} then makes exact. 158 */ 159 public static Set<String> legacyNames(Set<String> contested) { 160 Set<String> names = new HashSet<String>(); 161 for (String key : contested) { 162 names.add(key.substring(key.indexOf('\u0000') + 1).toUpperCase(Locale.ROOT)); 163 } 164 return names; 165 } 166 167 /** 168 * The normal name of one identifier segment, given the contested keys of its analysis. 169 * 170 * @param contested keys from {@link #scan} 171 * @param vendor vendor the legacy name was folded for 172 * @param type role of the segment 173 * @param origin the segment as written 174 * @param legacy its legacy normal name 175 * @return {@code legacy}, unless another name shares it and it does not denote {@code origin} 176 */ 177 public static String normalName(Set<String> contested, EDbVendor vendor, ESQLDataObjectType type, 178 String origin, String legacy) { 179 return normalName(contested, IdentifierService.defaultServiceFor(vendor), type, origin, legacy); 180 } 181 182 /** 183 * {@link #normalName(Set, EDbVendor, ESQLDataObjectType, String, String)} under the analysis's 184 * own identifier rules; pass the service {@link #scan(IdentifierService, Collection)} used. 185 */ 186 public static String normalName(Set<String> contested, IdentifierService service, ESQLDataObjectType type, 187 String origin, String legacy) { 188 IdentifierProfile profile = service.getProfile(); 189 if (!contested.contains(key(classOf(profile, type), type, legacy))) { 190 return legacy; 191 } 192 CanonKey identity = identity(service, type, origin); 193 if (identity == null || identity.equals(identity(service, type, legacy))) { 194 return legacy; 195 } 196 String stored; 197 try { 198 stored = IdentifierCodec.isQuoted(profile, type, origin) 199 ? IdentifierCodec.decodeLexical(profile, type, origin) 200 : service.normalizeSegment(origin, type); 201 } catch (RuntimeException e) { 202 return legacy; 203 } 204 if (stored == null || stored.isEmpty()) { 205 return legacy; 206 } 207 // The stored name as written when it reads back as this name, as lineage displays names 208 // (ClickHouse userId); else delimited (PostgreSQL "Amount"). Either reads back as SQL. 209 if (!IdentifierCodec.isQuoted(profile, type, stored) 210 && IdentifierCodec.isVendorValidSpelling(profile, type, stored) 211 && identity.equals(identity(service, type, stored))) { 212 return stored; 213 } 214 if (IdentifierCodec.hasQuotedForm(profile, type)) { 215 String delimited = IdentifierCodec.encodeStored(profile, type, stored); 216 if (identity.equals(identity(service, type, delimited))) { 217 return delimited; 218 } 219 } 220 return legacy; 221 } 222 223 /** 224 * The contested key of the legacy name {@code legacy} of a name of role {@code type}, as {@link #scan} returns 225 * them: for a caller that finds two names sharing one legacy name outside the SQL text a scan reads. 226 */ 227 public static String contestedKey(IdentifierService service, ESQLDataObjectType type, String legacy) { 228 return key(classOf(service.getProfile(), type), type, legacy); 229 } 230 231 /** Names in one class get the same legacy string for the same text and follow the same rules. */ 232 private static String classOf(IdentifierProfile profile, ESQLDataObjectType type) { 233 return SQLUtil.legacyFoldClass(type) + '/' + profile.groupOf(type); 234 } 235 236 /** 237 * The name classes lineage keeps apart: columns, and tables with the objects that share their 238 * rules. Routines, variables and schemas keep their legacy names, since code compares those 239 * across classes whose legacy folds differ (DB2 folds a routine name keeping case, and the same 240 * name folded as a variable to upper case), and one class changing alone would split them. 241 */ 242 private static Map<String, ESQLDataObjectType> nameClasses(IdentifierProfile profile) { 243 Map<String, ESQLDataObjectType> classes = new LinkedHashMap<String, ESQLDataObjectType>(); 244 for (ESQLDataObjectType type : new ESQLDataObjectType[] {ESQLDataObjectType.dotColumn, 245 ESQLDataObjectType.dotTable}) { 246 classes.put(classOf(profile, type), type); 247 } 248 return classes; 249 } 250 251 /** 252 * Column names are contested without case, since lineage matches column normal names 253 * case-insensitively; Oracle's fold keeps the case of a quoted name, so {@code "Amount"} gives 254 * {@code Amount} and a bare {@code amount} gives {@code AMOUNT}, two strings that lineage still 255 * matched. Table names are contested exactly. 256 */ 257 private static String key(String nameClass, ESQLDataObjectType type, String legacy) { 258 boolean caseless = "column".equals(SQLUtil.legacyFoldClass(type)); 259 return nameClass + '\u0000' + (caseless ? legacy.toUpperCase(Locale.ROOT) : legacy); 260 } 261 262 private static CanonKey identity(IdentifierService service, ESQLDataObjectType type, String spelling) { 263 try { 264 return service.canonKeyOf(spelling, type, IdentifierInputForm.SQL_LEXICAL); 265 } catch (RuntimeException e) { 266 // Not a name the rules can key (a malformed delimited spelling). 267 return null; 268 } 269 } 270 271 private static String looseText(String spelling) { 272 StringBuilder loose = new StringBuilder(spelling.length()); 273 for (int i = 0; i < spelling.length(); i++) { 274 char c = spelling.charAt(i); 275 if (DELIMITERS.indexOf(c) < 0) { 276 loose.append(c); 277 } 278 } 279 return loose.toString().toUpperCase(Locale.ROOT); 280 } 281}