Source code

001package gudusoft.gsqlparser.dlineage.dataflow.metadata.sqlenv;
002
003import gudusoft.gsqlparser.EDbVendor;
004import gudusoft.gsqlparser.dlineage.dataflow.metadata.MetadataAnalyzer;
005import gudusoft.gsqlparser.dlineage.dataflow.model.ModelBindingManager;
006import gudusoft.gsqlparser.dlineage.dataflow.model.xml.*;
007import gudusoft.gsqlparser.dlineage.util.DlineageUtil;
008import gudusoft.gsqlparser.dlineage.util.Pair3;
009import gudusoft.gsqlparser.sqlenv.*;
010import gudusoft.gsqlparser.util.SQLUtil;
011
012import java.util.*;
013
014public class SQLEnvMetadataAnalyzer implements MetadataAnalyzer<TSQLEnv> {
015
016    private Map<String, procedure> procedureMap = new LinkedHashMap<String, procedure>();
017    private Map<String, table> tableMap = new LinkedHashMap<String, table>();
018    private Map<String, oraclePackage> oraclePackageMap = new LinkedHashMap<String, oraclePackage>();
019    private Map<String, column> columnMap = new LinkedHashMap<String, column>();
020    private EDbVendor vendor;
021    private boolean lower = false;
022
023    public SQLEnvMetadataAnalyzer(boolean lower) {
024        this.lower = lower;
025    }
026
027    @Override
028    public synchronized dataflow analyzeMetadata(EDbVendor metadataVendor, TSQLEnv sqlenv) {
029        init(vendor);
030        dataflow dataflow = new dataflow();
031
032        String serverName = sqlenv.getDefaultServerName();
033        EDbVendor vendor = metadataVendor;
034        if (sqlenv.getDBVendor() != null) {
035            vendor = sqlenv.getDBVendor();
036        }
037
038        boolean supportsCatalogs = TSQLEnv.supportCatalog(vendor);
039        boolean supportsSchemas = TSQLEnv.supportSchema(vendor);
040
041        List<TSQLCatalog> databases = sqlenv.getCatalogList();
042        if (databases != null) {
043            for (int i = 0; i < databases.size(); i++) {
044                TSQLCatalog catalog = databases.get(i);
045                String databaseName = catalog.getName();
046                if (SQLUtil.parseNames(databaseName).size() > 1) {
047                    databaseName = "\"" + databaseName + "\"";
048                }
049                List<TSQLSchema> schemas = catalog.getSchemaList();
050                if (schemas == null) {
051                    continue;
052                }
053                for (int j = 0; j < schemas.size(); j++) {
054                    TSQLSchema jsonSchema = schemas.get(j);
055                    String schemaName = (String) jsonSchema.getName();
056                    if (SQLUtil.parseNames(schemaName).size() > 1) {
057                        schemaName = "\"" + schemaName + "\"";
058                    }
059                    List<TSQLSchemaObject> dbObjs = jsonSchema.getSchemaObjectList();
060                    if (dbObjs == null) {
061                        continue;
062                    }
063                    for (int k = 0; k < dbObjs.size(); k++) {
064                        TSQLSchemaObject jsonTable = dbObjs.get(k);
065                        if (jsonTable instanceof TSQLTable) {
066                            TSQLTable table = (TSQLTable) jsonTable;
067                            String tableName = table.getName();
068                            boolean isView = table.isView();
069                            List<TSQLColumn> columns = table.getColumnList();
070                            if (columns == null) {
071                                continue;
072                            }
073                            for (int l = 0; l < columns.size(); l++) {
074                                TSQLColumn jsonColumn = columns.get(l);
075                                String columnName = jsonColumn.getName();
076                                if (SQLUtil.isEmpty(columnName)) {
077                                    continue;
078                                }
079                                appendTable(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, databaseName, schemaName, tableName, isView, columnName,
080                                        storedTableIdentity(sqlenv, vendor, supportsCatalogs, supportsSchemas, catalog.getName(), jsonSchema.getName(), tableName));
081                                if ("*".equals(columnName) && !jsonColumn.isUnresolvedStar()) {
082                                    // a catalog column named *, not the star lineage writes back for an unresolved width
083                                    column published = columnMap.get(getFullName(serverName, databaseName, schemaName, tableName, columnName));
084                                    if (published != null) {
085                                        published.setStar(Boolean.FALSE);
086                                    }
087                                }
088                            }
089                        }
090                        if (jsonTable instanceof TSQLOraclePackage) {
091                            TSQLOraclePackage sqlOraclePackage = (TSQLOraclePackage) jsonTable;
092                            oraclePackage oraclePackage = appendOraclePackage(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, databaseName, schemaName, sqlOraclePackage);
093                            if (sqlOraclePackage.getSchemaObjectList() != null) {
094                                for (TSQLSchemaObject sqlProcedure : sqlOraclePackage.getSchemaObjectList()) {
095                                    appendProcedure(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, databaseName, schemaName, (TSQLSchemaObject) sqlProcedure, oraclePackage);
096                                }
097                            }
098                        }
099                        if(jsonTable instanceof TSQLProcedure){
100                            appendProcedure(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, databaseName, schemaName, (TSQLSchemaObject)jsonTable, null);
101                        }
102                    }
103                }
104            }
105        }
106        sortTableColumns(dataflow);
107        return dataflow;
108    }
109
110    private void appendProcedure(EDbVendor vendor, boolean supportsCatalogs, boolean supportsSchemas, dataflow dataflow, String serverName, String databaseName, String schemaName, TSQLSchemaObject sqlProcedure, oraclePackage oraclePackage) {
111        String qualifiedName = sqlProcedure.getQualifiedName();
112        if (DlineageUtil.isProcedureExcluded(qualifiedName)) {
113            return;
114        }
115        String procedureName = sqlProcedure.getName();
116        String procedureKey = getFullName(serverName, databaseName, schemaName, procedureName);
117        if (!procedureMap.containsKey(procedureKey)) {
118            procedure procedure = new procedure();
119            if (serverName != null && !TSQLEnv.DEFAULT_SERVER_NAME.equalsIgnoreCase(serverName)) {
120                procedure.setServer(serverName);
121            }
122            if(supportsCatalogs && supportsSchemas){
123                if(databaseName != null && !TSQLEnv.DEFAULT_DB_NAME.equalsIgnoreCase(databaseName)) {
124                    if (lower) {
125                        procedure.setDatabase(SQLUtil.trimColumnStringQuote(databaseName).toLowerCase(Locale.ROOT));
126                    } else {
127                        procedure.setDatabase(databaseName);
128                    }
129                }
130                if(schemaName != null && !TSQLEnv.DEFAULT_SCHEMA_NAME.equalsIgnoreCase(schemaName)) {
131                    if (lower) {
132                        procedure.setSchema(SQLUtil.trimColumnStringQuote(schemaName).toLowerCase(Locale.ROOT));
133                    } else {
134                        procedure.setSchema(schemaName);
135                    }
136                }
137            }
138            else if(supportsCatalogs){
139                if(databaseName != null && !TSQLEnv.DEFAULT_DB_NAME.equalsIgnoreCase(databaseName)) {
140                    if (lower) {
141                        procedure.setDatabase(SQLUtil.trimColumnStringQuote(databaseName).toLowerCase(Locale.ROOT));
142                    } else {
143                        procedure.setDatabase(databaseName);
144                    }
145                }
146            }
147            else if(supportsSchemas){
148                if(schemaName != null && !TSQLEnv.DEFAULT_SCHEMA_NAME.equalsIgnoreCase(schemaName)) {
149                    if (lower) {
150                        procedure.setSchema(SQLUtil.trimColumnStringQuote(schemaName).toLowerCase(Locale.ROOT));
151                    } else {
152                        procedure.setSchema(schemaName);
153                    }
154                }
155            }
156
157            if (lower) {
158                procedure.setName(SQLUtil.trimColumnStringQuote(procedureName).toLowerCase(Locale.ROOT));
159            } else {
160                procedure.setName(procedureName);
161            }
162            if (supportsSchemas && !SQLUtil.isEmpty(procedure.getSchema()) && !TSQLEnv.DEFAULT_SCHEMA_NAME.equalsIgnoreCase(procedure.getSchema())) {
163                procedure.setName(procedure.getSchema() + "." + procedure.getName());
164            }
165            if (supportsCatalogs && !SQLUtil.isEmpty(procedure.getDatabase()) && !TSQLEnv.DEFAULT_DB_NAME.equalsIgnoreCase(procedure.getDatabase())) {
166                procedure.setName(procedure.getDatabase() + "." + procedure.getName());
167            }
168            procedure.setId(String.valueOf(++ModelBindingManager.get().TABLE_COLUMN_ID));
169            if (ModelBindingManager.getGlobalSqlInfo() != null) {
170                procedure.setCoordinate(new Pair3<Long, Long, Integer>(-1L, -1L,
171                        ModelBindingManager.getGlobalSqlInfo().getIndexOf(ModelBindingManager.getGlobalHash())) + ","
172                        + new Pair3<Long, Long, Integer>(-1L, -1L,
173                        ModelBindingManager.getGlobalSqlInfo().getIndexOf(ModelBindingManager.getGlobalHash())));
174            }
175            procedure.setType(sqlProcedure.getDataObjectType().name().replace("dot","").toLowerCase(Locale.ROOT));
176            if(oraclePackage!=null) {
177                oraclePackage.getProcedures().add(procedure);
178            }
179            else{
180                dataflow.getProcedures().add(procedure);
181            }
182            procedureMap.put(procedureKey, procedure);
183        }
184    }
185
186    private oraclePackage appendOraclePackage(EDbVendor vendor, boolean supportsCatalogs, boolean supportsSchemas, dataflow dataflow, String serverName, String databaseName, String schemaName, TSQLOraclePackage sqlOraclePackage) {
187        String oraclePackageName = sqlOraclePackage.getName();
188        String oraclePackageKey = getFullName(serverName, databaseName, schemaName, oraclePackageName);
189        if (!oraclePackageMap.containsKey(oraclePackageKey)) {
190            oraclePackage oraclePackage = new oraclePackage();
191            if (serverName != null && !TSQLEnv.DEFAULT_SERVER_NAME.equalsIgnoreCase(serverName)) {
192                oraclePackage.setServer(serverName);
193            }
194            if(supportsCatalogs && supportsSchemas){
195                if(databaseName != null && !TSQLEnv.DEFAULT_DB_NAME.equalsIgnoreCase(databaseName)) {
196                    if (lower) {
197                        oraclePackage.setDatabase(SQLUtil.trimColumnStringQuote(databaseName).toLowerCase(Locale.ROOT));
198                    } else {
199                        oraclePackage.setDatabase(databaseName);
200                    }
201                }
202                if(schemaName != null && !TSQLEnv.DEFAULT_SCHEMA_NAME.equalsIgnoreCase(schemaName)) {
203                    if (lower) {
204                        oraclePackage.setSchema(SQLUtil.trimColumnStringQuote(schemaName).toLowerCase(Locale.ROOT));
205                    } else {
206                        oraclePackage.setSchema(schemaName);
207                    }
208                }
209            }
210            else if(supportsCatalogs){
211                if(databaseName != null && !TSQLEnv.DEFAULT_DB_NAME.equalsIgnoreCase(databaseName)) {
212                    if (lower) {
213                        oraclePackage.setDatabase(SQLUtil.trimColumnStringQuote(databaseName).toLowerCase(Locale.ROOT));
214                    } else {
215                        oraclePackage.setDatabase(databaseName);
216                    }
217                }
218            }
219            else if(supportsSchemas){
220                if(schemaName != null && !TSQLEnv.DEFAULT_SCHEMA_NAME.equalsIgnoreCase(schemaName)) {
221                    if (lower) {
222                        oraclePackage.setSchema(SQLUtil.trimColumnStringQuote(schemaName).toLowerCase(Locale.ROOT));
223                    } else {
224                        oraclePackage.setSchema(schemaName);
225                    }
226                }
227            }
228
229            if (lower) {
230                oraclePackage.setName(SQLUtil.trimColumnStringQuote(oraclePackageName).toLowerCase(Locale.ROOT));
231            } else {
232                oraclePackage.setName(oraclePackageName);
233            }
234            if (supportsSchemas && !SQLUtil.isEmpty(oraclePackage.getSchema()) && !TSQLEnv.DEFAULT_SCHEMA_NAME.equalsIgnoreCase(oraclePackage.getSchema())) {
235                oraclePackage.setName(oraclePackage.getSchema() + "." + oraclePackage.getName());
236            }
237            if (supportsCatalogs && !SQLUtil.isEmpty(oraclePackage.getDatabase()) && !TSQLEnv.DEFAULT_DB_NAME.equalsIgnoreCase(oraclePackage.getDatabase())) {
238                oraclePackage.setName(oraclePackage.getDatabase() + "." + oraclePackage.getName());
239            }
240            oraclePackage.setId(String.valueOf(++ModelBindingManager.get().TABLE_COLUMN_ID));
241            if (ModelBindingManager.getGlobalSqlInfo() != null) {
242                oraclePackage.setCoordinate(new Pair3<Long, Long, Integer>(-1L, -1L,
243                        ModelBindingManager.getGlobalSqlInfo().getIndexOf(ModelBindingManager.getGlobalHash())) + ","
244                        + new Pair3<Long, Long, Integer>(-1L, -1L,
245                        ModelBindingManager.getGlobalSqlInfo().getIndexOf(ModelBindingManager.getGlobalHash())));
246            }
247            dataflow.getPackages().add(oraclePackage);
248            oraclePackageMap.put(oraclePackageKey, oraclePackage);
249        }
250        return oraclePackageMap.get(oraclePackageKey);
251    }
252
253    private void init(EDbVendor vendor) {
254        this.vendor = vendor;
255        procedureMap.clear();
256        tableMap.clear();
257        columnMap.clear();
258        if (ModelBindingManager.get() == null) {
259            ModelBindingManager.set(new ModelBindingManager());
260        }
261    }
262
263    private void sortTableColumns(dataflow dataflow) {
264        if (dataflow.getTables() != null) {
265            for (table table : dataflow.getTables()) {
266                Collections.sort(table.getColumns(), new Comparator<column>() {
267                    public int compare(column t1, column t2) {
268                        if (t1.getName().equalsIgnoreCase("RelationRows"))
269                            return 1;
270                        if (t2.getName().equalsIgnoreCase("RelationRows"))
271                            return -1;
272                        return 0;
273                    }
274                });
275            }
276        }
277
278        if (dataflow.getResultsets() != null) {
279            for (table table : dataflow.getResultsets()) {
280                Collections.sort(table.getColumns(), new Comparator<column>() {
281                    public int compare(column t1, column t2) {
282                        if (t1.getName().equalsIgnoreCase("RelationRows"))
283                            return 1;
284                        if (t2.getName().equalsIgnoreCase("RelationRows"))
285                            return -1;
286                        return 0;
287                    }
288                });
289            }
290        }
291    }
292
293    /**
294     * The identity of a catalog table under the analysis's identifier rules, from the stored names the
295     * catalog holds, laid out as its published name is: the names this analyzer publishes are the bare
296     * stored names, which read as SQL would name another table ({@code "Orders"} published as
297     * {@code Orders}). Null when the rules give the name no identity.
298     */
299    private static QualifiedCanonKey storedTableIdentity(TSQLEnv sqlenv, EDbVendor vendor, boolean supportsCatalogs,
300            boolean supportsSchemas, String database, String schema, String table) {
301        List<String> segments = new ArrayList<String>();
302        if (supportsCatalogs && database != null && !TSQLEnv.DEFAULT_DB_NAME.equalsIgnoreCase(database)) { // non-identifier-compare: DEFAULT is the placeholder for a missing tier
303            segments.add(database);
304        }
305        if (supportsSchemas && schema != null && !TSQLEnv.DEFAULT_SCHEMA_NAME.equalsIgnoreCase(schema)) { // non-identifier-compare: DEFAULT is the placeholder for a missing tier
306            segments.add(schema);
307        }
308        segments.add(table);
309        // The catalog's own rules: the names are what it stored under them.
310        IdentifierService service = sqlenv.getIdentifierService();
311        if (service == null) {
312            service = IdentifierService.defaultServiceFor(vendor);
313        }
314        try {
315            return service.qualifiedCanonKeyFromStored(segments, ESQLDataObjectType.dotTable);
316        } catch (RuntimeException noIdentity) {
317            return null;
318        }
319    }
320
321    private void appendTable(EDbVendor vendor, boolean supportsCatalogs, boolean supportsSchemas, dataflow dataflow, String serverName, String databaseName, String schemaName, String tableName,
322                             boolean isView, String columnName, QualifiedCanonKey identity) {
323        String tableKey = getFullName(serverName, databaseName, schemaName, tableName);
324        if (!tableMap.containsKey(tableKey)) {
325            table table = new table();
326            table.setMergeIdentity(identity);
327            if (serverName != null && !TSQLEnv.DEFAULT_SERVER_NAME.equalsIgnoreCase(serverName)) {
328                table.setServer(serverName);
329            }
330            if(supportsCatalogs && supportsSchemas){
331                if(databaseName != null && !TSQLEnv.DEFAULT_DB_NAME.equalsIgnoreCase(databaseName)) {
332                    if (lower) {
333                        table.setDatabase(SQLUtil.trimColumnStringQuote(databaseName).toLowerCase(Locale.ROOT));
334                    } else {
335                        table.setDatabase(databaseName);
336                    }
337                }
338                if(schemaName != null && !TSQLEnv.DEFAULT_SCHEMA_NAME.equalsIgnoreCase(schemaName)) {
339                    if (lower) {
340                        table.setSchema(SQLUtil.trimColumnStringQuote(schemaName).toLowerCase(Locale.ROOT));
341                    } else {
342                        table.setSchema(schemaName);
343                    }
344                }
345            }
346            else if(supportsCatalogs){
347                if(databaseName != null && !TSQLEnv.DEFAULT_DB_NAME.equalsIgnoreCase(databaseName)) {
348                    if (lower) {
349                        table.setDatabase(SQLUtil.trimColumnStringQuote(databaseName).toLowerCase(Locale.ROOT));
350                    } else {
351                        table.setDatabase(databaseName);
352                    }
353                }
354            }
355            else if(supportsSchemas){
356                if(schemaName != null && !TSQLEnv.DEFAULT_SCHEMA_NAME.equalsIgnoreCase(schemaName)) {
357                    if (lower) {
358                        table.setSchema(SQLUtil.trimColumnStringQuote(schemaName).toLowerCase(Locale.ROOT));
359                    } else {
360                        table.setSchema(schemaName);
361                    }
362                }
363            }
364
365            if (lower) {
366                table.setName(SQLUtil.trimColumnStringQuote(tableName).toLowerCase(Locale.ROOT));
367            } else {
368                table.setName(tableName);
369            }
370            if (supportsSchemas && !SQLUtil.isEmpty(table.getSchema()) && !TSQLEnv.DEFAULT_SCHEMA_NAME.equalsIgnoreCase(table.getSchema())) {
371                table.setName(table.getSchema() + "." + table.getName());
372            }
373            if (supportsCatalogs && !SQLUtil.isEmpty(table.getDatabase()) && !TSQLEnv.DEFAULT_DB_NAME.equalsIgnoreCase(table.getDatabase())) {
374                table.setName(table.getDatabase() + "." + table.getName());
375            }
376            table.setId(String.valueOf(++ModelBindingManager.get().TABLE_COLUMN_ID));
377            if (ModelBindingManager.getGlobalSqlInfo() != null) {
378                table.setCoordinate(new Pair3<Long, Long, Integer>(-1L, -1L,
379                        ModelBindingManager.getGlobalSqlInfo().getIndexOf(ModelBindingManager.getGlobalHash())) + ","
380                        + new Pair3<Long, Long, Integer>(-1L, -1L,
381                        ModelBindingManager.getGlobalSqlInfo().getIndexOf(ModelBindingManager.getGlobalHash())));
382            }
383            if (isView) {
384                table.setType("view");
385                dataflow.getViews().add(table);
386            } else {
387                table.setType("table");
388                dataflow.getTables().add(table);
389            }
390            tableMap.put(tableKey, table);
391        }
392
393        table table = tableMap.get(tableKey);
394        String sourceColumnKey = getFullName(serverName, databaseName, schemaName, tableName, columnName);
395        if (!columnMap.containsKey(sourceColumnKey)) {
396            column column = new column();
397            column.setId(String.valueOf(++ModelBindingManager.get().TABLE_COLUMN_ID));
398            column.setName(columnName);
399            if ("RelationRows".equalsIgnoreCase(columnName)) {
400                column.setSource("system");
401            }
402            if (ModelBindingManager.getGlobalSqlInfo() != null) {
403                column.setCoordinate(new Pair3<Long, Long, Integer>(-1L, -1L,
404                        ModelBindingManager.getGlobalSqlInfo().getIndexOf(ModelBindingManager.getGlobalHash())) + ","
405                        + new Pair3<Long, Long, Integer>(-1L, -1L,
406                        ModelBindingManager.getGlobalSqlInfo().getIndexOf(ModelBindingManager.getGlobalHash())));
407            }
408            table.getColumns().add(column);
409            columnMap.put(sourceColumnKey, column);
410        }
411    }
412
413    private String getFullName(String... segments) {
414        StringBuilder builder = new StringBuilder();
415        for (int i = 0; i < segments.length; i++) {
416            builder.append(segments[i]);
417            if (i < segments.length - 1) {
418                builder.append(".");
419            }
420        }
421        return builder.toString();
422    }
423}