Source code

001package gudusoft.gsqlparser.dlineage.dataflow.metadata.sqlflow.sharded;
002
003import gudusoft.gsqlparser.EDbVendor;
004import gudusoft.gsqlparser.dlineage.dataflow.metadata.MetadataAnalyzer;
005import gudusoft.gsqlparser.dlineage.dataflow.metadata.MetadataReader;
006import gudusoft.gsqlparser.dlineage.dataflow.model.*;
007import gudusoft.gsqlparser.dlineage.dataflow.model.xml.*;
008import gudusoft.gsqlparser.dlineage.util.Pair3;
009import gudusoft.gsqlparser.sqlenv.TSQLEnv;
010import gudusoft.gsqlparser.util.SQLUtil;
011import gudusoft.gsqlparser.util.json.JSON;
012
013import java.io.File;
014import java.io.IOException;
015import java.nio.charset.StandardCharsets;
016import java.nio.file.Files;
017import java.nio.file.Path;
018import java.nio.file.Paths;
019import java.util.*;
020
021/**
022 * Extracts metadata information (tables, views, procedures, relationships) from a
023 * sqlflow-sharded manifest and its catalog shards.
024 *
025 * <p>This class does <b>NOT</b> analyze SQL source code in queries, nor does it produce
026 * lineage for query objects. The lineage analysis of SQL queries is performed in
027 * {@link gudusoft.gsqlparser.dlineage.DataFlowAnalyzer DataFlowAnalyzer}, which reads
028 * the source JSONL files through the {@code databaseMap} / {@code appendSqlInfo}
029 * mechanism and invokes the SQL parser to generate dataflow relationships.
030 */
031public class SqlflowShardedMetadataAnalyzer implements MetadataAnalyzer<String> {
032
033    private Map<String, procedure> procedureMap = new LinkedHashMap<String, procedure>();
034    private Map<String, oraclePackage> oraclePackageMap = new LinkedHashMap<String, oraclePackage>();
035    private Map<String, table> tableMap = new LinkedHashMap<String, table>();
036    private Map<String, column> columnMap = new LinkedHashMap<String, column>();
037    private EDbVendor vendor;
038    private TSQLEnv sqlenv;
039    private String baseDir;
040
041    public SqlflowShardedMetadataAnalyzer(String baseDir) {
042        if (baseDir == null || baseDir.trim().isEmpty()) {
043            throw new IllegalArgumentException("baseDir is required");
044        }
045        this.baseDir = baseDir;
046    }
047
048    public SqlflowShardedMetadataAnalyzer(TSQLEnv sqlenv, String baseDir) {
049        if (baseDir == null || baseDir.trim().isEmpty()) {
050            throw new IllegalArgumentException("baseDir is required");
051        }
052        this.sqlenv = sqlenv;
053        this.baseDir = baseDir;
054    }
055
056    @Override
057    public synchronized dataflow analyzeMetadata(EDbVendor metadataVendor, String metadata) {
058        init(metadataVendor);
059        dataflow dataflow = new dataflow();
060
061        String manifestContent;
062        try {
063            if (metadata.trim().startsWith("{") && metadata.trim().endsWith("}")) {
064                manifestContent = metadata.trim();
065            } else {
066                manifestContent = new String(Files.readAllBytes(Paths.get(metadata)), StandardCharsets.UTF_8);
067            }
068        } catch (IOException e) {
069            throw new RuntimeException("Failed to read manifest: " + metadata, e);
070        }
071
072        Map manifest = (Map) JSON.parseObject(manifestContent);
073        if (manifest == null) {
074            return dataflow;
075        }
076
077        if (!"sqlflow-sharded".equals(manifest.get("format"))) {
078            return dataflow;
079        }
080
081        // Version gate: the `format` field is authoritative, so consume any
082        // `sqlflow-sharded` manifest whose `formatVersion` we understand,
083        // regardless of the `createdBy` product brand (SQLdep, sqlflow, grabit
084        // all emit this format). Anything outside the supported range — a newer
085        // major version, a bogus 0/negative, or a non-numeric value — is refused
086        // with a visible error rather than mis-parsed. ONE authoritative check
087        // (MetadataReader) is used here and at every other consumer entry point.
088        if (!MetadataReader.isSupportedSqlflowSharded(manifestContent)) {
089            dataflow.getErrors().add(
090                    unsupportedVersionError(MetadataReader.shardedFormatVersion(manifestContent)));
091            return dataflow;
092        }
093
094        {
095            {
096                List servers = (List) manifest.get("servers");
097                if (servers == null) {
098                    return dataflow;
099                }
100
101                for (int x = 0; x < servers.size(); x++) {
102                    Map server = (Map) servers.get(x);
103                    String serverName = (String) server.get("name");
104                    String dbVendor = (String) server.get("dbVendor");
105
106                    EDbVendor vendor = metadataVendor;
107                    if (dbVendor != null) {
108                        vendor = EDbVendor.valueOf(dbVendor);
109                    }
110
111                    boolean supportsCatalogs = TSQLEnv.supportCatalog(vendor);
112                    boolean supportsSchemas = TSQLEnv.supportSchema(vendor);
113
114                    if (server.containsKey("supportsCatalogs")) {
115                        supportsCatalogs = (Boolean) server.get("supportsCatalogs");
116                    }
117                    if (server.containsKey("supportsSchemas")) {
118                        supportsSchemas = (Boolean) server.get("supportsSchemas");
119                    }
120
121                    List databases = (List) server.get("databases");
122                    List schemaShards = (List) server.get("schemas");
123
124                    // Route on the PHYSICAL shard container. Schema-topology
125                    // exports (e.g. Oracle) list shards under servers[].schemas[],
126                    // each pointing at a FLAT single-schema catalog file
127                    // ({name, tables, views, procedures}) rather than a catalog
128                    // file that nests a "schemas" array. Prefer schema shards
129                    // when present; treat an empty databases[] as absent so a
130                    // manifest with "databases":[] alongside schemas[] still
131                    // loads. Handling only `databases` left the dataflow with no
132                    // catalog table/view nodes for schema-topology exports.
133                    boolean hasSchemaShards = schemaShards != null && !schemaShards.isEmpty();
134                    boolean hasDatabaseShards = databases != null && !databases.isEmpty();
135
136                    if (hasSchemaShards) {
137                        for (int i = 0; i < schemaShards.size(); i++) {
138                            Map jsonSchemaShard = (Map) schemaShards.get(i);
139                            String schemaName = (String) jsonSchemaShard.get("name");
140                            if (schemaName != null && SQLUtil.parseNames(schemaName).size() > 1) {
141                                schemaName = "\"" + schemaName + "\"";
142                            }
143
144                            Map catalogInfo = (Map) jsonSchemaShard.get("catalog");
145                            if (catalogInfo == null) {
146                                continue;
147                            }
148                            String catalogPath = (String) catalogInfo.get("path");
149                            if (SQLUtil.isEmpty(catalogPath)) {
150                                continue;
151                            }
152
153                            String catalogContent = loadCatalogFile(catalogPath);
154                            if (catalogContent == null) {
155                                continue;
156                            }
157
158                            Map catalog = (Map) JSON.parseObject(catalogContent);
159                            if (catalog == null) {
160                                continue;
161                            }
162
163                            // Flat catalog IS the schema: read tables/views/
164                            // procedures directly off it, keyed by the shard's
165                            // schema name under the synthetic default catalog.
166                            processTablesAndViews(vendor, supportsCatalogs, supportsSchemas, dataflow,
167                                    serverName, TSQLEnv.DEFAULT_DB_NAME, schemaName, catalog);
168                            processProcedures(vendor, supportsCatalogs, supportsSchemas, dataflow,
169                                    serverName, TSQLEnv.DEFAULT_DB_NAME, schemaName, catalog);
170                        }
171                        continue;
172                    }
173
174                    if (!hasDatabaseShards) {
175                        continue;
176                    }
177
178                    for (int i = 0; i < databases.size(); i++) {
179                        Map jsonDatabase = (Map) databases.get(i);
180                        String databaseName = (String) jsonDatabase.get("name");
181                        if (SQLUtil.parseNames(databaseName).size() > 1) {
182                            databaseName = "\"" + databaseName + "\"";
183                        }
184
185                        Map catalogInfo = (Map) jsonDatabase.get("catalog");
186                        if (catalogInfo == null) {
187                            continue;
188                        }
189                        String catalogPath = (String) catalogInfo.get("path");
190
191                        String catalogContent = loadCatalogFile(catalogPath);
192                        if (catalogContent == null) {
193                            continue;
194                        }
195
196                        Map catalog = (Map) JSON.parseObject(catalogContent);
197                        if (catalog == null) {
198                            continue;
199                        }
200
201                        if (supportsCatalogs && supportsSchemas) {
202                            processCatalogWithSchemas(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, databaseName, catalog);
203                        } else if (supportsCatalogs) {
204                            processCatalogCatalogOnly(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, databaseName, catalog);
205                        } else if (supportsSchemas) {
206                            processCatalogSchemaOnly(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, catalog);
207                        }
208                    }
209                }
210            }
211        }
212
213        sortTableColumns(dataflow);
214        return dataflow;
215    }
216
217    /** A visible, machine-readable marker that a manifest was refused on version. */
218    private static error unsupportedVersionError(int formatVersion) {
219        error e = new error();
220        e.setErrorType(ErrorInfo.METADATA_ERROR);
221        e.setErrorMessage("Unsupported sqlflow-sharded formatVersion " + formatVersion
222                + "; this build supports up to " + MetadataReader.SUPPORTED_SHARDED_FORMAT_VERSION
223                + ". Metadata not loaded.");
224        return e;
225    }
226
227    private String loadCatalogFile(String catalogPath) {
228        try {
229            if (catalogPath.startsWith("/") || catalogPath.matches("^[A-Za-z]:.*")) {
230                return new String(Files.readAllBytes(Paths.get(catalogPath)), StandardCharsets.UTF_8);
231            }
232            String normalizedPath = catalogPath.replace("/", File.separator);
233            Path fullPath = Paths.get(baseDir, normalizedPath);
234            return new String(Files.readAllBytes(fullPath), StandardCharsets.UTF_8);
235        } catch (IOException e) {
236            return null;
237        }
238    }
239
240    private void processCatalogWithSchemas(EDbVendor vendor, boolean supportsCatalogs, boolean supportsSchemas,
241                                           dataflow dataflow, String serverName, String databaseName, Map catalog) {
242        List schemas = (List) catalog.get("schemas");
243        if (schemas == null) {
244            return;
245        }
246
247        for (int j = 0; j < schemas.size(); j++) {
248            Map jsonSchema = (Map) schemas.get(j);
249            String schemaName = (String) jsonSchema.get("name");
250            if (SQLUtil.parseNames(schemaName).size() > 1) {
251                schemaName = "\"" + schemaName + "\"";
252            }
253
254            processTablesAndViews(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, databaseName, schemaName, jsonSchema);
255            processProcedures(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, databaseName, schemaName, jsonSchema);
256        }
257    }
258
259    private void processCatalogCatalogOnly(EDbVendor vendor, boolean supportsCatalogs, boolean supportsSchemas,
260                                           dataflow dataflow, String serverName, String databaseName, Map catalog) {
261        processTablesAndViews(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, databaseName,
262                TSQLEnv.DEFAULT_SCHEMA_NAME, catalog);
263        processProcedures(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, databaseName,
264                TSQLEnv.DEFAULT_SCHEMA_NAME, catalog);
265    }
266
267    private void processCatalogSchemaOnly(EDbVendor vendor, boolean supportsCatalogs, boolean supportsSchemas,
268                                          dataflow dataflow, String serverName, Map catalog) {
269        String databaseName = TSQLEnv.DEFAULT_DB_NAME;
270        if (SQLUtil.parseNames(databaseName).size() > 1) {
271            databaseName = "\"" + databaseName + "\"";
272        }
273
274        List schemas = (List) catalog.get("schemas");
275        if (schemas == null) {
276            return;
277        }
278
279        for (int j = 0; j < schemas.size(); j++) {
280            Map jsonSchema = (Map) schemas.get(j);
281            String schemaName = (String) jsonSchema.get("name");
282            if (SQLUtil.parseNames(schemaName).size() > 1) {
283                schemaName = "\"" + schemaName + "\"";
284            }
285
286            processTablesAndViews(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, databaseName, schemaName, jsonSchema);
287            processProcedures(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, databaseName, schemaName, jsonSchema);
288        }
289    }
290
291    private void processTablesAndViews(EDbVendor vendor, boolean supportsCatalogs, boolean supportsSchemas,
292                                      dataflow dataflow, String serverName, String databaseName, String schemaName,
293                                      Map jsonSchema) {
294        List tables = (List) jsonSchema.get("tables");
295        List views = (List) jsonSchema.get("views");
296
297        List dbObjs = new ArrayList();
298        if (tables != null) {
299            dbObjs.addAll(tables);
300        }
301        if (views != null) {
302            dbObjs.addAll(views);
303        }
304
305        for (int k = 0; k < dbObjs.size(); k++) {
306            Map jsonTable = (Map) dbObjs.get(k);
307            String tableName = (String) jsonTable.get("name");
308            String type = (String) jsonTable.get("type");
309            String fromDDL = (String) jsonTable.get("fromDDL");
310            boolean isView = false;
311            if (type != null && type.toLowerCase().indexOf("view") != -1) {
312                isView = true;
313            }
314
315            List columns = (List) jsonTable.get("columns");
316            if (columns == null) {
317                continue;
318            }
319
320            for (int l = 0; l < columns.size(); l++) {
321                Map jsonColumn = (Map) columns.get(l);
322                String columnName = (String) jsonColumn.get("name");
323                String dataType = null;
324                if (jsonColumn.containsKey("dataType")) {
325                    dataType = (String) jsonColumn.get("dataType");
326                }
327                Boolean primaryKey = null;
328                if (jsonColumn.containsKey("primaryKey")) {
329                    primaryKey = (Boolean) jsonColumn.get("primaryKey");
330                }
331                Boolean unqiueKey = null;
332                if (jsonColumn.containsKey("unqiueKey")) {
333                    unqiueKey = (Boolean) jsonColumn.get("unqiueKey");
334                }
335                Boolean indexKey = null;
336                if (jsonColumn.containsKey("indexKey")) {
337                    indexKey = (Boolean) jsonColumn.get("indexKey");
338                }
339                Boolean foreignKey = null;
340                if (jsonColumn.containsKey("foreignKey")) {
341                    foreignKey = (Boolean) jsonColumn.get("foreignKey");
342                }
343
344                appendTable(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, databaseName,
345                        schemaName, tableName, isView, false, columnName, dataType, primaryKey, unqiueKey, indexKey, foreignKey, fromDDL);
346            }
347        }
348
349        List synonyms = (List) jsonSchema.get("synonyms");
350        if (synonyms != null) {
351            for (int k = 0; k < synonyms.size(); k++) {
352                Map jsonSynonym = (Map) synonyms.get(k);
353                String synonymName = (String) jsonSynonym.get("name");
354                String synonymSourceName = (String) jsonSynonym.get("sourceName");
355                String synonymSourceDb = (String) jsonSynonym.get("sourceDatabase");
356                String synonymSourceSchema = (String) jsonSynonym.get("sourceSchema");
357
358                if (SQLUtil.isEmpty(synonymSourceDb)) {
359                    synonymSourceDb = databaseName;
360                }
361                if (SQLUtil.isEmpty(synonymSourceSchema)) {
362                    synonymSourceSchema = schemaName;
363                }
364
365                String tableKey = getFullTableName(vendor, serverName, synonymSourceDb, synonymSourceSchema, synonymSourceName);
366                table sourceTable = tableMap.get(tableKey);
367                if (sourceTable != null) {
368                    List<column> columns = sourceTable.getColumns();
369                    String fromDDL = sourceTable.getFromDDL();
370                    for (int l = 0; l < columns.size(); l++) {
371                        column jsonColumn = columns.get(l);
372                        String columnName = jsonColumn.getName();
373                        String dataType = jsonColumn.getDataType();
374                        appendTable(vendor, supportsCatalogs, supportsSchemas, dataflow,
375                                serverName, databaseName, schemaName, synonymName, false, true,
376                                columnName, dataType, false, false, false, false, fromDDL);
377                    }
378
379                    String synonymTableKey = getFullTableName(vendor, serverName, databaseName, schemaName, synonymName);
380                    table synonymTable = tableMap.get(synonymTableKey);
381                    if (synonymTable != null) {
382                        List<column> synonymColumns = synonymTable.getColumns();
383
384                        for (int l = 0; l < synonymColumns.size(); l++) {
385                            relationship relationElement = new relationship();
386                            relationElement.setType(RelationshipType.fdd.name());
387                            relationElement.setEffectType(EffectType.synonym.name());
388
389                            long id = ++ModelBindingManager.get().RELATION_ID;
390                            relationElement.setId(String.valueOf(id));
391
392                            column targetColumn = synonymColumns.get(l);
393                            targetColumn target = new targetColumn();
394                            target.setId(String.valueOf(targetColumn.getId()));
395                            target.setColumn(targetColumn.getName());
396                            target.setParent_id(String.valueOf(synonymTable.getId()));
397                            target.setParent_name(synonymTable.getName());
398                            relationElement.setTarget(target);
399
400                            column sourceColumn = columns.get(l);
401                            sourceColumn source = new sourceColumn();
402                            source.setId(String.valueOf(sourceColumn.getId()));
403                            source.setColumn(sourceColumn.getName());
404                            source.setParent_id(String.valueOf(sourceTable.getId()));
405                            source.setParent_name(sourceTable.getName());
406                            relationElement.addSource(source);
407
408                            dataflow.getRelationships().add(relationElement);
409                        }
410                    }
411                }
412            }
413        }
414    }
415
416    private void processProcedures(EDbVendor vendor, boolean supportsCatalogs, boolean supportsSchemas,
417                                   dataflow dataflow, String serverName, String databaseName, String schemaName,
418                                   Map jsonSchema) {
419        List<Map> procedures = (List<Map>) jsonSchema.get("procedures");
420        if (procedures != null) {
421            for (Map procedure : procedures) {
422                appendProcedure(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, databaseName, schemaName, procedure, null);
423            }
424        }
425
426        List<Map> functions = (List<Map>) jsonSchema.get("functions");
427        if (functions != null) {
428            for (Map function : functions) {
429                appendProcedure(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, databaseName, schemaName, function, null);
430            }
431        }
432
433        List<Map> triggers = (List<Map>) jsonSchema.get("triggers");
434        if (triggers != null) {
435            for (Map trigger : triggers) {
436                appendProcedure(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, databaseName, schemaName, trigger, null);
437            }
438        }
439
440        List<Map> oraclePackages = (List<Map>) jsonSchema.get("packages");
441        if (oraclePackages != null) {
442            for (Map oraclePackageItem : oraclePackages) {
443                String oraclePackageName = (String) oraclePackageItem.get("name");
444                oraclePackage oraclePackage = appendOraclePackage(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, databaseName, schemaName, oraclePackageName);
445
446                List<Map> pkgProcedures = (List<Map>) oraclePackageItem.get("procedures");
447                if (pkgProcedures != null) {
448                    for (Map procedure : pkgProcedures) {
449                        appendProcedure(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, databaseName, schemaName, procedure, oraclePackage);
450                    }
451                }
452
453                List<Map> pkgFunctions = (List<Map>) oraclePackageItem.get("functions");
454                if (pkgFunctions != null) {
455                    for (Map function : pkgFunctions) {
456                        appendProcedure(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, databaseName, schemaName, function, oraclePackage);
457                    }
458                }
459
460                List<Map> pkgTriggers = (List<Map>) oraclePackageItem.get("triggers");
461                if (pkgTriggers != null) {
462                    for (Map trigger : pkgTriggers) {
463                        appendProcedure(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, databaseName, schemaName, trigger, oraclePackage);
464                    }
465                }
466            }
467        }
468    }
469
470    private void init(EDbVendor vendor) {
471        this.vendor = vendor;
472        procedureMap.clear();
473        oraclePackageMap.clear();
474        tableMap.clear();
475        columnMap.clear();
476        if (ModelBindingManager.get() == null) {
477            ModelBindingManager.set(new ModelBindingManager());
478        }
479    }
480
481    private void sortTableColumns(dataflow dataflow) {
482        if (dataflow.getTables() != null) {
483            for (table table : dataflow.getTables()) {
484                Collections.sort(table.getColumns(), new Comparator<column>() {
485                    public int compare(column t1, column t2) {
486                        if (t1.getName().equalsIgnoreCase("RelationRows"))
487                            return 1;
488                        if (t2.getName().equalsIgnoreCase("RelationRows"))
489                            return -1;
490                        return 0;
491                    }
492                });
493            }
494        }
495
496        if (dataflow.getResultsets() != null) {
497            for (table table : dataflow.getResultsets()) {
498                Collections.sort(table.getColumns(), new Comparator<column>() {
499                    public int compare(column t1, column t2) {
500                        if (t1.getName().equalsIgnoreCase("RelationRows"))
501                            return 1;
502                        if (t2.getName().equalsIgnoreCase("RelationRows"))
503                            return -1;
504                        return 0;
505                    }
506                });
507            }
508        }
509    }
510
511    private oraclePackage appendOraclePackage(EDbVendor vendor, boolean supportsCatalogs, boolean supportsSchemas,
512                                              dataflow dataflow, String serverName, String databaseName, String schemaName,
513                                              String oraclePackageName) {
514        String oraclePackageKey = getFullTableName(vendor, serverName, databaseName, schemaName, oraclePackageName);
515        if (!oraclePackageMap.containsKey(oraclePackageKey)) {
516            oraclePackage oraclePackage = new oraclePackage();
517            if (serverName != null) {
518                oraclePackage.setServer(serverName);
519            } else if (sqlenv != null && sqlenv.getDefaultServerName() != null
520                    && !TSQLEnv.DEFAULT_SERVER_NAME.equalsIgnoreCase(sqlenv.getDefaultServerName())) {
521                oraclePackage.setServer(sqlenv.getDefaultServerName());
522            }
523            if (supportsCatalogs && databaseName != null) {
524                oraclePackage.setDatabase(databaseName);
525            } else if (supportsCatalogs && sqlenv != null && sqlenv.getDefaultCatalogName() != null
526                    && !TSQLEnv.DEFAULT_DB_NAME.equalsIgnoreCase(sqlenv.getDefaultCatalogName())) {
527                oraclePackage.setDatabase(sqlenv.getDefaultCatalogName());
528            }
529
530            if (supportsSchemas && schemaName != null) {
531                oraclePackage.setSchema(schemaName);
532            } else if (supportsSchemas && sqlenv != null && sqlenv.getDefaultSchemaName() != null
533                    && !TSQLEnv.DEFAULT_SCHEMA_NAME.equalsIgnoreCase(sqlenv.getDefaultSchemaName())) {
534                oraclePackage.setSchema(sqlenv.getDefaultSchemaName());
535            }
536
537            oraclePackage.setName(oraclePackageName);
538            if (supportsSchemas && !SQLUtil.isEmpty(oraclePackage.getSchema())) {
539                oraclePackage.setName(oraclePackage.getSchema() + "." + oraclePackage.getName());
540            }
541            if (supportsCatalogs && !SQLUtil.isEmpty(oraclePackage.getDatabase())) {
542                oraclePackage.setName(oraclePackage.getDatabase() + "." + oraclePackage.getName());
543            }
544            oraclePackage.setId(String.valueOf(++ModelBindingManager.get().TABLE_COLUMN_ID));
545            if (ModelBindingManager.getGlobalSqlInfo() != null) {
546                oraclePackage.setCoordinate(new Pair3<Long, Long, Integer>(-1L, -1L,
547                        ModelBindingManager.getGlobalSqlInfo().getIndexOf(ModelBindingManager.getGlobalHash())) + ","
548                        + new Pair3<Long, Long, Integer>(-1L, -1L,
549                        ModelBindingManager.getGlobalSqlInfo().getIndexOf(ModelBindingManager.getGlobalHash())));
550            }
551            dataflow.getPackages().add(oraclePackage);
552            oraclePackageMap.put(oraclePackageKey, oraclePackage);
553        }
554        return oraclePackageMap.get(oraclePackageKey);
555    }
556
557    private void appendProcedure(EDbVendor vendor, boolean supportsCatalogs, boolean supportsSchemas,
558                                 dataflow dataflow, String serverName, String databaseName, String schemaName,
559                                 Map procedureObj, oraclePackage oraclePackage) {
560        String procedureName = (String) procedureObj.get("name");
561        if (procedureName == null) {
562            return;
563        }
564
565        String procedureKey = getFullTableName(vendor, serverName, databaseName, schemaName, procedureName);
566        if (!procedureMap.containsKey(procedureKey)) {
567            procedure procedure = new procedure();
568            if (serverName != null) {
569                procedure.setServer(serverName);
570            } else if (sqlenv != null && sqlenv.getDefaultServerName() != null
571                    && !TSQLEnv.DEFAULT_SERVER_NAME.equalsIgnoreCase(sqlenv.getDefaultServerName())) {
572                procedure.setServer(sqlenv.getDefaultServerName());
573            }
574            if (supportsCatalogs && databaseName != null) {
575                procedure.setDatabase(databaseName);
576            } else if (supportsCatalogs && sqlenv != null && sqlenv.getDefaultCatalogName() != null
577                    && !TSQLEnv.DEFAULT_DB_NAME.equalsIgnoreCase(sqlenv.getDefaultCatalogName())) {
578                procedure.setDatabase(sqlenv.getDefaultCatalogName());
579            }
580
581            if (supportsSchemas && schemaName != null) {
582                procedure.setSchema(schemaName);
583            } else if (supportsSchemas && sqlenv != null && sqlenv.getDefaultSchemaName() != null
584                    && !TSQLEnv.DEFAULT_SCHEMA_NAME.equalsIgnoreCase(sqlenv.getDefaultSchemaName())) {
585                procedure.setSchema(sqlenv.getDefaultSchemaName());
586            }
587
588            procedure.setName(procedureName);
589            if (supportsSchemas && !SQLUtil.isEmpty(procedure.getSchema())) {
590                procedure.setName(procedure.getSchema() + "." + procedure.getName());
591            }
592            if (supportsCatalogs && !SQLUtil.isEmpty(procedure.getDatabase())) {
593                procedure.setName(procedure.getDatabase() + "." + procedure.getName());
594            }
595            procedure.setId(String.valueOf(++ModelBindingManager.get().TABLE_COLUMN_ID));
596            if (ModelBindingManager.getGlobalSqlInfo() != null) {
597                procedure.setCoordinate(new Pair3<Long, Long, Integer>(-1L, -1L,
598                        ModelBindingManager.getGlobalSqlInfo().getIndexOf(ModelBindingManager.getGlobalHash())) + ","
599                        + new Pair3<Long, Long, Integer>(-1L, -1L,
600                        ModelBindingManager.getGlobalSqlInfo().getIndexOf(ModelBindingManager.getGlobalHash())));
601            }
602            procedure.setType((String) procedureObj.get("type"));
603            if (oraclePackage != null) {
604                oraclePackage.getProcedures().add(procedure);
605            } else {
606                dataflow.getProcedures().add(procedure);
607            }
608            procedureMap.put(procedureKey, procedure);
609        }
610    }
611
612    private void appendTable(EDbVendor vendor, boolean supportsCatalogs, boolean supportsSchemas,
613                            dataflow dataflow, String serverName, String databaseName, String schemaName,
614                            String tableName, boolean isView, boolean isSynonym, String columnName,
615                            String dataType, Boolean primaryKey, Boolean unqiueKey, Boolean indexKey,
616                            Boolean foreignKey, String fromDDL) {
617        String tableKey = getFullTableName(vendor, serverName, databaseName, schemaName, tableName);
618        if (!tableMap.containsKey(tableKey)) {
619            table table = new table();
620            if (serverName != null) {
621                table.setServer(serverName);
622            } else if (sqlenv != null && sqlenv.getDefaultServerName() != null
623                    && !TSQLEnv.DEFAULT_SERVER_NAME.equalsIgnoreCase(sqlenv.getDefaultServerName())) {
624                table.setServer(sqlenv.getDefaultServerName());
625            }
626            if (supportsCatalogs && databaseName != null) {
627                table.setDatabase(databaseName);
628            } else if (supportsCatalogs && sqlenv != null && sqlenv.getDefaultCatalogName() != null
629                    && !TSQLEnv.DEFAULT_DB_NAME.equalsIgnoreCase(sqlenv.getDefaultCatalogName())) {
630                table.setDatabase(sqlenv.getDefaultCatalogName());
631            }
632
633            if (supportsSchemas && schemaName != null) {
634                table.setSchema(schemaName);
635            } else if (supportsSchemas && sqlenv != null && sqlenv.getDefaultSchemaName() != null
636                    && !TSQLEnv.DEFAULT_SCHEMA_NAME.equalsIgnoreCase(sqlenv.getDefaultSchemaName())) {
637                table.setSchema(sqlenv.getDefaultSchemaName());
638            }
639
640            table.setName(tableName);
641            if (supportsSchemas && !SQLUtil.isEmpty(table.getSchema())) {
642                table.setName(table.getSchema() + "." + table.getName());
643            }
644            if (supportsCatalogs && !SQLUtil.isEmpty(table.getDatabase())) {
645                table.setName(table.getDatabase() + "." + table.getName());
646            }
647            table.setId(String.valueOf(++ModelBindingManager.get().TABLE_COLUMN_ID));
648            if (ModelBindingManager.getGlobalSqlInfo() != null) {
649                table.setCoordinate(new Pair3<Long, Long, Integer>(-1L, -1L,
650                        ModelBindingManager.getGlobalSqlInfo().getIndexOf(ModelBindingManager.getGlobalHash())) + ","
651                        + new Pair3<Long, Long, Integer>(-1L, -1L,
652                        ModelBindingManager.getGlobalSqlInfo().getIndexOf(ModelBindingManager.getGlobalHash())));
653            }
654
655            if (!SQLUtil.isEmpty(fromDDL)) {
656                table.setFromDDL(fromDDL);
657            }
658
659            if (isView) {
660                table.setType("view");
661                dataflow.getViews().add(table);
662            } else {
663                table.setType("table");
664                dataflow.getTables().add(table);
665            }
666            if (isSynonym) {
667                table.setSubType(SubType.synonym.name());
668            }
669
670            tableMap.put(tableKey, table);
671        }
672
673        table table = tableMap.get(tableKey);
674        String sourceColumnKey = getFullColumnName(vendor, serverName, databaseName, schemaName, tableName, columnName);
675        if (!columnMap.containsKey(sourceColumnKey)) {
676            column column = new column();
677            column.setId(String.valueOf(++ModelBindingManager.get().TABLE_COLUMN_ID));
678            column.setName(columnName);
679            column.setDataType(dataType);
680            column.setPrimaryKey(primaryKey);
681            column.setForeignKey(foreignKey);
682            column.setIndexKey(indexKey);
683            column.setUnqiueKey(unqiueKey);
684            if ("RelationRows".equalsIgnoreCase(columnName)) {
685                column.setSource("system");
686            }
687            if (ModelBindingManager.getGlobalSqlInfo() != null) {
688                column.setCoordinate(new Pair3<Long, Long, Integer>(-1L, -1L,
689                        ModelBindingManager.getGlobalSqlInfo().getIndexOf(ModelBindingManager.getGlobalHash())) + ","
690                        + new Pair3<Long, Long, Integer>(-1L, -1L,
691                        ModelBindingManager.getGlobalSqlInfo().getIndexOf(ModelBindingManager.getGlobalHash())));
692            }
693            table.getColumns().add(column);
694            columnMap.put(sourceColumnKey, column);
695        }
696    }
697
698    private String getFullColumnName(EDbVendor vendor, String... segments) {
699        StringBuilder builder = new StringBuilder();
700        for (int i = 1; i < segments.length; i++) {
701            if (segments[i] == null) {
702                continue;
703            }
704            builder.append(segments[i]);
705            if (i < segments.length - 1) {
706                builder.append(".");
707            }
708        }
709
710        return segments[0] + "." + SQLUtil.getIdentifierNormalColumnName(vendor, builder.toString());
711    }
712
713    private String getFullTableName(EDbVendor vendor, String... segments) {
714        StringBuilder builder = new StringBuilder();
715        for (int i = 1; i < segments.length; i++) {
716            if (segments[i] == null) {
717                continue;
718            }
719            builder.append(segments[i]);
720            if (i < segments.length - 1) {
721                builder.append(".");
722            }
723        }
724
725        return segments[0] + "." + SQLUtil.getIdentifierNormalTableName(vendor, builder.toString());
726    }
727
728    public static void main(String[] args) throws Exception {
729        String sampleManifestPath = "sqlflow-sharded/sample/manifest.json";
730        String baseDir = new File(sampleManifestPath).getParent();
731        dataflow dataflow = new SqlflowShardedMetadataAnalyzer(baseDir)
732                .analyzeMetadata(EDbVendor.dbvmssql, SQLUtil.getFileContent(new File(sampleManifestPath)));
733        System.out.println("Tables: " + dataflow.getTables().size());
734        System.out.println("Views: " + dataflow.getViews().size());
735        System.out.println("Procedures: " + dataflow.getProcedures().size());
736    }
737}