001package gudusoft.gsqlparser.dlineage.dataflow.metadata.sqlflow.sharded;
002
003import gudusoft.gsqlparser.EDbVendor;
004import gudusoft.gsqlparser.dlineage.dataflow.metadata.MetadataAnalyzer;
005import gudusoft.gsqlparser.dlineage.dataflow.model.EffectType;
006import gudusoft.gsqlparser.dlineage.dataflow.model.ModelBindingManager;
007import gudusoft.gsqlparser.dlineage.dataflow.model.RelationshipType;
008import gudusoft.gsqlparser.dlineage.dataflow.model.SubType;
009import gudusoft.gsqlparser.dlineage.dataflow.model.xml.*;
010import gudusoft.gsqlparser.dlineage.util.Pair3;
011import gudusoft.gsqlparser.sqlenv.TSQLEnv;
012import gudusoft.gsqlparser.util.SQLUtil;
013import gudusoft.gsqlparser.util.json.JSON;
014
015import java.io.File;
016import java.io.IOException;
017import java.nio.charset.StandardCharsets;
018import java.nio.file.Files;
019import java.nio.file.Path;
020import java.nio.file.Paths;
021import java.util.*;
022
023//autogenerate
024public class SqlflowShardedMetadataAnalyzer implements MetadataAnalyzer<String> {
025
026    private Map<String, procedure> procedureMap = new LinkedHashMap<String, procedure>();
027    private Map<String, oraclePackage> oraclePackageMap = new LinkedHashMap<String, oraclePackage>();
028    private Map<String, table> tableMap = new LinkedHashMap<String, table>();
029    private Map<String, column> columnMap = new LinkedHashMap<String, column>();
030    private EDbVendor vendor;
031    private TSQLEnv sqlenv;
032    private String baseDir;
033
034    public SqlflowShardedMetadataAnalyzer(String baseDir) {
035        if (baseDir == null || baseDir.trim().isEmpty()) {
036            throw new IllegalArgumentException("baseDir is required");
037        }
038        this.baseDir = baseDir;
039    }
040
041    public SqlflowShardedMetadataAnalyzer(TSQLEnv sqlenv, String baseDir) {
042        if (baseDir == null || baseDir.trim().isEmpty()) {
043            throw new IllegalArgumentException("baseDir is required");
044        }
045        this.sqlenv = sqlenv;
046        this.baseDir = baseDir;
047    }
048
049    @Override
050    public synchronized dataflow analyzeMetadata(EDbVendor metadataVendor, String metadata) {
051        init(metadataVendor);
052        dataflow dataflow = new dataflow();
053
054        String manifestContent;
055        try {
056            if (metadata.trim().startsWith("{") && metadata.trim().endsWith("}")) {
057                manifestContent = metadata.trim();
058            } else {
059                manifestContent = new String(Files.readAllBytes(Paths.get(metadata)), StandardCharsets.UTF_8);
060            }
061        } catch (IOException e) {
062            throw new RuntimeException("Failed to read manifest: " + metadata, e);
063        }
064
065        Map manifest = (Map) JSON.parseObject(manifestContent);
066        if (manifest == null) {
067            return dataflow;
068        }
069
070        if (!"sqlflow-sharded".equals(manifest.get("format"))) {
071            return dataflow;
072        }
073
074        if (manifest.containsKey("createdBy")) {
075            String createdBy = (String) manifest.get("createdBy");
076            if (createdBy.toLowerCase().indexOf("sqlflow") != -1) {
077                List servers = (List) manifest.get("servers");
078                if (servers == null) {
079                    return dataflow;
080                }
081
082                for (int x = 0; x < servers.size(); x++) {
083                    Map server = (Map) servers.get(x);
084                    String serverName = (String) server.get("name");
085                    String dbVendor = (String) server.get("dbVendor");
086
087                    EDbVendor vendor = metadataVendor;
088                    if (dbVendor != null) {
089                        vendor = EDbVendor.valueOf(dbVendor);
090                    }
091
092                    boolean supportsCatalogs = TSQLEnv.supportCatalog(vendor);
093                    boolean supportsSchemas = TSQLEnv.supportSchema(vendor);
094
095                    if (server.containsKey("supportsCatalogs")) {
096                        supportsCatalogs = (Boolean) server.get("supportsCatalogs");
097                    }
098                    if (server.containsKey("supportsSchemas")) {
099                        supportsSchemas = (Boolean) server.get("supportsSchemas");
100                    }
101
102                    List databases = (List) server.get("databases");
103                    if (databases == null) {
104                        continue;
105                    }
106
107                    for (int i = 0; i < databases.size(); i++) {
108                        Map jsonDatabase = (Map) databases.get(i);
109                        String databaseName = (String) jsonDatabase.get("name");
110                        if (SQLUtil.parseNames(databaseName).size() > 1) {
111                            databaseName = "\"" + databaseName + "\"";
112                        }
113
114                        Map catalogInfo = (Map) jsonDatabase.get("catalog");
115                        if (catalogInfo == null) {
116                            continue;
117                        }
118                        String catalogPath = (String) catalogInfo.get("path");
119
120                        String catalogContent = loadCatalogFile(catalogPath);
121                        if (catalogContent == null) {
122                            continue;
123                        }
124
125                        Map catalog = (Map) JSON.parseObject(catalogContent);
126                        if (catalog == null) {
127                            continue;
128                        }
129
130                        if (supportsCatalogs && supportsSchemas) {
131                            processCatalogWithSchemas(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, databaseName, catalog);
132                        } else if (supportsCatalogs) {
133                            processCatalogCatalogOnly(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, databaseName, catalog);
134                        } else if (supportsSchemas) {
135                            processCatalogSchemaOnly(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, catalog);
136                        }
137                    }
138                }
139            }
140        }
141
142        sortTableColumns(dataflow);
143        return dataflow;
144    }
145
146    private String loadCatalogFile(String catalogPath) {
147        try {
148            if (catalogPath.startsWith("/") || catalogPath.matches("^[A-Za-z]:.*")) {
149                return new String(Files.readAllBytes(Paths.get(catalogPath)), StandardCharsets.UTF_8);
150            }
151            String normalizedPath = catalogPath.replace("/", File.separator);
152            Path fullPath = Paths.get(baseDir, normalizedPath);
153            return new String(Files.readAllBytes(fullPath), StandardCharsets.UTF_8);
154        } catch (IOException e) {
155            return null;
156        }
157    }
158
159    private void processCatalogWithSchemas(EDbVendor vendor, boolean supportsCatalogs, boolean supportsSchemas,
160                                           dataflow dataflow, String serverName, String databaseName, Map catalog) {
161        List schemas = (List) catalog.get("schemas");
162        if (schemas == null) {
163            return;
164        }
165
166        for (int j = 0; j < schemas.size(); j++) {
167            Map jsonSchema = (Map) schemas.get(j);
168            String schemaName = (String) jsonSchema.get("name");
169            if (SQLUtil.parseNames(schemaName).size() > 1) {
170                schemaName = "\"" + schemaName + "\"";
171            }
172
173            processTablesAndViews(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, databaseName, schemaName, jsonSchema);
174            processProcedures(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, databaseName, schemaName, jsonSchema);
175        }
176    }
177
178    private void processCatalogCatalogOnly(EDbVendor vendor, boolean supportsCatalogs, boolean supportsSchemas,
179                                           dataflow dataflow, String serverName, String databaseName, Map catalog) {
180        processTablesAndViews(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, databaseName,
181                TSQLEnv.DEFAULT_SCHEMA_NAME, catalog);
182        processProcedures(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, databaseName,
183                TSQLEnv.DEFAULT_SCHEMA_NAME, catalog);
184    }
185
186    private void processCatalogSchemaOnly(EDbVendor vendor, boolean supportsCatalogs, boolean supportsSchemas,
187                                          dataflow dataflow, String serverName, Map catalog) {
188        String databaseName = TSQLEnv.DEFAULT_DB_NAME;
189        if (SQLUtil.parseNames(databaseName).size() > 1) {
190            databaseName = "\"" + databaseName + "\"";
191        }
192
193        List schemas = (List) catalog.get("schemas");
194        if (schemas == null) {
195            return;
196        }
197
198        for (int j = 0; j < schemas.size(); j++) {
199            Map jsonSchema = (Map) schemas.get(j);
200            String schemaName = (String) jsonSchema.get("name");
201            if (SQLUtil.parseNames(schemaName).size() > 1) {
202                schemaName = "\"" + schemaName + "\"";
203            }
204
205            processTablesAndViews(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, databaseName, schemaName, jsonSchema);
206            processProcedures(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, databaseName, schemaName, jsonSchema);
207        }
208    }
209
210    private void processTablesAndViews(EDbVendor vendor, boolean supportsCatalogs, boolean supportsSchemas,
211                                      dataflow dataflow, String serverName, String databaseName, String schemaName,
212                                      Map jsonSchema) {
213        List tables = (List) jsonSchema.get("tables");
214        List views = (List) jsonSchema.get("views");
215
216        List dbObjs = new ArrayList();
217        if (tables != null) {
218            dbObjs.addAll(tables);
219        }
220        if (views != null) {
221            dbObjs.addAll(views);
222        }
223
224        for (int k = 0; k < dbObjs.size(); k++) {
225            Map jsonTable = (Map) dbObjs.get(k);
226            String tableName = (String) jsonTable.get("name");
227            String type = (String) jsonTable.get("type");
228            String fromDDL = (String) jsonTable.get("fromDDL");
229            boolean isView = false;
230            if (type != null && type.toLowerCase().indexOf("view") != -1) {
231                isView = true;
232            }
233
234            List columns = (List) jsonTable.get("columns");
235            if (columns == null) {
236                continue;
237            }
238
239            for (int l = 0; l < columns.size(); l++) {
240                Map jsonColumn = (Map) columns.get(l);
241                String columnName = (String) jsonColumn.get("name");
242                String dataType = null;
243                if (jsonColumn.containsKey("dataType")) {
244                    dataType = (String) jsonColumn.get("dataType");
245                }
246                Boolean primaryKey = null;
247                if (jsonColumn.containsKey("primaryKey")) {
248                    primaryKey = (Boolean) jsonColumn.get("primaryKey");
249                }
250                Boolean unqiueKey = null;
251                if (jsonColumn.containsKey("unqiueKey")) {
252                    unqiueKey = (Boolean) jsonColumn.get("unqiueKey");
253                }
254                Boolean indexKey = null;
255                if (jsonColumn.containsKey("indexKey")) {
256                    indexKey = (Boolean) jsonColumn.get("indexKey");
257                }
258                Boolean foreignKey = null;
259                if (jsonColumn.containsKey("foreignKey")) {
260                    foreignKey = (Boolean) jsonColumn.get("foreignKey");
261                }
262
263                appendTable(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, databaseName,
264                        schemaName, tableName, isView, false, columnName, dataType, primaryKey, unqiueKey, indexKey, foreignKey, fromDDL);
265            }
266        }
267
268        List synonyms = (List) jsonSchema.get("synonyms");
269        if (synonyms != null) {
270            for (int k = 0; k < synonyms.size(); k++) {
271                Map jsonSynonym = (Map) synonyms.get(k);
272                String synonymName = (String) jsonSynonym.get("name");
273                String synonymSourceName = (String) jsonSynonym.get("sourceName");
274                String synonymSourceDb = (String) jsonSynonym.get("sourceDatabase");
275                String synonymSourceSchema = (String) jsonSynonym.get("sourceSchema");
276
277                if (SQLUtil.isEmpty(synonymSourceDb)) {
278                    synonymSourceDb = databaseName;
279                }
280                if (SQLUtil.isEmpty(synonymSourceSchema)) {
281                    synonymSourceSchema = schemaName;
282                }
283
284                String tableKey = getFullTableName(vendor, serverName, synonymSourceDb, synonymSourceSchema, synonymSourceName);
285                table sourceTable = tableMap.get(tableKey);
286                if (sourceTable != null) {
287                    List<column> columns = sourceTable.getColumns();
288                    String fromDDL = sourceTable.getFromDDL();
289                    for (int l = 0; l < columns.size(); l++) {
290                        column jsonColumn = columns.get(l);
291                        String columnName = jsonColumn.getName();
292                        String dataType = jsonColumn.getDataType();
293                        appendTable(vendor, supportsCatalogs, supportsSchemas, dataflow,
294                                serverName, databaseName, schemaName, synonymName, false, true,
295                                columnName, dataType, false, false, false, false, fromDDL);
296                    }
297
298                    String synonymTableKey = getFullTableName(vendor, serverName, databaseName, schemaName, synonymName);
299                    table synonymTable = tableMap.get(synonymTableKey);
300                    if (synonymTable != null) {
301                        List<column> synonymColumns = synonymTable.getColumns();
302
303                        for (int l = 0; l < synonymColumns.size(); l++) {
304                            relationship relationElement = new relationship();
305                            relationElement.setType(RelationshipType.fdd.name());
306                            relationElement.setEffectType(EffectType.synonym.name());
307
308                            long id = ++ModelBindingManager.get().RELATION_ID;
309                            relationElement.setId(String.valueOf(id));
310
311                            column targetColumn = synonymColumns.get(l);
312                            targetColumn target = new targetColumn();
313                            target.setId(String.valueOf(targetColumn.getId()));
314                            target.setColumn(targetColumn.getName());
315                            target.setParent_id(String.valueOf(synonymTable.getId()));
316                            target.setParent_name(synonymTable.getName());
317                            relationElement.setTarget(target);
318
319                            column sourceColumn = columns.get(l);
320                            sourceColumn source = new sourceColumn();
321                            source.setId(String.valueOf(sourceColumn.getId()));
322                            source.setColumn(sourceColumn.getName());
323                            source.setParent_id(String.valueOf(sourceTable.getId()));
324                            source.setParent_name(sourceTable.getName());
325                            relationElement.addSource(source);
326
327                            dataflow.getRelationships().add(relationElement);
328                        }
329                    }
330                }
331            }
332        }
333    }
334
335    private void processProcedures(EDbVendor vendor, boolean supportsCatalogs, boolean supportsSchemas,
336                                   dataflow dataflow, String serverName, String databaseName, String schemaName,
337                                   Map jsonSchema) {
338        List<Map> procedures = (List<Map>) jsonSchema.get("procedures");
339        if (procedures != null) {
340            for (Map procedure : procedures) {
341                appendProcedure(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, databaseName, schemaName, procedure, null);
342            }
343        }
344
345        List<Map> functions = (List<Map>) jsonSchema.get("functions");
346        if (functions != null) {
347            for (Map function : functions) {
348                appendProcedure(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, databaseName, schemaName, function, null);
349            }
350        }
351
352        List<Map> triggers = (List<Map>) jsonSchema.get("triggers");
353        if (triggers != null) {
354            for (Map trigger : triggers) {
355                appendProcedure(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, databaseName, schemaName, trigger, null);
356            }
357        }
358
359        List<Map> oraclePackages = (List<Map>) jsonSchema.get("packages");
360        if (oraclePackages != null) {
361            for (Map oraclePackageItem : oraclePackages) {
362                String oraclePackageName = (String) oraclePackageItem.get("name");
363                oraclePackage oraclePackage = appendOraclePackage(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, databaseName, schemaName, oraclePackageName);
364
365                List<Map> pkgProcedures = (List<Map>) oraclePackageItem.get("procedures");
366                if (pkgProcedures != null) {
367                    for (Map procedure : pkgProcedures) {
368                        appendProcedure(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, databaseName, schemaName, procedure, oraclePackage);
369                    }
370                }
371
372                List<Map> pkgFunctions = (List<Map>) oraclePackageItem.get("functions");
373                if (pkgFunctions != null) {
374                    for (Map function : pkgFunctions) {
375                        appendProcedure(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, databaseName, schemaName, function, oraclePackage);
376                    }
377                }
378
379                List<Map> pkgTriggers = (List<Map>) oraclePackageItem.get("triggers");
380                if (pkgTriggers != null) {
381                    for (Map trigger : pkgTriggers) {
382                        appendProcedure(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, databaseName, schemaName, trigger, oraclePackage);
383                    }
384                }
385            }
386        }
387    }
388
389    private void init(EDbVendor vendor) {
390        this.vendor = vendor;
391        procedureMap.clear();
392        oraclePackageMap.clear();
393        tableMap.clear();
394        columnMap.clear();
395        if (ModelBindingManager.get() == null) {
396            ModelBindingManager.set(new ModelBindingManager());
397        }
398    }
399
400    private void sortTableColumns(dataflow dataflow) {
401        if (dataflow.getTables() != null) {
402            for (table table : dataflow.getTables()) {
403                Collections.sort(table.getColumns(), new Comparator<column>() {
404                    public int compare(column t1, column t2) {
405                        if (t1.getName().equalsIgnoreCase("RelationRows"))
406                            return 1;
407                        if (t2.getName().equalsIgnoreCase("RelationRows"))
408                            return -1;
409                        return 0;
410                    }
411                });
412            }
413        }
414
415        if (dataflow.getResultsets() != null) {
416            for (table table : dataflow.getResultsets()) {
417                Collections.sort(table.getColumns(), new Comparator<column>() {
418                    public int compare(column t1, column t2) {
419                        if (t1.getName().equalsIgnoreCase("RelationRows"))
420                            return 1;
421                        if (t2.getName().equalsIgnoreCase("RelationRows"))
422                            return -1;
423                        return 0;
424                    }
425                });
426            }
427        }
428    }
429
430    private oraclePackage appendOraclePackage(EDbVendor vendor, boolean supportsCatalogs, boolean supportsSchemas,
431                                              dataflow dataflow, String serverName, String databaseName, String schemaName,
432                                              String oraclePackageName) {
433        String oraclePackageKey = getFullTableName(vendor, serverName, databaseName, schemaName, oraclePackageName);
434        if (!oraclePackageMap.containsKey(oraclePackageKey)) {
435            oraclePackage oraclePackage = new oraclePackage();
436            if (serverName != null) {
437                oraclePackage.setServer(serverName);
438            } else if (sqlenv != null && sqlenv.getDefaultServerName() != null
439                    && !TSQLEnv.DEFAULT_SERVER_NAME.equalsIgnoreCase(sqlenv.getDefaultServerName())) {
440                oraclePackage.setServer(sqlenv.getDefaultServerName());
441            }
442            if (supportsCatalogs && databaseName != null) {
443                oraclePackage.setDatabase(databaseName);
444            } else if (supportsCatalogs && sqlenv != null && sqlenv.getDefaultCatalogName() != null
445                    && !TSQLEnv.DEFAULT_DB_NAME.equalsIgnoreCase(sqlenv.getDefaultCatalogName())) {
446                oraclePackage.setDatabase(sqlenv.getDefaultCatalogName());
447            }
448
449            if (supportsSchemas && schemaName != null) {
450                oraclePackage.setSchema(schemaName);
451            } else if (supportsSchemas && sqlenv != null && sqlenv.getDefaultSchemaName() != null
452                    && !TSQLEnv.DEFAULT_SCHEMA_NAME.equalsIgnoreCase(sqlenv.getDefaultSchemaName())) {
453                oraclePackage.setSchema(sqlenv.getDefaultSchemaName());
454            }
455
456            oraclePackage.setName(oraclePackageName);
457            if (supportsSchemas && !SQLUtil.isEmpty(oraclePackage.getSchema())) {
458                oraclePackage.setName(oraclePackage.getSchema() + "." + oraclePackage.getName());
459            }
460            if (supportsCatalogs && !SQLUtil.isEmpty(oraclePackage.getDatabase())) {
461                oraclePackage.setName(oraclePackage.getDatabase() + "." + oraclePackage.getName());
462            }
463            oraclePackage.setId(String.valueOf(++ModelBindingManager.get().TABLE_COLUMN_ID));
464            if (ModelBindingManager.getGlobalSqlInfo() != null) {
465                oraclePackage.setCoordinate(new Pair3<Long, Long, Integer>(-1L, -1L,
466                        ModelBindingManager.getGlobalSqlInfo().getIndexOf(ModelBindingManager.getGlobalHash())) + ","
467                        + new Pair3<Long, Long, Integer>(-1L, -1L,
468                        ModelBindingManager.getGlobalSqlInfo().getIndexOf(ModelBindingManager.getGlobalHash())));
469            }
470            dataflow.getPackages().add(oraclePackage);
471            oraclePackageMap.put(oraclePackageKey, oraclePackage);
472        }
473        return oraclePackageMap.get(oraclePackageKey);
474    }
475
476    private void appendProcedure(EDbVendor vendor, boolean supportsCatalogs, boolean supportsSchemas,
477                                 dataflow dataflow, String serverName, String databaseName, String schemaName,
478                                 Map procedureObj, oraclePackage oraclePackage) {
479        String procedureName = (String) procedureObj.get("name");
480        if (procedureName == null) {
481            return;
482        }
483
484        String procedureKey = getFullTableName(vendor, serverName, databaseName, schemaName, procedureName);
485        if (!procedureMap.containsKey(procedureKey)) {
486            procedure procedure = new procedure();
487            if (serverName != null) {
488                procedure.setServer(serverName);
489            } else if (sqlenv != null && sqlenv.getDefaultServerName() != null
490                    && !TSQLEnv.DEFAULT_SERVER_NAME.equalsIgnoreCase(sqlenv.getDefaultServerName())) {
491                procedure.setServer(sqlenv.getDefaultServerName());
492            }
493            if (supportsCatalogs && databaseName != null) {
494                procedure.setDatabase(databaseName);
495            } else if (supportsCatalogs && sqlenv != null && sqlenv.getDefaultCatalogName() != null
496                    && !TSQLEnv.DEFAULT_DB_NAME.equalsIgnoreCase(sqlenv.getDefaultCatalogName())) {
497                procedure.setDatabase(sqlenv.getDefaultCatalogName());
498            }
499
500            if (supportsSchemas && schemaName != null) {
501                procedure.setSchema(schemaName);
502            } else if (supportsSchemas && sqlenv != null && sqlenv.getDefaultSchemaName() != null
503                    && !TSQLEnv.DEFAULT_SCHEMA_NAME.equalsIgnoreCase(sqlenv.getDefaultSchemaName())) {
504                procedure.setSchema(sqlenv.getDefaultSchemaName());
505            }
506
507            procedure.setName(procedureName);
508            if (supportsSchemas && !SQLUtil.isEmpty(procedure.getSchema())) {
509                procedure.setName(procedure.getSchema() + "." + procedure.getName());
510            }
511            if (supportsCatalogs && !SQLUtil.isEmpty(procedure.getDatabase())) {
512                procedure.setName(procedure.getDatabase() + "." + procedure.getName());
513            }
514            procedure.setId(String.valueOf(++ModelBindingManager.get().TABLE_COLUMN_ID));
515            if (ModelBindingManager.getGlobalSqlInfo() != null) {
516                procedure.setCoordinate(new Pair3<Long, Long, Integer>(-1L, -1L,
517                        ModelBindingManager.getGlobalSqlInfo().getIndexOf(ModelBindingManager.getGlobalHash())) + ","
518                        + new Pair3<Long, Long, Integer>(-1L, -1L,
519                        ModelBindingManager.getGlobalSqlInfo().getIndexOf(ModelBindingManager.getGlobalHash())));
520            }
521            procedure.setType((String) procedureObj.get("type"));
522            if (oraclePackage != null) {
523                oraclePackage.getProcedures().add(procedure);
524            } else {
525                dataflow.getProcedures().add(procedure);
526            }
527            procedureMap.put(procedureKey, procedure);
528        }
529    }
530
531    private void appendTable(EDbVendor vendor, boolean supportsCatalogs, boolean supportsSchemas,
532                            dataflow dataflow, String serverName, String databaseName, String schemaName,
533                            String tableName, boolean isView, boolean isSynonym, String columnName,
534                            String dataType, Boolean primaryKey, Boolean unqiueKey, Boolean indexKey,
535                            Boolean foreignKey, String fromDDL) {
536        String tableKey = getFullTableName(vendor, serverName, databaseName, schemaName, tableName);
537        if (!tableMap.containsKey(tableKey)) {
538            table table = new table();
539            if (serverName != null) {
540                table.setServer(serverName);
541            } else if (sqlenv != null && sqlenv.getDefaultServerName() != null
542                    && !TSQLEnv.DEFAULT_SERVER_NAME.equalsIgnoreCase(sqlenv.getDefaultServerName())) {
543                table.setServer(sqlenv.getDefaultServerName());
544            }
545            if (supportsCatalogs && databaseName != null) {
546                table.setDatabase(databaseName);
547            } else if (supportsCatalogs && sqlenv != null && sqlenv.getDefaultCatalogName() != null
548                    && !TSQLEnv.DEFAULT_DB_NAME.equalsIgnoreCase(sqlenv.getDefaultCatalogName())) {
549                table.setDatabase(sqlenv.getDefaultCatalogName());
550            }
551
552            if (supportsSchemas && schemaName != null) {
553                table.setSchema(schemaName);
554            } else if (supportsSchemas && sqlenv != null && sqlenv.getDefaultSchemaName() != null
555                    && !TSQLEnv.DEFAULT_SCHEMA_NAME.equalsIgnoreCase(sqlenv.getDefaultSchemaName())) {
556                table.setSchema(sqlenv.getDefaultSchemaName());
557            }
558
559            table.setName(tableName);
560            if (supportsSchemas && !SQLUtil.isEmpty(table.getSchema())) {
561                table.setName(table.getSchema() + "." + table.getName());
562            }
563            if (supportsCatalogs && !SQLUtil.isEmpty(table.getDatabase())) {
564                table.setName(table.getDatabase() + "." + table.getName());
565            }
566            table.setId(String.valueOf(++ModelBindingManager.get().TABLE_COLUMN_ID));
567            if (ModelBindingManager.getGlobalSqlInfo() != null) {
568                table.setCoordinate(new Pair3<Long, Long, Integer>(-1L, -1L,
569                        ModelBindingManager.getGlobalSqlInfo().getIndexOf(ModelBindingManager.getGlobalHash())) + ","
570                        + new Pair3<Long, Long, Integer>(-1L, -1L,
571                        ModelBindingManager.getGlobalSqlInfo().getIndexOf(ModelBindingManager.getGlobalHash())));
572            }
573
574            if (!SQLUtil.isEmpty(fromDDL)) {
575                table.setFromDDL(fromDDL);
576            }
577
578            if (isView) {
579                table.setType("view");
580                dataflow.getViews().add(table);
581            } else {
582                table.setType("table");
583                dataflow.getTables().add(table);
584            }
585            if (isSynonym) {
586                table.setSubType(SubType.synonym.name());
587            }
588
589            tableMap.put(tableKey, table);
590        }
591
592        table table = tableMap.get(tableKey);
593        String sourceColumnKey = getFullColumnName(vendor, serverName, databaseName, schemaName, tableName, columnName);
594        if (!columnMap.containsKey(sourceColumnKey)) {
595            column column = new column();
596            column.setId(String.valueOf(++ModelBindingManager.get().TABLE_COLUMN_ID));
597            column.setName(columnName);
598            column.setDataType(dataType);
599            column.setPrimaryKey(primaryKey);
600            column.setForeignKey(foreignKey);
601            column.setIndexKey(indexKey);
602            column.setUnqiueKey(unqiueKey);
603            if ("RelationRows".equalsIgnoreCase(columnName)) {
604                column.setSource("system");
605            }
606            if (ModelBindingManager.getGlobalSqlInfo() != null) {
607                column.setCoordinate(new Pair3<Long, Long, Integer>(-1L, -1L,
608                        ModelBindingManager.getGlobalSqlInfo().getIndexOf(ModelBindingManager.getGlobalHash())) + ","
609                        + new Pair3<Long, Long, Integer>(-1L, -1L,
610                        ModelBindingManager.getGlobalSqlInfo().getIndexOf(ModelBindingManager.getGlobalHash())));
611            }
612            table.getColumns().add(column);
613            columnMap.put(sourceColumnKey, column);
614        }
615    }
616
617    private String getFullColumnName(EDbVendor vendor, String... segments) {
618        StringBuilder builder = new StringBuilder();
619        for (int i = 1; i < segments.length; i++) {
620            if (segments[i] == null) {
621                continue;
622            }
623            builder.append(segments[i]);
624            if (i < segments.length - 1) {
625                builder.append(".");
626            }
627        }
628
629        return segments[0] + "." + SQLUtil.getIdentifierNormalColumnName(vendor, builder.toString());
630    }
631
632    private String getFullTableName(EDbVendor vendor, String... segments) {
633        StringBuilder builder = new StringBuilder();
634        for (int i = 1; i < segments.length; i++) {
635            if (segments[i] == null) {
636                continue;
637            }
638            builder.append(segments[i]);
639            if (i < segments.length - 1) {
640                builder.append(".");
641            }
642        }
643
644        return segments[0] + "." + SQLUtil.getIdentifierNormalTableName(vendor, builder.toString());
645    }
646
647    public static void main(String[] args) throws Exception {
648        String sampleManifestPath = "sqlflow-sharded/sample/manifest.json";
649        String baseDir = new File(sampleManifestPath).getParent();
650        dataflow dataflow = new SqlflowShardedMetadataAnalyzer(baseDir)
651                .analyzeMetadata(EDbVendor.dbvmssql, SQLUtil.getFileContent(new File(sampleManifestPath)));
652        System.out.println("Tables: " + dataflow.getTables().size());
653        System.out.println("Views: " + dataflow.getViews().size());
654        System.out.println("Procedures: " + dataflow.getProcedures().size());
655    }
656}