Source code

001package gudusoft.gsqlparser.dlineage.metadata;
002
003import gudusoft.gsqlparser.EDbVendor;
004import gudusoft.gsqlparser.dlineage.dataflow.metadata.MetadataReader;
005import gudusoft.gsqlparser.dlineage.dataflow.metadata.sqlflow.SqlflowMetadataAnalyzer;
006import gudusoft.gsqlparser.dlineage.dataflow.model.SubType;
007import gudusoft.gsqlparser.dlineage.dataflow.model.xml.*;
008import gudusoft.gsqlparser.dlineage.util.DataflowUtility;
009import gudusoft.gsqlparser.sqlenv.TSQLEnv;
010import gudusoft.gsqlparser.sqlenv.constant.SystemConstant;
011import gudusoft.gsqlparser.util.SQLUtil;
012import gudusoft.gsqlparser.util.json.JSON;
013
014import java.io.File;
015import java.util.ArrayList;
016import java.util.List;
017import java.util.Map;
018
019public class MetadataUtil {
020
021        public static Sqlflow convertMetadataJsonToSqlflow(EDbVendor vendor, String json) {
022                if (MetadataReader.isSqlflow(json)) {
023                        dataflow temp = new SqlflowMetadataAnalyzer().analyzeMetadata(vendor, json);
024                        Sqlflow sqlflow = MetadataUtil.convertDataflowToMetadata(vendor, temp);
025                        return sqlflow;
026                } else {
027                        throw new IllegalArgumentException("Not a sqlflow json.");
028                }
029        }
030
031        public static Sqlflow convertDataflowToMetadata(EDbVendor vendor, dataflow dataflow) {
032                if (dataflow == null) {
033                        return null;
034                }
035                Sqlflow sqlflow = new Sqlflow();
036                sqlflow.setCreatedBy(SystemConstant.name + " " + SystemConstant.version);
037                appendTables(sqlflow, vendor, dataflow);
038                appendPackages(sqlflow, vendor, dataflow);
039                appendProcedures(sqlflow, vendor, dataflow);
040                appendProcesses(sqlflow, vendor, dataflow);
041                appendErrorMessages(sqlflow, vendor, dataflow);
042                return sqlflow;
043        }
044
045        private static void appendErrorMessages(Sqlflow sqlflow, EDbVendor vendor, dataflow dataflow) {
046                if (dataflow.getErrors() != null && !dataflow.getErrors().isEmpty()) {
047                        for (error errorItem : dataflow.getErrors()) {
048                                Error error = new Error();
049                                error.setCoordinate(errorItem.getCoordinate());
050                                error.setErrorMessage(errorItem.getErrorMessage());
051                                error.setErrorType(errorItem.getErrorType());
052                                error.setFile(errorItem.getFile());
053                                error.setOriginCoordinate(errorItem.getOriginCoordinate());
054                                sqlflow.appendError(error);
055                        }
056                }
057        }
058
059        private static void appendProcedures(Sqlflow sqlflow, EDbVendor vendor, dataflow dataflow) {
060                if (dataflow.getProcedures() == null || dataflow.getProcedures().isEmpty()) {
061                        return;
062                }
063
064                for (procedure procedureItem : dataflow.getProcedures()) {
065                        Server server = new Server();
066                        server.setDbVendor(vendor.name());
067                        String serverName = procedureItem.getServer();
068                        if (serverName == null && !server.isSupportsCatalogs() && procedureItem.getDatabase() != null) {
069                                serverName = procedureItem.getDatabase();
070                        }
071                        if (serverName == null) {
072                                serverName = TSQLEnv.DEFAULT_SERVER_NAME;
073                        }
074                        server.setName(serverName);
075                        server = sqlflow.appendServer(server);
076
077                        Database database = null;
078
079                        if (server.isSupportsCatalogs()) {
080                                String databaseName = procedureItem.getDatabase();
081                                database = new Database();
082                                database.setName(databaseName == null ? TSQLEnv.DEFAULT_DB_NAME : databaseName);
083                                database = server.appendDatabase(database);
084                                database.setServer(server);
085                        }
086
087                        Schema schema = null;
088                        if (server.isSupportsSchemas()) {
089                                String schemaName = procedureItem.getSchema();
090                                schema = new Schema();
091                                schema.setName(schemaName == null ? TSQLEnv.DEFAULT_SCHEMA_NAME : schemaName);
092                                if (database != null) {
093                                        schema = database.appendSchema(server, schema);
094                                        schema.setParent(database);
095                                        schema.setServer(server);
096                                } else {
097                                        schema = server.appendSchema(schema);
098                                        schema.setServer(server);
099                                }
100                        }
101
102                        Procedure procedure = new Procedure();
103                        procedure.setId(procedureItem.getId());
104                        
105                        List<String> segments = SQLUtil.parseNames(procedureItem.getName());
106                        procedure.setName(segments.get(segments.size()-1));
107                        procedure.setServer(server);
108                        if (server.isSupportsSchemas()) {
109                                procedure.setParent(schema);
110                        }
111                        else if (server.isSupportsCatalogs()) {
112                                procedure.setParent(database);
113                        }
114                        
115                        procedure.setType(procedureItem.getType());
116                        procedure.setCoordinates(Coordinate.parse(procedureItem.getCoordinate()));
117                        if (procedureItem.getArguments() != null) {
118                                for (argument argumentItem : procedureItem.getArguments()) {
119                                        Argument argument = new Argument();
120                                        argument.setServer(server);
121                                        argument.setParent(procedure);
122                                        argument.setId(argumentItem.getId());
123                                        argument.setName(argumentItem.getName());
124                                        argument.setDataType(argumentItem.getDatatype());
125                                        argument.setInout(argumentItem.getInout());
126                                        argument.setCoordinates(Coordinate.parse(argumentItem.getCoordinate()));
127                                        procedure.appendArgument(argument);
128                                }
129                        }
130
131                        if (server.isSupportsSchemas()) {
132                                schema.appendProcedure(procedure);
133                        } else if (server.isSupportsCatalogs()) {
134                                database.appendProcedure(procedure);
135                        }
136                }
137        }
138
139        protected static void appendPackages(Sqlflow sqlflow, EDbVendor vendor, dataflow dataflow) {
140                if (dataflow.getPackages() == null || dataflow.getPackages().isEmpty()) {
141                        return;
142                }
143                for (oraclePackage oraclePackageItem : dataflow.getPackages()) {
144                        Server server = new Server();
145                        server.setDbVendor(vendor.name());
146                        String serverName = oraclePackageItem.getServer();
147                        if (serverName == null && !server.isSupportsCatalogs() && oraclePackageItem.getDatabase() != null) {
148                                serverName = oraclePackageItem.getDatabase();
149                        }
150                        if (serverName == null) {
151                                serverName = TSQLEnv.DEFAULT_SERVER_NAME;
152                        }
153                        server.setName(serverName);
154                        server = sqlflow.appendServer(server);
155
156                        Database database = null;
157
158                        if (server.isSupportsCatalogs()) {
159                                String databaseName = oraclePackageItem.getDatabase();
160                                database = new Database();
161                                database.setName(databaseName == null ? TSQLEnv.DEFAULT_DB_NAME : databaseName);
162                                database = server.appendDatabase(database);
163                                database.setServer(server);
164                        }
165
166                        Schema schema = null;
167                        if (server.isSupportsSchemas()) {
168                                String schemaName = oraclePackageItem.getSchema();
169                                schema = new Schema();
170                                schema.setName(schemaName == null ? TSQLEnv.DEFAULT_SCHEMA_NAME : schemaName);
171                                if (database != null) {
172                                        schema = database.appendSchema(server, schema);
173                                        schema.setParent(database);
174                                        schema.setServer(server);
175                                } else {
176                                        schema = server.appendSchema(schema);
177                                        schema.setServer(server);
178                                }
179                        }
180
181                        Package oraclePackage = new Package();
182                        oraclePackage.setId(oraclePackageItem.getId());
183                        List<String> segments = SQLUtil.parseNames(oraclePackageItem.getName());
184                        oraclePackage.setName(segments.get(segments.size()-1));
185                        oraclePackage.setServer(server);
186                        if (server.isSupportsSchemas()) {
187                                oraclePackage.setParent(schema);
188                        }
189                        else if (server.isSupportsCatalogs()) {
190                                oraclePackage.setParent(database);
191                        }
192                        
193                        oraclePackage.setCoordinates(Coordinate.parse(oraclePackageItem.getCoordinate()));
194                        if (server.isSupportsSchemas()) {
195                                schema.appendPackage(oraclePackage);
196                        }
197
198                        if (oraclePackageItem != null) {
199                                for (procedure procedureItem : oraclePackageItem.getProcedures()) {
200                                        Procedure procedure = new Procedure();
201                                        List<String> procedureSegments = SQLUtil.parseNames(procedureItem.getName());
202                                        procedure.setName(procedureSegments.get(procedureSegments.size() - 1));
203                                        procedure.setServer(server);
204                                        procedure.setParent(oraclePackage);
205                                        procedure.setId(procedureItem.getId());
206                                        procedure.setType(procedureItem.getType());
207                                        procedure.setCoordinates(Coordinate.parse(procedureItem.getCoordinate()));
208                                        if (procedureItem.getArguments() != null) {
209                                                for (argument argumentItem : procedureItem.getArguments()) {
210                                                        Argument argument = new Argument();
211                                                        argument.setServer(server);
212                                                        argument.setParent(procedure);
213                                                        argument.setId(argumentItem.getId());
214                                                        argument.setName(argumentItem.getName());
215                                                        argument.setDataType(argumentItem.getDatatype());
216                                                        argument.setInout(argumentItem.getInout());
217                                                        argument.setCoordinates(Coordinate.parse(argumentItem.getCoordinate()));
218                                                        procedure.appendArgument(argument);
219                                                }
220                                        }
221                                        oraclePackage.appendProcedure(procedure);
222                                }
223                        }
224                }
225        }
226
227        protected static void appendTables(Sqlflow sqlflow, EDbVendor vendor, dataflow dataflow) {
228                Map<String, table> tableMap = DataflowUtility.getDataflowDbObjMap(dataflow);
229                for (table tableItem : tableMap.values()) {
230                        Server server = new Server();
231                        server.setDbVendor(vendor.name());
232                        String serverName = tableItem.getServer();
233                        if (serverName == null && !server.isSupportsCatalogs() && tableItem.getDatabase() != null) {
234                                serverName = tableItem.getDatabase();
235                        }
236                        if (serverName == null) {
237                                serverName = TSQLEnv.DEFAULT_SERVER_NAME;
238                        }
239                        server.setName(serverName);
240                        server = sqlflow.appendServer(server);
241
242                        Database database = null;
243
244                        if (server.isSupportsCatalogs()) {
245                                String databaseName = tableItem.getDatabase();
246                                database = new Database();
247                                database.setName(databaseName == null ? TSQLEnv.DEFAULT_DB_NAME : databaseName);
248                                database = server.appendDatabase(database);
249                                database.setServer(server);
250                        }
251
252                        Schema schema = null;
253                        if (server.isSupportsSchemas()) {
254                                String schemaName = tableItem.getSchema();
255                                schema = new Schema();
256                                schema.setName(schemaName == null ? TSQLEnv.DEFAULT_SCHEMA_NAME : schemaName);
257                                if (database != null) {
258                                        schema = database.appendSchema(server, schema);
259                                        schema.setParent(database);
260                                        schema.setServer(server);
261                                } else {
262                                        schema = server.appendSchema(schema);
263                                        schema.setParent(schema);
264                                        schema.setServer(server);
265                                }
266                        }
267
268                        Table table = new Table();
269                        table.setId(tableItem.getId());
270                        table.setType(tableItem.getType());
271                        if (tableItem.getAlias() != null) {
272                                table.setAlias(tableItem.getAlias());
273                        }
274                        table.setCoordinates(Coordinate.parse(tableItem.getCoordinate()));
275                        table.setServer(server);
276                        if (server.isSupportsSchemas()) {
277                                table.setParent(schema);
278                        }
279                        else if (server.isSupportsCatalogs()) {
280                                table.setParent(database);
281                        }
282                        table.setSubType(tableItem.getSubType());
283                        table.setFromDDL(tableItem.getFromDDL());
284                        table.setDisplayName(tableItem.getName());
285                        if (SubType.dblink.name().equals(tableItem.getSubType())) {
286                                if (SQLUtil.trimColumnStringQuote(tableItem.getName().substring(tableItem.getName().lastIndexOf("@") + 1).trim())
287                                                .equals(SQLUtil.trimColumnStringQuote(tableItem.getDatabase()))) {
288                                        List<String> segments = SQLUtil
289                                                        .parseNames(tableItem.getName().substring(0, tableItem.getName().lastIndexOf("@")).trim());
290                                        table.setName(segments.get(segments.size() - 1));
291                                        table.setDbLink(tableItem.getDatabase());
292                                } else {
293                                        List<String> segments = SQLUtil.parseNames(tableItem.getName());
294                                        table.setName(segments.get(segments.size() - 1));
295                                }
296                        } else {
297                                List<String> segments = SQLUtil.parseNames(tableItem.getName());
298                                table.setName(segments.get(segments.size() - 1));
299                        }
300                        
301                        if (tableItem.getColumns() != null) {
302                                List<Column> columns = new ArrayList<>(tableItem.getColumns().size());
303                                for (column columnItem : tableItem.getColumns()) {
304                                        Column column = new Column();
305                                        column.setId(columnItem.getId());
306                                        column.setName(columnItem.getName());
307                                        column.setCoordinates(Coordinate.parse(columnItem.getCoordinate()));
308                                        column.setSource(columnItem.getSource());
309                                        column.setParent(table);
310                                        column.setServer(server);
311                                        column.setDataType(columnItem.getDataType());
312                                        column.setForeignKey(columnItem.isForeignKey());
313                                        column.setPrimaryKey(columnItem.isPrimaryKey());
314                                        column.setIndexKey(columnItem.isIndexKey());
315                                        column.setUnqiueKey(columnItem.isUnqiueKey());
316                                        columns.add(column);
317                                }
318                                table.setColumns(columns);
319                        }
320
321                        if (server.isSupportsSchemas()) {
322                                schema.appendTable(table);
323                        } else if (server.isSupportsCatalogs()) {
324                                database.appendTable(table);
325                        }
326                }
327        }
328
329        protected static void appendProcesses(Sqlflow sqlflow, EDbVendor vendor, dataflow dataflow) {
330                if (dataflow.getProcesses() == null || dataflow.getProcesses().isEmpty()) {
331                        return;
332                }
333                for (process processItem : dataflow.getProcesses()) {
334                        Server server = new Server();
335                        server.setDbVendor(vendor.name());
336                        String serverName = processItem.getServer();
337                        if (serverName == null && !server.isSupportsCatalogs() && processItem.getDatabase() != null) {
338                                serverName = processItem.getDatabase();
339                        }
340                        if (serverName == null) {
341                                serverName = TSQLEnv.DEFAULT_SERVER_NAME;
342                        }
343                        server.setName(serverName);
344                        server = sqlflow.appendServer(server);
345
346                        Database database = null;
347
348                        if (server.isSupportsCatalogs()) {
349                                String databaseName = processItem.getDatabase();
350                                database = new Database();
351                                database.setName(databaseName == null ? TSQLEnv.DEFAULT_DB_NAME : databaseName);
352                                database = server.appendDatabase(database);
353                                database.setServer(server);
354                        }
355
356                        Schema schema = null;
357                        if (server.isSupportsSchemas()) {
358                                String schemaName = processItem.getSchema();
359                                schema = new Schema();
360                                schema.setName(schemaName == null ? TSQLEnv.DEFAULT_SCHEMA_NAME : schemaName);
361                                if (database != null) {
362                                        schema = database.appendSchema(server, schema);
363                                        schema.setParent(database);
364                                        schema.setServer(server);
365                                } else {
366                                        schema = server.appendSchema(schema);
367                                        schema.setServer(server);
368                                }
369                        }
370
371                        Process process = new Process();
372                        process.setId(processItem.getId());
373                        process.setName(processItem.getName());
374                        process.setProcedureId(processItem.getProcedureId());
375                        process.setProcedureName(processItem.getProcedureName());
376                        process.setQueryHashId(processItem.getQueryHashId());
377                        process.setCoordinates(Coordinate.parse(processItem.getCoordinate()));
378                        process.setServer(server);
379                        if (server.isSupportsSchemas()) {
380                                process.setParent(schema);
381                        }
382                        else if (server.isSupportsCatalogs()) {
383                                process.setParent(database);
384                        }
385                        if (processItem.getTransforms() != null && !processItem.getTransforms().isEmpty()) {
386                                List<Transform> transforms = new ArrayList<Transform>(processItem.getTransforms().size());
387                                for(transform transformItem:processItem.getTransforms()) {
388                                        Transform transform = new Transform();
389                                        transform.setCode(transformItem.getCode());
390                                        transform.setType(transformItem.getType());
391                                        transform.setCoordinate(transformItem.getCoordinate());
392                                        transforms.add(transform);
393                                }
394                                process.setTransforms(transforms.toArray(new Transform[0]));
395                        }
396                        if (server.isSupportsSchemas()) {
397                                schema.appendProcess(process);
398                        } else if (server.isSupportsCatalogs()) {
399                                database.appendProcess(process);
400                        }
401                }
402        }
403
404        public static void main(String[] args) {
405//              dataflow dataflow = XML2Model.loadXML(dataflow.class,
406//                              SQLUtil.getFileContent("C:\\Users\\KK\\Desktop\\dataflow.xml"));
407//              Sqlflow sqlflow = MetadataUtil.convertDataflowToMetadata(EDbVendor.dbvoracle, dataflow);
408//              System.out.println(JSON.toJSONString(sqlflow));
409
410                String content = SQLUtil.getFileContent(new File("D:\\1.json"));
411                if (MetadataReader.isSqlflow(content)) {
412                        dataflow temp = new SqlflowMetadataAnalyzer().analyzeMetadata(EDbVendor.dbvpostgresql, content);
413                        Sqlflow sqlflow = MetadataUtil.convertDataflowToMetadata(EDbVendor.dbvpostgresql, temp);
414                        System.out.println(JSON.toJSONString(sqlflow));
415                }
416        }
417}