Source code

001package gudusoft.gsqlparser.dlineage.util;
002
003import gudusoft.gsqlparser.EDbVendor;
004import gudusoft.gsqlparser.dlineage.DataFlowAnalyzer;
005import gudusoft.gsqlparser.dlineage.ParallelDataFlowAnalyzer;
006import gudusoft.gsqlparser.dlineage.dataflow.metadata.MetadataReader;
007import gudusoft.gsqlparser.dlineage.dataflow.metadata.sqldep.SQLDepMetadataAnalyzer;
008import gudusoft.gsqlparser.dlineage.dataflow.model.ModelBindingManager;
009import gudusoft.gsqlparser.dlineage.dataflow.model.RelationshipType;
010import gudusoft.gsqlparser.dlineage.dataflow.model.json.Error;
011import gudusoft.gsqlparser.dlineage.dataflow.model.json.Process;
012import gudusoft.gsqlparser.dlineage.dataflow.model.json.*;
013import gudusoft.gsqlparser.dlineage.dataflow.model.xml.*;
014import gudusoft.gsqlparser.dlineage.metadata.Sqlflow;
015import gudusoft.gsqlparser.util.Logger;
016import gudusoft.gsqlparser.util.LoggerFactory;
017import gudusoft.gsqlparser.util.SQLUtil;
018import gudusoft.gsqlparser.util.json.JSON;
019
020import java.util.*;
021import java.util.concurrent.atomic.AtomicLong;
022import java.util.stream.Collectors;
023
024public class DataflowUtility {
025
026    private static final Logger logger = LoggerFactory.getLogger(FunctionUtility.class);
027
028    public static dataflow mergeFunctionCallDataflow(dataflow dataflow, EDbVendor dbVendor) {
029        if (ModelBindingManager.getGlobalVendor() == null) {
030            ModelBindingManager.setGlobalVendor(dbVendor);
031        }
032        dataflow instance = cloneDataflow(dataflow);
033        List<procedure> procedures = new ArrayList<>(instance.getProcedures());
034        if (instance.getPackages() != null) {
035            for (oraclePackage pkg : instance.getPackages()) {
036                if (pkg.getProcedures() != null) {
037                    for (procedure procedure : pkg.getProcedures()) {
038                        procedure.setOraclePackage(pkg);
039                        procedures.add(procedure);
040                    }
041                }
042            }
043        }
044
045        Map<String, procedure> procedureIdMap = procedures.stream().collect(Collectors.toMap(
046                t -> t.getId(),
047                t -> t,
048                (existingValue, newValue) -> newValue
049        ));
050        Map<String, table> functionIdMap = dataflow.getResultsets().stream().collect(Collectors.toMap(
051                t -> t.getId(),
052                t -> t,
053                (existingValue, newValue) -> newValue
054        ));
055
056        Map<String, table> functionMap = new HashMap<>();
057        Map<String, procedure> procedureMap = new HashMap<>();
058        Map<String, oraclePackage> oraclePackageMap = new HashMap<>();
059        Map<String, Set<String>> oraclePackageProcedureMap = new HashMap<>();
060        if (instance.getPackages() != null) {
061            for (oraclePackage pkg : instance.getPackages()) {
062                String qualifiedPackageName = DlineageUtil.getIdentifierOraclePackageNameWithArgNum(pkg);
063                if (!oraclePackageMap.containsKey(qualifiedPackageName)) {
064                    oraclePackageMap.put(qualifiedPackageName, pkg);
065                }
066                for (procedure procedure : pkg.getProcedures()) {
067                    String qualifiedProcedureName = qualifiedPackageName + "." + DlineageUtil.getIdentifierProcedureNameWithArgNum(procedure);
068                    if (!oraclePackageProcedureMap.containsKey(qualifiedPackageName)) {
069                        oraclePackageProcedureMap.put(qualifiedPackageName, new HashSet<>());
070                    }
071                    oraclePackageProcedureMap.get(qualifiedPackageName).add(qualifiedProcedureName);
072                }
073            }
074        }
075
076
077
078        for (procedure procedure : procedures) {
079            String qualifiedProcedureName = DlineageUtil.getIdentifierProcedureNameWithArgNum(procedure);
080            if (procedure.getOraclePackage() != null) {
081                String qualifiedPackageName = DlineageUtil.getIdentifierOraclePackageNameWithArgNum(procedure.getOraclePackage());
082                qualifiedProcedureName = qualifiedPackageName + "." + DlineageUtil.getIdentifierProcedureNameWithArgNum(procedure);
083            }
084            if (!procedureMap.containsKey(qualifiedProcedureName)) {
085                procedureMap.put(qualifiedProcedureName, procedure);
086            }
087        }
088
089        for (table function : instance.getResultsets()) {
090            String qualifiedFunctionName = DlineageUtil.getIdentifierFunctionName(function);
091            if (!functionMap.containsKey(qualifiedFunctionName)) {
092                functionMap.put(qualifiedFunctionName, function);
093            }
094        }
095
096        Set<oraclePackage> oraclePackages = new LinkedHashSet<>();
097        for (String key : oraclePackageMap.keySet()) {
098            oraclePackage standardPackage = oraclePackageMap.get(key);
099            oraclePackages.add(standardPackage);
100            Set<String> procedureNames = oraclePackageProcedureMap.get(key);
101            if (procedureNames != null) {
102                for (String procedureName : procedureNames) {
103                    procedureName = key + "." + procedureName;
104                    if (procedureMap.containsKey(procedureName)) {
105                        procedure standardProcedure = procedureMap.get(procedureName);
106                        if (!standardPackage.getProcedures().contains(standardProcedure)) {
107                            standardPackage.getProcedures().add(standardProcedure);
108                        }
109                        procedureMap.remove(procedureName);
110                    }
111                }
112            }
113        }
114
115        instance.setPackages(new ArrayList<>(oraclePackages));
116        instance.setProcedures(new ArrayList<>(procedureMap.values()));
117        instance.setResultsets(new ArrayList<>(functionMap.values()));
118
119        Map<String, relationship> mergeRelations = new LinkedHashMap<String, relationship>();
120
121        for (relationship relationship : instance.getRelationships()) {
122            if (relationship.getCaller() == null || relationship.getCallees() == null || relationship.getCallees().size() == 0) {
123                continue;
124            }
125
126            String targetId = relationship.getCaller().getId();
127            if (procedureIdMap.containsKey(targetId)) {
128                procedure procedure = procedureIdMap.get(targetId);
129                String procedureName = DlineageUtil.getIdentifierProcedureNameWithArgNum(procedure);
130                if (procedure.getOraclePackage() != null) {
131                    String qualifiedPackageName = DlineageUtil.getIdentifierOraclePackageNameWithArgNum(procedure.getOraclePackage());
132                    procedureName = qualifiedPackageName + "." + procedureName;
133                }
134                procedure standardProcedure = procedureMap.get(procedureName);
135                relationship.getCaller().setId(standardProcedure.getId());
136            } else if (functionIdMap.containsKey(targetId)) {
137                table function = functionIdMap.get(targetId);
138                String functionName = DlineageUtil.getIdentifierFunctionName(function);
139                table standardFunction = functionMap.get(functionName);
140                relationship.getCaller().setId(standardFunction.getId());
141            }
142
143            for (sourceColumn sourceColumn : relationship.getCallees()) {
144                String sourceId = sourceColumn.getId();
145                if (procedureIdMap.containsKey(sourceId)) {
146                    procedure procedure = procedureIdMap.get(sourceId);
147                    String procedureName = DlineageUtil.getIdentifierProcedureNameWithArgNum(procedure);
148                    if (procedure.getOraclePackage() != null) {
149                        String qualifiedPackageName = DlineageUtil.getIdentifierOraclePackageNameWithArgNum(procedure.getOraclePackage());
150                        procedureName = qualifiedPackageName + "." + procedureName;
151                    }
152                    procedure standardProcedure = procedureMap.get(procedureName);
153                    sourceColumn.setId(standardProcedure.getId());
154                } else if (functionIdMap.containsKey(sourceId)) {
155                    table function = functionIdMap.get(sourceId);
156                    String functionName = DlineageUtil.getIdentifierFunctionName(function);
157                    table standardFunction = functionMap.get(functionName);
158                    sourceColumn.setId(standardFunction.getId());
159                }
160            }
161
162            String jsonString = JSON.toJSONString(relationship).replaceAll("\"id\":\".+?\"", "");
163            String key = SHA256.getMd5(jsonString);
164            if (!mergeRelations.containsKey(key)) {
165                mergeRelations.put(key, relationship);
166            }
167        }
168
169        instance.setRelationships(new ArrayList<relationship>(mergeRelations.values()));
170
171        if (instance.getPackages() != null) {
172            for (oraclePackage pkg : instance.getPackages()) {
173                if (pkg.getProcedures() != null) {
174                    for (procedure procedure : pkg.getProcedures()) {
175                        procedure.setOraclePackage(null);
176                    }
177                }
178            }
179        }
180
181        return instance;
182    }
183
184    public static dataflow convertTableLevelToFunctionCallDataflow(dataflow dataflow, boolean showBuiltIn, EDbVendor dbVendor) {
185        if (ModelBindingManager.getGlobalVendor() == null) {
186            ModelBindingManager.setGlobalVendor(dbVendor);
187        }
188
189        dataflow instance = cloneDataflow(dataflow);
190
191        if (instance.getRelationships() == null) {
192            return instance;
193        }
194
195        List<relationship> callRelationships = instance.getRelationships().stream()
196                .filter(t -> RelationshipType.call.name().equals(t.getType()))
197                .filter(t -> !showBuiltIn ? !Boolean.TRUE.equals(t.getBuiltIn()): true).collect(Collectors.toList());
198        instance.setRelationships(callRelationships);
199
200        Set<String> ids = new HashSet<>();
201
202
203        callRelationships.stream().forEach(t -> {
204            ids.add(t.getCaller().getId());
205            t.getCallees().stream().forEach(t1 -> ids.add(t1.getId()));
206        });
207
208        Iterator<table> iterator = instance.getTables().iterator();
209        while (iterator.hasNext()) {
210            table t = iterator.next();
211            if (!ids.contains(t.getId())) {
212                iterator.remove();
213            }
214        }
215
216        iterator = instance.getResultsets().iterator();
217        while (iterator.hasNext()) {
218            table t = iterator.next();
219            if (!ids.contains(t.getId())) {
220                iterator.remove();
221            }
222        }
223
224        iterator = instance.getViews().iterator();
225        while (iterator.hasNext()) {
226            table t = iterator.next();
227            if (!ids.contains(t.getId())) {
228                iterator.remove();
229            }
230        }
231
232        iterator = instance.getStages().iterator();
233        while (iterator.hasNext()) {
234            table t = iterator.next();
235            if (!ids.contains(t.getId())) {
236                iterator.remove();
237            }
238        }
239
240        iterator = instance.getStreams().iterator();
241        while (iterator.hasNext()) {
242            table t = iterator.next();
243            if (!ids.contains(t.getId())) {
244                iterator.remove();
245            }
246        }
247
248        iterator = instance.getVariables().iterator();
249        while (iterator.hasNext()) {
250            table t = iterator.next();
251            if (!ids.contains(t.getId())) {
252                iterator.remove();
253            }
254        }
255
256        iterator = instance.getPaths().iterator();
257        while (iterator.hasNext()) {
258            table t = iterator.next();
259            if (!ids.contains(t.getId())) {
260                iterator.remove();
261            }
262        }
263
264        iterator = instance.getDatasources().iterator();
265        while (iterator.hasNext()) {
266            table t = iterator.next();
267            if (!ids.contains(t.getId())) {
268                iterator.remove();
269            }
270        }
271
272        iterator = instance.getDatabases().iterator();
273        while (iterator.hasNext()) {
274            table t = iterator.next();
275            if (!ids.contains(t.getId())) {
276                iterator.remove();
277            }
278        }
279
280        iterator = instance.getSchemas().iterator();
281        while (iterator.hasNext()) {
282            table t = iterator.next();
283            if (!ids.contains(t.getId())) {
284                iterator.remove();
285            }
286        }
287
288        iterator = instance.getSequences().iterator();
289        while (iterator.hasNext()) {
290            table t = iterator.next();
291            if (!ids.contains(t.getId())) {
292                iterator.remove();
293            }
294        }
295
296        return mergeFunctionCallDataflow(instance, dbVendor);
297    }
298
299    public static dataflow convertToTableLevelDataflow(dataflow dataflow) {
300        dataflow instance = cloneDataflow(dataflow);
301
302        if (instance.getRelationships() == null) {
303            return instance;
304        }
305
306        Map<String, LinkedHashSet<Pair<String, String>>> relationMap = new HashMap<String, LinkedHashSet<Pair<String, String>>>();
307        Map<String, LinkedHashSet<Pair3<String, String, relationship>>> callRelationMap = new HashMap<String, LinkedHashSet<Pair3<String, String, relationship>>>();
308        for (RelationshipType type : RelationshipType.values()) {
309            relationMap.put(type.name(), new LinkedHashSet<Pair<String, String>>());
310            callRelationMap.put(type.name(), new LinkedHashSet<Pair3<String, String, relationship>>());
311        }
312        for (relationship relationship : instance.getRelationships()) {
313            if (RelationshipType.call.name().equals(relationship.getType())) {
314                String targetId = relationship.getCaller().getId();
315                for (sourceColumn sourceColumn : relationship.getCallees()) {
316                    String sourceId = sourceColumn.getId();
317                    callRelationMap.get(relationship.getType()).add(new Pair3<>(targetId, sourceId, relationship));
318                }
319            } else {
320                String targetId = relationship.getTarget().getParent_id();
321                for (sourceColumn sourceColumn : relationship.getSources()) {
322                    String sourceId = sourceColumn.getParent_id();
323                    relationMap.get(relationship.getType()).add(new Pair<>(targetId, sourceId));
324                }
325            }
326        }
327
328        long maxId = 0;
329        Map<String, table> dbObjMap = getDataflowDbObjMap(instance);
330
331        List<procedure> procedures = new ArrayList<>(instance.getProcedures());
332        if (instance.getPackages() != null) {
333            for (oraclePackage pkg : instance.getPackages()) {
334                procedures.addAll(pkg.getProcedures());
335            }
336        }
337
338        Map<String, procedure> procedureMap = procedures.stream().collect(Collectors.toMap(
339                t -> t.getId(),
340                t -> t,
341                (existingValue, newValue) -> newValue
342        ));
343
344        for (table table : dbObjMap.values()) {
345            if (Long.valueOf(table.getId()) > maxId) {
346                maxId = Long.valueOf(table.getId());
347            }
348        }
349
350        AtomicLong id = new AtomicLong(maxId + 10000000);
351
352        for (table table : dbObjMap.values()) {
353            column column = new column();
354            column.setId(String.valueOf(id.incrementAndGet()));
355            column.setName(table.getType());
356            table.setColumns(Arrays.asList(column));
357        }
358
359        List<relationship> relations = new ArrayList<relationship>();
360        for (RelationshipType type : RelationshipType.values()) {
361            LinkedHashSet<Pair<String, String>> relationSet = relationMap.get(type.name());
362             LinkedHashSet<Pair3<String, String, relationship>> callRelationSet = callRelationMap.get(type.name());
363            if (RelationshipType.call.equals(type)) {
364                for (Pair3<String, String, relationship> pair : callRelationSet) {
365                    if ((dbObjMap.get(pair.first) == null && procedureMap.get(pair.first) == null)
366                            || (dbObjMap.get(pair.second) == null && procedureMap.get(pair.second) == null)) {
367                        continue;
368                    }
369
370                    relationship relationship = new relationship();
371                    relationship.setType(type.name());
372                    relationship.setId(String.valueOf(id.incrementAndGet()));
373                    relationship.setCallStmt(pair.third.getCallStmt());
374                    relationship.setCallCoordinate(pair.third.getCallCoordinate());
375
376                    targetColumn targetColumn = new targetColumn();
377                    if (dbObjMap.containsKey(pair.first)) {
378                        targetColumn.setId(dbObjMap.get(pair.first).getId());
379                        targetColumn.setName(dbObjMap.get(pair.first).getName());
380                        targetColumn.setCoordinate(dbObjMap.get(pair.first).getCoordinate());
381                    } else {
382                        targetColumn.setId(procedureMap.get(pair.first).getId());
383                        targetColumn.setName(procedureMap.get(pair.first).getName());
384                        targetColumn.setCoordinate(procedureMap.get(pair.first).getCoordinate());
385                    }
386
387                    sourceColumn sourceColumn = new sourceColumn();
388                    if (dbObjMap.containsKey(pair.second)) {
389                        sourceColumn.setId(dbObjMap.get(pair.second).getId());
390                        sourceColumn.setName(dbObjMap.get(pair.second).getName());
391                        sourceColumn.setCoordinate(dbObjMap.get(pair.second).getCoordinate());
392                    } else {
393                        sourceColumn.setId(procedureMap.get(pair.second).getId());
394                        sourceColumn.setName(procedureMap.get(pair.second).getName());
395                        sourceColumn.setCoordinate(procedureMap.get(pair.second).getCoordinate());
396                    }
397
398                    relationship.setCaller(targetColumn);
399                    relationship.setCallees(Arrays.asList(sourceColumn));
400                    relationship.setBuiltIn(pair.third.getBuiltIn());
401                    relations.add(relationship);
402                }
403            } else {
404                for (Pair<String, String> pair : relationSet) {
405                    if (dbObjMap.get(pair.first) == null || dbObjMap.get(pair.second) == null) {
406                        continue;
407                    }
408                    relationship relationship = new relationship();
409                    relationship.setType(type.name());
410                    relationship.setId(String.valueOf(id.incrementAndGet()));
411                    targetColumn targetColumn = new targetColumn();
412                    targetColumn.setId(dbObjMap.get(pair.first).getColumns().get(0).getId());
413                    targetColumn.setColumn(dbObjMap.get(pair.first).getColumns().get(0).getName());
414                    targetColumn.setParent_id(pair.first);
415                    targetColumn.setParent_name(dbObjMap.get(pair.first).getName());
416                    sourceColumn sourceColumn = new sourceColumn();
417                    sourceColumn.setId(dbObjMap.get(pair.second).getColumns().get(0).getId());
418                    sourceColumn.setColumn(dbObjMap.get(pair.second).getColumns().get(0).getName());
419                    sourceColumn.setParent_id(pair.second);
420                    sourceColumn.setParent_name(dbObjMap.get(pair.second).getName());
421                    relationship.setTarget(targetColumn);
422                    relationship.setSources(Arrays.asList(sourceColumn));
423                    relations.add(relationship);
424                }
425            }
426        }
427        instance.setRelationships(relations);
428        return instance;
429    }
430
431    public static dataflow convertToSchemaLevelDataflow(dataflow dataflow, EDbVendor dbVendor) throws Exception {
432        return convertToSchemaLevelDataflow(dataflow, dbVendor, false);
433    }
434
435    public static dataflow convertToSchemaLevelDataflow(dataflow dataflow, EDbVendor dbVendor, boolean isSimple) throws Exception {
436        dataflow instance = cloneDataflow(dataflow);
437
438        if (!isSimple) {
439            DataFlowAnalyzer analyzer = new DataFlowAnalyzer("", dbVendor, true);
440            instance = analyzer.getSimpleDataflow(instance, true, Arrays.asList(new String[]{"fdd", "fdr"}));
441        }
442
443        List<table> allTables = new ArrayList<table>();
444        allTables.addAll(instance.getTables());
445        allTables.addAll(instance.getViews());
446
447        ModelBindingManager.setGlobalVendor(dbVendor);
448
449        Map<String, String> tableIdSchameNameMap = allTables.stream().collect(Collectors.toMap(table -> table.getId(), table -> table.getFullSchemaName(), (existingValue, newValue) -> newValue));
450
451        if (instance.getRelationships() == null) {
452            return instance;
453        }
454
455        Map<String, LinkedHashSet<Pair<String, String>>> relationMap = new HashMap<String, LinkedHashSet<Pair<String, String>>>();
456        for (RelationshipType type : RelationshipType.values()) {
457            relationMap.put(type.name(), new LinkedHashSet<Pair<String, String>>());
458        }
459        for (relationship relationship : instance.getRelationships()) {
460            String targetId = relationship.getTarget().getParent_id();
461            for (sourceColumn sourceColumn : relationship.getSources()) {
462                String sourceId = sourceColumn.getParent_id();
463                relationMap.get(relationship.getType()).add(new Pair<>(tableIdSchameNameMap.get(targetId), tableIdSchameNameMap.get(sourceId)));
464            }
465        }
466
467        Map<String, table> dbObjMap = getDataflowDbObjMap(instance);
468        LinkedHashSet<String> schemaNameSet = new LinkedHashSet<String>();
469        schemaNameSet.addAll(tableIdSchameNameMap.values());
470        List<table> schemaTables = new ArrayList<table>();
471
472        AtomicLong id = new AtomicLong(0);
473
474        for (String schemaName : schemaNameSet) {
475            table table = new table();
476            table.setId(String.valueOf(id.incrementAndGet()));
477            table.setName(schemaName);
478            table.setType("schema");
479            schemaTables.add(table);
480            dbObjMap.put(schemaName, table);
481        }
482
483        for (table table : schemaTables) {
484            column column = new column();
485            column.setId(String.valueOf(id.incrementAndGet()));
486            column.setName(table.getType());
487            table.setColumns(Arrays.asList(column));
488        }
489
490        List<relationship> relations = new ArrayList<relationship>();
491        for (RelationshipType type : RelationshipType.values()) {
492            LinkedHashSet<Pair<String, String>> relationSet = relationMap.get(type.name());
493
494            for (Pair<String, String> pair : relationSet) {
495                if (dbObjMap.get(pair.first) == null || dbObjMap.get(pair.second) == null) {
496                    continue;
497                }
498                relationship relationship = new relationship();
499                relationship.setType(type.name());
500                relationship.setId(String.valueOf(id.incrementAndGet()));
501                targetColumn targetColumn = new targetColumn();
502                targetColumn.setId(dbObjMap.get(pair.first).getColumns().get(0).getId());
503                targetColumn.setColumn(dbObjMap.get(pair.first).getColumns().get(0).getName());
504                targetColumn.setParent_id(dbObjMap.get(pair.first).getId());
505                targetColumn.setParent_name(dbObjMap.get(pair.first).getName());
506                sourceColumn sourceColumn = new sourceColumn();
507                sourceColumn.setId(dbObjMap.get(pair.second).getColumns().get(0).getId());
508                sourceColumn.setColumn(dbObjMap.get(pair.second).getColumns().get(0).getName());
509                sourceColumn.setParent_id(dbObjMap.get(pair.second).getId());
510                sourceColumn.setParent_name(dbObjMap.get(pair.second).getName());
511                relationship.setTarget(targetColumn);
512                relationship.setSources(Arrays.asList(sourceColumn));
513                relations.add(relationship);
514            }
515        }
516
517        dataflow schemaDataflow = new dataflow();
518        schemaDataflow.setTables(schemaTables);
519        schemaDataflow.setRelationships(relations);
520        return schemaDataflow;
521    }
522
523    public static dataflow cloneDataflow(dataflow dataflow) {
524        dataflow dataflowCopy = new dataflow();
525        if (dataflow.getTables() != null) {
526            dataflowCopy.setTables(cloneTables(dataflow.getTables()));
527        }
528        if (dataflow.getViews() != null) {
529            dataflowCopy.setViews(cloneTables(dataflow.getViews()));
530        }
531        if (dataflow.getRelationships() != null) {
532            dataflowCopy.setRelationships(new ArrayList<>(dataflow.getRelationships()));
533        }
534        if (dataflow.getErrors() != null) {
535            dataflowCopy.setErrors(new ArrayList<>(dataflow.getErrors()));
536        }
537        if (dataflow.getPaths() != null) {
538            dataflowCopy.setPaths(cloneTables(dataflow.getPaths()));
539        }
540        if (dataflow.getPackages() != null) {
541            dataflowCopy.setPackages(new ArrayList<>(dataflow.getPackages()));
542        }
543        if (dataflow.getProcedures() != null) {
544            dataflowCopy.setProcedures(new ArrayList<>(dataflow.getProcedures()));
545        }
546        if (dataflow.getProcesses() != null) {
547            dataflowCopy.setProcesses(new ArrayList<>(dataflow.getProcesses()));
548        }
549        if (dataflow.getResultsets() != null) {
550            dataflowCopy.setResultsets(cloneTables(dataflow.getResultsets()));
551        }
552        if (dataflow.getVariables() != null) {
553            dataflowCopy.setVariables(cloneTables(dataflow.getVariables()));
554        }
555        if (dataflow.getStages() != null) {
556            dataflowCopy.setStages(cloneTables(dataflow.getStages()));
557        }
558        if (dataflow.getSequences() != null) {
559            dataflowCopy.setSequences(cloneTables(dataflow.getSequences()));
560        }
561        if (dataflow.getDatasources() != null) {
562            dataflowCopy.setDatasources(cloneTables(dataflow.getDatasources()));
563        }
564        if (dataflow.getDatabases() != null) {
565            dataflowCopy.setDatabases(cloneTables(dataflow.getDatabases()));
566        }
567        if (dataflow.getSchemas() != null) {
568            dataflowCopy.setSchemas(cloneTables(dataflow.getSchemas()));
569        }
570        if (dataflow.getStreams() != null) {
571            dataflowCopy.setStreams(cloneTables(dataflow.getStreams()));
572        }
573        dataflowCopy.setOrientation(dataflow.getOrientation());
574        return dataflowCopy;
575    }
576
577    private static List<table> cloneTables(List<table> tableList) {
578        if (tableList == null) {
579            return null;
580        }
581        List<table> tables = new ArrayList<>(tableList.size());
582        for (table item : tableList) {
583            try {
584                table table = (table) item.clone();
585                if (item.getColumns() != null) {
586                    table.setColumns(new ArrayList<column>(item.getColumns()));
587                }
588                tables.add(table);
589            } catch (CloneNotSupportedException e) {
590                logger.error("Clone table failed.", e);
591            }
592        }
593        return tables;
594    }
595
596    public static Map<String, table> getDataflowDbObjMap(dataflow dataflow) {
597        List<table> tables = new ArrayList<table>();
598        if (dataflow.getTables() != null) {
599            tables.addAll(dataflow.getTables());
600        }
601        if (dataflow.getViews() != null) {
602            tables.addAll(dataflow.getViews());
603        }
604        if (dataflow.getPaths() != null) {
605            tables.addAll(dataflow.getPaths());
606        }
607        if (dataflow.getResultsets() != null) {
608            tables.addAll(dataflow.getResultsets());
609        }
610        if (dataflow.getVariables() != null) {
611            tables.addAll(dataflow.getVariables());
612        }
613        if (dataflow.getStages() != null) {
614            tables.addAll(dataflow.getStages());
615        }
616        if (dataflow.getSequences() != null) {
617            tables.addAll(dataflow.getSequences());
618        }
619        if (dataflow.getDatasources() != null) {
620            tables.addAll(dataflow.getDatasources());
621        }
622        if (dataflow.getDatabases() != null) {
623            tables.addAll(dataflow.getDatabases());
624        }
625        if (dataflow.getSchemas() != null) {
626            tables.addAll(dataflow.getSchemas());
627        }
628        if (dataflow.getStreams() != null) {
629            tables.addAll(dataflow.getStreams());
630        }
631
632        Map<String, table> dbObjMap = new HashMap<>();
633        for (table table : tables) {
634            dbObjMap.put(table.getId(), table);
635        }
636        return dbObjMap;
637        }
638        public static Map<String, table> getDataflowDbObjNameMap(dataflow dataflow) {
639                List<table> tables = new ArrayList<table>();
640                if (dataflow.getTables() != null) {
641                        tables.addAll(dataflow.getTables());
642                }
643                if (dataflow.getViews() != null) {
644                        tables.addAll(dataflow.getViews());
645                }
646                if (dataflow.getPaths() != null) {
647                        tables.addAll(dataflow.getPaths());
648                }
649                if (dataflow.getResultsets() != null) {
650                        tables.addAll(dataflow.getResultsets());
651                }
652                if (dataflow.getVariables() != null) {
653                        tables.addAll(dataflow.getVariables());
654                }
655                if (dataflow.getStages() != null) {
656                        tables.addAll(dataflow.getStages());
657                }
658                if (dataflow.getSequences() != null) {
659                        tables.addAll(dataflow.getSequences());
660                }
661                if (dataflow.getDatasources() != null) {
662                        tables.addAll(dataflow.getDatasources());
663                }
664                if (dataflow.getDatabases() != null) {
665                        tables.addAll(dataflow.getDatabases());
666                }
667                if (dataflow.getSchemas() != null) {
668                        tables.addAll(dataflow.getSchemas());
669                }
670                if (dataflow.getStreams() != null) {
671                        tables.addAll(dataflow.getStreams());
672                }
673
674                Map<String, table> dbObjMap = new HashMap<>();
675                for(table table: tables){
676                        dbObjMap.put(table.getFullName(), table);
677                }
678                return dbObjMap;
679        }
680        public static dataflow mergeDataflows(Collection<dataflow> dataflows, EDbVendor vendor) {
681                return ParallelDataFlowAnalyzer.mergeDataFlows(dataflows, vendor);
682        }
683
684        public static dataflow readDataflowFromCsvMetadata(String csvMetadata, EDbVendor vendor) {
685                if (!MetadataReader.isMetadata(csvMetadata)) {
686                        throw new IllegalArgumentException("Illegal csv metadata.");
687                }
688                return new SQLDepMetadataAnalyzer().analyzeMetadata(vendor, csvMetadata);
689        }
690
691        public static Dataflow mergeDataflowsAndCsv(List<Pair<EDbVendor,dataflow>> pairs, String csvMetadata) {
692                //重置ID防止重复
693                Long index = 0L;
694                for(int i=0; i<pairs.size(); i++){
695                        dataflow dataflow = pairs.get(i).second;
696                        Map<String, table> objIDMap = getDataflowDbObjMap(dataflow);
697                        Map<String, String> idMaps = new HashMap<>();
698                        for (Map.Entry<String, table> entry : objIDMap.entrySet()) {
699                                index = index + 1;
700                                String id = index.toString();
701                                idMaps.put(entry.getKey(), id);
702                                table table = entry.getValue();
703                                table.setId(id);
704                                if(table.getColumns() != null){
705                                        for(column col: table.getColumns()){
706                                                index = index + 1;
707                                                id = index.toString();
708                                                idMaps.put(col.getId(), id);
709                                                col.setId(id);
710                                        }
711                                }
712                        }
713                        if(dataflow.getProcedures() != null){
714                                for(procedure procedure: dataflow.getProcedures()){
715                                        index = index + 1;
716                                        String id = index.toString();
717                                        idMaps.put(procedure.getId(), id);
718                                        procedure.setId(id);
719                                }
720                        }
721                        if(dataflow.getProcesses() != null){
722                                for(process process: dataflow.getProcesses()){
723                                        index = index + 1;
724                                        String id = index.toString();
725                                        idMaps.put(process.getId(), id);
726                                        process.setId(id);
727                                }
728                        }
729                        if(dataflow.getPackages() != null){
730                                for(oraclePackage oraclePackage: dataflow.getPackages()){
731                                        index = index + 1;
732                                        String id = index.toString();
733                                        idMaps.put(oraclePackage.getId(), id);
734                                        oraclePackage.setId(id);
735                                        if(oraclePackage.getProcedures() != null){
736                                                for(procedure procedure: oraclePackage.getProcedures()){
737                                                        index = index + 1;
738                                                        id = index.toString();
739                                                        idMaps.put(procedure.getId(), id);
740                                                        procedure.setId(id);
741                                                }
742                                        }
743                                }
744                        }
745                        if(dataflow.getRelationships() != null && dataflow.getRelationships().size()>0){
746                                for(relationship rel: dataflow.getRelationships()){
747                                        index = index + 1;
748                                        String id = index.toString();
749                                        rel.setId(id);
750                                        rel.setProcedureId(idMaps.get(rel.getProcedureId()));
751                                        rel.setProcessId(idMaps.get(rel.getProcessId()));
752                                        rel.getTarget().setParent_id(idMaps.get(rel.getTarget().getParent_id()));
753                                        rel.getTarget().setTarget_id(idMaps.get(rel.getTarget().getTarget_id()));
754                                        rel.getTarget().setId(idMaps.get(rel.getTarget().getId()));
755                                        for(sourceColumn column: rel.getSources()){
756                                                column.setParent_id(idMaps.get(column.getParent_id()));
757                                                column.setId(idMaps.get(column.getId()));
758                                        }
759                                }
760                        }
761                }
762                Dataflow mDataflow = DataFlowAnalyzer.getSqlflowJSONModel(pairs.get(0).first, pairs.get(0).second, false);
763                Map<String, table> objNameMap = getDataflowDbObjNameMap(pairs.get(0).second);
764                Sqlflow dbobjs = mDataflow.getDbobjs();
765                List<Relationship> mRelationshipList = new ArrayList<>();
766                if(mDataflow.getRelationships() != null && mDataflow.getRelationships().length>0){
767                        mRelationshipList = new LinkedList<>(Arrays.asList(mDataflow.getRelationships()));
768                }
769
770                List<Error> mErrorList = new ArrayList<>();
771                if(mDataflow.getErrors() != null && mDataflow.getErrors().length>0){
772                        mErrorList = new LinkedList<>(Arrays.asList(mDataflow.getErrors()));
773                }
774
775                List<Process> mProcessList = new ArrayList<>();
776                if(mDataflow.getProcesses() != null && mDataflow.getProcesses().length>0){
777                        mProcessList = new LinkedList<>(Arrays.asList(mDataflow.getProcesses()));
778                }
779
780                for(int i=1; i<pairs.size(); i++){
781                        objNameMap.putAll(getDataflowDbObjNameMap(pairs.get(i).second));
782                        Dataflow dataflow = DataFlowAnalyzer.getSqlflowJSONModel(pairs.get(i).first, pairs.get(i).second, false);
783                        if(dataflow.getDbobjs().getServers() != null && dataflow.getDbobjs().getServers().size()>0){
784                                if(dbobjs.getServers() == null){
785                                        dbobjs.setServers(new ArrayList<>());
786                                }
787                                dbobjs.getServers().addAll(dataflow.getDbobjs().getServers());
788                        }
789
790                        if(dataflow.getDbobjs().getErrorMessages() != null && dataflow.getDbobjs().getErrorMessages().size()>0){
791                                if(dbobjs.getErrorMessages() == null){
792                                        dbobjs.setErrorMessages(new ArrayList<>());
793                                }
794                                dbobjs.getErrorMessages().addAll(dataflow.getDbobjs().getErrorMessages());
795                        }
796
797                        if(dataflow.getRelationships() != null && dataflow.getRelationships().length>0){
798                                List<Relationship> relationshipList = new LinkedList<>(Arrays.asList(dataflow.getRelationships()));
799                                mRelationshipList.addAll(relationshipList);
800                        }
801
802                        if(dataflow.getErrors() != null && dataflow.getErrors().length>0){
803                                List<Error> errorList = new LinkedList<>(Arrays.asList(dataflow.getErrors()));
804                                mErrorList.addAll(errorList);
805                        }
806
807                        if(dataflow.getProcesses() != null && dataflow.getProcesses().length>0){
808                                List<Process> processList = new LinkedList<>(Arrays.asList(dataflow.getProcesses()));
809                                mProcessList.addAll(processList);
810                        }
811
812                }
813                if(!SQLUtil.isEmpty(csvMetadata)){
814                        /**
815                         * excel内的血缘 只合并关系 就是只做关联，如果找不到obj 就算了
816                         */
817                        dataflow df = readDataflowFromCsvMetadata(csvMetadata);
818                        Map<String, table> objIDMap = getDataflowDbObjMap(df);
819                        if(df.getRelationships() != null && df.getRelationships().size()>0){
820                                for(relationship rel: df.getRelationships()){
821                                        rel.setId("m"+rel.getId());
822                                        table sTable = objIDMap.get(rel.getTarget().getParent_id());
823                                        table tTable = objNameMap.get(sTable.getFullName());
824                                        rel.getTarget().setParent_id(tTable.getId());
825                                        for(column col: tTable.getColumns()){
826                                                if(col.getName().equalsIgnoreCase(rel.getTarget().getColumn())){
827                                                        rel.getTarget().setId(col.getId());
828                                                        break;
829                                                }
830                                        }
831                                        for(sourceColumn column: rel.getSources()){
832                                                sTable = objIDMap.get(column.getParent_id());
833                                                tTable = objNameMap.get(sTable.getFullName());
834                                                column.setParent_id(tTable.getId());
835                                                for(column col: tTable.getColumns()){
836                                                        if(col.getName().equalsIgnoreCase(column.getColumn())){
837                                                                column.setId(col.getId());
838                                                                break;
839                                                        }
840                                                }
841                                        }
842                                        mRelationshipList.add(toRelationship(rel));
843                                }
844                        }
845                }
846
847                mDataflow.setDbobjs(dbobjs);
848                mDataflow.setRelationships(mRelationshipList.toArray(new Relationship[mRelationshipList.size()]));
849                mDataflow.setProcesses(mProcessList.toArray(new Process[mProcessList.size()]));
850                mDataflow.setErrors(mErrorList.toArray(new Error[mErrorList.size()]));
851                return mDataflow;
852        }
853
854        public static dataflow readDataflowFromCsvMetadata(String csvMetadata) {
855                if (!MetadataReader.isMetadata(csvMetadata)) {
856                        throw new IllegalArgumentException("Illegal csv metadata.");
857                }
858                return new SQLDepMetadataAnalyzer().analyzeMetadata(null, csvMetadata);
859        }
860
861        private static Relationship toRelationship(relationship relation){
862                Relationship relationModel;
863                if (relation.getType().equals("join")) {
864                        JoinRelationship joinRelationModel = new JoinRelationship();
865                        joinRelationModel.setCondition(relation.getCondition());
866                        joinRelationModel.setJoinType(relation.getJoinType());
867                        joinRelationModel.setClause(relation.getClause());
868                        relationModel = joinRelationModel;
869                } else {
870                        relationModel = new Relationship();
871                }
872                relationModel.setId(relation.getId());
873                relationModel.setProcessId(relation.getProcessId());
874                relationModel.setProcessType(relation.getProcessType());
875                relationModel.setType(relation.getType());
876                relationModel.setEffectType(relation.getEffectType());
877                relationModel.setPartition(relation.getPartition());
878                relationModel.setFunction(relation.getFunction());
879                relationModel.setProcedureId(relation.getProcedureId());
880                relationModel.setSqlHash(relation.getSqlHash());
881                relationModel.setCondition(relation.getCondition());
882                relationModel.setSqlComment(relation.getSqlComment());
883                relationModel.setTimestampMax(relation.getTimestampMax());
884                relationModel.setTimestampMin(relation.getTimestampMin());
885
886                if (relation.getTarget() != null && relation.getSources() != null && !relation.getSources().isEmpty()) {
887                        RelationshipElement targetModel = new RelationshipElement();
888                        targetColumn target = relation.getTarget();
889                        targetModel.setColumn(target.getColumn());
890                        targetModel.setParentName(target.getParent_name());
891                        targetModel.setTargetName(target.getTarget_name());
892                        targetModel.setId(target.getId());
893                        targetModel.setTargetId(target.getTarget_id());
894                        targetModel.setParentId(target.getParent_id());
895                        targetModel.setCoordinates(Coordinate.parse(target.getCoordinate()));
896                        targetModel.setFunction(target.getFunction());
897                        targetModel.setType(target.getType());
898                        relationModel.setTarget(targetModel);
899
900                        List<RelationshipElement> sourceModels = new ArrayList<>();
901                        for (sourceColumn source : relation.getSources()) {
902                                RelationshipElement sourceModel = new RelationshipElement();
903                                sourceModel.setColumn(source.getColumn());
904                                sourceModel.setParentName(source.getParent_name());
905                                sourceModel.setSourceName(source.getSource_name());
906                                sourceModel.setColumnType(source.getColumn_type());
907                                sourceModel.setId(source.getId());
908                                sourceModel.setParentId(source.getParent_id());
909                                sourceModel.setSourceId(source.getSource_id());
910                                sourceModel.setCoordinates(Coordinate.parse(source.getCoordinate()));
911                                sourceModel.setClauseType(source.getClauseType());
912                                sourceModel.setType(source.getType());
913                                sourceModels.add(sourceModel);
914                                if (source.getTransforms() != null && !source.getTransforms().isEmpty()) {
915                                        List<Transform> transforms = new ArrayList<gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform>();
916                                        for (transform transform : source.getTransforms()) {
917                                                Transform item = new Transform();
918                                                item.setCode(transform.getCode());
919                                                item.setType(transform.getType());
920                                                item.setCoordinate(transform.getCoordinate(true));
921                                                transforms.add(item);
922                                        }
923                                        sourceModel.setTransforms(transforms.toArray(new Transform[0]));
924                                }
925                        }
926                        relationModel.setSources(sourceModels.toArray(new RelationshipElement[0]));
927                } else if (relation.getCaller() != null && relation.getCallees() != null
928                                && !relation.getCallees().isEmpty()) {
929                        RelationshipElement targetModel = new RelationshipElement();
930                        targetColumn target = relation.getCaller();
931                        targetModel.setName(target.getName());
932                        targetModel.setId(target.getId());
933                        targetModel.setCoordinates(Coordinate.parse(target.getCoordinate()));
934                        targetModel.setType(target.getType());
935                        relationModel.setCaller(targetModel);
936                        List<RelationshipElement> sourceModels = new ArrayList<gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement>();
937                        for (sourceColumn source : relation.getCallees()) {
938                                RelationshipElement sourceModel = new RelationshipElement();
939                                sourceModel.setName(source.getName());
940                                sourceModel.setId(source.getId());
941                                sourceModel.setCoordinates(Coordinate.parse(source.getCoordinate()));
942                                sourceModel.setType(source.getType());
943                                sourceModels.add(sourceModel);
944                        }
945                        relationModel.setCallees(sourceModels.toArray(new RelationshipElement[0]));
946                }
947                return relationModel;
948        }
949}