Source code

001package gudusoft.gsqlparser.dlineage.util;
002
003import gudusoft.gsqlparser.EDbVendor;
004import gudusoft.gsqlparser.dlineage.DataFlowAnalyzer;
005import gudusoft.gsqlparser.dlineage.ParallelDataFlowAnalyzer;
006import gudusoft.gsqlparser.dlineage.dataflow.metadata.MetadataReader;
007import gudusoft.gsqlparser.dlineage.dataflow.metadata.sqldep.SQLDepMetadataAnalyzer;
008import gudusoft.gsqlparser.dlineage.dataflow.model.ModelBindingManager;
009import gudusoft.gsqlparser.dlineage.dataflow.model.RelationshipType;
010import gudusoft.gsqlparser.dlineage.dataflow.model.json.*;
011import gudusoft.gsqlparser.dlineage.dataflow.model.json.Error;
012import gudusoft.gsqlparser.dlineage.dataflow.model.json.Process;
013import gudusoft.gsqlparser.dlineage.dataflow.model.xml.*;
014import gudusoft.gsqlparser.util.Logger;
015import gudusoft.gsqlparser.util.LoggerFactory;
016import gudusoft.gsqlparser.util.json.JSON;
017
018import gudusoft.gsqlparser.dlineage.metadata.Sqlflow;
019import gudusoft.gsqlparser.util.SQLUtil;
020import java.util.*;
021import java.util.concurrent.atomic.AtomicLong;
022import java.util.stream.Collectors;
023
024public class DataflowUtility {
025
026    private static final Logger logger = LoggerFactory.getLogger(FunctionUtility.class);
027
028    public static dataflow mergeFunctionCallDataflow(dataflow dataflow, EDbVendor dbVendor) {
029        if (ModelBindingManager.getGlobalVendor() == null) {
030            ModelBindingManager.setGlobalVendor(dbVendor);
031        }
032        dataflow instance = cloneDataflow(dataflow);
033        List<procedure> procedures = new ArrayList<>(instance.getProcedures());
034        if (instance.getPackages() != null) {
035            for (oraclePackage pkg : instance.getPackages()) {
036                if (pkg.getProcedures() != null) {
037                    for (procedure procedure : pkg.getProcedures()) {
038                        procedure.setOraclePackage(pkg);
039                        procedures.add(procedure);
040                    }
041                }
042            }
043        }
044
045        Map<String, procedure> procedureIdMap = procedures.stream().collect(Collectors.toMap(
046                t -> t.getId(),
047                t -> t,
048                (existingValue, newValue) -> newValue
049        ));
050        Map<String, table> functionIdMap = dataflow.getResultsets().stream().collect(Collectors.toMap(
051                t -> t.getId(),
052                t -> t,
053                (existingValue, newValue) -> newValue
054        ));
055
056        Map<String, table> functionMap = new HashMap<>();
057        Map<String, procedure> procedureMap = new HashMap<>();
058        Map<String, oraclePackage> oraclePackageMap = new HashMap<>();
059        Map<String, Set<String>> oraclePackageProcedureMap = new HashMap<>();
060        if (instance.getPackages() != null) {
061            for (oraclePackage pkg : instance.getPackages()) {
062                String qualifiedPackageName = DlineageUtil.getIdentifierOraclePackageNameWithArgNum(pkg);
063                if (!oraclePackageMap.containsKey(qualifiedPackageName)) {
064                    oraclePackageMap.put(qualifiedPackageName, pkg);
065                }
066                for (procedure procedure : pkg.getProcedures()) {
067                    String qualifiedProcedureName = qualifiedPackageName + "." + DlineageUtil.getIdentifierProcedureNameWithArgNum(procedure);
068                    if (!oraclePackageProcedureMap.containsKey(qualifiedPackageName)) {
069                        oraclePackageProcedureMap.put(qualifiedPackageName, new HashSet<>());
070                    }
071                    oraclePackageProcedureMap.get(qualifiedPackageName).add(qualifiedProcedureName);
072                }
073            }
074        }
075
076
077
078        for (procedure procedure : procedures) {
079            String qualifiedProcedureName = DlineageUtil.getIdentifierProcedureNameWithArgNum(procedure);
080            if (procedure.getOraclePackage() != null) {
081                String qualifiedPackageName = DlineageUtil.getIdentifierOraclePackageNameWithArgNum(procedure.getOraclePackage());
082                qualifiedProcedureName = qualifiedPackageName + "." + DlineageUtil.getIdentifierProcedureNameWithArgNum(procedure);
083            }
084            if (!procedureMap.containsKey(qualifiedProcedureName)) {
085                procedureMap.put(qualifiedProcedureName, procedure);
086            }
087        }
088
089        for (table function : instance.getResultsets()) {
090            String qualifiedFunctionName = DlineageUtil.getIdentifierFunctionName(function);
091            if (!functionMap.containsKey(qualifiedFunctionName)) {
092                functionMap.put(qualifiedFunctionName, function);
093            }
094        }
095
096        Set<oraclePackage> oraclePackages = new LinkedHashSet<>();
097        for (String key : oraclePackageMap.keySet()) {
098            oraclePackage standardPackage = oraclePackageMap.get(key);
099            oraclePackages.add(standardPackage);
100            Set<String> procedureNames = oraclePackageProcedureMap.get(key);
101            if (procedureNames != null) {
102                for (String procedureName : procedureNames) {
103                    procedureName = key + "." + procedureName;
104                    if (procedureMap.containsKey(procedureName)) {
105                        procedure standardProcedure = procedureMap.get(procedureName);
106                        if (!standardPackage.getProcedures().contains(standardProcedure)) {
107                            standardPackage.getProcedures().add(standardProcedure);
108                        }
109                        procedureMap.remove(procedureName);
110                    }
111                }
112            }
113        }
114
115        instance.setPackages(new ArrayList<>(oraclePackages));
116        instance.setProcedures(new ArrayList<>(procedureMap.values()));
117        instance.setResultsets(new ArrayList<>(functionMap.values()));
118
119        Map<String, relationship> mergeRelations = new LinkedHashMap<String, relationship>();
120
121        for (relationship relationship : instance.getRelationships()) {
122            if (relationship.getCaller() == null || relationship.getCallees() == null || relationship.getCallees().size() == 0) {
123                continue;
124            }
125
126            String targetId = relationship.getCaller().getId();
127            if (procedureIdMap.containsKey(targetId)) {
128                procedure procedure = procedureIdMap.get(targetId);
129                String procedureName = DlineageUtil.getIdentifierProcedureNameWithArgNum(procedure);
130                if (procedure.getOraclePackage() != null) {
131                    String qualifiedPackageName = DlineageUtil.getIdentifierOraclePackageNameWithArgNum(procedure.getOraclePackage());
132                    procedureName = qualifiedPackageName + "." + procedureName;
133                }
134                procedure standardProcedure = procedureMap.get(procedureName);
135                relationship.getCaller().setId(standardProcedure.getId());
136            } else if (functionIdMap.containsKey(targetId)) {
137                table function = functionIdMap.get(targetId);
138                String functionName = DlineageUtil.getIdentifierFunctionName(function);
139                table standardFunction = functionMap.get(functionName);
140                relationship.getCaller().setId(standardFunction.getId());
141            }
142
143            for (sourceColumn sourceColumn : relationship.getCallees()) {
144                String sourceId = sourceColumn.getId();
145                if (procedureIdMap.containsKey(sourceId)) {
146                    procedure procedure = procedureIdMap.get(sourceId);
147                    String procedureName = DlineageUtil.getIdentifierProcedureNameWithArgNum(procedure);
148                    if (procedure.getOraclePackage() != null) {
149                        String qualifiedPackageName = DlineageUtil.getIdentifierOraclePackageNameWithArgNum(procedure.getOraclePackage());
150                        procedureName = qualifiedPackageName + "." + procedureName;
151                    }
152                    procedure standardProcedure = procedureMap.get(procedureName);
153                    sourceColumn.setId(standardProcedure.getId());
154                } else if (functionIdMap.containsKey(sourceId)) {
155                    table function = functionIdMap.get(sourceId);
156                    String functionName = DlineageUtil.getIdentifierFunctionName(function);
157                    table standardFunction = functionMap.get(functionName);
158                    sourceColumn.setId(standardFunction.getId());
159                }
160            }
161
162            String jsonString = JSON.toJSONString(relationship).replaceAll("\"id\":\".+?\"", "");
163            String key = SHA256.getMd5(jsonString);
164            if (!mergeRelations.containsKey(key)) {
165                mergeRelations.put(key, relationship);
166            }
167        }
168
169        instance.setRelationships(new ArrayList<relationship>(mergeRelations.values()));
170
171        if (instance.getPackages() != null) {
172            for (oraclePackage pkg : instance.getPackages()) {
173                if (pkg.getProcedures() != null) {
174                    for (procedure procedure : pkg.getProcedures()) {
175                        procedure.setOraclePackage(null);
176                    }
177                }
178            }
179        }
180
181        return instance;
182    }
183
184    public static dataflow convertTableLevelToFunctionCallDataflow(dataflow dataflow, boolean showBuiltIn, EDbVendor dbVendor) {
185        if (ModelBindingManager.getGlobalVendor() == null) {
186            ModelBindingManager.setGlobalVendor(dbVendor);
187        }
188
189        dataflow instance = cloneDataflow(dataflow);
190
191        if (instance.getRelationships() == null) {
192            return instance;
193        }
194
195        List<relationship> callRelationships = instance.getRelationships().stream()
196                .filter(t -> RelationshipType.call.name().equals(t.getType()))
197                .filter(t -> !showBuiltIn ? !Boolean.TRUE.equals(t.getBuiltIn()): true).collect(Collectors.toList());
198        instance.setRelationships(callRelationships);
199
200        Set<String> ids = new HashSet<>();
201
202
203        callRelationships.stream().forEach(t -> {
204            ids.add(t.getCaller().getId());
205            t.getCallees().stream().forEach(t1 -> ids.add(t1.getId()));
206        });
207
208        Iterator<table> iterator = instance.getTables().iterator();
209        while (iterator.hasNext()) {
210            table t = iterator.next();
211            if (!ids.contains(t.getId())) {
212                iterator.remove();
213            }
214        }
215
216        iterator = instance.getResultsets().iterator();
217        while (iterator.hasNext()) {
218            table t = iterator.next();
219            if (!ids.contains(t.getId())) {
220                iterator.remove();
221            }
222        }
223
224        iterator = instance.getViews().iterator();
225        while (iterator.hasNext()) {
226            table t = iterator.next();
227            if (!ids.contains(t.getId())) {
228                iterator.remove();
229            }
230        }
231
232        iterator = instance.getStages().iterator();
233        while (iterator.hasNext()) {
234            table t = iterator.next();
235            if (!ids.contains(t.getId())) {
236                iterator.remove();
237            }
238        }
239
240        iterator = instance.getStreams().iterator();
241        while (iterator.hasNext()) {
242            table t = iterator.next();
243            if (!ids.contains(t.getId())) {
244                iterator.remove();
245            }
246        }
247
248        iterator = instance.getVariables().iterator();
249        while (iterator.hasNext()) {
250            table t = iterator.next();
251            if (!ids.contains(t.getId())) {
252                iterator.remove();
253            }
254        }
255
256        iterator = instance.getPaths().iterator();
257        while (iterator.hasNext()) {
258            table t = iterator.next();
259            if (!ids.contains(t.getId())) {
260                iterator.remove();
261            }
262        }
263
264        iterator = instance.getDatasources().iterator();
265        while (iterator.hasNext()) {
266            table t = iterator.next();
267            if (!ids.contains(t.getId())) {
268                iterator.remove();
269            }
270        }
271
272        iterator = instance.getDatabases().iterator();
273        while (iterator.hasNext()) {
274            table t = iterator.next();
275            if (!ids.contains(t.getId())) {
276                iterator.remove();
277            }
278        }
279
280        iterator = instance.getSchemas().iterator();
281        while (iterator.hasNext()) {
282            table t = iterator.next();
283            if (!ids.contains(t.getId())) {
284                iterator.remove();
285            }
286        }
287
288        iterator = instance.getSequences().iterator();
289        while (iterator.hasNext()) {
290            table t = iterator.next();
291            if (!ids.contains(t.getId())) {
292                iterator.remove();
293            }
294        }
295
296        return mergeFunctionCallDataflow(instance, dbVendor);
297    }
298
299    public static dataflow convertToTableLevelDataflow(dataflow dataflow) {
300        dataflow instance = cloneDataflow(dataflow);
301
302        if (instance.getRelationships() == null) {
303            return instance;
304        }
305
306        Map<String, LinkedHashSet<Pair<String, String>>> relationMap = new HashMap<String, LinkedHashSet<Pair<String, String>>>();
307        Map<String, LinkedHashSet<Pair3<String, String, relationship>>> callRelationMap = new HashMap<String, LinkedHashSet<Pair3<String, String, relationship>>>();
308        for (RelationshipType type : RelationshipType.values()) {
309            relationMap.put(type.name(), new LinkedHashSet<Pair<String, String>>());
310            callRelationMap.put(type.name(), new LinkedHashSet<Pair3<String, String, relationship>>());
311        }
312        for (relationship relationship : instance.getRelationships()) {
313            if (RelationshipType.call.name().equals(relationship.getType())) {
314                String targetId = relationship.getCaller().getId();
315                for (sourceColumn sourceColumn : relationship.getCallees()) {
316                    String sourceId = sourceColumn.getId();
317                    callRelationMap.get(relationship.getType()).add(new Pair3<>(targetId, sourceId, relationship));
318                }
319            } else {
320                String targetId = relationship.getTarget().getParent_id();
321                for (sourceColumn sourceColumn : relationship.getSources()) {
322                    String sourceId = sourceColumn.getParent_id();
323                    relationMap.get(relationship.getType()).add(new Pair<>(targetId, sourceId));
324                }
325            }
326        }
327
328        long maxId = 0;
329        Map<String, table> dbObjMap = getDataflowDbObjMap(instance);
330
331        List<procedure> procedures = new ArrayList<>(instance.getProcedures());
332        if (instance.getPackages() != null) {
333            for (oraclePackage pkg : instance.getPackages()) {
334                procedures.addAll(pkg.getProcedures());
335            }
336        }
337
338        Map<String, procedure> procedureMap = procedures.stream().collect(Collectors.toMap(
339                t -> t.getId(),
340                t -> t,
341                (existingValue, newValue) -> newValue
342        ));
343
344        for (table table : dbObjMap.values()) {
345            if (Long.valueOf(table.getId()) > maxId) {
346                maxId = Long.valueOf(table.getId());
347            }
348        }
349
350        AtomicLong id = new AtomicLong(maxId + 10000000);
351
352        for (table table : dbObjMap.values()) {
353            column column = new column();
354            column.setId(String.valueOf(id.incrementAndGet()));
355            column.setName(table.getType());
356            table.setColumns(Arrays.asList(column));
357        }
358
359        List<relationship> relations = new ArrayList<relationship>();
360        for (RelationshipType type : RelationshipType.values()) {
361            LinkedHashSet<Pair<String, String>> relationSet = relationMap.get(type.name());
362             LinkedHashSet<Pair3<String, String, relationship>> callRelationSet = callRelationMap.get(type.name());
363            if (RelationshipType.call.equals(type)) {
364                for (Pair3<String, String, relationship> pair : callRelationSet) {
365                    if ((dbObjMap.get(pair.first) == null && procedureMap.get(pair.first) == null)
366                            || (dbObjMap.get(pair.second) == null && procedureMap.get(pair.second) == null)) {
367                        continue;
368                    }
369
370                    relationship relationship = new relationship();
371                    relationship.setType(type.name());
372                    relationship.setId(String.valueOf(id.incrementAndGet()));
373                    relationship.setCallStmt(pair.third.getCallStmt());
374                    relationship.setCallCoordinate(pair.third.getCallCoordinate());
375
376                    targetColumn targetColumn = new targetColumn();
377                    if (dbObjMap.containsKey(pair.first)) {
378                        targetColumn.setId(dbObjMap.get(pair.first).getId());
379                        targetColumn.setName(dbObjMap.get(pair.first).getName());
380                        targetColumn.setCoordinate(dbObjMap.get(pair.first).getCoordinate());
381                    } else {
382                        targetColumn.setId(procedureMap.get(pair.first).getId());
383                        targetColumn.setName(procedureMap.get(pair.first).getName());
384                        targetColumn.setCoordinate(procedureMap.get(pair.first).getCoordinate());
385                    }
386
387                    sourceColumn sourceColumn = new sourceColumn();
388                    if (dbObjMap.containsKey(pair.second)) {
389                        sourceColumn.setId(dbObjMap.get(pair.second).getId());
390                        sourceColumn.setName(dbObjMap.get(pair.second).getName());
391                        sourceColumn.setCoordinate(dbObjMap.get(pair.second).getCoordinate());
392                    } else {
393                        sourceColumn.setId(procedureMap.get(pair.second).getId());
394                        sourceColumn.setName(procedureMap.get(pair.second).getName());
395                        sourceColumn.setCoordinate(procedureMap.get(pair.second).getCoordinate());
396                    }
397
398                    relationship.setCaller(targetColumn);
399                    relationship.setCallees(Arrays.asList(sourceColumn));
400                    relationship.setBuiltIn(pair.third.getBuiltIn());
401                    relations.add(relationship);
402                }
403            } else {
404                for (Pair<String, String> pair : relationSet) {
405                    if (dbObjMap.get(pair.first) == null || dbObjMap.get(pair.second) == null) {
406                        continue;
407                    }
408                    relationship relationship = new relationship();
409                    relationship.setType(type.name());
410                    relationship.setId(String.valueOf(id.incrementAndGet()));
411                    targetColumn targetColumn = new targetColumn();
412                    targetColumn.setId(dbObjMap.get(pair.first).getColumns().get(0).getId());
413                    targetColumn.setColumn(dbObjMap.get(pair.first).getColumns().get(0).getName());
414                    targetColumn.setParent_id(pair.first);
415                    targetColumn.setParent_name(dbObjMap.get(pair.first).getName());
416                    sourceColumn sourceColumn = new sourceColumn();
417                    sourceColumn.setId(dbObjMap.get(pair.second).getColumns().get(0).getId());
418                    sourceColumn.setColumn(dbObjMap.get(pair.second).getColumns().get(0).getName());
419                    sourceColumn.setParent_id(pair.second);
420                    sourceColumn.setParent_name(dbObjMap.get(pair.second).getName());
421                    relationship.setTarget(targetColumn);
422                    relationship.setSources(Arrays.asList(sourceColumn));
423                    relations.add(relationship);
424                }
425            }
426        }
427        instance.setRelationships(relations);
428        return instance;
429    }
430
431    public static dataflow cloneDataflow(dataflow dataflow) {
432        dataflow dataflowCopy = new dataflow();
433        if (dataflow.getTables() != null) {
434            dataflowCopy.setTables(cloneTables(dataflow.getTables()));
435        }
436        if (dataflow.getViews() != null) {
437            dataflowCopy.setViews(cloneTables(dataflow.getViews()));
438        }
439        if (dataflow.getRelationships() != null) {
440            dataflowCopy.setRelationships(new ArrayList<>(dataflow.getRelationships()));
441        }
442        if (dataflow.getErrors() != null) {
443            dataflowCopy.setErrors(new ArrayList<>(dataflow.getErrors()));
444        }
445        if (dataflow.getPaths() != null) {
446            dataflowCopy.setPaths(cloneTables(dataflow.getPaths()));
447        }
448        if (dataflow.getPackages() != null) {
449            dataflowCopy.setPackages(new ArrayList<>(dataflow.getPackages()));
450        }
451        if (dataflow.getProcedures() != null) {
452            dataflowCopy.setProcedures(new ArrayList<>(dataflow.getProcedures()));
453        }
454        if (dataflow.getProcesses() != null) {
455            dataflowCopy.setProcesses(new ArrayList<>(dataflow.getProcesses()));
456        }
457        if (dataflow.getResultsets() != null) {
458            dataflowCopy.setResultsets(cloneTables(dataflow.getResultsets()));
459        }
460        if (dataflow.getVariables() != null) {
461            dataflowCopy.setVariables(cloneTables(dataflow.getVariables()));
462        }
463        if (dataflow.getStages() != null) {
464            dataflowCopy.setStages(cloneTables(dataflow.getStages()));
465        }
466        if (dataflow.getSequences() != null) {
467            dataflowCopy.setSequences(cloneTables(dataflow.getSequences()));
468        }
469        if (dataflow.getDatasources() != null) {
470            dataflowCopy.setDatasources(cloneTables(dataflow.getDatasources()));
471        }
472        if (dataflow.getDatabases() != null) {
473            dataflowCopy.setDatabases(cloneTables(dataflow.getDatabases()));
474        }
475        if (dataflow.getSchemas() != null) {
476            dataflowCopy.setSchemas(cloneTables(dataflow.getSchemas()));
477        }
478        if (dataflow.getStreams() != null) {
479            dataflowCopy.setStreams(cloneTables(dataflow.getStreams()));
480        }
481        dataflowCopy.setOrientation(dataflow.getOrientation());
482        return dataflowCopy;
483    }
484
485    private static List<table> cloneTables(List<table> tableList) {
486        if (tableList == null) {
487            return null;
488        }
489        List<table> tables = new ArrayList<>(tableList.size());
490        for (table item : tableList) {
491            try {
492                table table = (table) item.clone();
493                if (item.getColumns() != null) {
494                    table.setColumns(new ArrayList<column>(item.getColumns()));
495                }
496                tables.add(table);
497            } catch (CloneNotSupportedException e) {
498                logger.error("Clone table failed.", e);
499            }
500        }
501        return tables;
502    }
503
504    public static Map<String, table> getDataflowDbObjMap(dataflow dataflow) {
505        List<table> tables = new ArrayList<table>();
506        if (dataflow.getTables() != null) {
507            tables.addAll(dataflow.getTables());
508        }
509        if (dataflow.getViews() != null) {
510            tables.addAll(dataflow.getViews());
511        }
512        if (dataflow.getPaths() != null) {
513            tables.addAll(dataflow.getPaths());
514        }
515        if (dataflow.getResultsets() != null) {
516            tables.addAll(dataflow.getResultsets());
517        }
518        if (dataflow.getVariables() != null) {
519            tables.addAll(dataflow.getVariables());
520        }
521        if (dataflow.getStages() != null) {
522            tables.addAll(dataflow.getStages());
523        }
524        if (dataflow.getSequences() != null) {
525            tables.addAll(dataflow.getSequences());
526        }
527        if (dataflow.getDatasources() != null) {
528            tables.addAll(dataflow.getDatasources());
529        }
530        if (dataflow.getDatabases() != null) {
531            tables.addAll(dataflow.getDatabases());
532        }
533        if (dataflow.getSchemas() != null) {
534            tables.addAll(dataflow.getSchemas());
535        }
536        if (dataflow.getStreams() != null) {
537            tables.addAll(dataflow.getStreams());
538        }
539
540        Map<String, table> dbObjMap = new HashMap<>();
541        for (table table : tables) {
542            dbObjMap.put(table.getId(), table);
543        }
544        return dbObjMap;
545        }
546        public static Map<String, table> getDataflowDbObjNameMap(dataflow dataflow) {
547                List<table> tables = new ArrayList<table>();
548                if (dataflow.getTables() != null) {
549                        tables.addAll(dataflow.getTables());
550                }
551                if (dataflow.getViews() != null) {
552                        tables.addAll(dataflow.getViews());
553                }
554                if (dataflow.getPaths() != null) {
555                        tables.addAll(dataflow.getPaths());
556                }
557                if (dataflow.getResultsets() != null) {
558                        tables.addAll(dataflow.getResultsets());
559                }
560                if (dataflow.getVariables() != null) {
561                        tables.addAll(dataflow.getVariables());
562                }
563                if (dataflow.getStages() != null) {
564                        tables.addAll(dataflow.getStages());
565                }
566                if (dataflow.getSequences() != null) {
567                        tables.addAll(dataflow.getSequences());
568                }
569                if (dataflow.getDatasources() != null) {
570                        tables.addAll(dataflow.getDatasources());
571                }
572                if (dataflow.getDatabases() != null) {
573                        tables.addAll(dataflow.getDatabases());
574                }
575                if (dataflow.getSchemas() != null) {
576                        tables.addAll(dataflow.getSchemas());
577                }
578                if (dataflow.getStreams() != null) {
579                        tables.addAll(dataflow.getStreams());
580                }
581
582                Map<String, table> dbObjMap = new HashMap<>();
583                for(table table: tables){
584                        dbObjMap.put(table.getFullName(), table);
585                }
586                return dbObjMap;
587        }
588        public static dataflow mergeDataflows(Collection<dataflow> dataflows, EDbVendor vendor) {
589                return ParallelDataFlowAnalyzer.mergeDataFlows(dataflows, vendor);
590        }
591
592        public static dataflow readDataflowFromCsvMetadata(String csvMetadata, EDbVendor vendor) {
593                if (!MetadataReader.isMetadata(csvMetadata)) {
594                        throw new IllegalArgumentException("Illegal csv metadata.");
595                }
596                return new SQLDepMetadataAnalyzer().analyzeMetadata(vendor, csvMetadata);
597        }
598
599        public static Dataflow mergeDataflowsAndCsv(List<Pair<EDbVendor,dataflow>> pairs, String csvMetadata) {
600                //重置ID防止重复
601                Long index = 0L;
602                for(int i=0; i<pairs.size(); i++){
603                        dataflow dataflow = pairs.get(i).second;
604                        Map<String, table> objIDMap = getDataflowDbObjMap(dataflow);
605                        Map<String, String> idMaps = new HashMap<>();
606                        for (Map.Entry<String, table> entry : objIDMap.entrySet()) {
607                                index = index + 1;
608                                String id = index.toString();
609                                idMaps.put(entry.getKey(), id);
610                                table table = entry.getValue();
611                                table.setId(id);
612                                if(table.getColumns() != null){
613                                        for(column col: table.getColumns()){
614                                                index = index + 1;
615                                                id = index.toString();
616                                                idMaps.put(col.getId(), id);
617                                                col.setId(id);
618                                        }
619                                }
620                        }
621                        if(dataflow.getProcedures() != null){
622                                for(procedure procedure: dataflow.getProcedures()){
623                                        index = index + 1;
624                                        String id = index.toString();
625                                        idMaps.put(procedure.getId(), id);
626                                        procedure.setId(id);
627                                }
628                        }
629                        if(dataflow.getProcesses() != null){
630                                for(process process: dataflow.getProcesses()){
631                                        index = index + 1;
632                                        String id = index.toString();
633                                        idMaps.put(process.getId(), id);
634                                        process.setId(id);
635                                }
636                        }
637                        if(dataflow.getPackages() != null){
638                                for(oraclePackage oraclePackage: dataflow.getPackages()){
639                                        index = index + 1;
640                                        String id = index.toString();
641                                        idMaps.put(oraclePackage.getId(), id);
642                                        oraclePackage.setId(id);
643                                        if(oraclePackage.getProcedures() != null){
644                                                for(procedure procedure: oraclePackage.getProcedures()){
645                                                        index = index + 1;
646                                                        id = index.toString();
647                                                        idMaps.put(procedure.getId(), id);
648                                                        procedure.setId(id);
649                                                }
650                                        }
651                                }
652                        }
653                        if(dataflow.getRelationships() != null && dataflow.getRelationships().size()>0){
654                                for(relationship rel: dataflow.getRelationships()){
655                                        index = index + 1;
656                                        String id = index.toString();
657                                        rel.setId(id);
658                                        rel.setProcedureId(idMaps.get(rel.getProcedureId()));
659                                        rel.setProcessId(idMaps.get(rel.getProcessId()));
660                                        rel.getTarget().setParent_id(idMaps.get(rel.getTarget().getParent_id()));
661                                        rel.getTarget().setTarget_id(idMaps.get(rel.getTarget().getTarget_id()));
662                                        rel.getTarget().setId(idMaps.get(rel.getTarget().getId()));
663                                        for(sourceColumn column: rel.getSources()){
664                                                column.setParent_id(idMaps.get(column.getParent_id()));
665                                                column.setId(idMaps.get(column.getId()));
666                                        }
667                                }
668                        }
669                }
670                Dataflow mDataflow = DataFlowAnalyzer.getSqlflowJSONModel(pairs.get(0).first, pairs.get(0).second, false);
671                Map<String, table> objNameMap = getDataflowDbObjNameMap(pairs.get(0).second);
672                Sqlflow dbobjs = mDataflow.getDbobjs();
673                List<Relationship> mRelationshipList = new ArrayList<>();
674                if(mDataflow.getRelationships() != null && mDataflow.getRelationships().length>0){
675                        mRelationshipList = new LinkedList<>(Arrays.asList(mDataflow.getRelationships()));
676                }
677
678                List<Error> mErrorList = new ArrayList<>();
679                if(mDataflow.getErrors() != null && mDataflow.getErrors().length>0){
680                        mErrorList = new LinkedList<>(Arrays.asList(mDataflow.getErrors()));
681                }
682
683                List<Process> mProcessList = new ArrayList<>();
684                if(mDataflow.getProcesses() != null && mDataflow.getProcesses().length>0){
685                        mProcessList = new LinkedList<>(Arrays.asList(mDataflow.getProcesses()));
686                }
687
688                for(int i=1; i<pairs.size(); i++){
689                        objNameMap.putAll(getDataflowDbObjNameMap(pairs.get(i).second));
690                        Dataflow dataflow = DataFlowAnalyzer.getSqlflowJSONModel(pairs.get(i).first, pairs.get(i).second, false);
691                        if(dataflow.getDbobjs().getServers() != null && dataflow.getDbobjs().getServers().size()>0){
692                                if(dbobjs.getServers() == null){
693                                        dbobjs.setServers(new ArrayList<>());
694                                }
695                                dbobjs.getServers().addAll(dataflow.getDbobjs().getServers());
696                        }
697
698                        if(dataflow.getDbobjs().getErrorMessages() != null && dataflow.getDbobjs().getErrorMessages().size()>0){
699                                if(dbobjs.getErrorMessages() == null){
700                                        dbobjs.setErrorMessages(new ArrayList<>());
701                                }
702                                dbobjs.getErrorMessages().addAll(dataflow.getDbobjs().getErrorMessages());
703                        }
704
705                        if(dataflow.getRelationships() != null && dataflow.getRelationships().length>0){
706                                List<Relationship> relationshipList = new LinkedList<>(Arrays.asList(dataflow.getRelationships()));
707                                mRelationshipList.addAll(relationshipList);
708                        }
709
710                        if(dataflow.getErrors() != null && dataflow.getErrors().length>0){
711                                List<Error> errorList = new LinkedList<>(Arrays.asList(dataflow.getErrors()));
712                                mErrorList.addAll(errorList);
713                        }
714
715                        if(dataflow.getProcesses() != null && dataflow.getProcesses().length>0){
716                                List<Process> processList = new LinkedList<>(Arrays.asList(dataflow.getProcesses()));
717                                mProcessList.addAll(processList);
718                        }
719
720                }
721                if(!SQLUtil.isEmpty(csvMetadata)){
722                        /**
723                         * excel内的血缘 只合并关系 就是只做关联，如果找不到obj 就算了
724                         */
725                        dataflow df = readDataflowFromCsvMetadata(csvMetadata);
726                        Map<String, table> objIDMap = getDataflowDbObjMap(df);
727                        if(df.getRelationships() != null && df.getRelationships().size()>0){
728                                for(relationship rel: df.getRelationships()){
729                                        rel.setId("m"+rel.getId());
730                                        table sTable = objIDMap.get(rel.getTarget().getParent_id());
731                                        table tTable = objNameMap.get(sTable.getFullName());
732                                        rel.getTarget().setParent_id(tTable.getId());
733                                        for(column col: tTable.getColumns()){
734                                                if(col.getName().equalsIgnoreCase(rel.getTarget().getColumn())){
735                                                        rel.getTarget().setId(col.getId());
736                                                        break;
737                                                }
738                                        }
739                                        for(sourceColumn column: rel.getSources()){
740                                                sTable = objIDMap.get(column.getParent_id());
741                                                tTable = objNameMap.get(sTable.getFullName());
742                                                column.setParent_id(tTable.getId());
743                                                for(column col: tTable.getColumns()){
744                                                        if(col.getName().equalsIgnoreCase(column.getColumn())){
745                                                                column.setId(col.getId());
746                                                                break;
747                                                        }
748                                                }
749                                        }
750                                        mRelationshipList.add(toRelationship(rel));
751                                }
752                        }
753                }
754
755                mDataflow.setDbobjs(dbobjs);
756                mDataflow.setRelationships(mRelationshipList.toArray(new Relationship[mRelationshipList.size()]));
757                mDataflow.setProcesses(mProcessList.toArray(new Process[mProcessList.size()]));
758                mDataflow.setErrors(mErrorList.toArray(new Error[mErrorList.size()]));
759                return mDataflow;
760        }
761
762        public static dataflow readDataflowFromCsvMetadata(String csvMetadata) {
763                if (!MetadataReader.isMetadata(csvMetadata)) {
764                        throw new IllegalArgumentException("Illegal csv metadata.");
765                }
766                return new SQLDepMetadataAnalyzer().analyzeMetadata(null, csvMetadata);
767        }
768
769        private static Relationship toRelationship(relationship relation){
770                Relationship relationModel;
771                if (relation.getType().equals("join")) {
772                        JoinRelationship joinRelationModel = new JoinRelationship();
773                        joinRelationModel.setCondition(relation.getCondition());
774                        joinRelationModel.setJoinType(relation.getJoinType());
775                        joinRelationModel.setClause(relation.getClause());
776                        relationModel = joinRelationModel;
777                } else {
778                        relationModel = new Relationship();
779                }
780                relationModel.setId(relation.getId());
781                relationModel.setProcessId(relation.getProcessId());
782                relationModel.setProcessType(relation.getProcessType());
783                relationModel.setType(relation.getType());
784                relationModel.setEffectType(relation.getEffectType());
785                relationModel.setPartition(relation.getPartition());
786                relationModel.setFunction(relation.getFunction());
787                relationModel.setProcedureId(relation.getProcedureId());
788                relationModel.setSqlHash(relation.getSqlHash());
789                relationModel.setCondition(relation.getCondition());
790                relationModel.setSqlComment(relation.getSqlComment());
791                relationModel.setTimestampMax(relation.getTimestampMax());
792                relationModel.setTimestampMin(relation.getTimestampMin());
793
794                if (relation.getTarget() != null && relation.getSources() != null && !relation.getSources().isEmpty()) {
795                        RelationshipElement targetModel = new RelationshipElement();
796                        targetColumn target = relation.getTarget();
797                        targetModel.setColumn(target.getColumn());
798                        targetModel.setParentName(target.getParent_name());
799                        targetModel.setTargetName(target.getTarget_name());
800                        targetModel.setId(target.getId());
801                        targetModel.setTargetId(target.getTarget_id());
802                        targetModel.setParentId(target.getParent_id());
803                        targetModel.setCoordinates(Coordinate.parse(target.getCoordinate()));
804                        targetModel.setFunction(target.getFunction());
805                        targetModel.setType(target.getType());
806                        relationModel.setTarget(targetModel);
807
808                        List<RelationshipElement> sourceModels = new ArrayList<>();
809                        for (sourceColumn source : relation.getSources()) {
810                                RelationshipElement sourceModel = new RelationshipElement();
811                                sourceModel.setColumn(source.getColumn());
812                                sourceModel.setParentName(source.getParent_name());
813                                sourceModel.setSourceName(source.getSource_name());
814                                sourceModel.setColumnType(source.getColumn_type());
815                                sourceModel.setId(source.getId());
816                                sourceModel.setParentId(source.getParent_id());
817                                sourceModel.setSourceId(source.getSource_id());
818                                sourceModel.setCoordinates(Coordinate.parse(source.getCoordinate()));
819                                sourceModel.setClauseType(source.getClauseType());
820                                sourceModel.setType(source.getType());
821                                sourceModels.add(sourceModel);
822                                if (source.getTransforms() != null && !source.getTransforms().isEmpty()) {
823                                        List<Transform> transforms = new ArrayList<gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform>();
824                                        for (transform transform : source.getTransforms()) {
825                                                Transform item = new Transform();
826                                                item.setCode(transform.getCode());
827                                                item.setType(transform.getType());
828                                                item.setCoordinate(transform.getCoordinate(true));
829                                                transforms.add(item);
830                                        }
831                                        sourceModel.setTransforms(transforms.toArray(new Transform[0]));
832                                }
833                        }
834                        relationModel.setSources(sourceModels.toArray(new RelationshipElement[0]));
835                } else if (relation.getCaller() != null && relation.getCallees() != null
836                                && !relation.getCallees().isEmpty()) {
837                        RelationshipElement targetModel = new RelationshipElement();
838                        targetColumn target = relation.getCaller();
839                        targetModel.setName(target.getName());
840                        targetModel.setId(target.getId());
841                        targetModel.setCoordinates(Coordinate.parse(target.getCoordinate()));
842                        targetModel.setType(target.getType());
843                        relationModel.setCaller(targetModel);
844                        List<RelationshipElement> sourceModels = new ArrayList<gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement>();
845                        for (sourceColumn source : relation.getCallees()) {
846                                RelationshipElement sourceModel = new RelationshipElement();
847                                sourceModel.setName(source.getName());
848                                sourceModel.setId(source.getId());
849                                sourceModel.setCoordinates(Coordinate.parse(source.getCoordinate()));
850                                sourceModel.setType(source.getType());
851                                sourceModels.add(sourceModel);
852                        }
853                        relationModel.setCallees(sourceModels.toArray(new RelationshipElement[0]));
854                }
855                return relationModel;
856        }
857}