Source code

001package gudusoft.gsqlparser.dlineage.util;
002
003import gudusoft.gsqlparser.EDbVendor;
004import gudusoft.gsqlparser.dlineage.DataFlowAnalyzer;
005import gudusoft.gsqlparser.dlineage.ParallelDataFlowAnalyzer;
006import gudusoft.gsqlparser.dlineage.dataflow.metadata.MetadataReader;
007import gudusoft.gsqlparser.dlineage.dataflow.metadata.sqldep.SQLDepMetadataAnalyzer;
008import gudusoft.gsqlparser.dlineage.dataflow.model.ModelBindingManager;
009import gudusoft.gsqlparser.dlineage.dataflow.model.Option;
010import gudusoft.gsqlparser.dlineage.dataflow.model.RelationshipType;
011import gudusoft.gsqlparser.dlineage.dataflow.model.json.Error;
012import gudusoft.gsqlparser.dlineage.dataflow.model.json.Process;
013import gudusoft.gsqlparser.dlineage.dataflow.model.json.*;
014import gudusoft.gsqlparser.dlineage.dataflow.model.xml.*;
015import gudusoft.gsqlparser.dlineage.metadata.Sqlflow;
016import gudusoft.gsqlparser.util.Logger;
017import gudusoft.gsqlparser.util.LoggerFactory;
018import gudusoft.gsqlparser.util.SQLUtil;
019import gudusoft.gsqlparser.util.json.JSON;
020
021import java.io.File;
022import java.util.*;
023import java.util.concurrent.atomic.AtomicLong;
024import java.util.stream.Collectors;
025
026public class DataflowUtility {
027
028    private static final Logger logger = LoggerFactory.getLogger(FunctionUtility.class);
029
030    public static dataflow mergeFunctionCallDataflow(dataflow dataflow, EDbVendor dbVendor) {
031        if (ModelBindingManager.getGlobalVendor() == null) {
032            ModelBindingManager.setGlobalVendor(dbVendor);
033        }
034        dataflow instance = cloneDataflow(dataflow);
035        List<procedure> procedures = new ArrayList<>(instance.getProcedures());
036        if (instance.getPackages() != null) {
037            for (oraclePackage pkg : instance.getPackages()) {
038                if (pkg.getProcedures() != null) {
039                    for (procedure procedure : pkg.getProcedures()) {
040                        procedure.setOraclePackage(pkg);
041                        procedures.add(procedure);
042                    }
043                }
044            }
045        }
046
047        Map<String, procedure> procedureIdMap = procedures.stream().collect(Collectors.toMap(
048                t -> t.getId(),
049                t -> t,
050                (existingValue, newValue) -> newValue
051        ));
052        Map<String, table> functionIdMap = dataflow.getResultsets().stream().collect(Collectors.toMap(
053                t -> t.getId(),
054                t -> t,
055                (existingValue, newValue) -> newValue
056        ));
057
058        Map<String, table> functionMap = new HashMap<>();
059        Map<String, procedure> procedureMap = new HashMap<>();
060        Map<String, oraclePackage> oraclePackageMap = new HashMap<>();
061        Map<String, Set<String>> oraclePackageProcedureMap = new HashMap<>();
062        if (instance.getPackages() != null) {
063            for (oraclePackage pkg : instance.getPackages()) {
064                String qualifiedPackageName = DlineageUtil.getIdentifierOraclePackageNameWithArgNum(pkg);
065                if (!oraclePackageMap.containsKey(qualifiedPackageName)) {
066                    oraclePackageMap.put(qualifiedPackageName, pkg);
067                }
068                for (procedure procedure : pkg.getProcedures()) {
069                    String qualifiedProcedureName = qualifiedPackageName + "." + DlineageUtil.getIdentifierProcedureNameWithArgNum(procedure);
070                    if (!oraclePackageProcedureMap.containsKey(qualifiedPackageName)) {
071                        oraclePackageProcedureMap.put(qualifiedPackageName, new HashSet<>());
072                    }
073                    oraclePackageProcedureMap.get(qualifiedPackageName).add(qualifiedProcedureName);
074                }
075            }
076        }
077
078
079
080        for (procedure procedure : procedures) {
081            String qualifiedProcedureName = DlineageUtil.getIdentifierProcedureNameWithArgNum(procedure);
082            if (procedure.getOraclePackage() != null) {
083                String qualifiedPackageName = DlineageUtil.getIdentifierOraclePackageNameWithArgNum(procedure.getOraclePackage());
084                qualifiedProcedureName = qualifiedPackageName + "." + DlineageUtil.getIdentifierProcedureNameWithArgNum(procedure);
085            }
086            if (!procedureMap.containsKey(qualifiedProcedureName)) {
087                procedureMap.put(qualifiedProcedureName, procedure);
088            }
089        }
090
091        for (table function : instance.getResultsets()) {
092            String qualifiedFunctionName = DlineageUtil.getIdentifierFunctionName(function);
093            if (!functionMap.containsKey(qualifiedFunctionName)) {
094                functionMap.put(qualifiedFunctionName, function);
095            }
096        }
097
098        Set<oraclePackage> oraclePackages = new LinkedHashSet<>();
099        for (String key : oraclePackageMap.keySet()) {
100            oraclePackage standardPackage = oraclePackageMap.get(key);
101            oraclePackages.add(standardPackage);
102            Set<String> procedureNames = oraclePackageProcedureMap.get(key);
103            if (procedureNames != null) {
104                for (String procedureName : procedureNames) {
105                    procedureName = key + "." + procedureName;
106                    if (procedureMap.containsKey(procedureName)) {
107                        procedure standardProcedure = procedureMap.get(procedureName);
108                        if (!standardPackage.getProcedures().contains(standardProcedure)) {
109                            standardPackage.getProcedures().add(standardProcedure);
110                        }
111                        procedureMap.remove(procedureName);
112                    }
113                }
114            }
115        }
116
117        instance.setPackages(new ArrayList<>(oraclePackages));
118        instance.setProcedures(new ArrayList<>(procedureMap.values()));
119        instance.setResultsets(new ArrayList<>(functionMap.values()));
120
121        Map<String, relationship> mergeRelations = new LinkedHashMap<String, relationship>();
122
123        for (relationship relationship : instance.getRelationships()) {
124            if (relationship.getCaller() == null || relationship.getCallees() == null || relationship.getCallees().size() == 0) {
125                continue;
126            }
127
128            String targetId = relationship.getCaller().getId();
129            if (procedureIdMap.containsKey(targetId)) {
130                procedure procedure = procedureIdMap.get(targetId);
131                String procedureName = DlineageUtil.getIdentifierProcedureNameWithArgNum(procedure);
132                if (procedure.getOraclePackage() != null) {
133                    String qualifiedPackageName = DlineageUtil.getIdentifierOraclePackageNameWithArgNum(procedure.getOraclePackage());
134                    procedureName = qualifiedPackageName + "." + procedureName;
135                }
136                procedure standardProcedure = procedureMap.get(procedureName);
137                relationship.getCaller().setId(standardProcedure.getId());
138            } else if (functionIdMap.containsKey(targetId)) {
139                table function = functionIdMap.get(targetId);
140                String functionName = DlineageUtil.getIdentifierFunctionName(function);
141                table standardFunction = functionMap.get(functionName);
142                relationship.getCaller().setId(standardFunction.getId());
143            }
144
145            for (sourceColumn sourceColumn : relationship.getCallees()) {
146                String sourceId = sourceColumn.getId();
147                if (procedureIdMap.containsKey(sourceId)) {
148                    procedure procedure = procedureIdMap.get(sourceId);
149                    String procedureName = DlineageUtil.getIdentifierProcedureNameWithArgNum(procedure);
150                    if (procedure.getOraclePackage() != null) {
151                        String qualifiedPackageName = DlineageUtil.getIdentifierOraclePackageNameWithArgNum(procedure.getOraclePackage());
152                        procedureName = qualifiedPackageName + "." + procedureName;
153                    }
154                    procedure standardProcedure = procedureMap.get(procedureName);
155                    sourceColumn.setId(standardProcedure.getId());
156                } else if (functionIdMap.containsKey(sourceId)) {
157                    table function = functionIdMap.get(sourceId);
158                    String functionName = DlineageUtil.getIdentifierFunctionName(function);
159                    table standardFunction = functionMap.get(functionName);
160                    sourceColumn.setId(standardFunction.getId());
161                }
162            }
163
164            String jsonString = JSON.toJSONString(relationship).replaceAll("\"id\":\".+?\"", "");
165            String key = SHA256.getMd5(jsonString);
166            if (!mergeRelations.containsKey(key)) {
167                mergeRelations.put(key, relationship);
168            }
169        }
170
171        instance.setRelationships(new ArrayList<relationship>(mergeRelations.values()));
172
173        if (instance.getPackages() != null) {
174            for (oraclePackage pkg : instance.getPackages()) {
175                if (pkg.getProcedures() != null) {
176                    for (procedure procedure : pkg.getProcedures()) {
177                        procedure.setOraclePackage(null);
178                    }
179                }
180            }
181        }
182
183        return instance;
184    }
185
186    public static dataflow convertTableLevelToFunctionCallDataflow(dataflow dataflow, boolean showBuiltIn, EDbVendor dbVendor) {
187        if (ModelBindingManager.getGlobalVendor() == null) {
188            ModelBindingManager.setGlobalVendor(dbVendor);
189        }
190
191        dataflow instance = cloneDataflow(dataflow);
192
193        if (instance.getRelationships() == null) {
194            return instance;
195        }
196
197        List<relationship> callRelationships = instance.getRelationships().stream()
198                .filter(t -> RelationshipType.call.name().equals(t.getType()))
199                .filter(t -> !showBuiltIn ? !Boolean.TRUE.equals(t.getBuiltIn()): true).collect(Collectors.toList());
200        instance.setRelationships(callRelationships);
201
202        Set<String> ids = new HashSet<>();
203
204
205        callRelationships.stream().forEach(t -> {
206            ids.add(t.getCaller().getId());
207            t.getCallees().stream().forEach(t1 -> ids.add(t1.getId()));
208        });
209
210        Iterator<table> iterator = instance.getTables().iterator();
211        while (iterator.hasNext()) {
212            table t = iterator.next();
213            if (!ids.contains(t.getId())) {
214                iterator.remove();
215            }
216        }
217
218        iterator = instance.getResultsets().iterator();
219        while (iterator.hasNext()) {
220            table t = iterator.next();
221            if (!ids.contains(t.getId())) {
222                iterator.remove();
223            }
224        }
225
226        iterator = instance.getViews().iterator();
227        while (iterator.hasNext()) {
228            table t = iterator.next();
229            if (!ids.contains(t.getId())) {
230                iterator.remove();
231            }
232        }
233
234        iterator = instance.getStages().iterator();
235        while (iterator.hasNext()) {
236            table t = iterator.next();
237            if (!ids.contains(t.getId())) {
238                iterator.remove();
239            }
240        }
241
242        iterator = instance.getStreams().iterator();
243        while (iterator.hasNext()) {
244            table t = iterator.next();
245            if (!ids.contains(t.getId())) {
246                iterator.remove();
247            }
248        }
249
250        iterator = instance.getVariables().iterator();
251        while (iterator.hasNext()) {
252            table t = iterator.next();
253            if (!ids.contains(t.getId())) {
254                iterator.remove();
255            }
256        }
257
258        iterator = instance.getPaths().iterator();
259        while (iterator.hasNext()) {
260            table t = iterator.next();
261            if (!ids.contains(t.getId())) {
262                iterator.remove();
263            }
264        }
265
266        iterator = instance.getDatasources().iterator();
267        while (iterator.hasNext()) {
268            table t = iterator.next();
269            if (!ids.contains(t.getId())) {
270                iterator.remove();
271            }
272        }
273
274        iterator = instance.getDatabases().iterator();
275        while (iterator.hasNext()) {
276            table t = iterator.next();
277            if (!ids.contains(t.getId())) {
278                iterator.remove();
279            }
280        }
281
282        iterator = instance.getSchemas().iterator();
283        while (iterator.hasNext()) {
284            table t = iterator.next();
285            if (!ids.contains(t.getId())) {
286                iterator.remove();
287            }
288        }
289
290        iterator = instance.getSequences().iterator();
291        while (iterator.hasNext()) {
292            table t = iterator.next();
293            if (!ids.contains(t.getId())) {
294                iterator.remove();
295            }
296        }
297
298        return mergeFunctionCallDataflow(instance, dbVendor);
299    }
300
301    public static dataflow convertToTableLevelDataflow(dataflow dataflow) {
302        dataflow instance = cloneDataflow(dataflow);
303
304        if (instance.getRelationships() == null) {
305            return instance;
306        }
307
308        Map<String, LinkedHashSet<Pair<String, String>>> relationMap = new HashMap<String, LinkedHashSet<Pair<String, String>>>();
309        Map<String, LinkedHashSet<Pair3<String, String, relationship>>> callRelationMap = new HashMap<String, LinkedHashSet<Pair3<String, String, relationship>>>();
310        for (RelationshipType type : RelationshipType.values()) {
311            relationMap.put(type.name(), new LinkedHashSet<Pair<String, String>>());
312            callRelationMap.put(type.name(), new LinkedHashSet<Pair3<String, String, relationship>>());
313        }
314        for (relationship relationship : instance.getRelationships()) {
315            if (RelationshipType.call.name().equals(relationship.getType())) {
316                String targetId = relationship.getCaller().getId();
317                for (sourceColumn sourceColumn : relationship.getCallees()) {
318                    String sourceId = sourceColumn.getId();
319                    callRelationMap.get(relationship.getType()).add(new Pair3<>(targetId, sourceId, relationship));
320                }
321            } else {
322                String targetId = relationship.getTarget().getParent_id();
323                for (sourceColumn sourceColumn : relationship.getSources()) {
324                    String sourceId = sourceColumn.getParent_id();
325                    relationMap.get(relationship.getType()).add(new Pair<>(targetId, sourceId));
326                }
327            }
328        }
329
330        long maxId = 0;
331        Map<String, table> dbObjMap = getDataflowDbObjMap(instance);
332
333        List<procedure> procedures = new ArrayList<>(instance.getProcedures());
334        if (instance.getPackages() != null) {
335            for (oraclePackage pkg : instance.getPackages()) {
336                procedures.addAll(pkg.getProcedures());
337            }
338        }
339
340        Map<String, procedure> procedureMap = procedures.stream().collect(Collectors.toMap(
341                t -> t.getId(),
342                t -> t,
343                (existingValue, newValue) -> newValue
344        ));
345
346        for (table table : dbObjMap.values()) {
347            if (Long.valueOf(table.getId()) > maxId) {
348                maxId = Long.valueOf(table.getId());
349            }
350        }
351
352        AtomicLong id = new AtomicLong(maxId + 10000000);
353
354        for (table table : dbObjMap.values()) {
355            column column = new column();
356            column.setId(String.valueOf(id.incrementAndGet()));
357            column.setName(table.getType());
358            table.setColumns(Arrays.asList(column));
359        }
360
361        List<relationship> relations = new ArrayList<relationship>();
362        for (RelationshipType type : RelationshipType.values()) {
363            LinkedHashSet<Pair<String, String>> relationSet = relationMap.get(type.name());
364             LinkedHashSet<Pair3<String, String, relationship>> callRelationSet = callRelationMap.get(type.name());
365            if (RelationshipType.call.equals(type)) {
366                for (Pair3<String, String, relationship> pair : callRelationSet) {
367                    if ((dbObjMap.get(pair.first) == null && procedureMap.get(pair.first) == null)
368                            || (dbObjMap.get(pair.second) == null && procedureMap.get(pair.second) == null)) {
369                        continue;
370                    }
371
372                    relationship relationship = new relationship();
373                    relationship.setType(type.name());
374                    relationship.setId(String.valueOf(id.incrementAndGet()));
375                    relationship.setCallStmt(pair.third.getCallStmt());
376                    relationship.setCallCoordinate(pair.third.getCallCoordinate());
377
378                    targetColumn targetColumn = new targetColumn();
379                    if (dbObjMap.containsKey(pair.first)) {
380                        targetColumn.setId(dbObjMap.get(pair.first).getId());
381                        targetColumn.setName(dbObjMap.get(pair.first).getName());
382                        targetColumn.setCoordinate(dbObjMap.get(pair.first).getCoordinate());
383                    } else {
384                        targetColumn.setId(procedureMap.get(pair.first).getId());
385                        targetColumn.setName(procedureMap.get(pair.first).getName());
386                        targetColumn.setCoordinate(procedureMap.get(pair.first).getCoordinate());
387                    }
388
389                    sourceColumn sourceColumn = new sourceColumn();
390                    if (dbObjMap.containsKey(pair.second)) {
391                        sourceColumn.setId(dbObjMap.get(pair.second).getId());
392                        sourceColumn.setName(dbObjMap.get(pair.second).getName());
393                        sourceColumn.setCoordinate(dbObjMap.get(pair.second).getCoordinate());
394                    } else {
395                        sourceColumn.setId(procedureMap.get(pair.second).getId());
396                        sourceColumn.setName(procedureMap.get(pair.second).getName());
397                        sourceColumn.setCoordinate(procedureMap.get(pair.second).getCoordinate());
398                    }
399
400                    relationship.setCaller(targetColumn);
401                    relationship.setCallees(Arrays.asList(sourceColumn));
402                    relationship.setBuiltIn(pair.third.getBuiltIn());
403                    relations.add(relationship);
404                }
405            } else {
406                for (Pair<String, String> pair : relationSet) {
407                    if (dbObjMap.get(pair.first) == null || dbObjMap.get(pair.second) == null) {
408                        continue;
409                    }
410                    relationship relationship = new relationship();
411                    relationship.setType(type.name());
412                    relationship.setId(String.valueOf(id.incrementAndGet()));
413                    targetColumn targetColumn = new targetColumn();
414                    targetColumn.setId(dbObjMap.get(pair.first).getColumns().get(0).getId());
415                    targetColumn.setColumn(dbObjMap.get(pair.first).getColumns().get(0).getName());
416                    targetColumn.setParent_id(pair.first);
417                    targetColumn.setParent_name(dbObjMap.get(pair.first).getName());
418                    sourceColumn sourceColumn = new sourceColumn();
419                    sourceColumn.setId(dbObjMap.get(pair.second).getColumns().get(0).getId());
420                    sourceColumn.setColumn(dbObjMap.get(pair.second).getColumns().get(0).getName());
421                    sourceColumn.setParent_id(pair.second);
422                    sourceColumn.setParent_name(dbObjMap.get(pair.second).getName());
423                    relationship.setTarget(targetColumn);
424                    relationship.setSources(Arrays.asList(sourceColumn));
425                    relations.add(relationship);
426                }
427            }
428        }
429        instance.setRelationships(relations);
430        return instance;
431    }
432
433    public static dataflow convertToSchemaLevelDataflow(dataflow dataflow, EDbVendor dbVendor) throws Exception {
434        return convertToSchemaLevelDataflow(dataflow, dbVendor, false);
435    }
436
437    public static dataflow convertToSchemaLevelDataflow(dataflow dataflow, EDbVendor dbVendor, boolean isSimple) throws Exception {
438        dataflow instance = cloneDataflow(dataflow);
439
440        if (!isSimple) {
441            DataFlowAnalyzer analyzer = new DataFlowAnalyzer("", dbVendor, true);
442            instance = analyzer.getSimpleDataflow(instance, true, Arrays.asList(new String[]{"fdd", "fdr"}));
443        }
444
445        List<table> allTables = new ArrayList<table>();
446        allTables.addAll(instance.getTables());
447        allTables.addAll(instance.getViews());
448
449        ModelBindingManager.setGlobalVendor(dbVendor);
450
451        Map<String, String> tableIdSchameNameMap = allTables.stream().collect(Collectors.toMap(table -> table.getId(), table -> table.getFullSchemaName(), (existingValue, newValue) -> newValue));
452
453        if (instance.getRelationships() == null) {
454            return instance;
455        }
456
457        Map<String, LinkedHashSet<Pair<String, String>>> relationMap = new HashMap<String, LinkedHashSet<Pair<String, String>>>();
458        for (RelationshipType type : RelationshipType.values()) {
459            relationMap.put(type.name(), new LinkedHashSet<Pair<String, String>>());
460        }
461        for (relationship relationship : instance.getRelationships()) {
462            String targetId = relationship.getTarget().getParent_id();
463            for (sourceColumn sourceColumn : relationship.getSources()) {
464                String sourceId = sourceColumn.getParent_id();
465                relationMap.get(relationship.getType()).add(new Pair<>(tableIdSchameNameMap.get(targetId), tableIdSchameNameMap.get(sourceId)));
466            }
467        }
468
469        Map<String, table> dbObjMap = getDataflowDbObjMap(instance);
470        LinkedHashSet<String> schemaNameSet = new LinkedHashSet<String>();
471        schemaNameSet.addAll(tableIdSchameNameMap.values());
472        List<table> schemaTables = new ArrayList<table>();
473
474        AtomicLong id = new AtomicLong(0);
475
476        for (String schemaName : schemaNameSet) {
477            table table = new table();
478            table.setId(String.valueOf(id.incrementAndGet()));
479            table.setName(schemaName);
480            table.setType("schema");
481            schemaTables.add(table);
482            dbObjMap.put(schemaName, table);
483        }
484
485        for (table table : schemaTables) {
486            column column = new column();
487            column.setId(String.valueOf(id.incrementAndGet()));
488            column.setName(table.getType());
489            table.setColumns(Arrays.asList(column));
490        }
491
492        List<relationship> relations = new ArrayList<relationship>();
493        for (RelationshipType type : RelationshipType.values()) {
494            LinkedHashSet<Pair<String, String>> relationSet = relationMap.get(type.name());
495
496            for (Pair<String, String> pair : relationSet) {
497                if (dbObjMap.get(pair.first) == null || dbObjMap.get(pair.second) == null) {
498                    continue;
499                }
500                relationship relationship = new relationship();
501                relationship.setType(type.name());
502                relationship.setId(String.valueOf(id.incrementAndGet()));
503                targetColumn targetColumn = new targetColumn();
504                targetColumn.setId(dbObjMap.get(pair.first).getColumns().get(0).getId());
505                targetColumn.setColumn(dbObjMap.get(pair.first).getColumns().get(0).getName());
506                targetColumn.setParent_id(dbObjMap.get(pair.first).getId());
507                targetColumn.setParent_name(dbObjMap.get(pair.first).getName());
508                sourceColumn sourceColumn = new sourceColumn();
509                sourceColumn.setId(dbObjMap.get(pair.second).getColumns().get(0).getId());
510                sourceColumn.setColumn(dbObjMap.get(pair.second).getColumns().get(0).getName());
511                sourceColumn.setParent_id(dbObjMap.get(pair.second).getId());
512                sourceColumn.setParent_name(dbObjMap.get(pair.second).getName());
513                relationship.setTarget(targetColumn);
514                relationship.setSources(Arrays.asList(sourceColumn));
515                relations.add(relationship);
516            }
517        }
518
519        dataflow schemaDataflow = new dataflow();
520        schemaDataflow.setTables(schemaTables);
521        schemaDataflow.setRelationships(relations);
522        return schemaDataflow;
523    }
524
525    public static dataflow cloneDataflow(dataflow dataflow) {
526        dataflow dataflowCopy = new dataflow();
527        if (dataflow.getTables() != null) {
528            dataflowCopy.setTables(cloneTables(dataflow.getTables()));
529        }
530        if (dataflow.getViews() != null) {
531            dataflowCopy.setViews(cloneTables(dataflow.getViews()));
532        }
533        if (dataflow.getRelationships() != null) {
534            dataflowCopy.setRelationships(new ArrayList<>(dataflow.getRelationships()));
535        }
536        if (dataflow.getErrors() != null) {
537            dataflowCopy.setErrors(new ArrayList<>(dataflow.getErrors()));
538        }
539        if (dataflow.getPaths() != null) {
540            dataflowCopy.setPaths(cloneTables(dataflow.getPaths()));
541        }
542        if (dataflow.getPackages() != null) {
543            dataflowCopy.setPackages(new ArrayList<>(dataflow.getPackages()));
544        }
545        if (dataflow.getProcedures() != null) {
546            dataflowCopy.setProcedures(new ArrayList<>(dataflow.getProcedures()));
547        }
548        if (dataflow.getProcesses() != null) {
549            dataflowCopy.setProcesses(new ArrayList<>(dataflow.getProcesses()));
550        }
551        if (dataflow.getResultsets() != null) {
552            dataflowCopy.setResultsets(cloneTables(dataflow.getResultsets()));
553        }
554        if (dataflow.getVariables() != null) {
555            dataflowCopy.setVariables(cloneTables(dataflow.getVariables()));
556        }
557        if (dataflow.getStages() != null) {
558            dataflowCopy.setStages(cloneTables(dataflow.getStages()));
559        }
560        if (dataflow.getSequences() != null) {
561            dataflowCopy.setSequences(cloneTables(dataflow.getSequences()));
562        }
563        if (dataflow.getDatasources() != null) {
564            dataflowCopy.setDatasources(cloneTables(dataflow.getDatasources()));
565        }
566        if (dataflow.getDatabases() != null) {
567            dataflowCopy.setDatabases(cloneTables(dataflow.getDatabases()));
568        }
569        if (dataflow.getSchemas() != null) {
570            dataflowCopy.setSchemas(cloneTables(dataflow.getSchemas()));
571        }
572        if (dataflow.getStreams() != null) {
573            dataflowCopy.setStreams(cloneTables(dataflow.getStreams()));
574        }
575        dataflowCopy.setOrientation(dataflow.getOrientation());
576        return dataflowCopy;
577    }
578
579    private static List<table> cloneTables(List<table> tableList) {
580        if (tableList == null) {
581            return null;
582        }
583        List<table> tables = new ArrayList<>(tableList.size());
584        for (table item : tableList) {
585            try {
586                table table = (table) item.clone();
587                if (item.getColumns() != null) {
588                    table.setColumns(new ArrayList<column>(item.getColumns()));
589                }
590                tables.add(table);
591            } catch (CloneNotSupportedException e) {
592                logger.error("Clone table failed.", e);
593            }
594        }
595        return tables;
596    }
597
598    public static Map<String, table> getDataflowDbObjMap(dataflow dataflow) {
599        List<table> tables = new ArrayList<table>();
600        if (dataflow.getTables() != null) {
601            tables.addAll(dataflow.getTables());
602        }
603        if (dataflow.getViews() != null) {
604            tables.addAll(dataflow.getViews());
605        }
606        if (dataflow.getPaths() != null) {
607            tables.addAll(dataflow.getPaths());
608        }
609        if (dataflow.getResultsets() != null) {
610            tables.addAll(dataflow.getResultsets());
611        }
612        if (dataflow.getVariables() != null) {
613            tables.addAll(dataflow.getVariables());
614        }
615        if (dataflow.getStages() != null) {
616            tables.addAll(dataflow.getStages());
617        }
618        if (dataflow.getSequences() != null) {
619            tables.addAll(dataflow.getSequences());
620        }
621        if (dataflow.getDatasources() != null) {
622            tables.addAll(dataflow.getDatasources());
623        }
624        if (dataflow.getDatabases() != null) {
625            tables.addAll(dataflow.getDatabases());
626        }
627        if (dataflow.getSchemas() != null) {
628            tables.addAll(dataflow.getSchemas());
629        }
630        if (dataflow.getStreams() != null) {
631            tables.addAll(dataflow.getStreams());
632        }
633
634        Map<String, table> dbObjMap = new HashMap<>();
635        for (table table : tables) {
636            dbObjMap.put(table.getId(), table);
637        }
638        return dbObjMap;
639        }
640        public static Map<String, table> getDataflowDbObjNameMap(dataflow dataflow) {
641                List<table> tables = new ArrayList<table>();
642                if (dataflow.getTables() != null) {
643                        tables.addAll(dataflow.getTables());
644                }
645                if (dataflow.getViews() != null) {
646                        tables.addAll(dataflow.getViews());
647                }
648                if (dataflow.getPaths() != null) {
649                        tables.addAll(dataflow.getPaths());
650                }
651                if (dataflow.getResultsets() != null) {
652                        tables.addAll(dataflow.getResultsets());
653                }
654                if (dataflow.getVariables() != null) {
655                        tables.addAll(dataflow.getVariables());
656                }
657                if (dataflow.getStages() != null) {
658                        tables.addAll(dataflow.getStages());
659                }
660                if (dataflow.getSequences() != null) {
661                        tables.addAll(dataflow.getSequences());
662                }
663                if (dataflow.getDatasources() != null) {
664                        tables.addAll(dataflow.getDatasources());
665                }
666                if (dataflow.getDatabases() != null) {
667                        tables.addAll(dataflow.getDatabases());
668                }
669                if (dataflow.getSchemas() != null) {
670                        tables.addAll(dataflow.getSchemas());
671                }
672                if (dataflow.getStreams() != null) {
673                        tables.addAll(dataflow.getStreams());
674                }
675
676                Map<String, table> dbObjMap = new HashMap<>();
677                for(table table: tables){
678                        dbObjMap.put(table.getFullName(), table);
679                }
680                return dbObjMap;
681        }
682        public static dataflow mergeDataflowsWithDifferentStartId(Collection<dataflow> dataflows, EDbVendor vendor) {
683                ModelBindingManager.setGlobalVendor(vendor);
684                try {
685                        return mergeDataflowsByStartId(dataflows, DATAFLOW_ID_RANGE * dataflows.size());
686                } finally {
687                        ModelBindingManager.removeGlobalVendor();
688                }
689        }
690
691        /**
692         * 内部合并实现：调用方需保证所有输入 dataflow 的 ID 空间不冲突，并且已提前设置好 ModelBindingManager。
693         * <p>
694         * 该方法代替之前位于 {@code ParallelDataFlowAnalyzer} 的同名实现，外部调用者应使用
695         * {@link #mergeDataflowsWithDifferentStartId(Collection, EDbVendor)} 或
696         * {@link #mergeDataflows(Collection, EDbVendor)}。
697         */
698        public static dataflow mergeDataflowsByStartId(Collection<dataflow> dataflows, long startId) {
699                dataflow mergeDataflow = new dataflow();
700
701                List<table> tableCopy = new ArrayList<table>();
702                List<table> viewCopy = new ArrayList<table>();
703                List<table> databaseCopy = new ArrayList<table>();
704                List<table> schemaCopy = new ArrayList<table>();
705                List<table> stageCopy = new ArrayList<table>();
706                List<table> dataSourceCopy = new ArrayList<table>();
707                List<table> streamCopy = new ArrayList<table>();
708                List<table> fileCopy = new ArrayList<table>();
709                List<table> variableCopy = new ArrayList<table>();
710                List<table> resultSetCopy = new ArrayList<table>();
711                List<table> sequenceCopy = new ArrayList<table>();
712
713                List<process> processCopy = new ArrayList<process>();
714                List<relationship> relationshipCopy = new ArrayList<relationship>();
715                List<procedure> procedureCopy = new ArrayList<procedure>();
716                List<oraclePackage> packageCopy = new ArrayList<oraclePackage>();
717                List<error> errorCopy = new ArrayList<error>();
718
719                List<table> tables = new ArrayList<table>();
720                for (dataflow dataflow : dataflows) {
721                        if (dataflow == null) {
722                                continue;
723                        }
724
725                        if (dataflow.getTables() != null) {
726                                tableCopy.addAll(dataflow.getTables());
727                        }
728                        mergeDataflow.setTables(tableCopy);
729                        if (dataflow.getViews() != null) {
730                                viewCopy.addAll(dataflow.getViews());
731                        }
732                        mergeDataflow.setViews(viewCopy);
733                        if (dataflow.getDatabases() != null) {
734                                databaseCopy.addAll(dataflow.getDatabases());
735                        }
736                        mergeDataflow.setDatabases(databaseCopy);
737                        if (dataflow.getSchemas() != null) {
738                                schemaCopy.addAll(dataflow.getSchemas());
739                        }
740                        mergeDataflow.setSchemas(schemaCopy);
741                        if (dataflow.getStages() != null) {
742                                stageCopy.addAll(dataflow.getStages());
743                        }
744                        mergeDataflow.setStages(stageCopy);
745                        if (dataflow.getDatasources() != null) {
746                                dataSourceCopy.addAll(dataflow.getDatasources());
747                        }
748                        mergeDataflow.setDatasources(dataSourceCopy);
749                        if (dataflow.getStreams() != null) {
750                                streamCopy.addAll(dataflow.getStreams());
751                        }
752                        mergeDataflow.setStreams(streamCopy);
753                        if (dataflow.getPaths() != null) {
754                                fileCopy.addAll(dataflow.getPaths());
755                        }
756                        mergeDataflow.setPaths(fileCopy);
757                        if (dataflow.getVariables() != null) {
758                                variableCopy.addAll(dataflow.getVariables());
759                        }
760                        mergeDataflow.setVariables(variableCopy);
761                        if (dataflow.getResultsets() != null) {
762                                resultSetCopy.addAll(dataflow.getResultsets());
763                        }
764                        mergeDataflow.setResultsets(resultSetCopy);
765                        if (dataflow.getSequences() != null) {
766                                sequenceCopy.addAll(dataflow.getSequences());
767                        }
768                        mergeDataflow.setSequences(sequenceCopy);
769                        if (dataflow.getProcesses() != null) {
770                                processCopy.addAll(dataflow.getProcesses());
771                        }
772                        mergeDataflow.setProcesses(processCopy);
773                        if (dataflow.getRelationships() != null) {
774                                relationshipCopy.addAll(dataflow.getRelationships());
775                        }
776                        mergeDataflow.setRelationships(relationshipCopy);
777                        if (dataflow.getProcedures() != null) {
778                                procedureCopy.addAll(dataflow.getProcedures());
779                        }
780                        mergeDataflow.setProcedures(procedureCopy);
781                        if (dataflow.getPackages() != null) {
782                                packageCopy.addAll(dataflow.getPackages());
783                        }
784                        mergeDataflow.setPackages(packageCopy);
785                        if (dataflow.getErrors() != null) {
786                                errorCopy.addAll(dataflow.getErrors());
787                        }
788                        if (errorCopy.size() > 10000) {
789                                errorCopy = errorCopy.subList(0, 10000);
790                        }
791                        mergeDataflow.setErrors(errorCopy);
792
793                        tables.addAll(dataflow.getTables());
794                        tables.addAll(dataflow.getViews());
795                        tables.addAll(dataflow.getDatabases());
796                        tables.addAll(dataflow.getSchemas());
797                        tables.addAll(dataflow.getStages());
798                        tables.addAll(dataflow.getDatasources());
799                        tables.addAll(dataflow.getStreams());
800                        tables.addAll(dataflow.getPaths());
801                        tables.addAll(dataflow.getResultsets());
802                        tables.addAll(dataflow.getVariables());
803                        if (dataflow.getSequences() != null) {
804                                tables.addAll(dataflow.getSequences());
805                        }
806                }
807
808                Map<String, List<table>> tableMap = new HashMap<String, List<table>>();
809                Map<String, String> tableTypeMap = new HashMap<String, String>();
810                Map<String, String> tableIdMap = new HashMap<String, String>();
811
812                Map<String, List<column>> columnMap = new HashMap<String, List<column>>();
813                Map<String, Set<String>> tableColumnMap = new HashMap<String, Set<String>>();
814                Map<String, String> columnIdMap = new HashMap<String, String>();
815                Map<String, column> columnMergeIdMap = new HashMap<String, column>();
816
817                List<procedure> procedures = new ArrayList<>(mergeDataflow.getProcedures());
818                if (mergeDataflow.getPackages() != null) {
819                        for (oraclePackage pkg : mergeDataflow.getPackages()) {
820                                procedures.addAll(pkg.getProcedures());
821                        }
822                }
823
824                Set<String> procedureIdSet = procedures.stream().map(t -> t.getId()).collect(Collectors.toSet());
825
826                for (table table : tables) {
827                        String qualifiedTableName = DlineageUtil.getQualifiedTableName(table);
828                        String tableFullName = DlineageUtil.getIdentifierNormalTableName(qualifiedTableName);
829                        if ("variable".endsWith(table.getType()) && !SQLUtil.isEmpty(table.getParent())) {
830                                tableFullName = table.getParent() + "." + tableFullName;
831                        }
832
833                        if (!tableMap.containsKey(tableFullName)) {
834                                tableMap.put(tableFullName, new ArrayList<table>());
835                        }
836
837                        tableMap.get(tableFullName).add(table);
838
839                        if (!tableTypeMap.containsKey(tableFullName)) {
840                                tableTypeMap.put(tableFullName, table.getType());
841                        } else if ("view".equals(table.getSubType())) {
842                                tableTypeMap.put(tableFullName, table.getType());
843                        } else if ("database".equals(table.getSubType())) {
844                                tableTypeMap.put(tableFullName, table.getType());
845                        } else if ("schema".equals(table.getSubType())) {
846                                tableTypeMap.put(tableFullName, table.getType());
847                        } else if ("stage".equals(table.getSubType())) {
848                                tableTypeMap.put(tableFullName, table.getType());
849                        } else if ("datasource".equals(table.getSubType())) {
850                                tableTypeMap.put(tableFullName, table.getType());
851                        } else if ("stream".equals(table.getSubType())) {
852                                tableTypeMap.put(tableFullName, table.getType());
853                        } else if ("file".equals(table.getSubType())) {
854                                tableTypeMap.put(tableFullName, table.getType());
855                        } else if ("sequence".equals(table.getSubType())) {
856                                tableTypeMap.put(tableFullName, table.getType());
857                        } else if ("table".equals(tableTypeMap.get(tableFullName))) {
858                                tableTypeMap.put(tableFullName, table.getType());
859                        } else if ("variable".equals(tableTypeMap.get(tableFullName))) {
860                                tableTypeMap.put(tableFullName, table.getType());
861                        }
862
863                        if (table.getColumns() != null) {
864                                if (!tableColumnMap.containsKey(tableFullName)) {
865                                        tableColumnMap.put(tableFullName, new LinkedHashSet<String>());
866                                }
867                                for (column column : table.getColumns()) {
868                                        String columnFullName = tableFullName + "."
869                                                        + DlineageUtil.getIdentifierNormalColumnName(column.getName());
870
871                                        if (!columnMap.containsKey(columnFullName)) {
872                                                columnMap.put(columnFullName, new ArrayList<column>());
873                                                tableColumnMap.get(tableFullName).add(columnFullName);
874                                        }
875
876                                        columnMap.get(columnFullName).add(column);
877                                }
878                        }
879                }
880
881                Iterator<String> tableNameIter = tableMap.keySet().iterator();
882                while (tableNameIter.hasNext()) {
883                        String tableName = tableNameIter.next();
884                        List<table> tableList = tableMap.get(tableName);
885                        table table;
886                        if (tableList.size() > 1) {
887                                table standardTable = tableList.get(0);
888                                //Function允许重名，不做合并处理
889                                if (standardTable.isFunction()) {
890                                        continue;
891                                }
892
893                                String type = tableTypeMap.get(tableName);
894                                table = new table();
895                                table.setId(String.valueOf(++startId));
896                                table.setServer(standardTable.getServer());
897                                table.setDatabase(standardTable.getDatabase());
898                                table.setSchema(standardTable.getSchema());
899                                table.setName(standardTable.getName());
900                                table.setDisplayName(standardTable.getDisplayName());
901                                table.setParent(standardTable.getParent());
902                                table.setColumns(new ArrayList<column>());
903                String subType = null;
904                for(table item: tableList){
905                    if (item.getSubType() != null) {
906                        subType = item.getSubType();
907                        break;
908                    }
909                }
910                if (subType != null) {
911                    table.setSubType(subType);
912                } else {
913                    table.setSubType(standardTable.getSubType());
914                }
915                                Set<String> processIds = new LinkedHashSet<String>();
916                                for (int k = 0; k < tableList.size(); k++) {
917                                        if (tableList.get(k).getProcessIds() != null) {
918                                                processIds.addAll(tableList.get(k).getProcessIds());
919                                        }
920                                }
921                                if (!processIds.isEmpty()) {
922                                        table.setProcessIds(new ArrayList<String>(processIds));
923                                }
924                                table.setType(type);
925                                for (table item : tableList) {
926                                        if (!SQLUtil.isEmpty(table.getCoordinate()) && !SQLUtil.isEmpty(item.getCoordinate())) {
927                                                if (table.getCoordinate().indexOf(item.getCoordinate()) == -1) {
928                                                        table.appendCoordinate(item.getCoordinate());
929                                                }
930                                        } else if (!SQLUtil.isEmpty(item.getCoordinate())) {
931                                                table.setCoordinate(item.getCoordinate());
932                                        }
933
934                                        if (!SQLUtil.isEmpty(table.getAlias()) && !SQLUtil.isEmpty(item.getAlias())) {
935                                                table.setAlias(table.getAlias() + "," + item.getAlias());
936                                        } else if (!SQLUtil.isEmpty(item.getAlias())) {
937                                                table.setAlias(item.getAlias());
938                                        }
939
940                                        tableIdMap.put(item.getId(), table.getId());
941
942                                        if (item.isView()) {
943                                                mergeDataflow.getViews().remove(item);
944                                        } else if (item.isDatabaseType()) {
945                                                mergeDataflow.getDatabases().remove(item);
946                                        } else if (item.isSchemaType()) {
947                                                mergeDataflow.getSchemas().remove(item);
948                                        } else if (item.isStage()) {
949                                                mergeDataflow.getStages().remove(item);
950                                        } else if (item.isDataSource()) {
951                                                mergeDataflow.getDatasources().remove(item);
952                                        } else if (item.isStream()) {
953                                                mergeDataflow.getStreams().remove(item);
954                                        } else if (item.isFile()) {
955                                                mergeDataflow.getPaths().remove(item);
956                                        } else if (item.isVariable()) {
957                                                mergeDataflow.getVariables().remove(item);
958                                        } else if (item.isTable()) {
959                                                mergeDataflow.getTables().remove(item);
960                                        } else if (item.isResultSet()) {
961                                                mergeDataflow.getResultsets().remove(item);
962                                        } else if (item.isSequence()) {
963                                                mergeDataflow.getSequences().remove(item);
964                                        }
965                                }
966
967                                if (table.isView()) {
968                                        mergeDataflow.getViews().add(table);
969                                } else if (table.isDatabaseType()) {
970                                        mergeDataflow.getDatabases().add(table);
971                                } else if (table.isSchemaType()) {
972                                        mergeDataflow.getSchemas().add(table);
973                                } else if (table.isStage()) {
974                                        mergeDataflow.getStages().add(table);
975                                } else if (table.isDataSource()) {
976                                        mergeDataflow.getDatasources().add(table);
977                                } else if (table.isStream()) {
978                                        mergeDataflow.getStreams().add(table);
979                                } else if (table.isFile()) {
980                                        mergeDataflow.getPaths().add(table);
981                                } else if (table.isVariable()) {
982                                        mergeDataflow.getVariables().add(table);
983                                } else if (table.isResultSet()) {
984                                        mergeDataflow.getResultsets().add(table);
985                                } else if (table.isSequence()) {
986                                        mergeDataflow.getSequences().add(table);
987                                } else {
988                                        mergeDataflow.getTables().add(table);
989                                }
990                        } else {
991                                table = tableList.get(0);
992                        }
993
994                        Set<String> columns = tableColumnMap.get(tableName);
995                        Iterator<String> columnIter = columns.iterator();
996                        List<column> mergeColumns = new ArrayList<column>();
997                        while (columnIter.hasNext()) {
998                                String columnName = columnIter.next();
999                                List<column> columnList = columnMap.get(columnName);
1000                                List<column> functions = new ArrayList<column>();
1001                                for (column t : columnList) {
1002                                        if (Boolean.TRUE.toString().equals(t.getIsFunction())) {
1003                                                functions.add(t);
1004                                        }
1005                                }
1006                                if (functions != null && !functions.isEmpty()) {
1007                                        for (column function : functions) {
1008                                                mergeColumns.add(function);
1009                                                columnIdMap.put(function.getId(), function.getId());
1010                                                columnMergeIdMap.put(function.getId(), function);
1011                                        }
1012
1013                                        columnList.removeAll(functions);
1014                                }
1015                                if (!columnList.isEmpty()) {
1016                                        column firstColumn = columnList.iterator().next();
1017                                        if (columnList.size() > 1) {
1018                                                column mergeColumn = new column();
1019                                                mergeColumn.setId(String.valueOf(++startId));
1020                                                mergeColumn.setName(firstColumn.getName());
1021                                                mergeColumn.setDisplayName(firstColumn.getDisplayName());
1022                                                mergeColumn.setSource(firstColumn.getSource());
1023                                                mergeColumn.setQualifiedTable(firstColumn.getQualifiedTable());
1024                                                mergeColumns.add(mergeColumn);
1025                                                for (column item : columnList) {
1026                                                        mergeColumn.appendCoordinate(item.getCoordinate());
1027                                                        columnIdMap.put(item.getId(), mergeColumn.getId());
1028                                                        //add by grq 2023.02.06 issue=I6DB5S
1029                                                        if (item.getDataType() != null) {
1030                                                                mergeColumn.setDataType(item.getDataType());
1031                                                        }
1032                                                        if (item.isForeignKey() != null) {
1033                                                                mergeColumn.setForeignKey(item.isForeignKey());
1034                                                        }
1035                                                        if (item.isUnqiueKey() != null) {
1036                                                                mergeColumn.setUnqiueKey(item.isUnqiueKey());
1037                                                        }
1038                                                        if (item.isIndexKey() != null) {
1039                                                                mergeColumn.setIndexKey(item.isIndexKey());
1040                                                        }
1041                                                        if (item.isPrimaryKey() != null) {
1042                                                                mergeColumn.setPrimaryKey(item.isPrimaryKey());
1043                                                        }
1044                                                        //end by grq
1045                                                }
1046                                                columnMergeIdMap.put(mergeColumn.getId(), mergeColumn);
1047                                        } else {
1048                                                mergeColumns.add(firstColumn);
1049                                                columnIdMap.put(firstColumn.getId(), firstColumn.getId());
1050                                                columnMergeIdMap.put(firstColumn.getId(), firstColumn);
1051                                        }
1052                                }
1053                        }
1054                        table.setColumns(mergeColumns);
1055                }
1056
1057                if (mergeDataflow.getRelationships() != null) {
1058                        Map<String, relationship> mergeRelations = new LinkedHashMap<String, relationship>();
1059                        for (int i = 0; i < mergeDataflow.getRelationships().size(); i++) {
1060                                relationship relation = mergeDataflow.getRelationships().get(i);
1061                                if (RelationshipType.call.name().equals(relation.getType())) {
1062                                        targetColumn target = relation.getCaller();
1063                                        if (target == null) {
1064                                                continue;
1065                                        }
1066                                        if (target != null && tableIdMap.containsKey(target.getId())) {
1067                                                target.setId(tableIdMap.get(target.getId()));
1068                                        }
1069
1070                                        List<sourceColumn> sources = relation.getCallees();
1071                                        Set<sourceColumn> sourceSet = new LinkedHashSet<sourceColumn>();
1072                                        if (sources != null) {
1073                                                for (sourceColumn source : sources) {
1074                                                        if (tableIdMap.containsKey(source.getId())) {
1075                                                                source.setId(tableIdMap.get(source.getId()));
1076                                                        }
1077                                                }
1078                                                sourceSet.addAll(sources);
1079                                                relation.setCallees(new ArrayList<sourceColumn>(sourceSet));
1080                                        }
1081
1082                                        String jsonString = JSON.toJSONString(relation, true);
1083                                        String key = SHA256.getMd5(jsonString);
1084                                        if (!mergeRelations.containsKey(key)) {
1085                                                mergeRelations.put(key, relation);
1086                                        }
1087                                } else {
1088                                        targetColumn target = relation.getTarget();
1089                                        if (target == null) {
1090                                                continue;
1091                                        }
1092                                        if (target != null && tableIdMap.containsKey(target.getParent_id())) {
1093                                                target.setParent_id(tableIdMap.get(target.getParent_id()));
1094                                        }
1095
1096                                        if (columnIdMap.containsKey(target.getId())) {
1097                                                target.setId(columnIdMap.get(target.getId()));
1098                                                target.setCoordinate(columnMergeIdMap.get(target.getId()).getCoordinate());
1099                                        }
1100
1101                                        List<sourceColumn> sources = relation.getSources();
1102                                        Set<sourceColumn> sourceSet = new LinkedHashSet<sourceColumn>();
1103                                        if (sources != null) {
1104                                                for (sourceColumn source : sources) {
1105                                                        if (tableIdMap.containsKey(source.getParent_id())) {
1106                                                                source.setParent_id(tableIdMap.get(source.getParent_id()));
1107                                                        }
1108                                                        if (tableIdMap.containsKey(source.getSource_id())) {
1109                                                                source.setSource_id(tableIdMap.get(source.getSource_id()));
1110                                                        }
1111                                                        if (columnIdMap.containsKey(source.getId())) {
1112                                                                source.setId(columnIdMap.get(source.getId()));
1113                                                                source.setCoordinate(columnMergeIdMap.get(source.getId()).getCoordinate());
1114                                                        }
1115                                                }
1116
1117                                                sourceSet.addAll(sources);
1118                                                relation.setSources(new ArrayList<sourceColumn>(sourceSet));
1119                                        }
1120
1121                                        String jsonString = JSON.toJSONString(relation, true);
1122                                        String key = SHA256.getMd5(jsonString);
1123                                        if (!mergeRelations.containsKey(key)) {
1124                                                mergeRelations.put(key, relation);
1125                                        }
1126                                }
1127                        }
1128
1129                        mergeDataflow.setRelationships(new ArrayList<relationship>(mergeRelations.values()));
1130                }
1131
1132                startId = addStarColumnBridgingRelations(mergeDataflow, startId);
1133
1134                tableMap.clear();
1135                tableTypeMap.clear();
1136                tableIdMap.clear();
1137                columnMap.clear();
1138                tableColumnMap.clear();
1139                columnIdMap.clear();
1140                columnMergeIdMap.clear();
1141                tables.clear();
1142
1143                return mergeDataflow;
1144        }
1145
1146        /**
1147         * 合并后同一张表可能同时拥有 * 列（来自只做 SELECT * 的来源）和具名列（来自显式引用列名的来源），
1148         * 但缺少 * -> 具名列 的 fdd 关系，会导致 上游.* -> 表.* 与 表.具名列 -> 下游.具名列 之间链路断开。
1149         * 本方法为这类表补齐 * -> 每个具名列 的 fdd 桥接（已存在则跳过；忽略 system / function 列）。
1150         */
1151        private static long addStarColumnBridgingRelations(dataflow df, long startId) {
1152                if (df == null) {
1153                        return startId;
1154                }
1155                List<relationship> newRelations = new ArrayList<>();
1156                for (table t : dataflow.getAllTables(df)) {
1157                        if (t.getColumns() == null || t.getColumns().size() < 2) {
1158                                continue;
1159                        }
1160                        column starColumn = null;
1161                        List<column> regularColumns = new ArrayList<>();
1162                        for (column c : t.getColumns()) {
1163                                if ("system".equals(c.getSource())) {
1164                                        continue;
1165                                }
1166                                if (Boolean.TRUE.toString().equals(c.getIsFunction())) {
1167                                        continue;
1168                                }
1169                                if ("*".equals(c.getName())) {
1170                                        if (starColumn == null) {
1171                                                starColumn = c;
1172                                        }
1173                                } else {
1174                                        regularColumns.add(c);
1175                                }
1176                        }
1177                        if (starColumn == null || regularColumns.isEmpty()) {
1178                                continue;
1179                        }
1180
1181                        Set<String> existingBridges = new HashSet<>();
1182                        if (df.getRelationships() != null) {
1183                                for (relationship rel : df.getRelationships()) {
1184                                        if (!RelationshipType.fdd.name().equals(rel.getType())) {
1185                                                continue;
1186                                        }
1187                                        if (rel.getTarget() == null || rel.getSources() == null) {
1188                                                continue;
1189                                        }
1190                                        if (!t.getId().equals(rel.getTarget().getParent_id())) {
1191                                                continue;
1192                                        }
1193                                        for (sourceColumn sc : rel.getSources()) {
1194                                                if (t.getId().equals(sc.getParent_id())) {
1195                                                        existingBridges.add(rel.getTarget().getId() + "<-" + sc.getId());
1196                                                }
1197                                        }
1198                                }
1199                        }
1200
1201                        String starId = starColumn.getId();
1202                        for (column target : regularColumns) {
1203                                if (existingBridges.contains(target.getId() + "<-" + starId)) {
1204                                        continue;
1205                                }
1206                                relationship rel = new relationship();
1207                                rel.setId(String.valueOf(++startId));
1208                                rel.setType(RelationshipType.fdd.name());
1209                                rel.setEffectType("expand_star");
1210
1211                                targetColumn tc = new targetColumn();
1212                                tc.setId(target.getId());
1213                                tc.setColumn(target.getName());
1214                                tc.setParent_id(t.getId());
1215                                tc.setParent_name(t.getName());
1216                                rel.setTarget(tc);
1217
1218                                sourceColumn sc = new sourceColumn();
1219                                sc.setId(starId);
1220                                sc.setColumn(starColumn.getName());
1221                                sc.setParent_id(t.getId());
1222                                sc.setParent_name(t.getName());
1223                                rel.setSources(Arrays.asList(sc));
1224
1225                                newRelations.add(rel);
1226                        }
1227                }
1228                if (!newRelations.isEmpty()) {
1229                        List<relationship> all = df.getRelationships() != null
1230                                        ? new ArrayList<>(df.getRelationships())
1231                                        : new ArrayList<>();
1232                        all.addAll(newRelations);
1233                        df.setRelationships(all);
1234                }
1235                return startId;
1236        }
1237
1238        /**
1239         * 单个 dataflow 的 ID 跨度（与 ParallelDataFlowAnalyzer 中的常量保持一致）。
1240         */
1241        private static final long DATAFLOW_ID_RANGE = 5000000L;
1242
1243        /**
1244         * 合并一组 startId 都为 0（即 ID 空间相互冲突）的 dataflow。
1245         * <p>
1246         * 标准合并 {@link #mergeDataflowsWithDifferentStartId(Collection, EDbVendor)} 的前提是
1247         * 每个 dataflow 在生成时已通过 {@code optionCopy.setStartId(5000000L * i)} 占用了
1248         * 不重叠的 ID 段；当外部调用方拿到的 dataflow 都是用默认 startId=0 生成时，直接合并
1249         * 会因为 ID 冲突而错乱。本方法会先按下标把每个 dataflow 的全部 ID 偏移
1250         * {@link #DATAFLOW_ID_RANGE} * index，再委托给标准合并逻辑。
1251         * <p>
1252         * 注意：此方法会原地修改入参 dataflow 的 ID（与标准合并行为一致）。
1253         */
1254        public static dataflow mergeDataflows(Collection<dataflow> dataflows, EDbVendor vendor) {
1255                if (dataflows == null || dataflows.isEmpty()) {
1256                        return null;
1257                }
1258                ModelBindingManager.setGlobalVendor(vendor);
1259                boolean optionSet = false;
1260                if (ModelBindingManager.getGlobalOption() == null) {
1261                        Option option = new Option();
1262                        option.setVendor(vendor);
1263                        ModelBindingManager.setGlobalOption(option);
1264                        optionSet = true;
1265                }
1266                try {
1267                        // ID 按下标 * DATAFLOW_ID_RANGE 偏移，避免 ID 冲突；
1268                        // coordinate 第三维 fileIdx 按前面所有 dataflow 累积的源文件数偏移，
1269                        // 使得合并后每个来源的 fileIdx 保持连续且不重叠。
1270                        long cumulativeFileIdx = 0L;
1271                        int i = 0;
1272                        for (dataflow df : dataflows) {
1273                                if (df != null) {
1274                                        int maxFileIdx = findMaxFileIdx(df);
1275                                        offsetDataflowIds(df, DATAFLOW_ID_RANGE * i, cumulativeFileIdx);
1276                                        if (maxFileIdx >= 0) {
1277                                                cumulativeFileIdx += (maxFileIdx + 1L);
1278                                        }
1279                                }
1280                                i++;
1281                        }
1282                        return mergeDataflowsWithDifferentStartId(dataflows, vendor);
1283                } finally {
1284                        ModelBindingManager.removeGlobalVendor();
1285                        if (optionSet) {
1286                                ModelBindingManager.removeGlobalOption();
1287                        }
1288                }
1289        }
1290
1291        /**
1292         * 从一组临时 XML 文件迭代加载并合并 dataflow，节约内存。
1293         * <p>
1294         * 与 {@link #iterativeMergeDataflowsFromFilesByStartId(List, long)} 不同的是，本方法假定文件中的
1295         * dataflow 是用默认 startId=0 生成的（ID 空间互相冲突），因此会按下标先做 ID 偏移再合并。
1296         * <p>
1297         * 注意：本方法按入参 {@code tempFiles} 顺序迭代合并，不做任何排序。
1298         * coordinate 第三维 fileIdx 的偏移也是严格按入参顺序累积的，
1299         * 调用方传入的顺序应与原始源文件的语义顺序保持一致。
1300         */
1301        public static dataflow iterativeMergeDataflowsFromFiles(List<File> tempFiles, EDbVendor vendor) {
1302                if (tempFiles == null || tempFiles.isEmpty()) {
1303                        return null;
1304                }
1305                ModelBindingManager.setGlobalVendor(vendor);
1306                boolean optionSet = false;
1307                if (ModelBindingManager.getGlobalOption() == null) {
1308                        Option option = new Option();
1309                        option.setVendor(vendor);
1310                        ModelBindingManager.setGlobalOption(option);
1311                        optionSet = true;
1312                }
1313                try {
1314                        return iterativeMergeFromFiles(tempFiles, DATAFLOW_ID_RANGE * tempFiles.size(), true);
1315                } finally {
1316                        ModelBindingManager.removeGlobalVendor();
1317                        if (optionSet) {
1318                                ModelBindingManager.removeGlobalOption();
1319                        }
1320                }
1321        }
1322
1323        /**
1324         * 从一组临时 XML 文件迭代加载并合并 dataflow，调用方需保证文件中的 dataflow ID 空间互不冲突，
1325         * 且已设置好 ModelBindingManager。
1326         * <p>
1327         * 主要供 {@code ParallelDataFlowAnalyzer} 等内部生成方使用：每个临时文件中的 dataflow 已通过
1328         * {@code optionCopy.setStartId(DATAFLOW_ID_RANGE * i)} 占用了不重叠的 ID 段。
1329         * <p>
1330         * 注意：本方法按入参 {@code tempFiles} 顺序迭代合并，不做任何排序。
1331         * coordinate 第三维 fileIdx 的偏移严格按入参顺序累积。
1332         */
1333        public static dataflow iterativeMergeDataflowsFromFilesByStartId(List<File> tempFiles, long startId) {
1334                return iterativeMergeFromFiles(tempFiles, startId, false);
1335        }
1336
1337        private static dataflow iterativeMergeFromFiles(List<File> tempFiles, long startId, boolean offsetIds) {
1338                if (tempFiles == null || tempFiles.isEmpty()) {
1339                        logger.warn("iterativeMergeDataflowsFromFiles: tempFiles is null or empty");
1340                        return null;
1341                }
1342
1343                // 注意：此处不再按文件大小排序，保持入参顺序。
1344                // 因为 coordinate 第三维 fileIdx 表示原始源文件下标，偏移必须与入参顺序一致，
1345                // 否则合并后 fileIdx 与原始源文件对不上。
1346                logger.info("iterativeMergeDataflowsFromFiles: total dataflows: " + tempFiles.size());
1347
1348                long startTime = System.currentTimeMillis();
1349                int totalIterations = tempFiles.size() - 1;
1350                logger.info("iterativeMergeDataflowsFromFiles: start merging, total iterations: " + totalIterations);
1351
1352                long loadStartTime = System.currentTimeMillis();
1353                dataflow mergedDataflow = XML2Model.loadXML(dataflow.class, tempFiles.get(0));
1354                long cumulativeFileIdx = 0L;
1355                if (mergedDataflow != null) {
1356                        // 即便 ID 不需偏移（byStartId 路径），coordinate 第三维 fileIdx 仍要偏移，
1357                        // 避免合并后不同来源的 fileIdx 撞在一起。
1358                        long idOffset = offsetIds ? 0L : 0L;
1359                        offsetDataflowIds(mergedDataflow, idOffset, 0L);
1360                        int maxFileIdx = findMaxFileIdx(mergedDataflow);
1361                        if (maxFileIdx >= 0) {
1362                                cumulativeFileIdx = maxFileIdx + 1L;
1363                        }
1364                }
1365                long loadEndTime = System.currentTimeMillis();
1366                int initialRelationCount = mergedDataflow != null && mergedDataflow.getRelationships() != null ? mergedDataflow.getRelationships().size() : 0;
1367                logger.info("iterativeMergeDataflowsFromFiles: loaded initial dataflow, relation count: " + initialRelationCount + ", time: " + (loadEndTime - loadStartTime) + "ms");
1368
1369                for (int i = 1; i < tempFiles.size(); i++) {
1370                        long iterationStartTime = System.currentTimeMillis();
1371                        int currentIteration = i;
1372                        int remainingIterations = totalIterations - currentIteration + 1;
1373
1374                        logger.info("iterativeMergeDataflowsFromFiles: iteration " + currentIteration + "/" + totalIterations + ", remaining: " + remainingIterations);
1375
1376                        long loadCurrentStartTime = System.currentTimeMillis();
1377                        dataflow currentDataflow = XML2Model.loadXML(dataflow.class, tempFiles.get(i));
1378                        if (currentDataflow != null) {
1379                                int maxFileIdx = findMaxFileIdx(currentDataflow);
1380                                long idOffset = offsetIds ? (DATAFLOW_ID_RANGE * i) : 0L;
1381                                offsetDataflowIds(currentDataflow, idOffset, cumulativeFileIdx);
1382                                if (maxFileIdx >= 0) {
1383                                        cumulativeFileIdx += (maxFileIdx + 1L);
1384                                }
1385                        }
1386                        long loadCurrentEndTime = System.currentTimeMillis();
1387                        int currentRelationCount = currentDataflow != null && currentDataflow.getRelationships() != null ? currentDataflow.getRelationships().size() : 0;
1388                        logger.info("iterativeMergeDataflowsFromFiles: loaded dataflow[" + i + "], relation count: " + currentRelationCount + ", time: " + (loadCurrentEndTime - loadCurrentStartTime) + "ms");
1389
1390                        List<dataflow> tempList = new ArrayList<>(2);
1391                        tempList.add(mergedDataflow);
1392                        tempList.add(currentDataflow);
1393
1394                        long mergeStartTime = System.currentTimeMillis();
1395                        mergedDataflow = mergeDataflowsByStartId(tempList, startId + DATAFLOW_ID_RANGE * i);
1396                        long mergeEndTime = System.currentTimeMillis();
1397                        int mergedRelationCount = mergedDataflow != null && mergedDataflow.getRelationships() != null ? mergedDataflow.getRelationships().size() : 0;
1398
1399                        long iterationEndTime = System.currentTimeMillis();
1400                        long iterationTime = iterationEndTime - iterationStartTime;
1401                        long mergeTime = mergeEndTime - mergeStartTime;
1402                        logger.info("iterativeMergeDataflowsFromFiles: iteration " + currentIteration + " completed, merged relation count: " + mergedRelationCount + ", merge time: " + mergeTime + "ms, total iteration time: " + iterationTime + "ms");
1403
1404                        currentDataflow = null;
1405                        tempList.clear();
1406                }
1407
1408                long endTime = System.currentTimeMillis();
1409                long totalTime = endTime - startTime;
1410                int finalRelationCount = mergedDataflow != null && mergedDataflow.getRelationships() != null ? mergedDataflow.getRelationships().size() : 0;
1411                logger.info("iterativeMergeDataflowsFromFiles: all iterations completed, final relation count: " + finalRelationCount + ", total time: " + totalTime + "ms (" + (totalTime / 1000.0) + "s)");
1412
1413                return mergedDataflow;
1414        }
1415
1416        private static String offsetId(String id, long offset) {
1417                if (id == null || id.isEmpty()) {
1418                        return id;
1419                }
1420                try {
1421                        return String.valueOf(Long.parseLong(id) + offset);
1422                } catch (NumberFormatException ignore) {
1423                        return id;
1424                }
1425        }
1426
1427        /**
1428         * 偏移 coordinate 第三维（fileIdx）。
1429         * <p>
1430         * coordinate 字符串格式为 {@code [line,col,fileIdx],[line,col,fileIdx]}，
1431         * 合并多个独立生成的 dataflow 时，它们的 fileIdx 都从 0 起始会互相冲突，
1432         * 通过给第三维加上 offset 来区分不同的来源。
1433         */
1434        private static String offsetCoordinate(String coordinate, long offset) {
1435                if (coordinate == null || coordinate.isEmpty() || offset == 0L) {
1436                        return coordinate;
1437                }
1438                StringBuilder sb = new StringBuilder(coordinate.length() + 8);
1439                int i = 0;
1440                int len = coordinate.length();
1441                while (i < len) {
1442                        int open = coordinate.indexOf('[', i);
1443                        if (open < 0) {
1444                                sb.append(coordinate, i, len);
1445                                break;
1446                        }
1447                        int close = coordinate.indexOf(']', open);
1448                        if (close < 0) {
1449                                sb.append(coordinate, i, len);
1450                                break;
1451                        }
1452                        sb.append(coordinate, i, open + 1);
1453                        String inner = coordinate.substring(open + 1, close);
1454                        String[] parts = inner.split(",", -1);
1455                        for (int k = 0; k < parts.length; k++) {
1456                                if (k > 0) sb.append(',');
1457                                if (k == 2) {
1458                                        String p = parts[k].trim();
1459                                        try {
1460                                                sb.append(Long.parseLong(p) + offset);
1461                                        } catch (NumberFormatException ignore) {
1462                                                sb.append(parts[k]);
1463                                        }
1464                                } else {
1465                                        sb.append(parts[k]);
1466                                }
1467                        }
1468                        sb.append(']');
1469                        i = close + 1;
1470                }
1471                return sb.toString();
1472        }
1473
1474        /**
1475         * 偏移 dataflow 中所有节点的 ID 和 coordinate 第三维 fileIdx。
1476         *
1477         * @param df            待偏移的 dataflow（原地修改）
1478         * @param idOffset      给所有 id / xxxId 字段加上的偏移量
1479         * @param fileIdxOffset 给所有 coordinate 第三维 (fileIdx) 加上的偏移量，
1480         *                      通常为合并时前面所有 dataflow 累积的源文件数，
1481         *                      以便合并后不同来源的 fileIdx 保持连续且不重叠。
1482         */
1483        private static void offsetDataflowIds(dataflow df, long idOffset, long fileIdxOffset) {
1484                if (idOffset == 0L && fileIdxOffset == 0L) {
1485                        return;
1486                }
1487
1488                // tables / views / databases / schemas / stages / datasources / streams / paths / variables / resultsets / sequences
1489                for (table t : dataflow.getAllTables(df)) {
1490                        offsetTableIds(t, idOffset);
1491                        t.setCoordinate(offsetCoordinate(t.getCoordinate(), fileIdxOffset));
1492                        if (t.getColumns() != null) {
1493                                for (column c : t.getColumns()) {
1494                                        c.setCoordinate(offsetCoordinate(c.getCoordinate(), fileIdxOffset));
1495                                }
1496                        }
1497                }
1498
1499                // processes
1500                if (df.getProcesses() != null) {
1501                        for (process p : df.getProcesses()) {
1502                                p.setId(offsetId(p.getId(), idOffset));
1503                                p.setProcedureId(offsetId(p.getProcedureId(), idOffset));
1504                                p.setCoordinate(offsetCoordinate(p.getCoordinate(), fileIdxOffset));
1505                        }
1506                }
1507
1508                // procedures
1509                if (df.getProcedures() != null) {
1510                        for (procedure p : df.getProcedures()) {
1511                                p.setId(offsetId(p.getId(), idOffset));
1512                                p.setCoordinate(offsetCoordinate(p.getCoordinate(), fileIdxOffset));
1513                                if (p.getArguments() != null) {
1514                                        for (argument arg : p.getArguments()) {
1515                                                arg.setId(offsetId(arg.getId(), idOffset));
1516                                                arg.setCoordinate(offsetCoordinate(arg.getCoordinate(), fileIdxOffset));
1517                                        }
1518                                }
1519                        }
1520                }
1521
1522                // oracle packages
1523                if (df.getPackages() != null) {
1524                        for (oraclePackage pkg : df.getPackages()) {
1525                                pkg.setId(offsetId(pkg.getId(), idOffset));
1526                                if (pkg.getProcedures() != null) {
1527                                        for (procedure pp : pkg.getProcedures()) {
1528                                                pp.setId(offsetId(pp.getId(), idOffset));
1529                                                pp.setCoordinate(offsetCoordinate(pp.getCoordinate(), fileIdxOffset));
1530                                                if (pp.getArguments() != null) {
1531                                                        for (argument arg : pp.getArguments()) {
1532                                                                arg.setId(offsetId(arg.getId(), idOffset));
1533                                                                arg.setCoordinate(offsetCoordinate(arg.getCoordinate(), fileIdxOffset));
1534                                                        }
1535                                                }
1536                                        }
1537                                }
1538                        }
1539                }
1540
1541                // relationships
1542                if (df.getRelationships() != null) {
1543                        for (relationship rel : df.getRelationships()) {
1544                                rel.setId(offsetId(rel.getId(), idOffset));
1545                                rel.setProcessId(offsetId(rel.getProcessId(), idOffset));
1546                                rel.setProcedureId(offsetId(rel.getProcedureId(), idOffset));
1547                                if (rel.getTarget() != null) {
1548                                        offsetTargetColumnIds(rel.getTarget(), idOffset);
1549                                }
1550                                if (rel.getCaller() != null) {
1551                                        offsetTargetColumnIds(rel.getCaller(), idOffset);
1552                                }
1553                                if (rel.getSources() != null) {
1554                                        for (sourceColumn sc : rel.getSources()) {
1555                                                offsetSourceColumnIds(sc, idOffset);
1556                                        }
1557                                }
1558                                if (rel.getCallees() != null) {
1559                                        for (sourceColumn sc : rel.getCallees()) {
1560                                                offsetSourceColumnIds(sc, idOffset);
1561                                        }
1562                                }
1563                        }
1564                }
1565        }
1566
1567        /**
1568         * 扫描 dataflow 中所有 coordinate，返回最大的第三维 (fileIdx) 值；
1569         * 若没有任何 coordinate 则返回 -1。
1570         */
1571        private static int findMaxFileIdx(dataflow df) {
1572                if (df == null) {
1573                        return -1;
1574                }
1575                int max = -1;
1576                for (table t : dataflow.getAllTables(df)) {
1577                        max = Math.max(max, maxFileIdxInCoordinate(t.getCoordinate()));
1578                        if (t.getColumns() != null) {
1579                                for (column c : t.getColumns()) {
1580                                        max = Math.max(max, maxFileIdxInCoordinate(c.getCoordinate()));
1581                                }
1582                        }
1583                }
1584                if (df.getProcesses() != null) {
1585                        for (process p : df.getProcesses()) {
1586                                max = Math.max(max, maxFileIdxInCoordinate(p.getCoordinate()));
1587                        }
1588                }
1589                if (df.getProcedures() != null) {
1590                        for (procedure p : df.getProcedures()) {
1591                                max = Math.max(max, maxFileIdxInCoordinate(p.getCoordinate()));
1592                                if (p.getArguments() != null) {
1593                                        for (argument arg : p.getArguments()) {
1594                                                max = Math.max(max, maxFileIdxInCoordinate(arg.getCoordinate()));
1595                                        }
1596                                }
1597                        }
1598                }
1599                if (df.getPackages() != null) {
1600                        for (oraclePackage pkg : df.getPackages()) {
1601                                if (pkg.getProcedures() != null) {
1602                                        for (procedure pp : pkg.getProcedures()) {
1603                                                max = Math.max(max, maxFileIdxInCoordinate(pp.getCoordinate()));
1604                                                if (pp.getArguments() != null) {
1605                                                        for (argument arg : pp.getArguments()) {
1606                                                                max = Math.max(max, maxFileIdxInCoordinate(arg.getCoordinate()));
1607                                                        }
1608                                                }
1609                                        }
1610                                }
1611                        }
1612                }
1613                return max;
1614        }
1615
1616        /**
1617         * 从 coordinate 字符串中解析出最大的第三维 (fileIdx) 值；
1618         * 若 coordinate 为空或没有任何可解析的数值则返回 -1。
1619         */
1620        private static int maxFileIdxInCoordinate(String coordinate) {
1621                if (coordinate == null || coordinate.isEmpty()) {
1622                        return -1;
1623                }
1624                int max = -1;
1625                int i = 0;
1626                int len = coordinate.length();
1627                while (i < len) {
1628                        int open = coordinate.indexOf('[', i);
1629                        if (open < 0) break;
1630                        int close = coordinate.indexOf(']', open);
1631                        if (close < 0) break;
1632                        String inner = coordinate.substring(open + 1, close);
1633                        String[] parts = inner.split(",", -1);
1634                        if (parts.length >= 3) {
1635                                try {
1636                                        int v = Integer.parseInt(parts[2].trim());
1637                                        if (v > max) max = v;
1638                                } catch (NumberFormatException ignore) {
1639                                        // 不是数字，跳过
1640                                }
1641                        }
1642                        i = close + 1;
1643                }
1644                return max;
1645        }
1646
1647        private static void offsetTableIds(table t, long offset) {
1648                if (t == null) {
1649                        return;
1650                }
1651                t.setId(offsetId(t.getId(), offset));
1652                if (t.getProcessIds() != null) {
1653                        List<String> newProcessIds = new ArrayList<>(t.getProcessIds().size());
1654                        for (String pid : t.getProcessIds()) {
1655                                newProcessIds.add(offsetId(pid, offset));
1656                        }
1657                        t.setProcessIds(newProcessIds);
1658                }
1659                if (t.getColumns() != null) {
1660                        for (column c : t.getColumns()) {
1661                                c.setId(offsetId(c.getId(), offset));
1662                        }
1663                }
1664        }
1665
1666        private static void offsetTargetColumnIds(targetColumn col, long offset) {
1667                col.setId(offsetId(col.getId(), offset));
1668                col.setParent_id(offsetId(col.getParent_id(), offset));
1669                col.setTarget_id(offsetId(col.getTarget_id(), offset));
1670        }
1671
1672        private static void offsetSourceColumnIds(sourceColumn col, long offset) {
1673                col.setId(offsetId(col.getId(), offset));
1674                col.setParent_id(offsetId(col.getParent_id(), offset));
1675                col.setSource_id(offsetId(col.getSource_id(), offset));
1676        }
1677
1678        public static dataflow readDataflowFromCsvMetadata(String csvMetadata, EDbVendor vendor) {
1679                if (!MetadataReader.isMetadata(csvMetadata)) {
1680                        throw new IllegalArgumentException("Illegal csv metadata.");
1681                }
1682                return new SQLDepMetadataAnalyzer().analyzeMetadata(vendor, csvMetadata);
1683        }
1684
1685        public static Dataflow mergeDataflowsAndCsv(List<Pair<EDbVendor,dataflow>> pairs, String csvMetadata) {
1686                //重置ID防止重复
1687                Long index = 0L;
1688                for(int i=0; i<pairs.size(); i++){
1689                        dataflow dataflow = pairs.get(i).second;
1690                        Map<String, table> objIDMap = getDataflowDbObjMap(dataflow);
1691                        Map<String, String> idMaps = new HashMap<>();
1692                        for (Map.Entry<String, table> entry : objIDMap.entrySet()) {
1693                                index = index + 1;
1694                                String id = index.toString();
1695                                idMaps.put(entry.getKey(), id);
1696                                table table = entry.getValue();
1697                                table.setId(id);
1698                                if(table.getColumns() != null){
1699                                        for(column col: table.getColumns()){
1700                                                index = index + 1;
1701                                                id = index.toString();
1702                                                idMaps.put(col.getId(), id);
1703                                                col.setId(id);
1704                                        }
1705                                }
1706                        }
1707                        if(dataflow.getProcedures() != null){
1708                                for(procedure procedure: dataflow.getProcedures()){
1709                                        index = index + 1;
1710                                        String id = index.toString();
1711                                        idMaps.put(procedure.getId(), id);
1712                                        procedure.setId(id);
1713                                }
1714                        }
1715                        if(dataflow.getProcesses() != null){
1716                                for(process process: dataflow.getProcesses()){
1717                                        index = index + 1;
1718                                        String id = index.toString();
1719                                        idMaps.put(process.getId(), id);
1720                                        process.setId(id);
1721                                }
1722                        }
1723                        if(dataflow.getPackages() != null){
1724                                for(oraclePackage oraclePackage: dataflow.getPackages()){
1725                                        index = index + 1;
1726                                        String id = index.toString();
1727                                        idMaps.put(oraclePackage.getId(), id);
1728                                        oraclePackage.setId(id);
1729                                        if(oraclePackage.getProcedures() != null){
1730                                                for(procedure procedure: oraclePackage.getProcedures()){
1731                                                        index = index + 1;
1732                                                        id = index.toString();
1733                                                        idMaps.put(procedure.getId(), id);
1734                                                        procedure.setId(id);
1735                                                }
1736                                        }
1737                                }
1738                        }
1739                        if(dataflow.getRelationships() != null && dataflow.getRelationships().size()>0){
1740                                for(relationship rel: dataflow.getRelationships()){
1741                                        index = index + 1;
1742                                        String id = index.toString();
1743                                        rel.setId(id);
1744                                        rel.setProcedureId(idMaps.get(rel.getProcedureId()));
1745                                        rel.setProcessId(idMaps.get(rel.getProcessId()));
1746                                        rel.getTarget().setParent_id(idMaps.get(rel.getTarget().getParent_id()));
1747                                        rel.getTarget().setTarget_id(idMaps.get(rel.getTarget().getTarget_id()));
1748                                        rel.getTarget().setId(idMaps.get(rel.getTarget().getId()));
1749                                        for(sourceColumn column: rel.getSources()){
1750                                                column.setParent_id(idMaps.get(column.getParent_id()));
1751                                                column.setId(idMaps.get(column.getId()));
1752                                        }
1753                                }
1754                        }
1755                }
1756                Dataflow mDataflow = DataFlowAnalyzer.getSqlflowJSONModel(pairs.get(0).first, pairs.get(0).second, false);
1757                Map<String, table> objNameMap = getDataflowDbObjNameMap(pairs.get(0).second);
1758                Sqlflow dbobjs = mDataflow.getDbobjs();
1759                List<Relationship> mRelationshipList = new ArrayList<>();
1760                if(mDataflow.getRelationships() != null && mDataflow.getRelationships().length>0){
1761                        mRelationshipList = new LinkedList<>(Arrays.asList(mDataflow.getRelationships()));
1762                }
1763
1764                List<Error> mErrorList = new ArrayList<>();
1765                if(mDataflow.getErrors() != null && mDataflow.getErrors().length>0){
1766                        mErrorList = new LinkedList<>(Arrays.asList(mDataflow.getErrors()));
1767                }
1768
1769                List<Process> mProcessList = new ArrayList<>();
1770                if(mDataflow.getProcesses() != null && mDataflow.getProcesses().length>0){
1771                        mProcessList = new LinkedList<>(Arrays.asList(mDataflow.getProcesses()));
1772                }
1773
1774                for(int i=1; i<pairs.size(); i++){
1775                        objNameMap.putAll(getDataflowDbObjNameMap(pairs.get(i).second));
1776                        Dataflow dataflow = DataFlowAnalyzer.getSqlflowJSONModel(pairs.get(i).first, pairs.get(i).second, false);
1777                        if(dataflow.getDbobjs().getServers() != null && dataflow.getDbobjs().getServers().size()>0){
1778                                if(dbobjs.getServers() == null){
1779                                        dbobjs.setServers(new ArrayList<>());
1780                                }
1781                                dbobjs.getServers().addAll(dataflow.getDbobjs().getServers());
1782                        }
1783
1784                        if(dataflow.getDbobjs().getErrorMessages() != null && dataflow.getDbobjs().getErrorMessages().size()>0){
1785                                if(dbobjs.getErrorMessages() == null){
1786                                        dbobjs.setErrorMessages(new ArrayList<>());
1787                                }
1788                                dbobjs.getErrorMessages().addAll(dataflow.getDbobjs().getErrorMessages());
1789                        }
1790
1791                        if(dataflow.getRelationships() != null && dataflow.getRelationships().length>0){
1792                                List<Relationship> relationshipList = new LinkedList<>(Arrays.asList(dataflow.getRelationships()));
1793                                mRelationshipList.addAll(relationshipList);
1794                        }
1795
1796                        if(dataflow.getErrors() != null && dataflow.getErrors().length>0){
1797                                List<Error> errorList = new LinkedList<>(Arrays.asList(dataflow.getErrors()));
1798                                mErrorList.addAll(errorList);
1799                        }
1800
1801                        if(dataflow.getProcesses() != null && dataflow.getProcesses().length>0){
1802                                List<Process> processList = new LinkedList<>(Arrays.asList(dataflow.getProcesses()));
1803                                mProcessList.addAll(processList);
1804                        }
1805
1806                }
1807                if(!SQLUtil.isEmpty(csvMetadata)){
1808                        /**
1809                         * excel内的血缘 只合并关系 就是只做关联，如果找不到obj 就算了
1810                         */
1811                        dataflow df = readDataflowFromCsvMetadata(csvMetadata);
1812                        Map<String, table> objIDMap = getDataflowDbObjMap(df);
1813                        if(df.getRelationships() != null && df.getRelationships().size()>0){
1814                                for(relationship rel: df.getRelationships()){
1815                                        rel.setId("m"+rel.getId());
1816                                        table sTable = objIDMap.get(rel.getTarget().getParent_id());
1817                                        table tTable = objNameMap.get(sTable.getFullName());
1818                                        rel.getTarget().setParent_id(tTable.getId());
1819                                        for(column col: tTable.getColumns()){
1820                                                if(col.getName().equalsIgnoreCase(rel.getTarget().getColumn())){
1821                                                        rel.getTarget().setId(col.getId());
1822                                                        break;
1823                                                }
1824                                        }
1825                                        for(sourceColumn column: rel.getSources()){
1826                                                sTable = objIDMap.get(column.getParent_id());
1827                                                tTable = objNameMap.get(sTable.getFullName());
1828                                                column.setParent_id(tTable.getId());
1829                                                for(column col: tTable.getColumns()){
1830                                                        if(col.getName().equalsIgnoreCase(column.getColumn())){
1831                                                                column.setId(col.getId());
1832                                                                break;
1833                                                        }
1834                                                }
1835                                        }
1836                                        mRelationshipList.add(toRelationship(rel));
1837                                }
1838                        }
1839                }
1840
1841                mDataflow.setDbobjs(dbobjs);
1842                mDataflow.setRelationships(mRelationshipList.toArray(new Relationship[mRelationshipList.size()]));
1843                mDataflow.setProcesses(mProcessList.toArray(new Process[mProcessList.size()]));
1844                mDataflow.setErrors(mErrorList.toArray(new Error[mErrorList.size()]));
1845                return mDataflow;
1846        }
1847
1848        public static dataflow readDataflowFromCsvMetadata(String csvMetadata) {
1849                if (!MetadataReader.isMetadata(csvMetadata)) {
1850                        throw new IllegalArgumentException("Illegal csv metadata.");
1851                }
1852                return new SQLDepMetadataAnalyzer().analyzeMetadata(null, csvMetadata);
1853        }
1854
1855        private static Relationship toRelationship(relationship relation){
1856                Relationship relationModel;
1857                if (relation.getType().equals("join")) {
1858                        JoinRelationship joinRelationModel = new JoinRelationship();
1859                        joinRelationModel.setCondition(relation.getCondition());
1860                        joinRelationModel.setJoinType(relation.getJoinType());
1861                        joinRelationModel.setClause(relation.getClause());
1862                        relationModel = joinRelationModel;
1863                } else {
1864                        relationModel = new Relationship();
1865                }
1866                relationModel.setId(relation.getId());
1867                relationModel.setProcessId(relation.getProcessId());
1868                relationModel.setProcessType(relation.getProcessType());
1869                relationModel.setType(relation.getType());
1870                relationModel.setEffectType(relation.getEffectType());
1871                relationModel.setPartition(relation.getPartition());
1872                relationModel.setFunction(relation.getFunction());
1873                relationModel.setProcedureId(relation.getProcedureId());
1874                relationModel.setSqlHash(relation.getSqlHash());
1875                relationModel.setCondition(relation.getCondition());
1876                relationModel.setSqlComment(relation.getSqlComment());
1877                relationModel.setTimestampMax(relation.getTimestampMax());
1878                relationModel.setTimestampMin(relation.getTimestampMin());
1879
1880                if (relation.getTarget() != null && relation.getSources() != null && !relation.getSources().isEmpty()) {
1881                        RelationshipElement targetModel = new RelationshipElement();
1882                        targetColumn target = relation.getTarget();
1883                        targetModel.setColumn(target.getColumn());
1884                        targetModel.setParentName(target.getParent_name());
1885                        targetModel.setTargetName(target.getTarget_name());
1886                        targetModel.setId(target.getId());
1887                        targetModel.setTargetId(target.getTarget_id());
1888                        targetModel.setParentId(target.getParent_id());
1889                        targetModel.setCoordinates(Coordinate.parse(target.getCoordinate()));
1890                        targetModel.setFunction(target.getFunction());
1891                        targetModel.setType(target.getType());
1892                        relationModel.setTarget(targetModel);
1893
1894                        List<RelationshipElement> sourceModels = new ArrayList<>();
1895                        for (sourceColumn source : relation.getSources()) {
1896                                RelationshipElement sourceModel = new RelationshipElement();
1897                                sourceModel.setColumn(source.getColumn());
1898                                sourceModel.setParentName(source.getParent_name());
1899                                sourceModel.setSourceName(source.getSource_name());
1900                                sourceModel.setColumnType(source.getColumn_type());
1901                                sourceModel.setId(source.getId());
1902                                sourceModel.setParentId(source.getParent_id());
1903                                sourceModel.setSourceId(source.getSource_id());
1904                                sourceModel.setCoordinates(Coordinate.parse(source.getCoordinate()));
1905                                sourceModel.setClauseType(source.getClauseType());
1906                                sourceModel.setType(source.getType());
1907                                sourceModels.add(sourceModel);
1908                                if (source.getTransforms() != null && !source.getTransforms().isEmpty()) {
1909                                        List<Transform> transforms = new ArrayList<gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform>();
1910                                        for (transform transform : source.getTransforms()) {
1911                                                Transform item = new Transform();
1912                                                item.setCode(transform.getCode());
1913                                                item.setType(transform.getType());
1914                                                item.setCoordinate(transform.getCoordinate(true));
1915                                                transforms.add(item);
1916                                        }
1917                                        sourceModel.setTransforms(transforms.toArray(new Transform[0]));
1918                                }
1919                        }
1920                        relationModel.setSources(sourceModels.toArray(new RelationshipElement[0]));
1921                } else if (relation.getCaller() != null && relation.getCallees() != null
1922                                && !relation.getCallees().isEmpty()) {
1923                        RelationshipElement targetModel = new RelationshipElement();
1924                        targetColumn target = relation.getCaller();
1925                        targetModel.setName(target.getName());
1926                        targetModel.setId(target.getId());
1927                        targetModel.setCoordinates(Coordinate.parse(target.getCoordinate()));
1928                        targetModel.setType(target.getType());
1929                        relationModel.setCaller(targetModel);
1930                        List<RelationshipElement> sourceModels = new ArrayList<gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement>();
1931                        for (sourceColumn source : relation.getCallees()) {
1932                                RelationshipElement sourceModel = new RelationshipElement();
1933                                sourceModel.setName(source.getName());
1934                                sourceModel.setId(source.getId());
1935                                sourceModel.setCoordinates(Coordinate.parse(source.getCoordinate()));
1936                                sourceModel.setType(source.getType());
1937                                sourceModels.add(sourceModel);
1938                        }
1939                        relationModel.setCallees(sourceModels.toArray(new RelationshipElement[0]));
1940                }
1941                return relationModel;
1942        }
1943
1944//    public static void main(String[] args) throws Exception {
1945//        // 基于文件的迭代合并（节约内存，适合大数据量）
1946//        dataflow dataflow = iterativeMergeDataflowsFromFiles(Arrays.asList(new File("C:\\Users\\KK\\xwechat_files\\wxid_z9ci6s8b7g0d21_9365\\msg\\file\\2026-06\\dataflow_item").listFiles()), EDbVendor.dbvoracle);
1947//        System.out.println(XML2Model.saveXML(dataflow));
1948//    }
1949//
1950//    public static void main(String[] args) throws Exception {
1951//        // 基于内存的合并（一次性加载所有文件到内存，适合小数据量）
1952//        File[] files = new File("C:\\Users\\KK\\xwechat_files\\wxid_z9ci6s8b7g0d21_9365\\msg\\file\\2026-06\\dataflow_item").listFiles();
1953//        if (files == null || files.length == 0) {
1954//            System.out.println("No dataflow files found.");
1955//            return;
1956//        }
1957//
1958//        List<dataflow> dataflows = new ArrayList<>();
1959//        for (File file : files) {
1960//            dataflow df = XML2Model.loadXML(dataflow.class, file);
1961//            if (df != null) {
1962//                dataflows.add(df);
1963//            }
1964//        }
1965//
1966//        dataflow mergedDataflow = mergeDataflows(dataflows, EDbVendor.dbvoracle);
1967//        System.out.println(XML2Model.saveXML(mergedDataflow));
1968//    }
1969//
1970//    public static void main(String[] args) throws Exception {
1971//        String sqlDirPath = "C:\\Users\\KK\\Desktop\\同义词验证";
1972//        File sqlDir = new File(sqlDirPath);
1973//
1974//        if (!sqlDir.exists() || !sqlDir.isDirectory()) {
1975//            System.err.println("目录不存在或不是目录: " + sqlDirPath);
1976//            return;
1977//        }
1978//
1979//        File[] sqlFiles = sqlDir.listFiles((dir, name) -> name.toLowerCase().endsWith(".sql"));
1980//
1981//        if (sqlFiles == null || sqlFiles.length == 0) {
1982//            System.err.println("目录下没有找到 SQL 文件: " + sqlDirPath);
1983//            return;
1984//        }
1985//
1986//        System.out.println("找到 " + sqlFiles.length + " 个 SQL 文件");
1987//        for (File file : sqlFiles) {
1988//            System.out.println("  - " + file.getName());
1989//        }
1990//
1991//        Option option = new Option();
1992//        option.setVendor(EDbVendor.dbvoracle);
1993//        option.setOutput(true);
1994//        option.setSimpleOutput(true);
1995//        option.setSimpleShowSynonym(true);
1996//        option.setParallel(4);
1997//
1998//        ParallelDataFlowAnalyzer analyzer = new ParallelDataFlowAnalyzer(
1999//                sqlFiles,
2000//                option
2001//        );
2002//
2003//        System.out.println("\n开始分析...");
2004//        long startTime = System.currentTimeMillis();
2005//        analyzer.generateDataFlow(false, true);
2006//        long endTime = System.currentTimeMillis();
2007//
2008//        dataflow dataflow = analyzer.getDataFlow();
2009//
2010//        analyzer.dispose();
2011//
2012//        if (dataflow == null) {
2013//            System.err.println("分析失败，未生成数据流");
2014//            return;
2015//        }
2016//
2017//        Option option1 = new Option();
2018//        option1.setVendor(EDbVendor.dbvoracle);
2019//        option1.setSimpleOutput(true);
2020//        dataflow simpleDataflow = new DataFlowAnalyzer("", option1).getSimpleDataflow(dataflow, true);
2021//
2022//
2023//        System.out.println("分析完成，耗时: " + (endTime - startTime) + " ms");
2024//        System.out.println("表数量: " + (simpleDataflow.getTables() != null ? simpleDataflow.getTables().size() : 0));
2025//        System.out.println("视图数量: " + (simpleDataflow.getViews() != null ? simpleDataflow.getViews().size() : 0));
2026//        System.out.println("关系数量: " + (simpleDataflow.getRelationships() != null ? simpleDataflow.getRelationships().size() : 0));
2027//        System.out.println("错误数量: " + (simpleDataflow.getErrors() != null ? simpleDataflow.getErrors().size() : 0));
2028//
2029//        System.out.println(XML2Model.saveXML(simpleDataflow));
2030//
2031//    }
2032}