Source code

001package gudusoft.gsqlparser.dlineage.util;
002
003import gudusoft.gsqlparser.EDbVendor;
004import gudusoft.gsqlparser.dlineage.DataFlowAnalyzer;
005import gudusoft.gsqlparser.dlineage.ParallelDataFlowAnalyzer;
006import gudusoft.gsqlparser.dlineage.dataflow.metadata.MetadataReader;
007import gudusoft.gsqlparser.dlineage.dataflow.metadata.sqldep.SQLDepMetadataAnalyzer;
008import gudusoft.gsqlparser.dlineage.dataflow.model.ModelBindingManager;
009import gudusoft.gsqlparser.dlineage.dataflow.model.Option;
010import gudusoft.gsqlparser.dlineage.dataflow.model.RelationshipType;
011import gudusoft.gsqlparser.dlineage.dataflow.model.json.Error;
012import gudusoft.gsqlparser.dlineage.dataflow.model.json.Process;
013import gudusoft.gsqlparser.dlineage.dataflow.model.json.*;
014import gudusoft.gsqlparser.dlineage.dataflow.model.xml.*;
015import gudusoft.gsqlparser.dlineage.metadata.Sqlflow;
016import gudusoft.gsqlparser.util.Logger;
017import gudusoft.gsqlparser.util.LoggerFactory;
018import gudusoft.gsqlparser.util.SQLUtil;
019
020import java.io.File;
021import java.util.*;
022import java.util.concurrent.atomic.AtomicLong;
023import java.util.stream.Collectors;
024
025public class DataflowUtility {
026
027    private static final Logger logger = LoggerFactory.getLogger(FunctionUtility.class);
028
029    public static dataflow mergeFunctionCallDataflow(dataflow dataflow, EDbVendor dbVendor) {
030        if (ModelBindingManager.getGlobalVendor() == null) {
031            ModelBindingManager.setGlobalVendor(dbVendor);
032        }
033        dataflow instance = cloneDataflow(dataflow);
034        List<procedure> procedures = new ArrayList<>(instance.getProcedures());
035        if (instance.getPackages() != null) {
036            for (oraclePackage pkg : instance.getPackages()) {
037                if (pkg.getProcedures() != null) {
038                    for (procedure procedure : pkg.getProcedures()) {
039                        procedure.setOraclePackage(pkg);
040                        procedures.add(procedure);
041                    }
042                }
043            }
044        }
045
046        Map<String, procedure> procedureIdMap = procedures.stream().collect(Collectors.toMap(
047                t -> t.getId(),
048                t -> t,
049                (existingValue, newValue) -> newValue
050        ));
051        Map<String, table> functionIdMap = dataflow.getResultsets().stream().collect(Collectors.toMap(
052                t -> t.getId(),
053                t -> t,
054                (existingValue, newValue) -> newValue
055        ));
056
057        Map<String, table> functionMap = new HashMap<>();
058        Map<String, procedure> procedureMap = new HashMap<>();
059        Map<String, oraclePackage> oraclePackageMap = new HashMap<>();
060        Map<String, Set<String>> oraclePackageProcedureMap = new HashMap<>();
061        if (instance.getPackages() != null) {
062            for (oraclePackage pkg : instance.getPackages()) {
063                String qualifiedPackageName = DlineageUtil.getIdentifierOraclePackageNameWithArgNum(pkg);
064                if (!oraclePackageMap.containsKey(qualifiedPackageName)) {
065                    oraclePackageMap.put(qualifiedPackageName, pkg);
066                }
067                for (procedure procedure : pkg.getProcedures()) {
068                    String qualifiedProcedureName = qualifiedPackageName + "." + DlineageUtil.getIdentifierProcedureNameWithArgNum(procedure);
069                    if (!oraclePackageProcedureMap.containsKey(qualifiedPackageName)) {
070                        oraclePackageProcedureMap.put(qualifiedPackageName, new HashSet<>());
071                    }
072                    oraclePackageProcedureMap.get(qualifiedPackageName).add(qualifiedProcedureName);
073                }
074            }
075        }
076
077
078
079        for (procedure procedure : procedures) {
080            String qualifiedProcedureName = DlineageUtil.getIdentifierProcedureNameWithArgNum(procedure);
081            if (procedure.getOraclePackage() != null) {
082                String qualifiedPackageName = DlineageUtil.getIdentifierOraclePackageNameWithArgNum(procedure.getOraclePackage());
083                qualifiedProcedureName = qualifiedPackageName + "." + DlineageUtil.getIdentifierProcedureNameWithArgNum(procedure);
084            }
085            if (!procedureMap.containsKey(qualifiedProcedureName)) {
086                procedureMap.put(qualifiedProcedureName, procedure);
087            }
088        }
089
090        for (table function : instance.getResultsets()) {
091            String qualifiedFunctionName = DlineageUtil.getIdentifierFunctionName(function);
092            if (!functionMap.containsKey(qualifiedFunctionName)) {
093                functionMap.put(qualifiedFunctionName, function);
094            }
095        }
096
097        Set<oraclePackage> oraclePackages = new LinkedHashSet<>();
098        for (String key : oraclePackageMap.keySet()) {
099            oraclePackage standardPackage = oraclePackageMap.get(key);
100            oraclePackages.add(standardPackage);
101            Set<String> procedureNames = oraclePackageProcedureMap.get(key);
102            if (procedureNames != null) {
103                for (String procedureName : procedureNames) {
104                    procedureName = key + "." + procedureName;
105                    if (procedureMap.containsKey(procedureName)) {
106                        procedure standardProcedure = procedureMap.get(procedureName);
107                        if (!standardPackage.getProcedures().contains(standardProcedure)) {
108                            standardPackage.getProcedures().add(standardProcedure);
109                        }
110                        procedureMap.remove(procedureName);
111                    }
112                }
113            }
114        }
115
116        instance.setPackages(new ArrayList<>(oraclePackages));
117        instance.setProcedures(new ArrayList<>(procedureMap.values()));
118        instance.setResultsets(new ArrayList<>(functionMap.values()));
119
120        Map<String, relationship> mergeRelations = new LinkedHashMap<String, relationship>();
121
122        for (relationship relationship : instance.getRelationships()) {
123            if (relationship.getCaller() == null || relationship.getCallees() == null || relationship.getCallees().size() == 0) {
124                continue;
125            }
126
127            String targetId = relationship.getCaller().getId();
128            if (procedureIdMap.containsKey(targetId)) {
129                procedure procedure = procedureIdMap.get(targetId);
130                String procedureName = DlineageUtil.getIdentifierProcedureNameWithArgNum(procedure);
131                if (procedure.getOraclePackage() != null) {
132                    String qualifiedPackageName = DlineageUtil.getIdentifierOraclePackageNameWithArgNum(procedure.getOraclePackage());
133                    procedureName = qualifiedPackageName + "." + procedureName;
134                }
135                procedure standardProcedure = procedureMap.get(procedureName);
136                relationship.getCaller().setId(standardProcedure.getId());
137            } else if (functionIdMap.containsKey(targetId)) {
138                table function = functionIdMap.get(targetId);
139                String functionName = DlineageUtil.getIdentifierFunctionName(function);
140                table standardFunction = functionMap.get(functionName);
141                relationship.getCaller().setId(standardFunction.getId());
142            }
143
144            for (sourceColumn sourceColumn : relationship.getCallees()) {
145                String sourceId = sourceColumn.getId();
146                if (procedureIdMap.containsKey(sourceId)) {
147                    procedure procedure = procedureIdMap.get(sourceId);
148                    String procedureName = DlineageUtil.getIdentifierProcedureNameWithArgNum(procedure);
149                    if (procedure.getOraclePackage() != null) {
150                        String qualifiedPackageName = DlineageUtil.getIdentifierOraclePackageNameWithArgNum(procedure.getOraclePackage());
151                        procedureName = qualifiedPackageName + "." + procedureName;
152                    }
153                    procedure standardProcedure = procedureMap.get(procedureName);
154                    sourceColumn.setId(standardProcedure.getId());
155                } else if (functionIdMap.containsKey(sourceId)) {
156                    table function = functionIdMap.get(sourceId);
157                    String functionName = DlineageUtil.getIdentifierFunctionName(function);
158                    table standardFunction = functionMap.get(functionName);
159                    sourceColumn.setId(standardFunction.getId());
160                }
161            }
162
163            String key = relationship.toDedupKey();
164            if (!mergeRelations.containsKey(key)) {
165                mergeRelations.put(key, relationship);
166            }
167        }
168
169        instance.setRelationships(new ArrayList<relationship>(mergeRelations.values()));
170
171        if (instance.getPackages() != null) {
172            for (oraclePackage pkg : instance.getPackages()) {
173                if (pkg.getProcedures() != null) {
174                    for (procedure procedure : pkg.getProcedures()) {
175                        procedure.setOraclePackage(null);
176                    }
177                }
178            }
179        }
180
181        return instance;
182    }
183
184    public static dataflow convertTableLevelToFunctionCallDataflow(dataflow dataflow, boolean showBuiltIn, EDbVendor dbVendor) {
185        if (ModelBindingManager.getGlobalVendor() == null) {
186            ModelBindingManager.setGlobalVendor(dbVendor);
187        }
188
189        dataflow instance = cloneDataflow(dataflow);
190
191        if (instance.getRelationships() == null) {
192            return instance;
193        }
194
195        List<relationship> callRelationships = instance.getRelationships().stream()
196                .filter(t -> RelationshipType.call.name().equals(t.getType()))
197                .filter(t -> !showBuiltIn ? !Boolean.TRUE.equals(t.getBuiltIn()): true).collect(Collectors.toList());
198        instance.setRelationships(callRelationships);
199
200        Set<String> ids = new HashSet<>();
201
202
203        callRelationships.stream().forEach(t -> {
204            ids.add(t.getCaller().getId());
205            t.getCallees().stream().forEach(t1 -> ids.add(t1.getId()));
206        });
207
208        Iterator<table> iterator = instance.getTables().iterator();
209        while (iterator.hasNext()) {
210            table t = iterator.next();
211            if (!ids.contains(t.getId())) {
212                iterator.remove();
213            }
214        }
215
216        iterator = instance.getResultsets().iterator();
217        while (iterator.hasNext()) {
218            table t = iterator.next();
219            if (!ids.contains(t.getId())) {
220                iterator.remove();
221            }
222        }
223
224        iterator = instance.getViews().iterator();
225        while (iterator.hasNext()) {
226            table t = iterator.next();
227            if (!ids.contains(t.getId())) {
228                iterator.remove();
229            }
230        }
231
232        iterator = instance.getStages().iterator();
233        while (iterator.hasNext()) {
234            table t = iterator.next();
235            if (!ids.contains(t.getId())) {
236                iterator.remove();
237            }
238        }
239
240        iterator = instance.getStreams().iterator();
241        while (iterator.hasNext()) {
242            table t = iterator.next();
243            if (!ids.contains(t.getId())) {
244                iterator.remove();
245            }
246        }
247
248        iterator = instance.getVariables().iterator();
249        while (iterator.hasNext()) {
250            table t = iterator.next();
251            if (!ids.contains(t.getId())) {
252                iterator.remove();
253            }
254        }
255
256        iterator = instance.getPaths().iterator();
257        while (iterator.hasNext()) {
258            table t = iterator.next();
259            if (!ids.contains(t.getId())) {
260                iterator.remove();
261            }
262        }
263
264        iterator = instance.getDatasources().iterator();
265        while (iterator.hasNext()) {
266            table t = iterator.next();
267            if (!ids.contains(t.getId())) {
268                iterator.remove();
269            }
270        }
271
272        iterator = instance.getDatabases().iterator();
273        while (iterator.hasNext()) {
274            table t = iterator.next();
275            if (!ids.contains(t.getId())) {
276                iterator.remove();
277            }
278        }
279
280        iterator = instance.getSchemas().iterator();
281        while (iterator.hasNext()) {
282            table t = iterator.next();
283            if (!ids.contains(t.getId())) {
284                iterator.remove();
285            }
286        }
287
288        iterator = instance.getSequences().iterator();
289        while (iterator.hasNext()) {
290            table t = iterator.next();
291            if (!ids.contains(t.getId())) {
292                iterator.remove();
293            }
294        }
295
296        return mergeFunctionCallDataflow(instance, dbVendor);
297    }
298
299    public static dataflow convertToTableLevelDataflow(dataflow dataflow) {
300        dataflow instance = cloneDataflow(dataflow);
301
302        if (instance.getRelationships() == null) {
303            return instance;
304        }
305
306        Map<String, LinkedHashSet<Pair<String, String>>> relationMap = new HashMap<String, LinkedHashSet<Pair<String, String>>>();
307        Map<String, LinkedHashSet<Pair3<String, String, relationship>>> callRelationMap = new HashMap<String, LinkedHashSet<Pair3<String, String, relationship>>>();
308        for (RelationshipType type : RelationshipType.values()) {
309            relationMap.put(type.name(), new LinkedHashSet<Pair<String, String>>());
310            callRelationMap.put(type.name(), new LinkedHashSet<Pair3<String, String, relationship>>());
311        }
312        for (relationship relationship : instance.getRelationships()) {
313            if (RelationshipType.call.name().equals(relationship.getType())) {
314                String targetId = relationship.getCaller().getId();
315                for (sourceColumn sourceColumn : relationship.getCallees()) {
316                    String sourceId = sourceColumn.getId();
317                    callRelationMap.get(relationship.getType()).add(new Pair3<>(targetId, sourceId, relationship));
318                }
319            } else {
320                String targetId = relationship.getTarget().getParent_id();
321                for (sourceColumn sourceColumn : relationship.getSources()) {
322                    String sourceId = sourceColumn.getParent_id();
323                    relationMap.get(relationship.getType()).add(new Pair<>(targetId, sourceId));
324                }
325            }
326        }
327
328        long maxId = 0;
329        Map<String, table> dbObjMap = getDataflowDbObjMap(instance);
330
331        List<procedure> procedures = new ArrayList<>(instance.getProcedures());
332        if (instance.getPackages() != null) {
333            for (oraclePackage pkg : instance.getPackages()) {
334                procedures.addAll(pkg.getProcedures());
335            }
336        }
337
338        Map<String, procedure> procedureMap = procedures.stream().collect(Collectors.toMap(
339                t -> t.getId(),
340                t -> t,
341                (existingValue, newValue) -> newValue
342        ));
343
344        for (table table : dbObjMap.values()) {
345            if (Long.valueOf(table.getId()) > maxId) {
346                maxId = Long.valueOf(table.getId());
347            }
348        }
349
350        AtomicLong id = new AtomicLong(maxId + 10000000);
351
352        for (table table : dbObjMap.values()) {
353            column column = new column();
354            column.setId(String.valueOf(id.incrementAndGet()));
355            column.setName(table.getType());
356            table.setColumns(Arrays.asList(column));
357        }
358
359        List<relationship> relations = new ArrayList<relationship>();
360        for (RelationshipType type : RelationshipType.values()) {
361            LinkedHashSet<Pair<String, String>> relationSet = relationMap.get(type.name());
362             LinkedHashSet<Pair3<String, String, relationship>> callRelationSet = callRelationMap.get(type.name());
363            if (RelationshipType.call.equals(type)) {
364                for (Pair3<String, String, relationship> pair : callRelationSet) {
365                    if ((dbObjMap.get(pair.first) == null && procedureMap.get(pair.first) == null)
366                            || (dbObjMap.get(pair.second) == null && procedureMap.get(pair.second) == null)) {
367                        continue;
368                    }
369
370                    relationship relationship = new relationship();
371                    relationship.setType(type.name());
372                    relationship.setId(String.valueOf(id.incrementAndGet()));
373                    relationship.setCallStmt(pair.third.getCallStmt());
374                    relationship.setCallCoordinate(pair.third.getCallCoordinate());
375
376                    targetColumn targetColumn = new targetColumn();
377                    if (dbObjMap.containsKey(pair.first)) {
378                        targetColumn.setId(dbObjMap.get(pair.first).getId());
379                        targetColumn.setName(dbObjMap.get(pair.first).getName());
380                        targetColumn.setCoordinate(dbObjMap.get(pair.first).getCoordinate());
381                    } else {
382                        targetColumn.setId(procedureMap.get(pair.first).getId());
383                        targetColumn.setName(procedureMap.get(pair.first).getName());
384                        targetColumn.setCoordinate(procedureMap.get(pair.first).getCoordinate());
385                    }
386
387                    sourceColumn sourceColumn = new sourceColumn();
388                    if (dbObjMap.containsKey(pair.second)) {
389                        sourceColumn.setId(dbObjMap.get(pair.second).getId());
390                        sourceColumn.setName(dbObjMap.get(pair.second).getName());
391                        sourceColumn.setCoordinate(dbObjMap.get(pair.second).getCoordinate());
392                    } else {
393                        sourceColumn.setId(procedureMap.get(pair.second).getId());
394                        sourceColumn.setName(procedureMap.get(pair.second).getName());
395                        sourceColumn.setCoordinate(procedureMap.get(pair.second).getCoordinate());
396                    }
397
398                    relationship.setCaller(targetColumn);
399                    relationship.setCallees(Arrays.asList(sourceColumn));
400                    relationship.setBuiltIn(pair.third.getBuiltIn());
401                    relations.add(relationship);
402                }
403            } else {
404                for (Pair<String, String> pair : relationSet) {
405                    if (dbObjMap.get(pair.first) == null || dbObjMap.get(pair.second) == null) {
406                        continue;
407                    }
408                    relationship relationship = new relationship();
409                    relationship.setType(type.name());
410                    relationship.setId(String.valueOf(id.incrementAndGet()));
411                    targetColumn targetColumn = new targetColumn();
412                    targetColumn.setId(dbObjMap.get(pair.first).getColumns().get(0).getId());
413                    targetColumn.setColumn(dbObjMap.get(pair.first).getColumns().get(0).getName());
414                    targetColumn.setParent_id(pair.first);
415                    targetColumn.setParent_name(dbObjMap.get(pair.first).getName());
416                    sourceColumn sourceColumn = new sourceColumn();
417                    sourceColumn.setId(dbObjMap.get(pair.second).getColumns().get(0).getId());
418                    sourceColumn.setColumn(dbObjMap.get(pair.second).getColumns().get(0).getName());
419                    sourceColumn.setParent_id(pair.second);
420                    sourceColumn.setParent_name(dbObjMap.get(pair.second).getName());
421                    relationship.setTarget(targetColumn);
422                    relationship.setSources(Arrays.asList(sourceColumn));
423                    relations.add(relationship);
424                }
425            }
426        }
427        instance.setRelationships(relations);
428        return instance;
429    }
430
431    public static dataflow convertToSchemaLevelDataflow(dataflow dataflow, EDbVendor dbVendor) throws Exception {
432        return convertToSchemaLevelDataflow(dataflow, dbVendor, false);
433    }
434
435    public static dataflow convertToSchemaLevelDataflow(dataflow dataflow, EDbVendor dbVendor, boolean isSimple) throws Exception {
436        dataflow instance = cloneDataflow(dataflow);
437
438        if (!isSimple) {
439            DataFlowAnalyzer analyzer = new DataFlowAnalyzer("", dbVendor, true);
440            instance = analyzer.getSimpleDataflow(instance, true, Arrays.asList(new String[]{"fdd", "fdr"}));
441        }
442
443        List<table> allTables = new ArrayList<table>();
444        allTables.addAll(instance.getTables());
445        allTables.addAll(instance.getViews());
446
447        ModelBindingManager.setGlobalVendor(dbVendor);
448
449        Map<String, String> tableIdSchameNameMap = allTables.stream().collect(Collectors.toMap(table -> table.getId(), table -> table.getFullSchemaName(), (existingValue, newValue) -> newValue));
450
451        if (instance.getRelationships() == null) {
452            return instance;
453        }
454
455        Map<String, LinkedHashSet<Pair<String, String>>> relationMap = new HashMap<String, LinkedHashSet<Pair<String, String>>>();
456        for (RelationshipType type : RelationshipType.values()) {
457            relationMap.put(type.name(), new LinkedHashSet<Pair<String, String>>());
458        }
459        for (relationship relationship : instance.getRelationships()) {
460            String targetId = relationship.getTarget().getParent_id();
461            for (sourceColumn sourceColumn : relationship.getSources()) {
462                String sourceId = sourceColumn.getParent_id();
463                relationMap.get(relationship.getType()).add(new Pair<>(tableIdSchameNameMap.get(targetId), tableIdSchameNameMap.get(sourceId)));
464            }
465        }
466
467        Map<String, table> dbObjMap = getDataflowDbObjMap(instance);
468        LinkedHashSet<String> schemaNameSet = new LinkedHashSet<String>();
469        schemaNameSet.addAll(tableIdSchameNameMap.values());
470        List<table> schemaTables = new ArrayList<table>();
471
472        AtomicLong id = new AtomicLong(0);
473
474        for (String schemaName : schemaNameSet) {
475            table table = new table();
476            table.setId(String.valueOf(id.incrementAndGet()));
477            table.setName(schemaName);
478            table.setType("schema");
479            schemaTables.add(table);
480            dbObjMap.put(schemaName, table);
481        }
482
483        for (table table : schemaTables) {
484            column column = new column();
485            column.setId(String.valueOf(id.incrementAndGet()));
486            column.setName(table.getType());
487            table.setColumns(Arrays.asList(column));
488        }
489
490        List<relationship> relations = new ArrayList<relationship>();
491        for (RelationshipType type : RelationshipType.values()) {
492            LinkedHashSet<Pair<String, String>> relationSet = relationMap.get(type.name());
493
494            for (Pair<String, String> pair : relationSet) {
495                if (dbObjMap.get(pair.first) == null || dbObjMap.get(pair.second) == null) {
496                    continue;
497                }
498                relationship relationship = new relationship();
499                relationship.setType(type.name());
500                relationship.setId(String.valueOf(id.incrementAndGet()));
501                targetColumn targetColumn = new targetColumn();
502                targetColumn.setId(dbObjMap.get(pair.first).getColumns().get(0).getId());
503                targetColumn.setColumn(dbObjMap.get(pair.first).getColumns().get(0).getName());
504                targetColumn.setParent_id(dbObjMap.get(pair.first).getId());
505                targetColumn.setParent_name(dbObjMap.get(pair.first).getName());
506                sourceColumn sourceColumn = new sourceColumn();
507                sourceColumn.setId(dbObjMap.get(pair.second).getColumns().get(0).getId());
508                sourceColumn.setColumn(dbObjMap.get(pair.second).getColumns().get(0).getName());
509                sourceColumn.setParent_id(dbObjMap.get(pair.second).getId());
510                sourceColumn.setParent_name(dbObjMap.get(pair.second).getName());
511                relationship.setTarget(targetColumn);
512                relationship.setSources(Arrays.asList(sourceColumn));
513                relations.add(relationship);
514            }
515        }
516
517        dataflow schemaDataflow = new dataflow();
518        schemaDataflow.setTables(schemaTables);
519        schemaDataflow.setRelationships(relations);
520        return schemaDataflow;
521    }
522
523    private static final int PARALLEL_CLONE_THRESHOLD = 1000;
524
525    public static dataflow cloneDataflow(dataflow dataflow) {
526        dataflow dataflowCopy = new dataflow();
527        if (dataflow.getTables() != null) {
528            dataflowCopy.setTables(cloneTables(dataflow.getTables()));
529        }
530        if (dataflow.getViews() != null) {
531            dataflowCopy.setViews(cloneTables(dataflow.getViews()));
532        }
533        if (dataflow.getRelationships() != null) {
534            dataflowCopy.setRelationships(new ArrayList<>(dataflow.getRelationships()));
535        }
536        if (dataflow.getErrors() != null) {
537            dataflowCopy.setErrors(new ArrayList<>(dataflow.getErrors()));
538        }
539        if (dataflow.getPaths() != null) {
540            dataflowCopy.setPaths(cloneTables(dataflow.getPaths()));
541        }
542        if (dataflow.getPackages() != null) {
543            dataflowCopy.setPackages(new ArrayList<>(dataflow.getPackages()));
544        }
545        if (dataflow.getProcedures() != null) {
546            dataflowCopy.setProcedures(new ArrayList<>(dataflow.getProcedures()));
547        }
548        if (dataflow.getProcesses() != null) {
549            dataflowCopy.setProcesses(new ArrayList<>(dataflow.getProcesses()));
550        }
551        if (dataflow.getResultsets() != null) {
552            dataflowCopy.setResultsets(cloneTables(dataflow.getResultsets()));
553        }
554        if (dataflow.getVariables() != null) {
555            dataflowCopy.setVariables(cloneTables(dataflow.getVariables()));
556        }
557        if (dataflow.getStages() != null) {
558            dataflowCopy.setStages(cloneTables(dataflow.getStages()));
559        }
560        if (dataflow.getSequences() != null) {
561            dataflowCopy.setSequences(cloneTables(dataflow.getSequences()));
562        }
563        if (dataflow.getDatasources() != null) {
564            dataflowCopy.setDatasources(cloneTables(dataflow.getDatasources()));
565        }
566        if (dataflow.getDatabases() != null) {
567            dataflowCopy.setDatabases(cloneTables(dataflow.getDatabases()));
568        }
569        if (dataflow.getSchemas() != null) {
570            dataflowCopy.setSchemas(cloneTables(dataflow.getSchemas()));
571        }
572        if (dataflow.getStreams() != null) {
573            dataflowCopy.setStreams(cloneTables(dataflow.getStreams()));
574        }
575        dataflowCopy.setOrientation(dataflow.getOrientation());
576        return dataflowCopy;
577    }
578
579    private static List<table> cloneTables(List<table> tableList) {
580        if (tableList == null) {
581            return null;
582        }
583        int size = tableList.size();
584        if (size >= PARALLEL_CLONE_THRESHOLD) {
585            return tableList.parallelStream()
586                    .map(DataflowUtility::cloneTable)
587                    .collect(Collectors.toCollection(() -> new ArrayList<>(size)));
588        } else {
589            List<table> tables = new ArrayList<>(size);
590            for (int i = 0; i < size; i++) {
591                tables.add(cloneTable(tableList.get(i)));
592            }
593            return tables;
594        }
595    }
596
597    private static table cloneTable(table item) {
598        try {
599            table cloned = (table) item.clone();
600            List<column> columns = item.getColumns();
601            if (columns != null) {
602                cloned.setColumns(new ArrayList<>(columns));
603            }
604            return cloned;
605        } catch (CloneNotSupportedException e) {
606            throw new RuntimeException("Clone table failed.", e);
607        }
608    }
609
610    public static Map<String, table> getDataflowDbObjMap(dataflow dataflow) {
611        List<table> tables = new ArrayList<table>();
612        if (dataflow.getTables() != null) {
613            tables.addAll(dataflow.getTables());
614        }
615        if (dataflow.getViews() != null) {
616            tables.addAll(dataflow.getViews());
617        }
618        if (dataflow.getPaths() != null) {
619            tables.addAll(dataflow.getPaths());
620        }
621        if (dataflow.getResultsets() != null) {
622            tables.addAll(dataflow.getResultsets());
623        }
624        if (dataflow.getVariables() != null) {
625            tables.addAll(dataflow.getVariables());
626        }
627        if (dataflow.getStages() != null) {
628            tables.addAll(dataflow.getStages());
629        }
630        if (dataflow.getSequences() != null) {
631            tables.addAll(dataflow.getSequences());
632        }
633        if (dataflow.getDatasources() != null) {
634            tables.addAll(dataflow.getDatasources());
635        }
636        if (dataflow.getDatabases() != null) {
637            tables.addAll(dataflow.getDatabases());
638        }
639        if (dataflow.getSchemas() != null) {
640            tables.addAll(dataflow.getSchemas());
641        }
642        if (dataflow.getStreams() != null) {
643            tables.addAll(dataflow.getStreams());
644        }
645
646        Map<String, table> dbObjMap = new HashMap<>();
647        for (table table : tables) {
648            dbObjMap.put(table.getId(), table);
649        }
650        return dbObjMap;
651        }
652        public static Map<String, table> getDataflowDbObjNameMap(dataflow dataflow) {
653                List<table> tables = new ArrayList<table>();
654                if (dataflow.getTables() != null) {
655                        tables.addAll(dataflow.getTables());
656                }
657                if (dataflow.getViews() != null) {
658                        tables.addAll(dataflow.getViews());
659                }
660                if (dataflow.getPaths() != null) {
661                        tables.addAll(dataflow.getPaths());
662                }
663                if (dataflow.getResultsets() != null) {
664                        tables.addAll(dataflow.getResultsets());
665                }
666                if (dataflow.getVariables() != null) {
667                        tables.addAll(dataflow.getVariables());
668                }
669                if (dataflow.getStages() != null) {
670                        tables.addAll(dataflow.getStages());
671                }
672                if (dataflow.getSequences() != null) {
673                        tables.addAll(dataflow.getSequences());
674                }
675                if (dataflow.getDatasources() != null) {
676                        tables.addAll(dataflow.getDatasources());
677                }
678                if (dataflow.getDatabases() != null) {
679                        tables.addAll(dataflow.getDatabases());
680                }
681                if (dataflow.getSchemas() != null) {
682                        tables.addAll(dataflow.getSchemas());
683                }
684                if (dataflow.getStreams() != null) {
685                        tables.addAll(dataflow.getStreams());
686                }
687
688                Map<String, table> dbObjMap = new HashMap<>();
689                for(table table: tables){
690                        dbObjMap.put(table.getFullName(), table);
691                }
692                return dbObjMap;
693        }
694        public static dataflow mergeDataflowsWithDifferentStartId(Collection<dataflow> dataflows, EDbVendor vendor) {
695                ModelBindingManager.setGlobalVendor(vendor);
696                try {
697                        return mergeDataflowsByStartId(dataflows, DATAFLOW_ID_RANGE * dataflows.size());
698                } finally {
699                        ModelBindingManager.removeGlobalVendor();
700                }
701        }
702
703        /**
704         * 内部合并实现：调用方需保证所有输入 dataflow 的 ID 空间不冲突，并且已提前设置好 ModelBindingManager。
705         * <p>
706         * 该方法代替之前位于 {@code ParallelDataFlowAnalyzer} 的同名实现，外部调用者应使用
707         * {@link #mergeDataflowsWithDifferentStartId(Collection, EDbVendor)} 或
708         * {@link #mergeDataflows(Collection, EDbVendor)}。
709         */
710        public static dataflow mergeDataflowsByStartId(Collection<dataflow> dataflows, long startId) {
711                dataflow mergeDataflow = new dataflow();
712
713                List<table> tableCopy = new ArrayList<table>();
714                List<table> viewCopy = new ArrayList<table>();
715                List<table> databaseCopy = new ArrayList<table>();
716                List<table> schemaCopy = new ArrayList<table>();
717                List<table> stageCopy = new ArrayList<table>();
718                List<table> dataSourceCopy = new ArrayList<table>();
719                List<table> streamCopy = new ArrayList<table>();
720                List<table> fileCopy = new ArrayList<table>();
721                List<table> variableCopy = new ArrayList<table>();
722                List<table> resultSetCopy = new ArrayList<table>();
723                List<table> sequenceCopy = new ArrayList<table>();
724
725                List<process> processCopy = new ArrayList<process>();
726                List<relationship> relationshipCopy = new ArrayList<relationship>();
727                List<procedure> procedureCopy = new ArrayList<procedure>();
728                List<oraclePackage> packageCopy = new ArrayList<oraclePackage>();
729                List<error> errorCopy = new ArrayList<error>();
730
731                List<table> tables = new ArrayList<table>();
732                int dfIdx = 0;
733                for (dataflow dataflow : dataflows) {
734                        if (dataflow == null) {
735                                continue;
736                        }
737
738                        if (dataflow.getTables() != null) {
739                                tableCopy.addAll(dataflow.getTables());
740                        }
741                        mergeDataflow.setTables(tableCopy);
742                        if (dataflow.getViews() != null) {
743                                viewCopy.addAll(dataflow.getViews());
744                        }
745                        mergeDataflow.setViews(viewCopy);
746                        if (dataflow.getDatabases() != null) {
747                                databaseCopy.addAll(dataflow.getDatabases());
748                        }
749                        mergeDataflow.setDatabases(databaseCopy);
750                        if (dataflow.getSchemas() != null) {
751                                schemaCopy.addAll(dataflow.getSchemas());
752                        }
753                        mergeDataflow.setSchemas(schemaCopy);
754                        if (dataflow.getStages() != null) {
755                                stageCopy.addAll(dataflow.getStages());
756                        }
757                        mergeDataflow.setStages(stageCopy);
758                        if (dataflow.getDatasources() != null) {
759                                dataSourceCopy.addAll(dataflow.getDatasources());
760                        }
761                        mergeDataflow.setDatasources(dataSourceCopy);
762                        if (dataflow.getStreams() != null) {
763                                streamCopy.addAll(dataflow.getStreams());
764                        }
765                        mergeDataflow.setStreams(streamCopy);
766                        if (dataflow.getPaths() != null) {
767                                fileCopy.addAll(dataflow.getPaths());
768                        }
769                        mergeDataflow.setPaths(fileCopy);
770                        if (dataflow.getVariables() != null) {
771                                variableCopy.addAll(dataflow.getVariables());
772                        }
773                        mergeDataflow.setVariables(variableCopy);
774                        if (dataflow.getResultsets() != null) {
775                                resultSetCopy.addAll(dataflow.getResultsets());
776                        }
777                        mergeDataflow.setResultsets(resultSetCopy);
778                        if (dataflow.getSequences() != null) {
779                                sequenceCopy.addAll(dataflow.getSequences());
780                        }
781                        mergeDataflow.setSequences(sequenceCopy);
782
783                        dfIdx++;
784
785                        if (dataflow.getProcesses() != null) {
786                                processCopy.addAll(dataflow.getProcesses());
787                        }
788                        mergeDataflow.setProcesses(processCopy);
789                        if (dataflow.getRelationships() != null) {
790                                relationshipCopy.addAll(dataflow.getRelationships());
791                        }
792                        mergeDataflow.setRelationships(relationshipCopy);
793                        if (dataflow.getProcedures() != null) {
794                                procedureCopy.addAll(dataflow.getProcedures());
795                        }
796                        mergeDataflow.setProcedures(procedureCopy);
797                        if (dataflow.getPackages() != null) {
798                                packageCopy.addAll(dataflow.getPackages());
799                        }
800                        mergeDataflow.setPackages(packageCopy);
801                        if (dataflow.getErrors() != null) {
802                                errorCopy.addAll(dataflow.getErrors());
803                        }
804                        if (errorCopy.size() > 10000) {
805                                errorCopy = errorCopy.subList(0, 10000);
806                        }
807                        mergeDataflow.setErrors(errorCopy);
808
809                        tables.addAll(dataflow.getTables());
810                        tables.addAll(dataflow.getViews());
811                        tables.addAll(dataflow.getDatabases());
812                        tables.addAll(dataflow.getSchemas());
813                        tables.addAll(dataflow.getStages());
814                        tables.addAll(dataflow.getDatasources());
815                        tables.addAll(dataflow.getStreams());
816                        tables.addAll(dataflow.getPaths());
817                        tables.addAll(dataflow.getResultsets());
818                        tables.addAll(dataflow.getVariables());
819                        if (dataflow.getSequences() != null) {
820                                tables.addAll(dataflow.getSequences());
821                        }
822                }
823
824                Map<String, List<table>> tableMap = new HashMap<String, List<table>>();
825                Map<String, String> tableTypeMap = new HashMap<String, String>();
826                Map<String, String> tableIdMap = new HashMap<String, String>();
827
828                Map<String, List<column>> columnMap = new HashMap<String, List<column>>();
829                Map<String, Set<String>> tableColumnMap = new HashMap<String, Set<String>>();
830                Map<String, String> columnIdMap = new HashMap<String, String>();
831                Map<String, column> columnMergeIdMap = new HashMap<String, column>();
832
833                List<procedure> procedures = new ArrayList<>(mergeDataflow.getProcedures());
834                if (mergeDataflow.getPackages() != null) {
835                        for (oraclePackage pkg : mergeDataflow.getPackages()) {
836                                procedures.addAll(pkg.getProcedures());
837                        }
838                }
839
840                Set<String> procedureIdSet = procedures.stream().map(t -> t.getId()).collect(Collectors.toSet());
841
842                for (table table : tables) {
843                        String qualifiedTableName = DlineageUtil.getQualifiedTableName(table);
844                        String tableFullName = DlineageUtil.getIdentifierNormalTableName(qualifiedTableName);
845                        if ("variable".endsWith(table.getType()) && !SQLUtil.isEmpty(table.getParent())) {
846                                tableFullName = table.getParent() + "." + tableFullName;
847                        }
848
849                        if (!tableMap.containsKey(tableFullName)) {
850                                tableMap.put(tableFullName, new ArrayList<table>());
851                        }
852
853                        tableMap.get(tableFullName).add(table);
854
855                        if (!tableTypeMap.containsKey(tableFullName)) {
856                                tableTypeMap.put(tableFullName, table.getType());
857                        } else if ("view".equals(table.getSubType())) {
858                                tableTypeMap.put(tableFullName, table.getType());
859                        } else if ("database".equals(table.getSubType())) {
860                                tableTypeMap.put(tableFullName, table.getType());
861                        } else if ("schema".equals(table.getSubType())) {
862                                tableTypeMap.put(tableFullName, table.getType());
863                        } else if ("stage".equals(table.getSubType())) {
864                                tableTypeMap.put(tableFullName, table.getType());
865                        } else if ("datasource".equals(table.getSubType())) {
866                                tableTypeMap.put(tableFullName, table.getType());
867                        } else if ("stream".equals(table.getSubType())) {
868                                tableTypeMap.put(tableFullName, table.getType());
869                        } else if ("file".equals(table.getSubType())) {
870                                tableTypeMap.put(tableFullName, table.getType());
871                        } else if ("sequence".equals(table.getSubType())) {
872                                tableTypeMap.put(tableFullName, table.getType());
873                        } else if ("table".equals(tableTypeMap.get(tableFullName))) {
874                                tableTypeMap.put(tableFullName, table.getType());
875                        } else if ("variable".equals(tableTypeMap.get(tableFullName))) {
876                                tableTypeMap.put(tableFullName, table.getType());
877                        }
878
879                        if (table.getColumns() != null) {
880                                if (!tableColumnMap.containsKey(tableFullName)) {
881                                        tableColumnMap.put(tableFullName, new LinkedHashSet<String>());
882                                }
883                                for (column column : table.getColumns()) {
884                                        String columnFullName = tableFullName + "."
885                                                        + DlineageUtil.getIdentifierNormalColumnName(column.getName());
886
887                                        if (!columnMap.containsKey(columnFullName)) {
888                                                columnMap.put(columnFullName, new ArrayList<column>());
889                                                tableColumnMap.get(tableFullName).add(columnFullName);
890                                        }
891
892                                        columnMap.get(columnFullName).add(column);
893                                }
894                        }
895                }
896
897                Iterator<String> tableNameIter = tableMap.keySet().iterator();
898                while (tableNameIter.hasNext()) {
899                        String tableName = tableNameIter.next();
900                        List<table> tableList = tableMap.get(tableName);
901                        table table;
902                        if (tableList.size() > 1) {
903                                table standardTable = tableList.get(0);
904                                //Function允许重名，不做合并处理
905                                //检查列表中任意一个表是否为function，防止function表被合并丢失
906                                boolean hasFunction = false;
907                                for (table t : tableList) {
908                                        if (t.isFunction()) {
909                                                hasFunction = true;
910                                                break;
911                                        }
912                                }
913                                if (hasFunction) {
914                                        continue;
915                                }
916
917                                String type = tableTypeMap.get(tableName);
918                                table = new table();
919                                table.setId(String.valueOf(++startId));
920                                table.setServer(standardTable.getServer());
921                                table.setDatabase(standardTable.getDatabase());
922                                table.setSchema(standardTable.getSchema());
923                                table.setName(standardTable.getName());
924                                table.setDisplayName(standardTable.getDisplayName());
925                                table.setParent(standardTable.getParent());
926                                table.setColumns(new ArrayList<column>());
927                String subType = null;
928                for(table item: tableList){
929                    if (item.getSubType() != null) {
930                        subType = item.getSubType();
931                        break;
932                    }
933                }
934                if (subType != null) {
935                    table.setSubType(subType);
936                } else {
937                    table.setSubType(standardTable.getSubType());
938                }
939                                mergeEndpointClassification(table, tableList);
940                                Set<String> processIds = new LinkedHashSet<String>();
941                                for (int k = 0; k < tableList.size(); k++) {
942                                        if (tableList.get(k).getProcessIds() != null) {
943                                                processIds.addAll(tableList.get(k).getProcessIds());
944                                        }
945                                }
946                                if (!processIds.isEmpty()) {
947                                        table.setProcessIds(new ArrayList<String>(processIds));
948                                }
949                                table.setType(type);
950                                for (table item : tableList) {
951                                        if (!SQLUtil.isEmpty(table.getCoordinate()) && !SQLUtil.isEmpty(item.getCoordinate())) {
952                                                if (table.getCoordinate().indexOf(item.getCoordinate()) == -1) {
953                                                        table.appendCoordinate(item.getCoordinate());
954                                                }
955                                        } else if (!SQLUtil.isEmpty(item.getCoordinate())) {
956                                                table.setCoordinate(item.getCoordinate());
957                                        }
958
959                                        if (!SQLUtil.isEmpty(table.getAlias()) && !SQLUtil.isEmpty(item.getAlias())) {
960                                                table.setAlias(table.getAlias() + "," + item.getAlias());
961                                        } else if (!SQLUtil.isEmpty(item.getAlias())) {
962                                                table.setAlias(item.getAlias());
963                                        }
964
965                                        tableIdMap.put(item.getId(), table.getId());
966
967                                        if (item.isView()) {
968                                                mergeDataflow.getViews().remove(item);
969                                        } else if (item.isDatabaseType()) {
970                                                mergeDataflow.getDatabases().remove(item);
971                                        } else if (item.isSchemaType()) {
972                                                mergeDataflow.getSchemas().remove(item);
973                                        } else if (item.isStage()) {
974                                                mergeDataflow.getStages().remove(item);
975                                        } else if (item.isDataSource()) {
976                                                mergeDataflow.getDatasources().remove(item);
977                                        } else if (item.isStream()) {
978                                                mergeDataflow.getStreams().remove(item);
979                                        } else if (item.isFile()) {
980                                                mergeDataflow.getPaths().remove(item);
981                                        } else if (item.isVariable()) {
982                                                mergeDataflow.getVariables().remove(item);
983                                        } else if (item.isTable()) {
984                                                mergeDataflow.getTables().remove(item);
985                                        } else if (item.isResultSet()) {
986                                                mergeDataflow.getResultsets().remove(item);
987                                        } else if (item.isSequence()) {
988                                                mergeDataflow.getSequences().remove(item);
989                                        }
990                                }
991
992                                if (table.isView()) {
993                                        mergeDataflow.getViews().add(table);
994                                } else if (table.isDatabaseType()) {
995                                        mergeDataflow.getDatabases().add(table);
996                                } else if (table.isSchemaType()) {
997                                        mergeDataflow.getSchemas().add(table);
998                                } else if (table.isStage()) {
999                                        mergeDataflow.getStages().add(table);
1000                                } else if (table.isDataSource()) {
1001                                        mergeDataflow.getDatasources().add(table);
1002                                } else if (table.isStream()) {
1003                                        mergeDataflow.getStreams().add(table);
1004                                } else if (table.isFile()) {
1005                                        mergeDataflow.getPaths().add(table);
1006                                } else if (table.isVariable()) {
1007                                        mergeDataflow.getVariables().add(table);
1008                                } else if (table.isResultSet()) {
1009                                        mergeDataflow.getResultsets().add(table);
1010                                } else if (table.isSequence()) {
1011                                        mergeDataflow.getSequences().add(table);
1012                                } else {
1013                                        mergeDataflow.getTables().add(table);
1014                                }
1015                        } else {
1016                                table = tableList.get(0);
1017                        }
1018
1019                        Set<String> columns = tableColumnMap.get(tableName);
1020                        Iterator<String> columnIter = columns.iterator();
1021                        List<column> mergeColumns = new ArrayList<column>();
1022                        while (columnIter.hasNext()) {
1023                                String columnName = columnIter.next();
1024                                List<column> columnList = columnMap.get(columnName);
1025                                List<column> functions = new ArrayList<column>();
1026                                for (column t : columnList) {
1027                                        if (Boolean.TRUE.toString().equals(t.getIsFunction())) {
1028                                                functions.add(t);
1029                                        }
1030                                }
1031                                if (functions != null && !functions.isEmpty()) {
1032                                        for (column function : functions) {
1033                                                mergeColumns.add(function);
1034                                                columnIdMap.put(function.getId(), function.getId());
1035                                                columnMergeIdMap.put(function.getId(), function);
1036                                        }
1037
1038                                        columnList.removeAll(functions);
1039                                }
1040                                if (!columnList.isEmpty()) {
1041                                        column firstColumn = columnList.iterator().next();
1042                                        if (columnList.size() > 1) {
1043                                                column mergeColumn = new column();
1044                                                mergeColumn.setId(String.valueOf(++startId));
1045                                                mergeColumn.setName(firstColumn.getName());
1046                                                mergeColumn.setDisplayName(firstColumn.getDisplayName());
1047                                                mergeColumn.setSource(firstColumn.getSource());
1048                                                mergeColumn.setQualifiedTable(firstColumn.getQualifiedTable());
1049                                                mergeColumns.add(mergeColumn);
1050                                                for (column item : columnList) {
1051                                                        mergeColumn.appendCoordinate(item.getCoordinate());
1052                                                        columnIdMap.put(item.getId(), mergeColumn.getId());
1053                                                        //add by grq 2023.02.06 issue=I6DB5S
1054                                                        if (item.getDataType() != null) {
1055                                                                mergeColumn.setDataType(item.getDataType());
1056                                                        }
1057                                                        if (item.isForeignKey() != null) {
1058                                                                mergeColumn.setForeignKey(item.isForeignKey());
1059                                                        }
1060                                                        if (item.isUnqiueKey() != null) {
1061                                                                mergeColumn.setUnqiueKey(item.isUnqiueKey());
1062                                                        }
1063                                                        if (item.isIndexKey() != null) {
1064                                                                mergeColumn.setIndexKey(item.isIndexKey());
1065                                                        }
1066                                                        if (item.isPrimaryKey() != null) {
1067                                                                mergeColumn.setPrimaryKey(item.isPrimaryKey());
1068                                                        }
1069                                                        //end by grq
1070                                                }
1071                                                columnMergeIdMap.put(mergeColumn.getId(), mergeColumn);
1072                                        } else {
1073                                                mergeColumns.add(firstColumn);
1074                                                columnIdMap.put(firstColumn.getId(), firstColumn.getId());
1075                                                columnMergeIdMap.put(firstColumn.getId(), firstColumn);
1076                                        }
1077                                }
1078                        }
1079                        table.setColumns(mergeColumns);
1080                }
1081
1082                Map<String, String> procedureNameToResultSetId = new HashMap<>();
1083                for (dataflow df : dataflows) {
1084                        if (df == null || df.getProcedures() == null) {
1085                                continue;
1086                        }
1087
1088                        Map<String, String> rsIdToProcId = new HashMap<>();
1089                        if (df.getResultsets() != null) {
1090                                for (table t : df.getResultsets()) {
1091                                        if (!t.isFunction() && t.getProcedureId() != null) {
1092                                                rsIdToProcId.put(t.getId(), t.getProcedureId());
1093                                        }
1094                                }
1095                        }
1096                        if (df.getTables() != null) {
1097                                for (table t : df.getTables()) {
1098                                        if (t.getProcedureId() != null) {
1099                                                rsIdToProcId.put(t.getId(), t.getProcedureId());
1100                                        }
1101                                }
1102                        }
1103                        if (df.getVariables() != null) {
1104                                for (table t : df.getVariables()) {
1105                                        if (t.getProcedureId() != null) {
1106                                                rsIdToProcId.put(t.getId(), t.getProcedureId());
1107                                        }
1108                                }
1109                        }
1110                        if (rsIdToProcId.isEmpty()) {
1111                                continue;
1112                        }
1113
1114                        Map<String, String> procIdToName = new HashMap<>();
1115                        for (procedure proc : df.getProcedures()) {
1116                                procIdToName.put(proc.getId(), DlineageUtil.getIdentifierNormalTableName(proc.getName()));
1117                        }
1118
1119                        for (Map.Entry<String, String> entry : rsIdToProcId.entrySet()) {
1120                                String procName = procIdToName.get(entry.getValue());
1121                                if (procName != null) {
1122                                        String mergedRsId = tableIdMap.get(entry.getKey());
1123                                        String resultSetId = mergedRsId != null ? mergedRsId : entry.getKey();
1124                                        procedureNameToResultSetId.put(procName, resultSetId);
1125                                }
1126                        }
1127                }
1128
1129                List<table> allTables = dataflow.getAllTables(mergeDataflow);
1130                Map<String, table> allTableIdMap = new HashMap<>(allTables.size() * 4 / 3 + 1);
1131                for (table t : allTables) {
1132                        allTableIdMap.put(t.getId(), t);
1133                }
1134
1135                Map<String, String> functionIdToResultSetId = new HashMap<>();
1136                Map<String, column> funcColIdToResultSetCol = new HashMap<>();
1137                for (table t : allTableIdMap.values()) {
1138                        if (t.isFunction()) {
1139                                String functionName = DlineageUtil.getIdentifierNormalTableName(
1140                                                DlineageUtil.getTableFullName(t.getName()));
1141                                String resultSetId = procedureNameToResultSetId.get(functionName);
1142                                if (resultSetId != null) {
1143                                        functionIdToResultSetId.put(t.getId(), resultSetId);
1144                                        table resultSet = allTableIdMap.get(resultSetId);
1145                                        if (resultSet != null && t.getColumns() != null && resultSet.getColumns() != null) {
1146                                                List<column> funcCols = t.getColumns();
1147                                                List<column> rsCols = resultSet.getColumns();
1148                                                int minSize = Math.min(funcCols.size(), rsCols.size());
1149                                                for (int i = 0; i < minSize; i++) {
1150                                                        funcColIdToResultSetCol.put(funcCols.get(i).getId(), rsCols.get(i));
1151                                                }
1152                                        }
1153                                }
1154                        }
1155                }
1156
1157                if (mergeDataflow.getRelationships() != null) {
1158                        Map<String, relationship> mergeRelations = new LinkedHashMap<String, relationship>();
1159                        for (int i = 0; i < mergeDataflow.getRelationships().size(); i++) {
1160                                relationship relation = mergeDataflow.getRelationships().get(i);
1161                                if (RelationshipType.call.name().equals(relation.getType())) {
1162                                        targetColumn target = relation.getCaller();
1163                                        List<sourceColumn> sources = relation.getCallees();
1164                                        if (target == null) {
1165                                                continue;
1166                                        }
1167                                        String mappedTargetId = tableIdMap.get(target.getId());
1168                                        if (mappedTargetId != null) {
1169                                                target.setId(mappedTargetId);
1170                                        }
1171
1172                                        Set<sourceColumn> sourceSet = new LinkedHashSet<sourceColumn>();
1173                                        if (sources != null) {
1174                                                for (sourceColumn source : sources) {
1175                                                        String mappedSourceId = tableIdMap.get(source.getId());
1176                                                        if (mappedSourceId != null) {
1177                                                                source.setId(mappedSourceId);
1178                                                        }
1179                                                }
1180                                                sourceSet.addAll(sources);
1181                                                relation.setCallees(new ArrayList<sourceColumn>(sourceSet));
1182                                        }
1183
1184                                        String key = relation.toDedupKey();
1185                                        mergeRelations.putIfAbsent(key, relation);
1186                                } else {
1187                                        targetColumn target = relation.getTarget();
1188                                        if (target == null) {
1189                                                continue;
1190                                        }
1191                                        String mappedTargetParentId = tableIdMap.get(target.getParent_id());
1192                                        if (mappedTargetParentId != null) {
1193                                                target.setParent_id(mappedTargetParentId);
1194                                        }
1195
1196                                        String mappedTargetColId = columnIdMap.get(target.getId());
1197                                        if (mappedTargetColId != null) {
1198                                                target.setId(mappedTargetColId);
1199                                                column mergedCol = columnMergeIdMap.get(mappedTargetColId);
1200                                                if (mergedCol != null) {
1201                                                        target.setCoordinate(mergedCol.getCoordinate());
1202                                                }
1203                                        }
1204
1205                                        List<sourceColumn> sources = relation.getSources();
1206                                        Set<sourceColumn> sourceSet = new LinkedHashSet<sourceColumn>();
1207                                        // Same contract as DataFlowAnalyzer.mergeTables: dedup runs
1208                                        // under the merged column's stamped coordinate, but a source's
1209                                        // own usable reference-site coordinate is restored on the
1210                                        // surviving relation.
1211                                        Map<sourceColumn, String> referenceCoordinates = new IdentityHashMap<sourceColumn, String>();
1212                                        if (sources != null) {
1213                                                for (sourceColumn source : sources) {
1214                                                        String mappedParentId = tableIdMap.get(source.getParent_id());
1215                                                        if (mappedParentId != null) {
1216                                                                source.setParent_id(mappedParentId);
1217                                                        }
1218                                                        String mappedSourceId = tableIdMap.get(source.getSource_id());
1219                                                        if (mappedSourceId != null) {
1220                                                                source.setSource_id(mappedSourceId);
1221                                                        }
1222                                                        String originalParentId = source.getParent_id();
1223                                                        if (!functionIdToResultSetId.isEmpty()) {
1224                                                                String resultSetId = functionIdToResultSetId.get(originalParentId);
1225                                                                if (resultSetId != null) {
1226                                                                        source.setParent_id(resultSetId);
1227                                                                        column matched = funcColIdToResultSetCol.get(source.getId());
1228                                                                        if (matched != null) {
1229                                                                                source.setId(matched.getId());
1230                                                                                source.setColumn(matched.getName());
1231                                                                        }
1232                                                                        table resultSetTable = allTableIdMap.get(source.getParent_id());
1233                                                                        if (resultSetTable != null) {
1234                                                                                source.setParent_name(resultSetTable.getName());
1235                                                                        }
1236                                                                }
1237                                                        }
1238                                                        String mappedColId = columnIdMap.get(source.getId());
1239                                                        if (mappedColId != null) {
1240                                                                source.setId(mappedColId);
1241                                                                column mergedCol = columnMergeIdMap.get(mappedColId);
1242                                                                if (mergedCol != null) {
1243                                                                        String ownCoordinate = source.getCoordinate();
1244                                                                        source.setCoordinate(mergedCol.getCoordinate());
1245                                                                        if (DlineageUtil.hasUsableCoordinate(ownCoordinate)) {
1246                                                                                referenceCoordinates.put(source, ownCoordinate);
1247                                                                        }
1248                                                                }
1249                                                        }
1250                                                }
1251
1252                                                sourceSet.addAll(sources);
1253                                                relation.setSources(new ArrayList<sourceColumn>(sourceSet));
1254                                        }
1255
1256                                        String key = relation.toDedupKey();
1257                                        if (mergeRelations.putIfAbsent(key, relation) == null) {
1258                                                for (Map.Entry<sourceColumn, String> entry : referenceCoordinates.entrySet()) {
1259                                                        entry.getKey().setCoordinate(entry.getValue());
1260                                                }
1261                                        }
1262                                }
1263                        }
1264
1265                        mergeDataflow.setRelationships(new ArrayList<relationship>(mergeRelations.values()));
1266                }
1267
1268                startId = addStarColumnBridgingRelations(mergeDataflow, startId);
1269
1270                tableMap.clear();
1271                tableTypeMap.clear();
1272                tableIdMap.clear();
1273                columnMap.clear();
1274                tableColumnMap.clear();
1275                columnIdMap.clear();
1276                columnMergeIdMap.clear();
1277                tables.clear();
1278
1279                return mergeDataflow;
1280        }
1281
1282        /**
1283         * 合并后同一张表可能同时拥有 * 列（来自只做 SELECT * 的来源）和具名列（来自显式引用列名的来源），
1284         * 但缺少 * -> 具名列 的 fdd 关系，会导致 上游.* -> 表.* 与 表.具名列 -> 下游.具名列 之间链路断开。
1285         * 本方法为这类表补齐 * -> 每个具名列 的 fdd 桥接（已存在则跳过；忽略 system / function 列）。
1286         */
1287        private static long addStarColumnBridgingRelations(dataflow df, long startId) {
1288                if (df == null) {
1289                        return startId;
1290                }
1291                Map<String, Set<String>> bridgeIndex = new HashMap<>();
1292                if (df.getRelationships() != null) {
1293                        for (relationship rel : df.getRelationships()) {
1294                                if (!RelationshipType.fdd.name().equals(rel.getType())) {
1295                                        continue;
1296                                }
1297                                if (rel.getTarget() == null || rel.getSources() == null) {
1298                                        continue;
1299                                }
1300                                String parentId = rel.getTarget().getParent_id();
1301                                if (parentId == null) {
1302                                        continue;
1303                                }
1304                                for (sourceColumn sc : rel.getSources()) {
1305                                        if (parentId.equals(sc.getParent_id())) {
1306                                                bridgeIndex.computeIfAbsent(parentId, k -> new HashSet<>())
1307                                                                .add(rel.getTarget().getId() + "<-" + sc.getId());
1308                                        }
1309                                }
1310                        }
1311                }
1312
1313                List<relationship> newRelations = new ArrayList<>();
1314                for (table t : dataflow.getAllTables(df)) {
1315                        if (t.getColumns() == null || t.getColumns().size() < 2) {
1316                                continue;
1317                        }
1318                        column starColumn = null;
1319                        List<column> regularColumns = new ArrayList<>();
1320                        for (column c : t.getColumns()) {
1321                                if ("system".equals(c.getSource())) {
1322                                        continue;
1323                                }
1324                                if (Boolean.TRUE.toString().equals(c.getIsFunction())) {
1325                                        continue;
1326                                }
1327                                if ("*".equals(c.getName())) {
1328                                        if (starColumn == null) {
1329                                                starColumn = c;
1330                                        }
1331                                } else {
1332                                        regularColumns.add(c);
1333                                }
1334                        }
1335                        if (starColumn == null || regularColumns.isEmpty()) {
1336                                continue;
1337                        }
1338
1339                        Set<String> existingBridges = bridgeIndex.getOrDefault(t.getId(), Collections.emptySet());
1340
1341                        String starId = starColumn.getId();
1342                        for (column target : regularColumns) {
1343                                if (existingBridges.contains(target.getId() + "<-" + starId)) {
1344                                        continue;
1345                                }
1346                                relationship rel = new relationship();
1347                                rel.setId(String.valueOf(++startId));
1348                                rel.setType(RelationshipType.fdd.name());
1349                                rel.setEffectType("expand_star");
1350
1351                                targetColumn tc = new targetColumn();
1352                                tc.setId(target.getId());
1353                                tc.setColumn(target.getName());
1354                                tc.setParent_id(t.getId());
1355                                tc.setParent_name(t.getName());
1356                                rel.setTarget(tc);
1357
1358                                sourceColumn sc = new sourceColumn();
1359                                sc.setId(starId);
1360                                sc.setColumn(starColumn.getName());
1361                                sc.setParent_id(t.getId());
1362                                sc.setParent_name(t.getName());
1363                                rel.setSources(Arrays.asList(sc));
1364
1365                                newRelations.add(rel);
1366                        }
1367                }
1368                if (!newRelations.isEmpty()) {
1369                        List<relationship> all = df.getRelationships() != null
1370                                        ? new ArrayList<>(df.getRelationships())
1371                                        : new ArrayList<>();
1372                        all.addAll(newRelations);
1373                        df.setRelationships(all);
1374                }
1375                return startId;
1376        }
1377
1378        /**
1379         * 单个 dataflow 的 ID 跨度（与 ParallelDataFlowAnalyzer 中的常量保持一致）。
1380         */
1381        private static final long DATAFLOW_ID_RANGE = 5000000L;
1382
1383        /**
1384         * 合并一组 startId 都为 0（即 ID 空间相互冲突）的 dataflow。
1385         * <p>
1386         * 标准合并 {@link #mergeDataflowsWithDifferentStartId(Collection, EDbVendor)} 的前提是
1387         * 每个 dataflow 在生成时已通过 {@code optionCopy.setStartId(5000000L * i)} 占用了
1388         * 不重叠的 ID 段；当外部调用方拿到的 dataflow 都是用默认 startId=0 生成时，直接合并
1389         * 会因为 ID 冲突而错乱。本方法会先按下标把每个 dataflow 的全部 ID 偏移
1390         * {@link #DATAFLOW_ID_RANGE} * index，再委托给标准合并逻辑。
1391         * <p>
1392         * 注意：此方法会原地修改入参 dataflow 的 ID（与标准合并行为一致）。
1393         */
1394        public static dataflow mergeDataflows(Collection<dataflow> dataflows, EDbVendor vendor) {
1395                if (dataflows == null || dataflows.isEmpty()) {
1396                        return null;
1397                }
1398                ModelBindingManager.setGlobalVendor(vendor);
1399                boolean optionSet = false;
1400                if (ModelBindingManager.getGlobalOption() == null) {
1401                        Option option = new Option();
1402                        option.setVendor(vendor);
1403                        ModelBindingManager.setGlobalOption(option);
1404                        optionSet = true;
1405                }
1406                try {
1407                        // ID 按下标 * DATAFLOW_ID_RANGE 偏移，避免 ID 冲突；
1408                        // coordinate 第三维 fileIdx 按前面所有 dataflow 累积的源文件数偏移，
1409                        // 使得合并后每个来源的 fileIdx 保持连续且不重叠。
1410                        long cumulativeFileIdx = 0L;
1411                        int i = 0;
1412                        for (dataflow df : dataflows) {
1413                                if (df != null) {
1414                                        int maxFileIdx = findMaxFileIdx(df);
1415                                        offsetDataflowIds(df, DATAFLOW_ID_RANGE * i, cumulativeFileIdx);
1416                                        if (maxFileIdx >= 0) {
1417                                                cumulativeFileIdx += (maxFileIdx + 1L);
1418                                        }
1419                                }
1420                                i++;
1421                        }
1422                        return mergeDataflowsWithDifferentStartId(dataflows, vendor);
1423                } finally {
1424                        ModelBindingManager.removeGlobalVendor();
1425                        if (optionSet) {
1426                                ModelBindingManager.removeGlobalOption();
1427                        }
1428                }
1429        }
1430
1431        /**
1432         * 从一组临时 XML 文件迭代加载并合并 dataflow，节约内存。
1433         * <p>
1434         * 与 {@link #iterativeMergeDataflowsFromFilesByStartId(List, long)} 不同的是，本方法假定文件中的
1435         * dataflow 是用默认 startId=0 生成的（ID 空间互相冲突），因此会按下标先做 ID 偏移再合并。
1436         * <p>
1437         * 注意：本方法按入参 {@code tempFiles} 顺序迭代合并，不做任何排序。
1438         * coordinate 第三维 fileIdx 的偏移也是严格按入参顺序累积的，
1439         * 调用方传入的顺序应与原始源文件的语义顺序保持一致。
1440         */
1441        public static dataflow iterativeMergeDataflowsFromFiles(List<File> tempFiles, EDbVendor vendor) {
1442                if (tempFiles == null || tempFiles.isEmpty()) {
1443                        return null;
1444                }
1445                ModelBindingManager.setGlobalVendor(vendor);
1446                boolean optionSet = false;
1447                if (ModelBindingManager.getGlobalOption() == null) {
1448                        Option option = new Option();
1449                        option.setVendor(vendor);
1450                        ModelBindingManager.setGlobalOption(option);
1451                        optionSet = true;
1452                }
1453                try {
1454                        return iterativeMergeFromFiles(tempFiles, DATAFLOW_ID_RANGE * tempFiles.size(), true);
1455                } finally {
1456                        ModelBindingManager.removeGlobalVendor();
1457                        if (optionSet) {
1458                                ModelBindingManager.removeGlobalOption();
1459                        }
1460                }
1461        }
1462
1463        /**
1464         * 从一组临时 XML 文件迭代加载并合并 dataflow，调用方需保证文件中的 dataflow ID 空间互不冲突，
1465         * 且已设置好 ModelBindingManager。
1466         * <p>
1467         * 主要供 {@code ParallelDataFlowAnalyzer} 等内部生成方使用：每个临时文件中的 dataflow 已通过
1468         * {@code optionCopy.setStartId(DATAFLOW_ID_RANGE * i)} 占用了不重叠的 ID 段。
1469         * <p>
1470         * 注意：本方法按入参 {@code tempFiles} 顺序迭代合并，不做任何排序。
1471         * coordinate 第三维 fileIdx 的偏移严格按入参顺序累积。
1472         */
1473        public static dataflow iterativeMergeDataflowsFromFilesByStartId(List<File> tempFiles, long startId) {
1474                return iterativeMergeFromFiles(tempFiles, startId, false);
1475        }
1476
1477        private static dataflow iterativeMergeFromFiles(List<File> tempFiles, long startId, boolean offsetIds) {
1478                if (tempFiles == null || tempFiles.isEmpty()) {
1479                        logger.warn("iterativeMergeDataflowsFromFiles: tempFiles is null or empty");
1480                        return null;
1481                }
1482
1483                logger.info("iterativeMergeDataflowsFromFiles: total dataflows: " + tempFiles.size());
1484
1485                long startTime = System.currentTimeMillis();
1486                int totalIterations = tempFiles.size() - 1;
1487                logger.info("iterativeMergeDataflowsFromFiles: start merging, total iterations: " + totalIterations);
1488
1489                long loadStartTime = System.currentTimeMillis();
1490                dataflow mergedDataflow = XML2Model.loadXML(dataflow.class, tempFiles.get(0));
1491                long cumulativeFileIdx = 0L;
1492                if (mergedDataflow != null) {
1493                        long idOffset = offsetIds ? 0L : 0L;
1494                        offsetDataflowIds(mergedDataflow, idOffset, 0L);
1495                        int maxFileIdx = findMaxFileIdx(mergedDataflow);
1496                        if (maxFileIdx >= 0) {
1497                                cumulativeFileIdx = maxFileIdx + 1L;
1498                        }
1499                }
1500                long loadEndTime = System.currentTimeMillis();
1501                int initialRelationCount = mergedDataflow != null && mergedDataflow.getRelationships() != null ? mergedDataflow.getRelationships().size() : 0;
1502                logger.info("iterativeMergeDataflowsFromFiles: loaded initial dataflow, relation count: " + initialRelationCount + ", time: " + (loadEndTime - loadStartTime) + "ms");
1503
1504                Set<Long> mergedDedupKeys = new HashSet<>(Math.max(initialRelationCount, 16));
1505                if (mergedDataflow != null && mergedDataflow.getRelationships() != null) {
1506                        for (relationship rel : mergedDataflow.getRelationships()) {
1507                                mergedDedupKeys.add(rel.toDedupHash());
1508                        }
1509                }
1510
1511                long currentMaxId = startId;
1512                for (int i = 1; i < tempFiles.size(); i++) {
1513                        long iterationStartTime = System.currentTimeMillis();
1514                        int currentIteration = i;
1515                        int remainingIterations = totalIterations - currentIteration + 1;
1516
1517                        logger.info("iterativeMergeDataflowsFromFiles: iteration " + currentIteration + "/" + totalIterations + ", remaining: " + remainingIterations);
1518
1519                        long loadCurrentStartTime = System.currentTimeMillis();
1520                        dataflow currentDataflow = XML2Model.loadXML(dataflow.class, tempFiles.get(i));
1521                        if (currentDataflow != null) {
1522                                int maxFileIdx = findMaxFileIdx(currentDataflow);
1523                                long idOffset = offsetIds ? (DATAFLOW_ID_RANGE * i) : 0L;
1524                                offsetDataflowIds(currentDataflow, idOffset, cumulativeFileIdx);
1525                                if (maxFileIdx >= 0) {
1526                                        cumulativeFileIdx += (maxFileIdx + 1L);
1527                                }
1528                        }
1529                        long loadCurrentEndTime = System.currentTimeMillis();
1530                        int currentRelationCount = currentDataflow != null && currentDataflow.getRelationships() != null ? currentDataflow.getRelationships().size() : 0;
1531                        logger.info("iterativeMergeDataflowsFromFiles: loaded dataflow[" + i + "], relation count: " + currentRelationCount + ", time: " + (loadCurrentEndTime - loadCurrentStartTime) + "ms");
1532
1533                        long mergeStartTime = System.currentTimeMillis();
1534                        //autogenerate 首个 dataflow 加载失败(mergedDataflow==null)时，以首个可用 dataflow 作为合并基准并初始化去重索引
1535                        if (mergedDataflow == null && currentDataflow != null) {
1536                                mergedDataflow = currentDataflow;
1537                                if (mergedDataflow.getRelationships() != null) {
1538                                        for (relationship rel : mergedDataflow.getRelationships()) {
1539                                                mergedDedupKeys.add(rel.toDedupHash());
1540                                        }
1541                                }
1542                        } else {
1543                                currentMaxId = incrementalMergeDataflow(mergedDataflow, currentDataflow, currentMaxId, mergedDedupKeys);
1544                        }
1545                        long mergeEndTime = System.currentTimeMillis();
1546                        int mergedRelationCount = mergedDataflow != null && mergedDataflow.getRelationships() != null ? mergedDataflow.getRelationships().size() : 0;
1547
1548                        long iterationEndTime = System.currentTimeMillis();
1549                        long iterationTime = iterationEndTime - iterationStartTime;
1550                        long mergeTime = mergeEndTime - mergeStartTime;
1551                        logger.info("iterativeMergeDataflowsFromFiles: iteration " + currentIteration + " completed, merged relation count: " + mergedRelationCount + ", merge time: " + mergeTime + "ms, total iteration time: " + iterationTime + "ms");
1552
1553                        currentDataflow = null;
1554                }
1555
1556                long endTime = System.currentTimeMillis();
1557                long totalTime = endTime - startTime;
1558                int finalRelationCount = mergedDataflow != null && mergedDataflow.getRelationships() != null ? mergedDataflow.getRelationships().size() : 0;
1559                logger.info("iterativeMergeDataflowsFromFiles: all iterations completed, final relation count: " + finalRelationCount + ", total time: " + totalTime + "ms (" + (totalTime / 1000.0) + "s)");
1560
1561                return mergedDataflow;
1562        }
1563
1564        /**
1565         * 增量合并：将 {@code currentDataflow} 合并到 {@code mergedDataflow} 中，原地修改 mergedDataflow。
1566         * <p>
1567         * 与 {@link #mergeDataflowsByStartId} 不同，本方法保持 mergedDataflow 中已有表/列的 ID 不变，
1568         * 只将 currentDataflow 中同名表/列的属性追加到已有表/列上，已有关系不做任何处理。
1569         * 仅对 currentDataflow 的关系做 ID 重映射和去重后追加到 mergedDataflow。
1570         *
1571         * @param mergedDataflow  已合并的 dataflow（原地修改）
1572         * @param currentDataflow 新加载的 dataflow
1573         * @param currentMaxId    当前最大 ID，用于生成新 ID
1574         * @param mergedDedupKeys 已有关系的去重 hash 集合（原地修改）
1575         * @return 更新后的最大 ID
1576         */
1577        private static long incrementalMergeDataflow(dataflow mergedDataflow, dataflow currentDataflow,
1578                                                                                                  long currentMaxId, Set<Long> mergedDedupKeys) {
1579                //autogenerate mergedDataflow 为 null 时不可解引用，避免首个 dataflow 加载失败引发 NPE
1580                if (currentDataflow == null || mergedDataflow == null) {
1581                        return currentMaxId;
1582                }
1583
1584                // 1. 构建 mergedDataflow 已有表的名称索引
1585                Map<String, table> existingTableMap = new LinkedHashMap<>();
1586                for (table t : dataflow.getAllTables(mergedDataflow)) {
1587                        String tableFullName = getTableFullNameKey(t);
1588                        if (!existingTableMap.containsKey(tableFullName)) {
1589                                existingTableMap.put(tableFullName, t);
1590                        }
1591                }
1592
1593                // 2. 处理 currentDataflow 的表和列，构建 ID 映射（只映射新ID → 已有ID）
1594                Map<String, String> tableIdMap = new HashMap<>();
1595                Map<String, String> columnIdMap = new HashMap<>();
1596                Map<String, column> columnMergeIdMap = new HashMap<>();
1597                Set<String> newOrModifiedTableIds = new HashSet<>();
1598
1599                for (table t : dataflow.getAllTables(currentDataflow)) {
1600                        String tableFullName = getTableFullNameKey(t);
1601                        table existingTable = existingTableMap.get(tableFullName);
1602
1603                        if (existingTable != null && !t.isFunction()) {
1604                                // 同名表：合并到已有表，已有表 ID 保持不变
1605                                tableIdMap.put(t.getId(), existingTable.getId());
1606                                mergeTableProperties(existingTable, t);
1607                                mergeColumnsIncremental(existingTable, t, columnIdMap, columnMergeIdMap);
1608                                newOrModifiedTableIds.add(existingTable.getId());
1609                        } else {
1610                                // 新表：直接追加到 mergedDataflow
1611                                addTableToMergedDataflow(mergedDataflow, t);
1612                                existingTableMap.put(tableFullName, t);
1613                                tableIdMap.put(t.getId(), t.getId());
1614                                newOrModifiedTableIds.add(t.getId());
1615                                if (t.getColumns() != null) {
1616                                        for (column c : t.getColumns()) {
1617                                                columnIdMap.put(c.getId(), c.getId());
1618                                                columnMergeIdMap.put(c.getId(), c);
1619                                        }
1620                                }
1621                        }
1622                }
1623
1624                // 3. 追加其他实体
1625                appendOtherEntities(mergedDataflow, currentDataflow);
1626
1627                // 4. 构建 function→resultSet 映射（基于合并后的完整数据）
1628                Map<String, String> functionIdToResultSetId = new HashMap<>();
1629                Map<String, column> funcColIdToResultSetCol = new HashMap<>();
1630                Map<String, table> allTableIdMap = new HashMap<>();
1631                buildFunctionResultSetMapping(mergedDataflow, functionIdToResultSetId,
1632                                funcColIdToResultSetCol, allTableIdMap);
1633
1634                // 5. 只处理 currentDataflow 的关系
1635                if (currentDataflow.getRelationships() != null) {
1636                        List<relationship> newRelations = new ArrayList<>();
1637                        for (relationship rel : currentDataflow.getRelationships()) {
1638                                remapAndDedupRelation(rel, tableIdMap, columnIdMap, columnMergeIdMap,
1639                                                functionIdToResultSetId, funcColIdToResultSetCol, allTableIdMap,
1640                                                mergedDedupKeys, newRelations);
1641                        }
1642                        if (!newRelations.isEmpty()) {
1643                                if (mergedDataflow.getRelationships() == null) {
1644                                        mergedDataflow.setRelationships(new ArrayList<relationship>());
1645                                }
1646                                mergedDataflow.getRelationships().addAll(newRelations);
1647                        }
1648                }
1649
1650                // 6. 为新表或修改过的表补齐 * 列桥接关系
1651                if (!newOrModifiedTableIds.isEmpty()) {
1652                        currentMaxId = addStarColumnBridgingRelationsForTables(mergedDataflow, currentMaxId,
1653                                        newOrModifiedTableIds);
1654                }
1655
1656                return currentMaxId;
1657        }
1658
1659        /**
1660         * 获取表的全名 key，用于同名表匹配。
1661         */
1662        private static String getTableFullNameKey(table table) {
1663                String qualifiedTableName = DlineageUtil.getQualifiedTableName(table);
1664                String tableFullName = DlineageUtil.getIdentifierNormalTableName(qualifiedTableName);
1665                if ("variable".endsWith(table.getType()) && !SQLUtil.isEmpty(table.getParent())) {
1666                        tableFullName = table.getParent() + "." + tableFullName;
1667                }
1668                return tableFullName;
1669        }
1670
1671        /**
1672         * 将 newTable 的属性合并到 existingTable 上（已有表 ID 保持不变）。
1673         */
1674        private static void mergeTableProperties(table existingTable, table newTable) {
1675                if (!SQLUtil.isEmpty(newTable.getCoordinate())) {
1676                        if (SQLUtil.isEmpty(existingTable.getCoordinate())) {
1677                                existingTable.setCoordinate(newTable.getCoordinate());
1678                        } else if (existingTable.getCoordinate().indexOf(newTable.getCoordinate()) == -1) {
1679                                existingTable.appendCoordinate(newTable.getCoordinate());
1680                        }
1681                }
1682                if (!SQLUtil.isEmpty(newTable.getAlias())) {
1683                        if (SQLUtil.isEmpty(existingTable.getAlias())) {
1684                                existingTable.setAlias(newTable.getAlias());
1685                        } else if (!existingTable.getAlias().contains(newTable.getAlias())) {
1686                                existingTable.setAlias(existingTable.getAlias() + "," + newTable.getAlias());
1687                        }
1688                }
1689                if (newTable.getProcessIds() != null) {
1690                        if (existingTable.getProcessIds() == null) {
1691                                existingTable.setProcessIds(new ArrayList<>(newTable.getProcessIds()));
1692                        } else {
1693                                Set<String> merged = new LinkedHashSet<>(existingTable.getProcessIds());
1694                                merged.addAll(newTable.getProcessIds());
1695                                existingTable.setProcessIds(new ArrayList<>(merged));
1696                        }
1697                }
1698                if (newTable.getProcedureId() != null && existingTable.getProcedureId() == null) {
1699                        existingTable.setProcedureId(newTable.getProcedureId());
1700                }
1701                mergeEndpointClassificationPair(existingTable, newTable);
1702        }
1703
1704        /** Canonical outermost-first order of {@code templatedParts} labels. */
1705        private static final String[] TEMPLATED_PART_ORDER = { "server", "db", "schema", "name" };
1706
1707        private static final String DYNAMIC_TEMPLATE_KIND = "DYNAMIC_TEMPLATE";
1708
1709        /**
1710         * Carries the authoritative endpoint classification through a same-name merge instead of
1711         * silently dropping it (docs/tmp/dynamic-template-endpoint-provenance.md, merge finding).
1712         * The name-derived facts (kind, normalizedName) and the create-effect facts
1713         * (createdInSql, endpointIntroduction) survive as "any occurrence states it".
1714         *
1715         * <p>Template provenance is stricter, and "templated" means {@code templatedParts} is
1716         * present — NOT {@code endpointKind == DYNAMIC_TEMPLATE}, because a templated literal-temp
1717         * ({@code [#t_' + @X + ']}) keeps its temp kind while carrying parts. Parts survive only
1718         * when EVERY occurrence is templated; one plain occurrence means a STATIC statement also
1719         * references this name, so the merged node keeps the PLAIN occurrence's kind (making the
1720         * outcome independent of merge order and merge API) and drops the template claim — the
1721         * shared-node contract.
1722         */
1723        private static void mergeEndpointClassification(table merged, List<table> occurrences) {
1724                boolean created = false;
1725                String introduction = null;
1726                String normalized = null;
1727                String templatedKind = null;
1728                String plainKind = null;
1729                boolean anyTemplate = false;
1730                boolean allTemplate = true;
1731                Set<String> parts = new LinkedHashSet<String>();
1732                for (table occurrence : occurrences) {
1733                        // non-identifier-compare: fixed enum-name/flag wire values, not DB object names
1734                        if ("true".equals(occurrence.getCreatedInSql())) {
1735                                created = true;
1736                        }
1737                        if (introduction == null) {
1738                                introduction = occurrence.getEndpointIntroduction();
1739                        }
1740                        if (normalized == null) {
1741                                normalized = occurrence.getNormalizedName();
1742                        }
1743                        if (occurrence.getTemplatedParts() != null) {
1744                                anyTemplate = true;
1745                                addTemplatedParts(parts, occurrence.getTemplatedParts());
1746                                if (templatedKind == null) {
1747                                        templatedKind = occurrence.getEndpointKind();
1748                                }
1749                        } else {
1750                                allTemplate = false;
1751                                if (plainKind == null) {
1752                                        plainKind = occurrence.getEndpointKind();
1753                                }
1754                        }
1755                }
1756                if (created) {
1757                        merged.setCreatedInSql("true");
1758                }
1759                if (introduction != null) {
1760                        merged.setEndpointIntroduction(introduction);
1761                }
1762                if (normalized != null) {
1763                        merged.setNormalizedName(normalized);
1764                }
1765                if (anyTemplate && allTemplate) {
1766                        merged.setTemplatedParts(joinTemplatedParts(parts));
1767                        if (templatedKind != null) {
1768                                merged.setEndpointKind(templatedKind);
1769                        }
1770                } else if (plainKind != null) {
1771                        // No template, or mixed static/templated: the plain occurrence's kind stands.
1772                        merged.setEndpointKind(plainKind);
1773                }
1774        }
1775
1776        /** Pairwise form of {@link #mergeEndpointClassification} for the incremental merge. */
1777        private static void mergeEndpointClassificationPair(table existingTable, table newTable) {
1778                // non-identifier-compare: fixed enum-name/flag wire values, not DB object names
1779                if ("true".equals(newTable.getCreatedInSql()) && existingTable.getCreatedInSql() == null) {
1780                        existingTable.setCreatedInSql("true");
1781                }
1782                if (existingTable.getEndpointIntroduction() == null) {
1783                        existingTable.setEndpointIntroduction(newTable.getEndpointIntroduction());
1784                }
1785                if (existingTable.getNormalizedName() == null) {
1786                        existingTable.setNormalizedName(newTable.getNormalizedName());
1787                }
1788                boolean existingTemplate = existingTable.getTemplatedParts() != null;
1789                boolean incomingTemplate = newTable.getTemplatedParts() != null;
1790                if (existingTemplate && incomingTemplate) {
1791                        Set<String> parts = new LinkedHashSet<String>();
1792                        addTemplatedParts(parts, existingTable.getTemplatedParts());
1793                        addTemplatedParts(parts, newTable.getTemplatedParts());
1794                        existingTable.setTemplatedParts(joinTemplatedParts(parts));
1795                        if (existingTable.getEndpointKind() == null) {
1796                                existingTable.setEndpointKind(newTable.getEndpointKind());
1797                        }
1798                } else if (existingTemplate) {
1799                        // A plain occurrence of the same name arrived: mixed static/templated. The
1800                        // template claim is withdrawn and the PLAIN side's kind stands — the same
1801                        // outcome the reverse arrival order and the batch merge produce. Monotone:
1802                        // once withdrawn, a later templated occurrence stays out.
1803                        existingTable.setTemplatedParts(null);
1804                        existingTable.setEndpointKind(newTable.getEndpointKind());
1805                } else if (incomingTemplate) {
1806                        // Mixed, other order: keep the existing (plain) side's kind; never adopt the
1807                        // templated side's kind or parts.
1808                } else if (existingTable.getEndpointKind() == null) {
1809                        existingTable.setEndpointKind(newTable.getEndpointKind());
1810                }
1811        }
1812
1813        private static void addTemplatedParts(Set<String> parts, String joined) {
1814                if (SQLUtil.isEmpty(joined)) {
1815                        return;
1816                }
1817                for (String part : joined.split("\\|")) {
1818                        if (!part.isEmpty()) {
1819                                parts.add(part);
1820                        }
1821                }
1822        }
1823
1824        private static String joinTemplatedParts(Set<String> parts) {
1825                StringBuilder sb = new StringBuilder();
1826                for (String part : TEMPLATED_PART_ORDER) {
1827                        if (parts.contains(part)) {
1828                                if (sb.length() > 0) {
1829                                        sb.append('|');
1830                                }
1831                                sb.append(part);
1832                        }
1833                }
1834                return sb.length() == 0 ? null : sb.toString();
1835        }
1836
1837        /**
1838         * 将 newTable 的列合并到 existingTable 上（已有列 ID 保持不变）。
1839         */
1840        private static void mergeColumnsIncremental(table existingTable, table newTable,
1841                                                                                                 Map<String, String> columnIdMap,
1842                                                                                                 Map<String, column> columnMergeIdMap) {
1843                if (newTable.getColumns() == null) {
1844                        return;
1845                }
1846                if (existingTable.getColumns() == null) {
1847                        existingTable.setColumns(new ArrayList<column>());
1848                }
1849
1850                Map<String, column> existingColMap = new LinkedHashMap<>();
1851                for (column c : existingTable.getColumns()) {
1852                        existingColMap.put(c.getName(), c);
1853                }
1854
1855                for (column newCol : newTable.getColumns()) {
1856                        if (Boolean.TRUE.toString().equals(newCol.getIsFunction())) {
1857                                // function 列直接追加，不做合并
1858                                existingTable.getColumns().add(newCol);
1859                                columnIdMap.put(newCol.getId(), newCol.getId());
1860                                columnMergeIdMap.put(newCol.getId(), newCol);
1861                                continue;
1862                        }
1863
1864                        column existingCol = existingColMap.get(newCol.getName());
1865                        if (existingCol != null) {
1866                                // 同名列：合并到已有列，已有列 ID 保持不变
1867                                columnIdMap.put(newCol.getId(), existingCol.getId());
1868                                if (!SQLUtil.isEmpty(newCol.getCoordinate())) {
1869                                        if (SQLUtil.isEmpty(existingCol.getCoordinate())) {
1870                                                existingCol.setCoordinate(newCol.getCoordinate());
1871                                        } else if (existingCol.getCoordinate().indexOf(newCol.getCoordinate()) == -1) {
1872                                                existingCol.appendCoordinate(newCol.getCoordinate());
1873                                        }
1874                                }
1875                                if (newCol.getDataType() != null) {
1876                                        existingCol.setDataType(newCol.getDataType());
1877                                }
1878                                if (newCol.isForeignKey() != null) {
1879                                        existingCol.setForeignKey(newCol.isForeignKey());
1880                                }
1881                                if (newCol.isUnqiueKey() != null) {
1882                                        existingCol.setUnqiueKey(newCol.isUnqiueKey());
1883                                }
1884                                if (newCol.isIndexKey() != null) {
1885                                        existingCol.setIndexKey(newCol.isIndexKey());
1886                                }
1887                                if (newCol.isPrimaryKey() != null) {
1888                                        existingCol.setPrimaryKey(newCol.isPrimaryKey());
1889                                }
1890                        } else {
1891                                // 新列：直接追加
1892                                existingTable.getColumns().add(newCol);
1893                                existingColMap.put(newCol.getName(), newCol);
1894                                columnIdMap.put(newCol.getId(), newCol.getId());
1895                                columnMergeIdMap.put(newCol.getId(), newCol);
1896                        }
1897                }
1898        }
1899
1900        /**
1901         * 将 table 按类型追加到 mergedDataflow 对应的列表中。
1902         */
1903        private static void addTableToMergedDataflow(dataflow df, table t) {
1904                if (t.isView()) {
1905                        if (df.getViews() == null) df.setViews(new ArrayList<table>());
1906                        df.getViews().add(t);
1907                } else if (t.isDatabaseType()) {
1908                        if (df.getDatabases() == null) df.setDatabases(new ArrayList<table>());
1909                        df.getDatabases().add(t);
1910                } else if (t.isSchemaType()) {
1911                        if (df.getSchemas() == null) df.setSchemas(new ArrayList<table>());
1912                        df.getSchemas().add(t);
1913                } else if (t.isStage()) {
1914                        if (df.getStages() == null) df.setStages(new ArrayList<table>());
1915                        df.getStages().add(t);
1916                } else if (t.isDataSource()) {
1917                        if (df.getDatasources() == null) df.setDatasources(new ArrayList<table>());
1918                        df.getDatasources().add(t);
1919                } else if (t.isStream()) {
1920                        if (df.getStreams() == null) df.setStreams(new ArrayList<table>());
1921                        df.getStreams().add(t);
1922                } else if (t.isFile()) {
1923                        if (df.getPaths() == null) df.setPaths(new ArrayList<table>());
1924                        df.getPaths().add(t);
1925                } else if (t.isVariable()) {
1926                        if (df.getVariables() == null) df.setVariables(new ArrayList<table>());
1927                        df.getVariables().add(t);
1928                } else if (t.isResultSet()) {
1929                        if (df.getResultsets() == null) df.setResultsets(new ArrayList<table>());
1930                        df.getResultsets().add(t);
1931                } else if (t.isSequence()) {
1932                        if (df.getSequences() == null) df.setSequences(new ArrayList<table>());
1933                        df.getSequences().add(t);
1934                } else {
1935                        if (df.getTables() == null) df.setTables(new ArrayList<table>());
1936                        df.getTables().add(t);
1937                }
1938        }
1939
1940        /**
1941         * 将 currentDataflow 的非表实体追加到 mergedDataflow。
1942         */
1943        private static void appendOtherEntities(dataflow mergedDataflow, dataflow currentDataflow) {
1944                if (currentDataflow.getProcesses() != null) {
1945                        if (mergedDataflow.getProcesses() == null) {
1946                                mergedDataflow.setProcesses(new ArrayList<process>());
1947                        }
1948                        mergedDataflow.getProcesses().addAll(currentDataflow.getProcesses());
1949                }
1950                if (currentDataflow.getProcedures() != null) {
1951                        if (mergedDataflow.getProcedures() == null) {
1952                                mergedDataflow.setProcedures(new ArrayList<procedure>());
1953                        }
1954                        mergedDataflow.getProcedures().addAll(currentDataflow.getProcedures());
1955                }
1956                if (currentDataflow.getPackages() != null) {
1957                        if (mergedDataflow.getPackages() == null) {
1958                                mergedDataflow.setPackages(new ArrayList<oraclePackage>());
1959                        }
1960                        mergedDataflow.getPackages().addAll(currentDataflow.getPackages());
1961                }
1962                if (currentDataflow.getErrors() != null) {
1963                        if (mergedDataflow.getErrors() == null) {
1964                                mergedDataflow.setErrors(new ArrayList<error>());
1965                        }
1966                        mergedDataflow.getErrors().addAll(currentDataflow.getErrors());
1967                        if (mergedDataflow.getErrors().size() > 10000) {
1968                                mergedDataflow.setErrors(mergedDataflow.getErrors().subList(0, 10000));
1969                        }
1970                }
1971        }
1972
1973        /**
1974         * 基于合并后的 dataflow 构建 function→resultSet 映射。
1975         */
1976        private static void buildFunctionResultSetMapping(dataflow df,
1977                                                                                                          Map<String, String> functionIdToResultSetId,
1978                                                                                                          Map<String, column> funcColIdToResultSetCol,
1979                                                                                                          Map<String, table> allTableIdMap) {
1980                // 收集所有 procedure
1981                List<procedure> allProcedures = new ArrayList<>();
1982                if (df.getProcedures() != null) {
1983                        allProcedures.addAll(df.getProcedures());
1984                }
1985                if (df.getPackages() != null) {
1986                        for (oraclePackage pkg : df.getPackages()) {
1987                                if (pkg.getProcedures() != null) {
1988                                        allProcedures.addAll(pkg.getProcedures());
1989                                }
1990                        }
1991                }
1992
1993                // 构建 procedure name → id 映射
1994                Map<String, String> procNameToId = new HashMap<>();
1995                for (procedure proc : allProcedures) {
1996                        procNameToId.put(DlineageUtil.getIdentifierNormalTableName(proc.getName()), proc.getId());
1997                }
1998
1999                // 构建 resultSet/procedureId → procedure name 映射
2000                Map<String, String> procedureNameToResultSetId = new HashMap<>();
2001                List<table> allTables = dataflow.getAllTables(df);
2002                for (table t : allTables) {
2003                        if (t.getProcedureId() != null && !t.isFunction()) {
2004                                String procName = null;
2005                                for (procedure proc : allProcedures) {
2006                                        if (proc.getId().equals(t.getProcedureId())) {
2007                                                procName = DlineageUtil.getIdentifierNormalTableName(proc.getName());
2008                                                break;
2009                                        }
2010                                }
2011                                if (procName != null) {
2012                                        procedureNameToResultSetId.put(procName, t.getId());
2013                                }
2014                        }
2015                }
2016
2017                // 构建 allTableIdMap
2018                for (table t : allTables) {
2019                        allTableIdMap.put(t.getId(), t);
2020                }
2021
2022                // 构建 functionId → resultSetId 映射
2023                for (table t : allTables) {
2024                        if (t.isFunction()) {
2025                                String functionName = DlineageUtil.getIdentifierNormalTableName(
2026                                                DlineageUtil.getTableFullName(t.getName()));
2027                                String resultSetId = procedureNameToResultSetId.get(functionName);
2028                                if (resultSetId != null) {
2029                                        functionIdToResultSetId.put(t.getId(), resultSetId);
2030                                        table resultSet = allTableIdMap.get(resultSetId);
2031                                        if (resultSet != null && t.getColumns() != null && resultSet.getColumns() != null) {
2032                                                List<column> funcCols = t.getColumns();
2033                                                List<column> rsCols = resultSet.getColumns();
2034                                                int minSize = Math.min(funcCols.size(), rsCols.size());
2035                                                for (int j = 0; j < minSize; j++) {
2036                                                        funcColIdToResultSetCol.put(funcCols.get(j).getId(), rsCols.get(j));
2037                                                }
2038                                        }
2039                                }
2040                        }
2041                }
2042        }
2043
2044        /**
2045         * 对单条关系做 ID 重映射和去重，如果未重复则加入 newRelations 列表。
2046         */
2047        private static void remapAndDedupRelation(relationship rel,
2048                                                                                          Map<String, String> tableIdMap,
2049                                                                                          Map<String, String> columnIdMap,
2050                                                                                          Map<String, column> columnMergeIdMap,
2051                                                                                          Map<String, String> functionIdToResultSetId,
2052                                                                                          Map<String, column> funcColIdToResultSetCol,
2053                                                                                          Map<String, table> allTableIdMap,
2054                                                                                          Set<Long> mergedDedupKeys,
2055                                                                                          List<relationship> newRelations) {
2056                if (RelationshipType.call.name().equals(rel.getType())) {
2057                        // call 类型关系
2058                        targetColumn caller = rel.getCaller();
2059                        if (caller == null) {
2060                                return;
2061                        }
2062                        String mappedCallerId = tableIdMap.get(caller.getId());
2063                        if (mappedCallerId != null) {
2064                                caller.setId(mappedCallerId);
2065                        }
2066
2067                        List<sourceColumn> callees = rel.getCallees();
2068                        if (callees != null) {
2069                                Set<sourceColumn> calleeSet = new LinkedHashSet<>();
2070                                for (sourceColumn callee : callees) {
2071                                        String mappedSourceId = tableIdMap.get(callee.getId());
2072                                        if (mappedSourceId != null) {
2073                                                callee.setId(mappedSourceId);
2074                                        }
2075                                }
2076                                calleeSet.addAll(callees);
2077                                rel.setCallees(new ArrayList<>(calleeSet));
2078                        }
2079
2080                        if (mergedDedupKeys.add(rel.toDedupHash())) {
2081                                newRelations.add(rel);
2082                        }
2083                } else {
2084                        // fdd/frd 等类型关系
2085                        targetColumn target = rel.getTarget();
2086                        if (target == null) {
2087                                return;
2088                        }
2089                        String mappedTargetParentId = tableIdMap.get(target.getParent_id());
2090                        if (mappedTargetParentId != null) {
2091                                target.setParent_id(mappedTargetParentId);
2092                        }
2093                        String mappedTargetColId = columnIdMap.get(target.getId());
2094                        if (mappedTargetColId != null) {
2095                                target.setId(mappedTargetColId);
2096                                column mergedCol = columnMergeIdMap.get(mappedTargetColId);
2097                                if (mergedCol != null) {
2098                                        target.setCoordinate(mergedCol.getCoordinate());
2099                                }
2100                        }
2101
2102                        List<sourceColumn> sources = rel.getSources();
2103                        // Same contract as DataFlowAnalyzer.mergeTables: dedup runs under the
2104                        // merged column's stamped coordinate, but a source's own usable
2105                        // reference-site coordinate is restored on the surviving relation.
2106                        Map<sourceColumn, String> referenceCoordinates = new IdentityHashMap<>();
2107                        if (sources != null) {
2108                                Set<sourceColumn> sourceSet = new LinkedHashSet<>();
2109                                for (sourceColumn source : sources) {
2110                                        String mappedParentId = tableIdMap.get(source.getParent_id());
2111                                        if (mappedParentId != null) {
2112                                                source.setParent_id(mappedParentId);
2113                                        }
2114                                        String mappedSourceId = tableIdMap.get(source.getSource_id());
2115                                        if (mappedSourceId != null) {
2116                                                source.setSource_id(mappedSourceId);
2117                                        }
2118                                        String originalParentId = source.getParent_id();
2119                                        if (!functionIdToResultSetId.isEmpty()) {
2120                                                String resultSetId = functionIdToResultSetId.get(originalParentId);
2121                                                if (resultSetId != null) {
2122                                                        source.setParent_id(resultSetId);
2123                                                        column matched = funcColIdToResultSetCol.get(source.getId());
2124                                                        if (matched != null) {
2125                                                                source.setId(matched.getId());
2126                                                                source.setColumn(matched.getName());
2127                                                        }
2128                                                        table resultSetTable = allTableIdMap.get(source.getParent_id());
2129                                                        if (resultSetTable != null) {
2130                                                                source.setParent_name(resultSetTable.getName());
2131                                                        }
2132                                                }
2133                                        }
2134                                        String mappedColId = columnIdMap.get(source.getId());
2135                                        if (mappedColId != null) {
2136                                                source.setId(mappedColId);
2137                                                column mergedCol = columnMergeIdMap.get(mappedColId);
2138                                                if (mergedCol != null) {
2139                                                        String ownCoordinate = source.getCoordinate();
2140                                                        source.setCoordinate(mergedCol.getCoordinate());
2141                                                        if (DlineageUtil.hasUsableCoordinate(ownCoordinate)) {
2142                                                                referenceCoordinates.put(source, ownCoordinate);
2143                                                        }
2144                                                }
2145                                        }
2146                                }
2147                                sourceSet.addAll(sources);
2148                                rel.setSources(new ArrayList<>(sourceSet));
2149                        }
2150
2151                        if (mergedDedupKeys.add(rel.toDedupHash())) {
2152                                newRelations.add(rel);
2153                                for (Map.Entry<sourceColumn, String> entry : referenceCoordinates.entrySet()) {
2154                                        entry.getKey().setCoordinate(entry.getValue());
2155                                }
2156                        }
2157                }
2158        }
2159
2160        /**
2161         * 只对指定表做 * 列桥接，避免全量扫描。
2162         */
2163        private static long addStarColumnBridgingRelationsForTables(dataflow df, long startId,
2164                                                                                                                                 Set<String> targetTableIds) {
2165                if (df == null || targetTableIds.isEmpty()) {
2166                        return startId;
2167                }
2168
2169                // 构建已有桥接关系索引
2170                Map<String, Set<String>> bridgeIndex = new HashMap<>();
2171                if (df.getRelationships() != null) {
2172                        for (relationship rel : df.getRelationships()) {
2173                                if (!RelationshipType.fdd.name().equals(rel.getType())) {
2174                                        continue;
2175                                }
2176                                if (rel.getTarget() == null || rel.getSources() == null) {
2177                                        continue;
2178                                }
2179                                String parentId = rel.getTarget().getParent_id();
2180                                if (parentId == null) {
2181                                        continue;
2182                                }
2183                                for (sourceColumn sc : rel.getSources()) {
2184                                        if (parentId.equals(sc.getParent_id())) {
2185                                                bridgeIndex.computeIfAbsent(parentId, k -> new HashSet<>())
2186                                                                .add(rel.getTarget().getId() + "<-" + sc.getId());
2187                                        }
2188                                }
2189                        }
2190                }
2191
2192                List<relationship> newRelations = new ArrayList<>();
2193                for (table t : dataflow.getAllTables(df)) {
2194                        if (!targetTableIds.contains(t.getId())) {
2195                                continue;
2196                        }
2197                        if (t.getColumns() == null || t.getColumns().size() < 2) {
2198                                continue;
2199                        }
2200                        column starColumn = null;
2201                        List<column> regularColumns = new ArrayList<>();
2202                        for (column c : t.getColumns()) {
2203                                if ("system".equals(c.getSource())) {
2204                                        continue;
2205                                }
2206                                if (Boolean.TRUE.toString().equals(c.getIsFunction())) {
2207                                        continue;
2208                                }
2209                                if ("*".equals(c.getName())) {
2210                                        if (starColumn == null) {
2211                                                starColumn = c;
2212                                        }
2213                                } else {
2214                                        regularColumns.add(c);
2215                                }
2216                        }
2217                        if (starColumn == null || regularColumns.isEmpty()) {
2218                                continue;
2219                        }
2220
2221                        Set<String> existingBridges = bridgeIndex.getOrDefault(t.getId(), Collections.emptySet());
2222                        String starId = starColumn.getId();
2223                        for (column target : regularColumns) {
2224                                if (existingBridges.contains(target.getId() + "<-" + starId)) {
2225                                        continue;
2226                                }
2227                                relationship rel = new relationship();
2228                                rel.setId(String.valueOf(++startId));
2229                                rel.setType(RelationshipType.fdd.name());
2230                                rel.setEffectType("expand_star");
2231
2232                                targetColumn tc = new targetColumn();
2233                                tc.setId(target.getId());
2234                                tc.setColumn(target.getName());
2235                                tc.setParent_id(t.getId());
2236                                tc.setParent_name(t.getName());
2237                                rel.setTarget(tc);
2238
2239                                sourceColumn sc = new sourceColumn();
2240                                sc.setId(starId);
2241                                sc.setColumn(starColumn.getName());
2242                                sc.setParent_id(t.getId());
2243                                sc.setParent_name(t.getName());
2244                                rel.setSources(Arrays.asList(sc));
2245
2246                                newRelations.add(rel);
2247                        }
2248                }
2249
2250                if (!newRelations.isEmpty()) {
2251                        if (df.getRelationships() == null) {
2252                                df.setRelationships(new ArrayList<relationship>());
2253                        }
2254                        df.getRelationships().addAll(newRelations);
2255                }
2256                return startId;
2257        }
2258
2259        private static String offsetId(String id, long offset) {
2260                if (id == null || id.isEmpty()) {
2261                        return id;
2262                }
2263                try {
2264                        return String.valueOf(Long.parseLong(id) + offset);
2265                } catch (NumberFormatException ignore) {
2266                        return id;
2267                }
2268        }
2269
2270        /**
2271         * 偏移 coordinate 第三维（fileIdx）。
2272         * <p>
2273         * coordinate 字符串格式为 {@code [line,col,fileIdx],[line,col,fileIdx]}，
2274         * 合并多个独立生成的 dataflow 时，它们的 fileIdx 都从 0 起始会互相冲突，
2275         * 通过给第三维加上 offset 来区分不同的来源。
2276         */
2277        private static String offsetCoordinate(String coordinate, long offset) {
2278                if (coordinate == null || coordinate.isEmpty() || offset == 0L) {
2279                        return coordinate;
2280                }
2281                StringBuilder sb = new StringBuilder(coordinate.length() + 8);
2282                int i = 0;
2283                int len = coordinate.length();
2284                while (i < len) {
2285                        int open = coordinate.indexOf('[', i);
2286                        if (open < 0) {
2287                                sb.append(coordinate, i, len);
2288                                break;
2289                        }
2290                        int close = coordinate.indexOf(']', open);
2291                        if (close < 0) {
2292                                sb.append(coordinate, i, len);
2293                                break;
2294                        }
2295                        sb.append(coordinate, i, open + 1);
2296                        String inner = coordinate.substring(open + 1, close);
2297                        String[] parts = inner.split(",", -1);
2298                        for (int k = 0; k < parts.length; k++) {
2299                                if (k > 0) sb.append(',');
2300                                if (k == 2) {
2301                                        String p = parts[k].trim();
2302                                        try {
2303                                                sb.append(Long.parseLong(p) + offset);
2304                                        } catch (NumberFormatException ignore) {
2305                                                sb.append(parts[k]);
2306                                        }
2307                                } else {
2308                                        sb.append(parts[k]);
2309                                }
2310                        }
2311                        sb.append(']');
2312                        i = close + 1;
2313                }
2314                return sb.toString();
2315        }
2316
2317        /**
2318         * 偏移 dataflow 中所有节点的 ID 和 coordinate 第三维 fileIdx。
2319         *
2320         * @param df            待偏移的 dataflow（原地修改）
2321         * @param idOffset      给所有 id / xxxId 字段加上的偏移量
2322         * @param fileIdxOffset 给所有 coordinate 第三维 (fileIdx) 加上的偏移量，
2323         *                      通常为合并时前面所有 dataflow 累积的源文件数，
2324         *                      以便合并后不同来源的 fileIdx 保持连续且不重叠。
2325         */
2326        private static void offsetDataflowIds(dataflow df, long idOffset, long fileIdxOffset) {
2327                if (idOffset == 0L && fileIdxOffset == 0L) {
2328                        return;
2329                }
2330
2331                // tables / views / databases / schemas / stages / datasources / streams / paths / variables / resultsets / sequences
2332                for (table t : dataflow.getAllTables(df)) {
2333                        offsetTableIds(t, idOffset);
2334                        t.setCoordinate(offsetCoordinate(t.getCoordinate(), fileIdxOffset));
2335                        if (t.getColumns() != null) {
2336                                for (column c : t.getColumns()) {
2337                                        c.setCoordinate(offsetCoordinate(c.getCoordinate(), fileIdxOffset));
2338                                }
2339                        }
2340                }
2341
2342                // processes
2343                if (df.getProcesses() != null) {
2344                        for (process p : df.getProcesses()) {
2345                                p.setId(offsetId(p.getId(), idOffset));
2346                                p.setProcedureId(offsetId(p.getProcedureId(), idOffset));
2347                                p.setCoordinate(offsetCoordinate(p.getCoordinate(), fileIdxOffset));
2348                        }
2349                }
2350
2351                // procedures
2352                if (df.getProcedures() != null) {
2353                        for (procedure p : df.getProcedures()) {
2354                                p.setId(offsetId(p.getId(), idOffset));
2355                                p.setCoordinate(offsetCoordinate(p.getCoordinate(), fileIdxOffset));
2356                                if (p.getArguments() != null) {
2357                                        for (argument arg : p.getArguments()) {
2358                                                arg.setId(offsetId(arg.getId(), idOffset));
2359                                                arg.setCoordinate(offsetCoordinate(arg.getCoordinate(), fileIdxOffset));
2360                                        }
2361                                }
2362                        }
2363                }
2364
2365                // oracle packages
2366                if (df.getPackages() != null) {
2367                        for (oraclePackage pkg : df.getPackages()) {
2368                                pkg.setId(offsetId(pkg.getId(), idOffset));
2369                                if (pkg.getProcedures() != null) {
2370                                        for (procedure pp : pkg.getProcedures()) {
2371                                                pp.setId(offsetId(pp.getId(), idOffset));
2372                                                pp.setCoordinate(offsetCoordinate(pp.getCoordinate(), fileIdxOffset));
2373                                                if (pp.getArguments() != null) {
2374                                                        for (argument arg : pp.getArguments()) {
2375                                                                arg.setId(offsetId(arg.getId(), idOffset));
2376                                                                arg.setCoordinate(offsetCoordinate(arg.getCoordinate(), fileIdxOffset));
2377                                                        }
2378                                                }
2379                                        }
2380                                }
2381                        }
2382                }
2383
2384                // relationships
2385                if (df.getRelationships() != null) {
2386                        for (relationship rel : df.getRelationships()) {
2387                                rel.setId(offsetId(rel.getId(), idOffset));
2388                                rel.setProcessId(offsetId(rel.getProcessId(), idOffset));
2389                                rel.setProcedureId(offsetId(rel.getProcedureId(), idOffset));
2390                                if (rel.getTarget() != null) {
2391                                        offsetTargetColumnIds(rel.getTarget(), idOffset);
2392                                }
2393                                if (rel.getCaller() != null) {
2394                                        offsetTargetColumnIds(rel.getCaller(), idOffset);
2395                                }
2396                                if (rel.getSources() != null) {
2397                                        for (sourceColumn sc : rel.getSources()) {
2398                                                offsetSourceColumnIds(sc, idOffset);
2399                                        }
2400                                }
2401                                if (rel.getCallees() != null) {
2402                                        for (sourceColumn sc : rel.getCallees()) {
2403                                                offsetSourceColumnIds(sc, idOffset);
2404                                        }
2405                                }
2406                        }
2407                }
2408        }
2409
2410        /**
2411         * 扫描 dataflow 中所有 coordinate，返回最大的第三维 (fileIdx) 值；
2412         * 若没有任何 coordinate 则返回 -1。
2413         */
2414        private static int findMaxFileIdx(dataflow df) {
2415                if (df == null) {
2416                        return -1;
2417                }
2418                int max = -1;
2419                for (table t : dataflow.getAllTables(df)) {
2420                        max = Math.max(max, maxFileIdxInCoordinate(t.getCoordinate()));
2421                        if (t.getColumns() != null) {
2422                                for (column c : t.getColumns()) {
2423                                        max = Math.max(max, maxFileIdxInCoordinate(c.getCoordinate()));
2424                                }
2425                        }
2426                }
2427                if (df.getProcesses() != null) {
2428                        for (process p : df.getProcesses()) {
2429                                max = Math.max(max, maxFileIdxInCoordinate(p.getCoordinate()));
2430                        }
2431                }
2432                if (df.getProcedures() != null) {
2433                        for (procedure p : df.getProcedures()) {
2434                                max = Math.max(max, maxFileIdxInCoordinate(p.getCoordinate()));
2435                                if (p.getArguments() != null) {
2436                                        for (argument arg : p.getArguments()) {
2437                                                max = Math.max(max, maxFileIdxInCoordinate(arg.getCoordinate()));
2438                                        }
2439                                }
2440                        }
2441                }
2442                if (df.getPackages() != null) {
2443                        for (oraclePackage pkg : df.getPackages()) {
2444                                if (pkg.getProcedures() != null) {
2445                                        for (procedure pp : pkg.getProcedures()) {
2446                                                max = Math.max(max, maxFileIdxInCoordinate(pp.getCoordinate()));
2447                                                if (pp.getArguments() != null) {
2448                                                        for (argument arg : pp.getArguments()) {
2449                                                                max = Math.max(max, maxFileIdxInCoordinate(arg.getCoordinate()));
2450                                                        }
2451                                                }
2452                                        }
2453                                }
2454                        }
2455                }
2456                return max;
2457        }
2458
2459        /**
2460         * 从 coordinate 字符串中解析出最大的第三维 (fileIdx) 值；
2461         * 若 coordinate 为空或没有任何可解析的数值则返回 -1。
2462         */
2463        private static int maxFileIdxInCoordinate(String coordinate) {
2464                if (coordinate == null || coordinate.isEmpty()) {
2465                        return -1;
2466                }
2467                int max = -1;
2468                int i = 0;
2469                int len = coordinate.length();
2470                while (i < len) {
2471                        int open = coordinate.indexOf('[', i);
2472                        if (open < 0) break;
2473                        int close = coordinate.indexOf(']', open);
2474                        if (close < 0) break;
2475                        String inner = coordinate.substring(open + 1, close);
2476                        String[] parts = inner.split(",", -1);
2477                        if (parts.length >= 3) {
2478                                try {
2479                                        int v = Integer.parseInt(parts[2].trim());
2480                                        if (v > max) max = v;
2481                                } catch (NumberFormatException ignore) {
2482                                        // 不是数字，跳过
2483                                }
2484                        }
2485                        i = close + 1;
2486                }
2487                return max;
2488        }
2489
2490        private static void offsetTableIds(table t, long offset) {
2491                if (t == null) {
2492                        return;
2493                }
2494                t.setId(offsetId(t.getId(), offset));
2495                if (t.getProcessIds() != null) {
2496                        List<String> newProcessIds = new ArrayList<>(t.getProcessIds().size());
2497                        for (String pid : t.getProcessIds()) {
2498                                newProcessIds.add(offsetId(pid, offset));
2499                        }
2500                        t.setProcessIds(newProcessIds);
2501                }
2502                if (t.getColumns() != null) {
2503                        for (column c : t.getColumns()) {
2504                                c.setId(offsetId(c.getId(), offset));
2505                        }
2506                }
2507        }
2508
2509        private static void offsetTargetColumnIds(targetColumn col, long offset) {
2510                col.setId(offsetId(col.getId(), offset));
2511                col.setParent_id(offsetId(col.getParent_id(), offset));
2512                col.setTarget_id(offsetId(col.getTarget_id(), offset));
2513        }
2514
2515        private static void offsetSourceColumnIds(sourceColumn col, long offset) {
2516                col.setId(offsetId(col.getId(), offset));
2517                col.setParent_id(offsetId(col.getParent_id(), offset));
2518                col.setSource_id(offsetId(col.getSource_id(), offset));
2519        }
2520
2521        public static dataflow readDataflowFromCsvMetadata(String csvMetadata, EDbVendor vendor) {
2522                if (!MetadataReader.isMetadata(csvMetadata)) {
2523                        throw new IllegalArgumentException("Illegal csv metadata.");
2524                }
2525                return new SQLDepMetadataAnalyzer().analyzeMetadata(vendor, csvMetadata);
2526        }
2527
2528        public static Dataflow mergeDataflowsAndCsv(List<Pair<EDbVendor,dataflow>> pairs, String csvMetadata) {
2529                //重置ID防止重复
2530                Long index = 0L;
2531                for(int i=0; i<pairs.size(); i++){
2532                        dataflow dataflow = pairs.get(i).second;
2533                        Map<String, table> objIDMap = getDataflowDbObjMap(dataflow);
2534                        Map<String, String> idMaps = new HashMap<>();
2535                        for (Map.Entry<String, table> entry : objIDMap.entrySet()) {
2536                                index = index + 1;
2537                                String id = index.toString();
2538                                idMaps.put(entry.getKey(), id);
2539                                table table = entry.getValue();
2540                                table.setId(id);
2541                                if(table.getColumns() != null){
2542                                        for(column col: table.getColumns()){
2543                                                index = index + 1;
2544                                                id = index.toString();
2545                                                idMaps.put(col.getId(), id);
2546                                                col.setId(id);
2547                                        }
2548                                }
2549                        }
2550                        if(dataflow.getProcedures() != null){
2551                                for(procedure procedure: dataflow.getProcedures()){
2552                                        index = index + 1;
2553                                        String id = index.toString();
2554                                        idMaps.put(procedure.getId(), id);
2555                                        procedure.setId(id);
2556                                }
2557                        }
2558                        if(dataflow.getProcesses() != null){
2559                                for(process process: dataflow.getProcesses()){
2560                                        index = index + 1;
2561                                        String id = index.toString();
2562                                        idMaps.put(process.getId(), id);
2563                                        process.setId(id);
2564                                }
2565                        }
2566                        if(dataflow.getPackages() != null){
2567                                for(oraclePackage oraclePackage: dataflow.getPackages()){
2568                                        index = index + 1;
2569                                        String id = index.toString();
2570                                        idMaps.put(oraclePackage.getId(), id);
2571                                        oraclePackage.setId(id);
2572                                        if(oraclePackage.getProcedures() != null){
2573                                                for(procedure procedure: oraclePackage.getProcedures()){
2574                                                        index = index + 1;
2575                                                        id = index.toString();
2576                                                        idMaps.put(procedure.getId(), id);
2577                                                        procedure.setId(id);
2578                                                }
2579                                        }
2580                                }
2581                        }
2582                        if(dataflow.getRelationships() != null && dataflow.getRelationships().size()>0){
2583                                for(relationship rel: dataflow.getRelationships()){
2584                                        index = index + 1;
2585                                        String id = index.toString();
2586                                        rel.setId(id);
2587                                        rel.setProcedureId(idMaps.get(rel.getProcedureId()));
2588                                        rel.setProcessId(idMaps.get(rel.getProcessId()));
2589                                        rel.getTarget().setParent_id(idMaps.get(rel.getTarget().getParent_id()));
2590                                        rel.getTarget().setTarget_id(idMaps.get(rel.getTarget().getTarget_id()));
2591                                        rel.getTarget().setId(idMaps.get(rel.getTarget().getId()));
2592                                        for(sourceColumn column: rel.getSources()){
2593                                                column.setParent_id(idMaps.get(column.getParent_id()));
2594                                                column.setId(idMaps.get(column.getId()));
2595                                        }
2596                                }
2597                        }
2598                }
2599                Dataflow mDataflow = DataFlowAnalyzer.getSqlflowJSONModel(pairs.get(0).first, pairs.get(0).second, false);
2600                Map<String, table> objNameMap = getDataflowDbObjNameMap(pairs.get(0).second);
2601                Sqlflow dbobjs = mDataflow.getDbobjs();
2602                List<Relationship> mRelationshipList = new ArrayList<>();
2603                if(mDataflow.getRelationships() != null && mDataflow.getRelationships().length>0){
2604                        mRelationshipList = new LinkedList<>(Arrays.asList(mDataflow.getRelationships()));
2605                }
2606
2607                List<Error> mErrorList = new ArrayList<>();
2608                if(mDataflow.getErrors() != null && mDataflow.getErrors().length>0){
2609                        mErrorList = new LinkedList<>(Arrays.asList(mDataflow.getErrors()));
2610                }
2611
2612                List<Process> mProcessList = new ArrayList<>();
2613                if(mDataflow.getProcesses() != null && mDataflow.getProcesses().length>0){
2614                        mProcessList = new LinkedList<>(Arrays.asList(mDataflow.getProcesses()));
2615                }
2616
2617                for(int i=1; i<pairs.size(); i++){
2618                        objNameMap.putAll(getDataflowDbObjNameMap(pairs.get(i).second));
2619                        Dataflow dataflow = DataFlowAnalyzer.getSqlflowJSONModel(pairs.get(i).first, pairs.get(i).second, false);
2620                        if(dataflow.getDbobjs().getServers() != null && dataflow.getDbobjs().getServers().size()>0){
2621                                if(dbobjs.getServers() == null){
2622                                        dbobjs.setServers(new ArrayList<>());
2623                                }
2624                                dbobjs.getServers().addAll(dataflow.getDbobjs().getServers());
2625                        }
2626
2627                        if(dataflow.getDbobjs().getErrorMessages() != null && dataflow.getDbobjs().getErrorMessages().size()>0){
2628                                if(dbobjs.getErrorMessages() == null){
2629                                        dbobjs.setErrorMessages(new ArrayList<>());
2630                                }
2631                                dbobjs.getErrorMessages().addAll(dataflow.getDbobjs().getErrorMessages());
2632                        }
2633
2634                        if(dataflow.getRelationships() != null && dataflow.getRelationships().length>0){
2635                                List<Relationship> relationshipList = new LinkedList<>(Arrays.asList(dataflow.getRelationships()));
2636                                mRelationshipList.addAll(relationshipList);
2637                        }
2638
2639                        if(dataflow.getErrors() != null && dataflow.getErrors().length>0){
2640                                List<Error> errorList = new LinkedList<>(Arrays.asList(dataflow.getErrors()));
2641                                mErrorList.addAll(errorList);
2642                        }
2643
2644                        if(dataflow.getProcesses() != null && dataflow.getProcesses().length>0){
2645                                List<Process> processList = new LinkedList<>(Arrays.asList(dataflow.getProcesses()));
2646                                mProcessList.addAll(processList);
2647                        }
2648
2649                }
2650                if(!SQLUtil.isEmpty(csvMetadata)){
2651                        /**
2652                         * excel内的血缘 只合并关系 就是只做关联，如果找不到obj 就算了
2653                         */
2654                        dataflow df = readDataflowFromCsvMetadata(csvMetadata);
2655                        Map<String, table> objIDMap = getDataflowDbObjMap(df);
2656                        if(df.getRelationships() != null && df.getRelationships().size()>0){
2657                                for(relationship rel: df.getRelationships()){
2658                                        rel.setId("m"+rel.getId());
2659                                        table sTable = objIDMap.get(rel.getTarget().getParent_id());
2660                                        table tTable = objNameMap.get(sTable.getFullName());
2661                                        rel.getTarget().setParent_id(tTable.getId());
2662                                        for(column col: tTable.getColumns()){
2663                                                if(col.getName().equalsIgnoreCase(rel.getTarget().getColumn())){
2664                                                        rel.getTarget().setId(col.getId());
2665                                                        break;
2666                                                }
2667                                        }
2668                                        for(sourceColumn column: rel.getSources()){
2669                                                sTable = objIDMap.get(column.getParent_id());
2670                                                tTable = objNameMap.get(sTable.getFullName());
2671                                                column.setParent_id(tTable.getId());
2672                                                for(column col: tTable.getColumns()){
2673                                                        if(col.getName().equalsIgnoreCase(column.getColumn())){
2674                                                                column.setId(col.getId());
2675                                                                break;
2676                                                        }
2677                                                }
2678                                        }
2679                                        mRelationshipList.add(toRelationship(rel));
2680                                }
2681                        }
2682                }
2683
2684                mDataflow.setDbobjs(dbobjs);
2685                mDataflow.setRelationships(mRelationshipList.toArray(new Relationship[mRelationshipList.size()]));
2686                mDataflow.setProcesses(mProcessList.toArray(new Process[mProcessList.size()]));
2687                mDataflow.setErrors(mErrorList.toArray(new Error[mErrorList.size()]));
2688                return mDataflow;
2689        }
2690
2691        public static dataflow readDataflowFromCsvMetadata(String csvMetadata) {
2692                if (!MetadataReader.isMetadata(csvMetadata)) {
2693                        throw new IllegalArgumentException("Illegal csv metadata.");
2694                }
2695                return new SQLDepMetadataAnalyzer().analyzeMetadata(null, csvMetadata);
2696        }
2697
2698        private static Relationship toRelationship(relationship relation){
2699                Relationship relationModel;
2700                if (relation.getType().equals("join")) {
2701                        JoinRelationship joinRelationModel = new JoinRelationship();
2702                        joinRelationModel.setCondition(relation.getCondition());
2703                        joinRelationModel.setJoinType(relation.getJoinType());
2704                        joinRelationModel.setClause(relation.getClause());
2705                        relationModel = joinRelationModel;
2706                } else {
2707                        relationModel = new Relationship();
2708                }
2709                relationModel.setId(relation.getId());
2710                relationModel.setProcessId(relation.getProcessId());
2711                relationModel.setProcessType(relation.getProcessType());
2712                relationModel.setType(relation.getType());
2713                relationModel.setEffectType(relation.getEffectType());
2714                relationModel.setPartition(relation.getPartition());
2715                relationModel.setFunction(relation.getFunction());
2716                relationModel.setProcedureId(relation.getProcedureId());
2717                relationModel.setSqlHash(relation.getSqlHash());
2718                relationModel.setCondition(relation.getCondition());
2719                relationModel.setSqlComment(relation.getSqlComment());
2720                relationModel.setTimestampMax(relation.getTimestampMax());
2721                relationModel.setTimestampMin(relation.getTimestampMin());
2722
2723                if (relation.getTarget() != null && relation.getSources() != null && !relation.getSources().isEmpty()) {
2724                        RelationshipElement targetModel = new RelationshipElement();
2725                        targetColumn target = relation.getTarget();
2726                        targetModel.setColumn(target.getColumn());
2727                        targetModel.setParentName(target.getParent_name());
2728                        targetModel.setTargetName(target.getTarget_name());
2729                        targetModel.setId(target.getId());
2730                        targetModel.setTargetId(target.getTarget_id());
2731                        targetModel.setParentId(target.getParent_id());
2732                        targetModel.setCoordinates(Coordinate.parse(target.getCoordinate()));
2733                        targetModel.setFunction(target.getFunction());
2734                        targetModel.setType(target.getType());
2735                        relationModel.setTarget(targetModel);
2736
2737                        List<RelationshipElement> sourceModels = new ArrayList<>();
2738                        for (sourceColumn source : relation.getSources()) {
2739                                RelationshipElement sourceModel = new RelationshipElement();
2740                                sourceModel.setColumn(source.getColumn());
2741                                sourceModel.setParentName(source.getParent_name());
2742                                sourceModel.setSourceName(source.getSource_name());
2743                                sourceModel.setColumnType(source.getColumn_type());
2744                                sourceModel.setId(source.getId());
2745                                sourceModel.setParentId(source.getParent_id());
2746                                sourceModel.setSourceId(source.getSource_id());
2747                                sourceModel.setCoordinates(Coordinate.parse(source.getCoordinate()));
2748                                sourceModel.setClauseType(source.getClauseType());
2749                                sourceModel.setType(source.getType());
2750                                sourceModels.add(sourceModel);
2751                                if (source.getTransforms() != null && !source.getTransforms().isEmpty()) {
2752                                        List<Transform> transforms = new ArrayList<gudusoft.gsqlparser.dlineage.dataflow.model.json.Transform>();
2753                                        for (transform transform : source.getTransforms()) {
2754                                                Transform item = new Transform();
2755                                                item.setCode(transform.getCode());
2756                                                item.setType(transform.getType());
2757                                                item.setCoordinate(transform.getCoordinate(true));
2758                                                transforms.add(item);
2759                                        }
2760                                        sourceModel.setTransforms(transforms.toArray(new Transform[0]));
2761                                }
2762                        }
2763                        relationModel.setSources(sourceModels.toArray(new RelationshipElement[0]));
2764                } else if (relation.getCaller() != null && relation.getCallees() != null
2765                                && !relation.getCallees().isEmpty()) {
2766                        RelationshipElement targetModel = new RelationshipElement();
2767                        targetColumn target = relation.getCaller();
2768                        targetModel.setName(target.getName());
2769                        targetModel.setId(target.getId());
2770                        targetModel.setCoordinates(Coordinate.parse(target.getCoordinate()));
2771                        targetModel.setType(target.getType());
2772                        relationModel.setCaller(targetModel);
2773                        List<RelationshipElement> sourceModels = new ArrayList<gudusoft.gsqlparser.dlineage.dataflow.model.json.RelationshipElement>();
2774                        for (sourceColumn source : relation.getCallees()) {
2775                                RelationshipElement sourceModel = new RelationshipElement();
2776                                sourceModel.setName(source.getName());
2777                                sourceModel.setId(source.getId());
2778                                sourceModel.setCoordinates(Coordinate.parse(source.getCoordinate()));
2779                                sourceModel.setType(source.getType());
2780                                sourceModels.add(sourceModel);
2781                        }
2782                        relationModel.setCallees(sourceModels.toArray(new RelationshipElement[0]));
2783                }
2784                return relationModel;
2785        }
2786
2787//    public static void main(String[] args) throws Exception {
2788//        // 基于文件的迭代合并（节约内存，适合大数据量）
2789//        dataflow dataflow = iterativeMergeDataflowsFromFiles(Arrays.asList(new File("C:\\Users\\KK\\xwechat_files\\wxid_z9ci6s8b7g0d21_9365\\msg\\file\\2026-06\\dataflow_item").listFiles()), EDbVendor.dbvoracle);
2790//        System.out.println(XML2Model.saveXML(dataflow));
2791//    }
2792//
2793//    public static void main(String[] args) throws Exception {
2794//        // 基于内存的合并（一次性加载所有文件到内存，适合小数据量）
2795//        File[] files = new File("C:\\Users\\KK\\xwechat_files\\wxid_z9ci6s8b7g0d21_9365\\msg\\file\\2026-06\\dataflow_item").listFiles();
2796//        if (files == null || files.length == 0) {
2797//            System.out.println("No dataflow files found.");
2798//            return;
2799//        }
2800//
2801//        List<dataflow> dataflows = new ArrayList<>();
2802//        for (File file : files) {
2803//            dataflow df = XML2Model.loadXML(dataflow.class, file);
2804//            if (df != null) {
2805//                dataflows.add(df);
2806//            }
2807//        }
2808//
2809//        dataflow mergedDataflow = mergeDataflows(dataflows, EDbVendor.dbvoracle);
2810//        System.out.println(XML2Model.saveXML(mergedDataflow));
2811//    }
2812//
2813//    public static void main(String[] args) throws Exception {
2814//        String sqlDirPath = "C:\\Users\\KK\\Desktop\\同义词验证";
2815//        File sqlDir = new File(sqlDirPath);
2816//
2817//        if (!sqlDir.exists() || !sqlDir.isDirectory()) {
2818//            System.err.println("目录不存在或不是目录: " + sqlDirPath);
2819//            return;
2820//        }
2821//
2822//        File[] sqlFiles = sqlDir.listFiles((dir, name) -> name.toLowerCase().endsWith(".sql"));
2823//
2824//        if (sqlFiles == null || sqlFiles.length == 0) {
2825//            System.err.println("目录下没有找到 SQL 文件: " + sqlDirPath);
2826//            return;
2827//        }
2828//
2829//        System.out.println("找到 " + sqlFiles.length + " 个 SQL 文件");
2830//        for (File file : sqlFiles) {
2831//            System.out.println("  - " + file.getName());
2832//        }
2833//
2834//        Option option = new Option();
2835//        option.setVendor(EDbVendor.dbvoracle);
2836//        option.setOutput(true);
2837//        option.setSimpleOutput(true);
2838//        option.setSimpleShowSynonym(true);
2839//        option.setParallel(4);
2840//
2841//        ParallelDataFlowAnalyzer analyzer = new ParallelDataFlowAnalyzer(
2842//                sqlFiles,
2843//                option
2844//        );
2845//
2846//        System.out.println("\n开始分析...");
2847//        long startTime = System.currentTimeMillis();
2848//        analyzer.generateDataFlow(false, true);
2849//        long endTime = System.currentTimeMillis();
2850//
2851//        dataflow dataflow = analyzer.getDataFlow();
2852//
2853//        analyzer.dispose();
2854//
2855//        if (dataflow == null) {
2856//            System.err.println("分析失败，未生成数据流");
2857//            return;
2858//        }
2859//
2860//        Option option1 = new Option();
2861//        option1.setVendor(EDbVendor.dbvoracle);
2862//        option1.setSimpleOutput(true);
2863//        dataflow simpleDataflow = new DataFlowAnalyzer("", option1).getSimpleDataflow(dataflow, true);
2864//
2865//
2866//        System.out.println("分析完成，耗时: " + (endTime - startTime) + " ms");
2867//        System.out.println("表数量: " + (simpleDataflow.getTables() != null ? simpleDataflow.getTables().size() : 0));
2868//        System.out.println("视图数量: " + (simpleDataflow.getViews() != null ? simpleDataflow.getViews().size() : 0));
2869//        System.out.println("关系数量: " + (simpleDataflow.getRelationships() != null ? simpleDataflow.getRelationships().size() : 0));
2870//        System.out.println("错误数量: " + (simpleDataflow.getErrors() != null ? simpleDataflow.getErrors().size() : 0));
2871//
2872//        System.out.println(XML2Model.saveXML(simpleDataflow));
2873//
2874//    }
2875}