001package gudusoft.gsqlparser.dlineage.util;
002
003import gudusoft.gsqlparser.EDbVendor;
004import gudusoft.gsqlparser.dlineage.dataflow.model.DataflowRemoveOption;
005import gudusoft.gsqlparser.dlineage.dataflow.model.RelationshipType;
006import gudusoft.gsqlparser.dlineage.dataflow.model.ResultSetType;
007import gudusoft.gsqlparser.dlineage.dataflow.model.xml.*;
008import gudusoft.gsqlparser.util.Logger;
009import gudusoft.gsqlparser.util.LoggerFactory;
010import gudusoft.gsqlparser.util.SQLUtil;
011
012import java.util.*;
013
014public class DataflowRemoveHelper {
015
016    private static final Logger logger = LoggerFactory.getLogger(DataflowRemoveHelper.class);
017
018    private Map<String, Boolean> isTables = new HashMap<String, Boolean>();
019    private Map<String, Boolean> targetTables = new HashMap<String, Boolean>();
020
021    public dataflow remove(dataflow instance, EDbVendor dbVendor, DataflowRemoveOption option) {
022        try {
023            targetTables.clear();
024            isTables.clear();
025            dataflow simple = new dataflow();
026            List<relationship> simpleRelations = new ArrayList<relationship>();
027            List<relationship> relations = instance.getRelationships();
028            if (relations != null && relations.size() > 0) {
029                Map<String, Set<relationship>> targetIdRelationMap = new HashMap<String, Set<relationship>>();
030                for (relationship relation : relations) {
031                    if (relation.getTarget() != null) {
032                        String key = relation.getTarget().getParent_id() + "." + relation.getTarget().getId();
033                        if (!targetIdRelationMap.containsKey(key)) {
034                            targetIdRelationMap.put(key, new HashSet<relationship>());
035                        }
036                        targetIdRelationMap.get(key).add(relation);
037                    }
038                }
039
040                long maxId = Long
041                        .parseLong(relations.get(relations.size() - 1).getId().split("\\-")[0].replace("_", "")) * 100;
042                for (int i = 0; i < relations.size(); i++) {
043                    relationship relationElem = relations.get(i);
044                    if (RelationshipType.call.name().equals(relationElem.getType())) {
045                        continue;
046                    }
047                    if (RelationshipType.er.name().equals(relationElem.getType())) {
048                        continue;
049                    }
050                    targetColumn target = relationElem.getTarget();
051                    String targetParent = target.getParent_id();
052                    if (isTarget(option, instance, targetParent)) {
053                        List<Pair<sourceColumn, List<String>>> relationSources = new ArrayList<Pair<sourceColumn, List<String>>>();
054                        findSourceRelations(option, dbVendor, target, instance, targetIdRelationMap, relationElem, relationSources,
055                                new String[]{relationElem.getType()});
056                        if (relationSources.size() > 0) {
057                            Map<sourceColumn, List<Pair<sourceColumn, List<String>>>> columnMap = new HashMap<sourceColumn, List<Pair<sourceColumn, List<String>>>>();
058                            for (Pair<sourceColumn, List<String>> item : relationSources) {
059                                if (!columnMap.containsKey(item.first)) {
060                                    columnMap.put(item.first, new ArrayList<Pair<sourceColumn, List<String>>>());
061                                }
062                                columnMap.get(item.first).add(item);
063                            }
064
065                            Iterator<sourceColumn> iter = columnMap.keySet().iterator();
066                            Map<String, List<sourceColumn>> relationSourceMap = new HashMap<String, List<sourceColumn>>();
067                            while (iter.hasNext()) {
068                                sourceColumn column = iter.next();
069                                String relationType = mergeRelationType(columnMap.get(column));
070                                if (!relationSourceMap.containsKey(relationType)) {
071                                    relationSourceMap.put(relationType, new ArrayList<sourceColumn>());
072                                }
073                                relationSourceMap.get(relationType).add(column);
074                            }
075
076                            Iterator<String> sourceIter = relationSourceMap.keySet().iterator();
077                            while (sourceIter.hasNext()) {
078                                String relationType = sourceIter.next();
079                                relationship simpleRelation = (relationship) relationElem.clone();
080                                simpleRelation.setSources(relationSourceMap.get(relationType));
081                                simpleRelation.setType(relationType);
082                                simpleRelation.setId(String.valueOf(++maxId));
083                                simpleRelations.add(simpleRelation);
084                            }
085                        }
086                    }
087                }
088            }
089            simple.setErrors(instance.getErrors());
090            simple.setPackages(instance.getPackages());
091            simple.setProcedures(instance.getProcedures());
092            simple.setProcesses(instance.getProcesses());
093            simple.setTables(instance.getTables());
094            simple.setViews(instance.getViews());
095            simple.setDatabases(instance.getDatabases());
096            simple.setSchemas(instance.getSchemas());
097            simple.setStages(instance.getStages());
098            simple.setSequences(instance.getSequences());
099            simple.setDatasources(instance.getDatasources());
100            simple.setStreams(instance.getStreams());
101            simple.setPaths(instance.getPaths());
102
103            if (instance.getVariables() != null) {
104                List<table> variables = new ArrayList<table>();
105                for (int i = 0; i < instance.getVariables().size(); i++) {
106                    table variable = instance.getVariables().get(i);
107                    if (!isRemoveVariable(option, variable)) {
108                        variables.add(variable);
109                    }
110                }
111                simple.setVariables(variables);
112            }
113            if (instance.getResultsets() != null) {
114                List<table> resultSets = new ArrayList<table>();
115                for (int i = 0; i < instance.getResultsets().size(); i++) {
116                    table resultSet = instance.getResultsets().get(i);
117                    if (!isRemoveResultSet(option, resultSet)) {
118                        resultSets.add(resultSet);
119                    }
120                }
121                simple.setResultsets(resultSets);
122            }
123            simple.setRelationships(simpleRelations);
124            simple.setOrientation(instance.getOrientation());
125            return simple;
126        } catch (Exception e) {
127            logger.error("Remove dataflow resultSet failed.", e);
128        }
129        return instance;
130    }
131
132    private boolean isRemoveVariable(DataflowRemoveOption option, table variable) {
133        if (option.isRemoveVariable()) {
134            if ("variable".equals(variable.getType())) {
135                return true;
136            }
137        } else if (option.isRemoveCursor()) {
138            if ("variable".equals(variable.getType()) && "cursor".equals(variable.getSubType())) {
139                return true;
140            }
141        }
142        return false;
143    }
144
145    private boolean isRemoveResultSet(DataflowRemoveOption option, table resultSet) {
146        ResultSetType resultSetType = ResultSetType.of(resultSet.getType());
147        if (option.getRemoveResultSetTypes().contains(resultSetType)) {
148            return true;
149        }
150        if (option.getRemoveResultSetTypes().contains(ResultSetType.function)) {
151            if ("function".equals(resultSet.getSubType())) {
152                return true;
153            }
154        }
155        return false;
156    }
157
158    private void findSourceRelations(DataflowRemoveOption option, EDbVendor dbVendor, targetColumn target, dataflow instance, Map<String, Set<relationship>> sourceIdRelationMap,
159                                     relationship targetRelation, List<Pair<sourceColumn, List<String>>> relationSources, String[] pathTypes) {
160        findStarSourceRelations(option, dbVendor, target, instance, null, sourceIdRelationMap, targetRelation, relationSources, pathTypes,
161                new HashSet<String>(), new LinkedHashSet<transform>(), new LinkedHashSet<candidateTable>());
162    }
163
164    private void findStarSourceRelations(DataflowRemoveOption option, EDbVendor dbVendor, targetColumn target, dataflow instance, targetColumn starRelationTarget, Map<String, Set<relationship>> sourceIdRelationMap,
165                                         relationship targetRelation, List<Pair<sourceColumn, List<String>>> relationSources, String[] pathTypes,
166                                         Set<String> paths, Set<transform> transforms, Set<candidateTable> candidateTables) {
167        if (targetRelation != null && targetRelation.getSources() != null) {
168            for (int i = 0; i < targetRelation.getSources().size(); i++) {
169                sourceColumn source = targetRelation.getSources().get(i);
170
171                if (starRelationTarget != null && !"*".equals(source.getColumn())
172                        && !DlineageUtil.getIdentifierNormalColumnName(starRelationTarget.getColumn(), dbVendor)
173                        .equals(DlineageUtil.getIdentifierNormalColumnName(source.getColumn(), dbVendor))) {
174                    continue;
175                }
176
177                String sourceColumnId = source.getId();
178                String sourceParentId = source.getParent_id();
179                if (sourceParentId == null || sourceColumnId == null) {
180                    continue;
181                }
182                if (isTarget(option, instance, sourceParentId)) {
183                    List<transform> transforms2 = new ArrayList<transform>(transforms.size());
184                    transforms2.addAll(transforms);
185                    Collections.reverse(transforms2);
186                    
187                    List<candidateTable> candidateTables2 = new ArrayList<candidateTable>(candidateTables.size());
188                    candidateTables2.addAll(candidateTables);
189                    
190                    sourceColumn sourceColumnCopy = DlineageUtil.copySourceColumn(source);
191                    for (transform t : transforms2) {
192                        sourceColumnCopy.addTransform(t);
193                    }
194                    for (candidateTable t : candidateTables2) {
195                        sourceColumnCopy.addCandidateParent(t);
196                    }
197                    
198                    if (Boolean.TRUE.equals(target.isStruct()) && Boolean.TRUE.equals(source.isStruct())) {
199                        List<String> targetColumns = SQLUtil.parseNames(target.getColumn());
200                        List<String> sourceColumns = SQLUtil.parseNames(source.getColumn());
201                        if (!DlineageUtil.getIdentifierNormalColumnName(targetColumns.get(targetColumns.size() - 1), dbVendor)
202                                .equals(DlineageUtil.getIdentifierNormalColumnName(sourceColumns.get(sourceColumns.size() - 1), dbVendor))) {
203                            continue;
204                        }
205                    }
206                    relationSources.add(new Pair<sourceColumn, List<String>>(sourceColumnCopy, Arrays.asList(pathTypes)));
207                } else {
208                    Set<relationship> sourceRelations = sourceIdRelationMap
209                            .get(source.getParent_id() + "." + source.getId());
210                    if (sourceRelations != null) {
211                        if (paths.contains(source.getParent_id() + "." + source.getId())) {
212                            continue;
213                        } else {
214                            paths.add(source.getParent_id() + "." + source.getId());
215                            if (source.getTransforms() != null) {
216                                transforms.addAll(source.getTransforms());
217                            }
218                            if (source.getCandidateParents() != null) {
219                                candidateTables.addAll(source.getCandidateParents());
220                            }
221                        }
222                        if (sourceRelations != null) {
223                            for (relationship relation : sourceRelations) {
224                                LinkedHashSet<transform> transforms2 = new LinkedHashSet<transform>(transforms.size());
225                                transforms2.addAll(transforms);
226                                LinkedHashSet<candidateTable> candidateTables2 = new LinkedHashSet<candidateTable>(candidateTables.size());
227                                candidateTables2.addAll(candidateTables);
228                                String[] types = new String[pathTypes.length + 1];
229                                types[0] = relation.getType();
230                                System.arraycopy(pathTypes, 0, types, 1, pathTypes.length);
231                                if (!"*".equals(source.getColumn())) {
232                                    findStarSourceRelations(option, dbVendor, target, instance, null, sourceIdRelationMap, relation, relationSources,
233                                            types, paths, transforms2, candidateTables2);
234                                } else {
235                                    findStarSourceRelations(option, dbVendor, target, instance,
236                                            starRelationTarget == null ? targetRelation.getTarget() : starRelationTarget,
237                                            sourceIdRelationMap, relation, relationSources, types, paths, transforms, candidateTables2);
238                                }
239                            }
240                        }
241                    }
242                }
243            }
244        }
245    }
246
247    private boolean isTarget(DataflowRemoveOption option, dataflow instance, String targetParentId) {
248        if (targetTables.containsKey(targetParentId))
249            return targetTables.get(targetParentId);
250        if (isTable(instance, targetParentId)) {
251            targetTables.put(targetParentId, true);
252            return true;
253        } else if (isView(instance, targetParentId)) {
254            targetTables.put(targetParentId, true);
255            return true;
256        } else if (isStage(instance, targetParentId)) {
257            targetTables.put(targetParentId, true);
258            return true;
259        } else if (isSequence(instance, targetParentId)) {
260            targetTables.put(targetParentId, true);
261            return true;
262        } else if (isDataSource(instance, targetParentId)) {
263            targetTables.put(targetParentId, true);
264            return true;
265        } else if (isDatabase(instance, targetParentId)) {
266            targetTables.put(targetParentId, true);
267            return true;
268        } else if (isSchema(instance, targetParentId)) {
269            targetTables.put(targetParentId, true);
270            return true;
271        } else if (isStream(instance, targetParentId)) {
272            targetTables.put(targetParentId, true);
273            return true;
274        } else if (isFile(instance, targetParentId)) {
275            targetTables.put(targetParentId, true);
276            return true;
277        } else if (isVariable(option, instance, targetParentId)) {
278            targetTables.put(targetParentId, true);
279            return true;
280        } else if (isTargetResultSet(option, instance, targetParentId)) {
281            targetTables.put(targetParentId, true);
282            return true;
283        }
284        targetTables.put(targetParentId, false);
285        return false;
286    }
287
288    private boolean isTable(dataflow instance, String targetParentId) {
289        if (isTables.containsKey(targetParentId)) {
290            return isTables.get(targetParentId);
291        }
292        if (instance.getTables() != null) {
293            for (int i = 0; i < instance.getTables().size(); i++) {
294                table resultSet = instance.getTables().get(i);
295                isTables.put(resultSet.getId(), true);
296                if (resultSet.getId().equalsIgnoreCase(targetParentId)) {
297                    return true;
298                }
299            }
300        }
301        isTables.put(targetParentId, false);
302        return false;
303    }
304
305    private boolean isView(dataflow instance, String targetParent) {
306        if (instance.getViews() != null) {
307            for (int i = 0; i < instance.getViews().size(); i++) {
308                table resultSet = instance.getViews().get(i);
309                if (resultSet.getId().equalsIgnoreCase(targetParent)) {
310                    return true;
311                }
312            }
313        }
314        return false;
315    }
316
317    private boolean isStage(dataflow instance, String targetParent) {
318        if (instance.getStages() != null) {
319            for (int i = 0; i < instance.getStages().size(); i++) {
320                table resultSet = instance.getStages().get(i);
321                if (resultSet.getId().equalsIgnoreCase(targetParent)) {
322                    return true;
323                }
324            }
325        }
326        return false;
327    }
328
329    private boolean isSequence(dataflow instance, String targetParent) {
330        if (instance.getSequences() != null) {
331            for (int i = 0; i < instance.getSequences().size(); i++) {
332                table resultSet = instance.getSequences().get(i);
333                if (resultSet.getId().equalsIgnoreCase(targetParent)) {
334                    return true;
335                }
336            }
337        }
338        return false;
339    }
340
341    private boolean isDataSource(dataflow instance, String targetParent) {
342        if (instance.getDatasources() != null) {
343            for (int i = 0; i < instance.getDatasources().size(); i++) {
344                table resultSet = instance.getDatasources().get(i);
345                if (resultSet.getId().equalsIgnoreCase(targetParent)) {
346                    return true;
347                }
348            }
349        }
350        return false;
351    }
352
353    private boolean isDatabase(dataflow instance, String targetParent) {
354        if (instance.getDatabases() != null) {
355            for (int i = 0; i < instance.getDatabases().size(); i++) {
356                table resultSet = instance.getDatabases().get(i);
357                if (resultSet.getId().equalsIgnoreCase(targetParent)) {
358                    return true;
359                }
360            }
361        }
362        return false;
363    }
364
365    private boolean isSchema(dataflow instance, String targetParent) {
366        if (instance.getSchemas() != null) {
367            for (int i = 0; i < instance.getSchemas().size(); i++) {
368                table resultSet = instance.getSchemas().get(i);
369                if (resultSet.getId().equalsIgnoreCase(targetParent)) {
370                    return true;
371                }
372            }
373        }
374        return false;
375    }
376
377    private boolean isStream(dataflow instance, String targetParent) {
378        if (instance.getStreams() != null) {
379            for (int i = 0; i < instance.getStreams().size(); i++) {
380                table resultSet = instance.getStreams().get(i);
381                if (resultSet.getId().equalsIgnoreCase(targetParent)) {
382                    return true;
383                }
384            }
385        }
386        return false;
387    }
388
389    private boolean isFile(dataflow instance, String targetParent) {
390        if (instance.getPaths() != null) {
391            for (int i = 0; i < instance.getPaths().size(); i++) {
392                table resultSet = instance.getPaths().get(i);
393                if (resultSet.getId().equalsIgnoreCase(targetParent)) {
394                    return true;
395                }
396            }
397        }
398        return false;
399    }
400
401    private boolean isVariable(DataflowRemoveOption option, dataflow instance, String targetParent) {
402        if (instance.getVariables() != null) {
403            for (int i = 0; i < instance.getVariables().size(); i++) {
404                table variable = instance.getVariables().get(i);
405                if (variable.getId().equalsIgnoreCase(targetParent)) {
406                    return !isRemoveVariable(option, variable);
407                }
408            }
409        }
410        return false;
411    }
412
413    private boolean isTargetResultSet(DataflowRemoveOption option, dataflow instance, String targetParent) {
414        if (instance.getResultsets() != null) {
415            for (int i = 0; i < instance.getResultsets().size(); i++) {
416                table resultSet = instance.getResultsets().get(i);
417                if (resultSet.getId().equalsIgnoreCase(targetParent)) {
418                    return !isRemoveResultSet(option, resultSet);
419                }
420            }
421        }
422        return false;
423    }
424
425    private String mergeRelationType(List<Pair<sourceColumn, List<String>>> typePaths) {
426        RelationshipType relationType = RelationshipType.join;
427        for (int i = 0; i < typePaths.size(); i++) {
428            List<String> path = typePaths.get(i).second;
429            RelationshipType type = RelationshipType.valueOf(getRelationType(path));
430            if (type.ordinal() < relationType.ordinal()) {
431                relationType = type;
432            }
433        }
434        return relationType.name();
435    }
436
437    private String getRelationType(List<String> typePaths) {
438        if (typePaths.contains("join"))
439            return "join";
440        if (typePaths.contains("fdr"))
441            return "fdr";
442        if (typePaths.contains("frd"))
443            return "frd";
444        if (typePaths.contains("fddi"))
445            return "fddi";
446        return "fdd";
447    }
448
449}