001package gudusoft.gsqlparser.dlineage.util;
002
003import gudusoft.gsqlparser.EDbVendor;
004import gudusoft.gsqlparser.dlineage.dataflow.model.DataflowRemoveOption;
005import gudusoft.gsqlparser.dlineage.dataflow.model.RelationshipType;
006import gudusoft.gsqlparser.dlineage.dataflow.model.ResultSetType;
007import gudusoft.gsqlparser.dlineage.dataflow.model.xml.*;
008import gudusoft.gsqlparser.util.Logger;
009import gudusoft.gsqlparser.util.LoggerFactory;
010import gudusoft.gsqlparser.sqlenv.ESQLDataObjectType;
011
012import java.util.*;
013
014public class DataflowRemoveHelper {
015
016    private static final Logger logger = LoggerFactory.getLogger(DataflowRemoveHelper.class);
017
018    private Map<String, Boolean> isTables = new HashMap<String, Boolean>();
019    private Map<String, Boolean> targetTables = new HashMap<String, Boolean>();
020
021    /** The document's published columns by id, for {@link DlineageUtil#isStarEndpoint}. */
022    private java.util.Map<String, column> columnsById = java.util.Collections.emptyMap();
023
024    public dataflow remove(dataflow instance, EDbVendor dbVendor, DataflowRemoveOption option) {
025        try {
026            columnsById = DlineageUtil.outputColumnsById(instance);
027            targetTables.clear();
028            isTables.clear();
029            dataflow simple = new dataflow();
030            List<relationship> simpleRelations = new ArrayList<relationship>();
031            List<relationship> relations = instance.getRelationships();
032            if (relations != null && relations.size() > 0) {
033                Map<String, Set<relationship>> targetIdRelationMap = new HashMap<String, Set<relationship>>();
034                for (relationship relation : relations) {
035                    if (relation.getTarget() != null) {
036                        String key = relation.getTarget().getParent_id() + "." + relation.getTarget().getId();
037                        if (!targetIdRelationMap.containsKey(key)) {
038                            targetIdRelationMap.put(key, new HashSet<relationship>());
039                        }
040                        targetIdRelationMap.get(key).add(relation);
041                    }
042                }
043
044                long maxId = Long
045                        .parseLong(relations.get(relations.size() - 1).getId().split("\\-")[0].replace("_", "")) * 100;
046                for (int i = 0; i < relations.size(); i++) {
047                    relationship relationElem = relations.get(i);
048                    if (RelationshipType.call.name().equals(relationElem.getType())) {
049                        simpleRelations.add(relationElem);
050                        continue;
051                    }
052                    if (RelationshipType.er.name().equals(relationElem.getType())) {
053                        simpleRelations.add(relationElem);
054                        continue;
055                    }
056                    targetColumn target = relationElem.getTarget();
057                    String targetParent = target.getParent_id();
058                    if (isTarget(option, instance, targetParent)) {
059                        List<Pair<sourceColumn, List<String>>> relationSources = new ArrayList<Pair<sourceColumn, List<String>>>();
060                        findSourceRelations(option, dbVendor, target, instance, targetIdRelationMap, relationElem, relationSources,
061                                new String[]{relationElem.getType()});
062                        if (relationSources.size() > 0) {
063                            Map<sourceColumn, List<Pair<sourceColumn, List<String>>>> columnMap = new HashMap<sourceColumn, List<Pair<sourceColumn, List<String>>>>();
064                            for (Pair<sourceColumn, List<String>> item : relationSources) {
065                                if (!columnMap.containsKey(item.first)) {
066                                    columnMap.put(item.first, new ArrayList<Pair<sourceColumn, List<String>>>());
067                                }
068                                columnMap.get(item.first).add(item);
069                            }
070
071                            Iterator<sourceColumn> iter = columnMap.keySet().iterator();
072                            Map<String, List<sourceColumn>> relationSourceMap = new HashMap<String, List<sourceColumn>>();
073                            while (iter.hasNext()) {
074                                sourceColumn column = iter.next();
075                                String relationType = mergeRelationType(columnMap.get(column));
076                                if (!relationSourceMap.containsKey(relationType)) {
077                                    relationSourceMap.put(relationType, new ArrayList<sourceColumn>());
078                                }
079                                relationSourceMap.get(relationType).add(column);
080                            }
081
082                            Iterator<String> sourceIter = relationSourceMap.keySet().iterator();
083                            while (sourceIter.hasNext()) {
084                                String relationType = sourceIter.next();
085                                relationship simpleRelation = (relationship) relationElem.clone();
086                                simpleRelation.setSources(relationSourceMap.get(relationType));
087                                simpleRelation.setType(relationType);
088                                simpleRelation.setId(String.valueOf(++maxId));
089                                simpleRelations.add(simpleRelation);
090                            }
091                        }
092                    }
093                }
094            }
095            simple.setErrors(instance.getErrors());
096            simple.setPackages(instance.getPackages());
097            simple.setProcedures(instance.getProcedures());
098            simple.setProcesses(instance.getProcesses());
099            simple.setTables(instance.getTables());
100            simple.setViews(instance.getViews());
101            simple.setDatabases(instance.getDatabases());
102            simple.setSchemas(instance.getSchemas());
103            simple.setStages(instance.getStages());
104            simple.setSequences(instance.getSequences());
105            simple.setDatasources(instance.getDatasources());
106            simple.setStreams(instance.getStreams());
107            simple.setPaths(instance.getPaths());
108
109            if (instance.getVariables() != null) {
110                List<table> variables = new ArrayList<table>();
111                for (int i = 0; i < instance.getVariables().size(); i++) {
112                    table variable = instance.getVariables().get(i);
113                    if (!isRemoveVariable(option, variable)) {
114                        variables.add(variable);
115                    }
116                }
117                simple.setVariables(variables);
118            }
119            if (instance.getResultsets() != null) {
120                List<table> resultSets = new ArrayList<table>();
121                for (int i = 0; i < instance.getResultsets().size(); i++) {
122                    table resultSet = instance.getResultsets().get(i);
123                    if (!isRemoveResultSet(option, resultSet)) {
124                        resultSets.add(resultSet);
125                    }
126                }
127                simple.setResultsets(resultSets);
128            }
129            simple.setRelationships(simpleRelations);
130            simple.setOrientation(instance.getOrientation());
131            return simple;
132        } catch (Exception e) {
133            logger.error("Remove dataflow resultSet failed.", e);
134        }
135        return instance;
136    }
137
138    private boolean isRemoveVariable(DataflowRemoveOption option, table variable) {
139        if (option.isRemoveVariable()) {
140            if ("variable".equals(variable.getType())) {
141                return !variable.isTarget();
142            }
143        } else if (option.isRemoveCursor()) {
144            if ("variable".equals(variable.getType()) && "cursor".equals(variable.getSubType())) {
145                return !variable.isTarget();
146            }
147        }
148        return false;
149    }
150
151    private boolean isRemoveResultSet(DataflowRemoveOption option, table resultSet) {
152        ResultSetType resultSetType = ResultSetType.of(resultSet.getType());
153        if (option.getRemoveResultSetTypes().contains(resultSetType)) {
154            return true;
155        }
156        if (option.getRemoveResultSetTypes().contains(ResultSetType.function)) {
157            if ("function".equals(resultSet.getSubType())) {
158                return true;
159            }
160        }
161        return false;
162    }
163
164    private void findSourceRelations(DataflowRemoveOption option, EDbVendor dbVendor, targetColumn target, dataflow instance, Map<String, Set<relationship>> sourceIdRelationMap,
165                                     relationship targetRelation, List<Pair<sourceColumn, List<String>>> relationSources, String[] pathTypes) {
166        findStarSourceRelations(option, dbVendor, target, instance, null, false, sourceIdRelationMap, targetRelation, relationSources,
167                pathTypes, new HashSet<String>(), new LinkedHashSet<transform>(), new LinkedHashSet<candidateTable>());
168    }
169
170    /**
171     * @param positionUnproven whether the {@code *} the walk follows is read at a position its members' unknown order
172     *        cannot tell ({@link sourceColumn#isPositionUnproven()}): the {@code *} of a table that fills it stands for
173     *        it, never a column of that table named like {@code starRelationTarget} (Mantis 4860, as the simple output
174     *        of DataFlowAnalyzer does)
175     */
176    private void findStarSourceRelations(DataflowRemoveOption option, EDbVendor dbVendor, targetColumn target, dataflow instance, targetColumn starRelationTarget,
177                                         boolean positionUnproven, Map<String, Set<relationship>> sourceIdRelationMap,
178                                         relationship targetRelation, List<Pair<sourceColumn, List<String>>> relationSources, String[] pathTypes,
179                                         Set<String> paths, Set<transform> transforms, Set<candidateTable> candidateTables) {
180        if (targetRelation != null && targetRelation.getSources() != null) {
181            Set<String> starTableIds = starTableIds(instance, targetRelation, positionUnproven);
182            for (int i = 0; i < targetRelation.getSources().size(); i++) {
183                sourceColumn source = targetRelation.getSources().get(i);
184
185                if (starTableIds.contains(source.getParent_id())
186                        && !DlineageUtil.isStarEndpoint(columnsById, source.getId(), source.getColumn())) {
187                    continue;
188                }
189                if (starRelationTarget != null && !DlineageUtil.isStarEndpoint(columnsById, source.getId(), source.getColumn())
190                        && !sameColumn(dbVendor, starRelationTarget.getColumn(), source.getColumn())) {
191                    continue;
192                }
193
194                String sourceColumnId = source.getId();
195                String sourceParentId = source.getParent_id();
196                if (sourceParentId == null || sourceColumnId == null) {
197                    continue;
198                }
199                if (isTarget(option, instance, sourceParentId)) {
200                    List<transform> transforms2 = new ArrayList<transform>(transforms.size());
201                    transforms2.addAll(transforms);
202                    Collections.reverse(transforms2);
203                    
204                    List<candidateTable> candidateTables2 = new ArrayList<candidateTable>(candidateTables.size());
205                    candidateTables2.addAll(candidateTables);
206                    
207                    sourceColumn sourceColumnCopy = DlineageUtil.copySourceColumn(source);
208                    for (transform t : transforms2) {
209                        sourceColumnCopy.addTransform(t);
210                    }
211                    for (candidateTable t : candidateTables2) {
212                        sourceColumnCopy.addCandidateParent(t);
213                    }
214                    
215                    // a STRUCT field node folds onto the source its path reaches, whatever either one is named
216                    // (STRUCT(customer.name AS who): buyer.who <- customer.name), as DataFlowAnalyzer.getSimpleDataflow
217                    relationSources.add(new Pair<sourceColumn, List<String>>(sourceColumnCopy, Arrays.asList(pathTypes)));
218                } else {
219                    Set<relationship> sourceRelations = sourceIdRelationMap
220                            .get(source.getParent_id() + "." + source.getId());
221                    if (sourceRelations != null) {
222                        if (paths.contains(source.getParent_id() + "." + source.getId())) {
223                            continue;
224                        } else {
225                            paths.add(source.getParent_id() + "." + source.getId());
226                            if (source.getTransforms() != null) {
227                                transforms.addAll(source.getTransforms());
228                            }
229                            if (source.getCandidateParents() != null) {
230                                candidateTables.addAll(source.getCandidateParents());
231                            }
232                        }
233                        if (sourceRelations != null) {
234                            for (relationship relation : sourceRelations) {
235                                LinkedHashSet<transform> transforms2 = new LinkedHashSet<transform>(transforms.size());
236                                transforms2.addAll(transforms);
237                                LinkedHashSet<candidateTable> candidateTables2 = new LinkedHashSet<candidateTable>(candidateTables.size());
238                                candidateTables2.addAll(candidateTables);
239                                String[] types = new String[pathTypes.length + 1];
240                                types[0] = relation.getType();
241                                System.arraycopy(pathTypes, 0, types, 1, pathTypes.length);
242                                if (!DlineageUtil.isStarEndpoint(columnsById, source.getId(), source.getColumn())) {
243                                    findStarSourceRelations(option, dbVendor, target, instance, null, false, sourceIdRelationMap, relation,
244                                            relationSources, types, paths, transforms2, candidateTables2);
245                                } else {
246                                    findStarSourceRelations(option, dbVendor, target, instance,
247                                            starRelationTarget == null ? targetRelation.getTarget() : starRelationTarget,
248                                            positionUnproven || source.isPositionUnproven(), sourceIdRelationMap, relation,
249                                            relationSources, types, paths, transforms, candidateTables2);
250                                }
251                            }
252                        }
253                    }
254                }
255            }
256        }
257    }
258
259    /** The tables whose {@code *} {@code relation} reads, when the walk's {@code *} has no member proven at the position. */
260    private Set<String> starTableIds(dataflow instance, relationship relation, boolean positionUnproven) {
261        Set<String> ids = new HashSet<String>();
262        if (positionUnproven) {
263            for (sourceColumn source : relation.getSources()) {
264                if (source.getParent_id() != null && DlineageUtil.isStarEndpoint(columnsById, source.getId(), source.getColumn())
265                        && isTable(instance, source.getParent_id())) {
266                    ids.add(source.getParent_id());
267                }
268            }
269        }
270        return ids;
271    }
272
273    private boolean isTarget(DataflowRemoveOption option, dataflow instance, String targetParentId) {
274        if (targetTables.containsKey(targetParentId))
275            return targetTables.get(targetParentId);
276        if (isTable(instance, targetParentId)) {
277            targetTables.put(targetParentId, true);
278            return true;
279        } else if (isView(instance, targetParentId)) {
280            targetTables.put(targetParentId, true);
281            return true;
282        } else if (isStage(instance, targetParentId)) {
283            targetTables.put(targetParentId, true);
284            return true;
285        } else if (isSequence(instance, targetParentId)) {
286            targetTables.put(targetParentId, true);
287            return true;
288        } else if (isDataSource(instance, targetParentId)) {
289            targetTables.put(targetParentId, true);
290            return true;
291        } else if (isDatabase(instance, targetParentId)) {
292            targetTables.put(targetParentId, true);
293            return true;
294        } else if (isSchema(instance, targetParentId)) {
295            targetTables.put(targetParentId, true);
296            return true;
297        } else if (isStream(instance, targetParentId)) {
298            targetTables.put(targetParentId, true);
299            return true;
300        } else if (isFile(instance, targetParentId)) {
301            targetTables.put(targetParentId, true);
302            return true;
303        } else if (isVariable(option, instance, targetParentId)) {
304            targetTables.put(targetParentId, true);
305            return true;
306        } else if (isTargetResultSet(option, instance, targetParentId)) {
307            targetTables.put(targetParentId, true);
308            return true;
309        }
310        targetTables.put(targetParentId, false);
311        return false;
312    }
313
314    private boolean isTable(dataflow instance, String targetParentId) {
315        if (isTables.containsKey(targetParentId)) {
316            return isTables.get(targetParentId);
317        }
318        if (instance.getTables() != null) {
319            for (int i = 0; i < instance.getTables().size(); i++) {
320                table resultSet = instance.getTables().get(i);
321                isTables.put(resultSet.getId(), true);
322                if (resultSet.getId().equalsIgnoreCase(targetParentId)) {
323                    return true;
324                }
325            }
326        }
327        isTables.put(targetParentId, false);
328        return false;
329    }
330
331    private boolean isView(dataflow instance, String targetParent) {
332        if (instance.getViews() != null) {
333            for (int i = 0; i < instance.getViews().size(); i++) {
334                table resultSet = instance.getViews().get(i);
335                if (resultSet.getId().equalsIgnoreCase(targetParent)) {
336                    return true;
337                }
338            }
339        }
340        return false;
341    }
342
343    private boolean isStage(dataflow instance, String targetParent) {
344        if (instance.getStages() != null) {
345            for (int i = 0; i < instance.getStages().size(); i++) {
346                table resultSet = instance.getStages().get(i);
347                if (resultSet.getId().equalsIgnoreCase(targetParent)) {
348                    return true;
349                }
350            }
351        }
352        return false;
353    }
354
355    private boolean isSequence(dataflow instance, String targetParent) {
356        if (instance.getSequences() != null) {
357            for (int i = 0; i < instance.getSequences().size(); i++) {
358                table resultSet = instance.getSequences().get(i);
359                if (resultSet.getId().equalsIgnoreCase(targetParent)) {
360                    return true;
361                }
362            }
363        }
364        return false;
365    }
366
367    private boolean isDataSource(dataflow instance, String targetParent) {
368        if (instance.getDatasources() != null) {
369            for (int i = 0; i < instance.getDatasources().size(); i++) {
370                table resultSet = instance.getDatasources().get(i);
371                if (resultSet.getId().equalsIgnoreCase(targetParent)) {
372                    return true;
373                }
374            }
375        }
376        return false;
377    }
378
379    private boolean isDatabase(dataflow instance, String targetParent) {
380        if (instance.getDatabases() != null) {
381            for (int i = 0; i < instance.getDatabases().size(); i++) {
382                table resultSet = instance.getDatabases().get(i);
383                if (resultSet.getId().equalsIgnoreCase(targetParent)) {
384                    return true;
385                }
386            }
387        }
388        return false;
389    }
390
391    private boolean isSchema(dataflow instance, String targetParent) {
392        if (instance.getSchemas() != null) {
393            for (int i = 0; i < instance.getSchemas().size(); i++) {
394                table resultSet = instance.getSchemas().get(i);
395                if (resultSet.getId().equalsIgnoreCase(targetParent)) {
396                    return true;
397                }
398            }
399        }
400        return false;
401    }
402
403    private boolean isStream(dataflow instance, String targetParent) {
404        if (instance.getStreams() != null) {
405            for (int i = 0; i < instance.getStreams().size(); i++) {
406                table resultSet = instance.getStreams().get(i);
407                if (resultSet.getId().equalsIgnoreCase(targetParent)) {
408                    return true;
409                }
410            }
411        }
412        return false;
413    }
414
415    private boolean isFile(dataflow instance, String targetParent) {
416        if (instance.getPaths() != null) {
417            for (int i = 0; i < instance.getPaths().size(); i++) {
418                table resultSet = instance.getPaths().get(i);
419                if (resultSet.getId().equalsIgnoreCase(targetParent)) {
420                    return true;
421                }
422            }
423        }
424        return false;
425    }
426
427    private boolean isVariable(DataflowRemoveOption option, dataflow instance, String targetParent) {
428        if (instance.getVariables() != null) {
429            for (int i = 0; i < instance.getVariables().size(); i++) {
430                table variable = instance.getVariables().get(i);
431                if (variable.getId().equalsIgnoreCase(targetParent)) {
432                    return !isRemoveVariable(option, variable);
433                }
434            }
435        }
436        return false;
437    }
438
439    private boolean isTargetResultSet(DataflowRemoveOption option, dataflow instance, String targetParent) {
440        if (instance.getResultsets() != null) {
441            for (int i = 0; i < instance.getResultsets().size(); i++) {
442                table resultSet = instance.getResultsets().get(i);
443                if (resultSet.getId().equalsIgnoreCase(targetParent)) {
444                    return !isRemoveResultSet(option, resultSet);
445                }
446            }
447        }
448        return false;
449    }
450
451    private String mergeRelationType(List<Pair<sourceColumn, List<String>>> typePaths) {
452        RelationshipType relationType = RelationshipType.join;
453        for (int i = 0; i < typePaths.size(); i++) {
454            List<String> path = typePaths.get(i).second;
455            RelationshipType type = RelationshipType.valueOf(getRelationType(path));
456            if (type.ordinal() < relationType.ordinal()) {
457                relationType = type;
458            }
459        }
460        return relationType.name();
461    }
462
463    private String getRelationType(List<String> typePaths) {
464        if (typePaths.contains("join"))
465            return "join";
466        if (typePaths.contains("fdr"))
467            return "fdr";
468        if (typePaths.contains("frd"))
469            return "frd";
470        if (typePaths.contains("fddi"))
471            return "fddi";
472        return "fdd";
473    }
474
475
476    /** Two column names of the output model, compared as column paths by the vendor's rules. */
477    private static boolean sameColumn(EDbVendor dbVendor, String a, String b) {
478        if (a == null || b == null) {
479            return false;
480        }
481        return DlineageUtil.sameColumnPath(dbVendor, a, b);
482    }
483}