001package gudusoft.gsqlparser.dlineage.util;
002
003import java.util.*;
004
005import gudusoft.gsqlparser.EDbVendor;
006import gudusoft.gsqlparser.dlineage.dataflow.model.RelationshipType;
007import gudusoft.gsqlparser.dlineage.dataflow.model.xml.*;
008import gudusoft.gsqlparser.util.Logger;
009import gudusoft.gsqlparser.util.LoggerFactory;
010import gudusoft.gsqlparser.sqlenv.ESQLDataObjectType;
011
012public class RemoveDataflowFunction {
013
014    private static final Logger logger = LoggerFactory.getLogger(DataflowRemoveHelper.class);
015    private Map<String, Boolean> isTables = new HashMap<String, Boolean>();
016    private Map<String, Boolean> targetTables = new HashMap<String, Boolean>();
017
018    /** The document's published columns by id, for {@link DlineageUtil#isStarEndpoint}. */
019    private java.util.Map<String, column> columnsById = java.util.Collections.emptyMap();
020
021    public dataflow removeFunction(dataflow instance, EDbVendor dbVendor) {
022        if (instance.getResultsets() == null || instance.getResultsets().isEmpty()) {
023            return instance;
024        }
025        try {
026            columnsById = DlineageUtil.outputColumnsById(instance);
027            targetTables.clear();
028            isTables.clear();
029            dataflow simple = new dataflow();
030            simple.setCommentBindings(instance.getCommentBindings());
031            List<relationship> simpleRelations = new ArrayList<relationship>();
032            List<relationship> relations = instance.getRelationships();
033            if (relations != null && relations.size() > 0) {
034                Map<String, Set<relationship>> targetIdRelationMap = new HashMap<String, Set<relationship>>();
035                for (relationship relation : relations) {
036                    if (relation.getTarget() != null) {
037                        String key = relation.getTarget().getParent_id() + "." + relation.getTarget().getId();
038                        if (!targetIdRelationMap.containsKey(key)) {
039                            targetIdRelationMap.put(key, new HashSet<relationship>());
040                        }
041                        targetIdRelationMap.get(key).add(relation);
042                    }
043                }
044
045                long maxId = Long
046                        .parseLong(relations.get(relations.size() - 1).getId().split("\\-")[0].replace("_", "")) * 100;
047                for (int i = 0; i < relations.size(); i++) {
048                    relationship relationElem = relations.get(i);
049                    if (RelationshipType.call.name().equals(relationElem.getType())) {
050                        simpleRelations.add(relationElem);
051                        continue;
052                    }
053                    if (RelationshipType.er.name().equals(relationElem.getType())) {
054                        simpleRelations.add(relationElem);
055                        continue;
056                    }
057                    targetColumn target = relationElem.getTarget();
058                    String targetParent = target.getParent_id();
059                    if (isTarget(instance, targetParent)) {
060                        List<Pair<sourceColumn, List<String>>> relationSources = new ArrayList<Pair<sourceColumn, List<String>>>();
061                        findSourceRelations(dbVendor, target, instance, targetIdRelationMap, relationElem, relationSources,
062                                new String[]{relationElem.getType()});
063                        if (relationSources.size() > 0) {
064                            Map<sourceColumn, List<Pair<sourceColumn, List<String>>>> columnMap = new HashMap<sourceColumn, List<Pair<sourceColumn, List<String>>>>();
065                            for (Pair<sourceColumn, List<String>> item : relationSources) {
066                                if (!columnMap.containsKey(item.first)) {
067                                    columnMap.put(item.first, new ArrayList<Pair<sourceColumn, List<String>>>());
068                                }
069                                columnMap.get(item.first).add(item);
070                            }
071
072                            Iterator<sourceColumn> iter = columnMap.keySet().iterator();
073                            Map<String, List<sourceColumn>> relationSourceMap = new HashMap<String, List<sourceColumn>>();
074                            while (iter.hasNext()) {
075                                sourceColumn column = iter.next();
076                                String relationType = mergeRelationType(columnMap.get(column));
077                                if (!relationSourceMap.containsKey(relationType)) {
078                                    relationSourceMap.put(relationType, new ArrayList<sourceColumn>());
079                                }
080                                relationSourceMap.get(relationType).add(column);
081                            }
082
083                            Iterator<String> sourceIter = relationSourceMap.keySet().iterator();
084                            while (sourceIter.hasNext()) {
085                                String relationType = sourceIter.next();
086                                relationship simpleRelation = (relationship) relationElem.clone();
087                                simpleRelation.setSources(relationSourceMap.get(relationType));
088                                simpleRelation.setType(relationType);
089                                simpleRelation.setId(String.valueOf(++maxId));
090                                simpleRelations.add(simpleRelation);
091                            }
092                        }
093                    }
094                }
095            }
096            simple.setErrors(instance.getErrors());
097            simple.setPackages(instance.getPackages());
098            simple.setProcedures(instance.getProcedures());
099            simple.setProcesses(instance.getProcesses());
100            simple.setTables(instance.getTables());
101            simple.setViews(instance.getViews());
102            simple.setDatabases(instance.getDatabases());
103            simple.setSchemas(instance.getSchemas());
104            simple.setStages(instance.getStages());
105            simple.setSequences(instance.getSequences());
106            simple.setDatasources(instance.getDatasources());
107            simple.setStreams(instance.getStreams());
108            simple.setPaths(instance.getPaths());
109            simple.setVariables(instance.getVariables());
110            if (instance.getResultsets() != null) {
111                List<table> resultSets = new ArrayList<table>();
112                for (int i = 0; i < instance.getResultsets().size(); i++) {
113                    table resultSet = instance.getResultsets().get(i);
114                    if (!isFunction(resultSet)) {
115                        resultSets.add(resultSet);
116                    }
117                }
118                simple.setResultsets(resultSets);
119            }
120            simple.setRelationships(simpleRelations);
121            simple.setOrientation(instance.getOrientation());
122            gudusoft.gsqlparser.dlineage.comment.CommentBindingProjection.retain(simple);
123            return simple;
124        } catch (Exception e) {
125            logger.error("Remove dataflow function failed.", e);
126        }
127        return instance;
128    }
129
130    private boolean isFunction(table resultSet) {
131        if ("function".equals(resultSet.getType())) {
132            return true;
133        } else if ("resultset".equals(resultSet.getType()) && "function".equals(resultSet.getSubType())) {
134            return true;
135        }
136        return false;
137    }
138
139        private void findSourceRelations(EDbVendor dbVendor, targetColumn target, dataflow instance, Map<String, Set<relationship>> sourceIdRelationMap,
140                                                                         relationship targetRelation, List<Pair<sourceColumn, List<String>>> relationSources, String[] pathTypes) {
141                findStarSourceRelations(dbVendor, target, instance, null, false, sourceIdRelationMap, targetRelation, relationSources,
142                                pathTypes, new HashSet<String>(), new LinkedHashSet<transform>(), new LinkedHashSet<candidateTable>());
143        }
144
145    /**
146     * @param positionUnproven whether the {@code *} the walk follows is read at a position its members' unknown order
147     *        cannot tell ({@link sourceColumn#isPositionUnproven()}): the {@code *} of a table that fills it stands for
148     *        it, never a column of that table named like {@code starRelationTarget} (Mantis 4860, as the simple output
149     *        of DataFlowAnalyzer does)
150     */
151    private void findStarSourceRelations(EDbVendor dbVendor, targetColumn target, dataflow instance, targetColumn starRelationTarget,
152                                         boolean positionUnproven, Map<String, Set<relationship>> sourceIdRelationMap,
153                                         relationship targetRelation, List<Pair<sourceColumn, List<String>>> relationSources, String[] pathTypes,
154                                                                                 Set<String> paths, Set<transform> transforms, Set<candidateTable> candidateTables) {
155        if (targetRelation != null && targetRelation.getSources() != null) {
156            Set<String> starTableIds = starTableIds(instance, targetRelation, positionUnproven);
157            for (int i = 0; i < targetRelation.getSources().size(); i++) {
158                sourceColumn source = targetRelation.getSources().get(i);
159
160                if (starTableIds.contains(source.getParent_id())
161                        && !DlineageUtil.isStarEndpoint(columnsById, source.getId(), source.getColumn())) {
162                    continue;
163                }
164                if (starRelationTarget != null && !DlineageUtil.isStarEndpoint(columnsById, source.getId(), source.getColumn())
165                        && !sameColumn(dbVendor, starRelationTarget.getColumn(), source.getColumn())) {
166                    continue;
167                }
168
169                String sourceColumnId = source.getId();
170                String sourceParentId = source.getParent_id();
171                if (sourceParentId == null || sourceColumnId == null) {
172                    continue;
173                }
174                if (isTarget(instance, sourceParentId)) {
175                    List<transform> transforms2 = new ArrayList<transform>(transforms.size());
176                    transforms2.addAll(transforms);
177                    Collections.reverse(transforms2);
178                    
179                    List<candidateTable> candidateTables2 = new ArrayList<candidateTable>(candidateTables.size());
180                    candidateTables2.addAll(candidateTables);
181                    
182                    sourceColumn sourceColumnCopy = DlineageUtil.copySourceColumn(source);
183                    for (transform t : transforms2) {
184                        sourceColumnCopy.addTransform(t);
185                    }
186                    for (candidateTable t : candidateTables2) {
187                        sourceColumnCopy.addCandidateParent(t);
188                    }
189                    
190                    // a STRUCT field node folds onto the source its path reaches, whatever either one is named
191                    // (STRUCT(customer.name AS who): buyer.who <- customer.name), as DataFlowAnalyzer.getSimpleDataflow
192                    relationSources.add(new Pair<sourceColumn, List<String>>(sourceColumnCopy, Arrays.asList(pathTypes)));
193                } else {
194                    Set<relationship> sourceRelations = sourceIdRelationMap
195                            .get(source.getParent_id() + "." + source.getId());
196                    if (sourceRelations != null) {
197                        if (paths.contains(source.getParent_id() + "." + source.getId())) {
198                            continue;
199                        } else {
200                            paths.add(source.getParent_id() + "." + source.getId());
201                            if (source.getTransforms() != null) {
202                                transforms.addAll(source.getTransforms());
203                            }
204                            if (source.getCandidateParents() != null) {
205                                candidateTables.addAll(source.getCandidateParents());
206                            }
207                        }
208                        if (sourceRelations != null) {
209                            for (relationship relation : sourceRelations) {
210                                LinkedHashSet<transform> transforms2 = new LinkedHashSet<transform>(transforms.size());
211                                transforms2.addAll(transforms);
212                                LinkedHashSet<candidateTable> candidateTables2 = new LinkedHashSet<candidateTable>(candidateTables.size());
213                                candidateTables2.addAll(candidateTables);
214                                String[] types = new String[pathTypes.length + 1];
215                                types[0] = relation.getType();
216                                System.arraycopy(pathTypes, 0, types, 1, pathTypes.length);
217                                if (!DlineageUtil.isStarEndpoint(columnsById, source.getId(), source.getColumn())) {
218                                    findStarSourceRelations(dbVendor, target, instance, null, false, sourceIdRelationMap, relation,
219                                            relationSources, types, paths, transforms2, candidateTables2);
220                                } else {
221                                    findStarSourceRelations(dbVendor, target, instance,
222                                            starRelationTarget == null ? targetRelation.getTarget() : starRelationTarget,
223                                            positionUnproven || source.isPositionUnproven(), sourceIdRelationMap, relation,
224                                            relationSources, types, paths, transforms, candidateTables2);
225                                }
226                            }
227                        }
228                    }
229                }
230            }
231        }
232    }
233
234    /** The tables whose {@code *} {@code relation} reads, when the walk's {@code *} has no member proven at the position. */
235    private Set<String> starTableIds(dataflow instance, relationship relation, boolean positionUnproven) {
236        Set<String> ids = new HashSet<String>();
237        if (positionUnproven) {
238            for (sourceColumn source : relation.getSources()) {
239                if (source.getParent_id() != null && DlineageUtil.isStarEndpoint(columnsById, source.getId(), source.getColumn())
240                        && isTable(instance, source.getParent_id())) {
241                    ids.add(source.getParent_id());
242                }
243            }
244        }
245        return ids;
246    }
247
248    private boolean isTarget(dataflow instance, String targetParentId) {
249        if (targetTables.containsKey(targetParentId))
250            return targetTables.get(targetParentId);
251        if (isTable(instance, targetParentId)) {
252            targetTables.put(targetParentId, true);
253            return true;
254        } else if (isView(instance, targetParentId)) {
255            targetTables.put(targetParentId, true);
256            return true;
257        } else if (isStage(instance, targetParentId)) {
258            targetTables.put(targetParentId, true);
259            return true;
260        } else if (isSequence(instance, targetParentId)) {
261            targetTables.put(targetParentId, true);
262            return true;
263        } else if (isDataSource(instance, targetParentId)) {
264            targetTables.put(targetParentId, true);
265            return true;
266        } else if (isDatabase(instance, targetParentId)) {
267            targetTables.put(targetParentId, true);
268            return true;
269        } else if (isSchema(instance, targetParentId)) {
270            targetTables.put(targetParentId, true);
271            return true;
272        } else if (isStream(instance, targetParentId)) {
273            targetTables.put(targetParentId, true);
274            return true;
275        } else if (isFile(instance, targetParentId)) {
276            targetTables.put(targetParentId, true);
277            return true;
278        } else if (isVariable(instance, targetParentId)) {
279            targetTables.put(targetParentId, true);
280            return true;
281        } else if (isTargetResultSet(instance, targetParentId)) {
282            targetTables.put(targetParentId, true);
283            return true;
284        }
285        targetTables.put(targetParentId, false);
286        return false;
287    }
288
289    private boolean isTable(dataflow instance, String targetParentId) {
290        if (isTables.containsKey(targetParentId)) {
291            return isTables.get(targetParentId);
292        }
293        if (instance.getTables() != null) {
294            for (int i = 0; i < instance.getTables().size(); i++) {
295                table resultSet = instance.getTables().get(i);
296                isTables.put(resultSet.getId(), true);
297                if (resultSet.getId().equalsIgnoreCase(targetParentId)) {
298                    return true;
299                }
300            }
301        }
302        isTables.put(targetParentId, false);
303        return false;
304    }
305
306    private boolean isView(dataflow instance, String targetParent) {
307        if (instance.getViews() != null) {
308            for (int i = 0; i < instance.getViews().size(); i++) {
309                table resultSet = instance.getViews().get(i);
310                if (resultSet.getId().equalsIgnoreCase(targetParent)) {
311                    return true;
312                }
313            }
314        }
315        return false;
316    }
317
318    private boolean isStage(dataflow instance, String targetParent) {
319        if (instance.getStages() != null) {
320            for (int i = 0; i < instance.getStages().size(); i++) {
321                table resultSet = instance.getStages().get(i);
322                if (resultSet.getId().equalsIgnoreCase(targetParent)) {
323                    return true;
324                }
325            }
326        }
327        return false;
328    }
329
330    private boolean isSequence(dataflow instance, String targetParent) {
331        if (instance.getSequences() != null) {
332            for (int i = 0; i < instance.getSequences().size(); i++) {
333                table resultSet = instance.getSequences().get(i);
334                if (resultSet.getId().equalsIgnoreCase(targetParent)) {
335                    return true;
336                }
337            }
338        }
339        return false;
340    }
341
342    private boolean isDataSource(dataflow instance, String targetParent) {
343        if (instance.getDatasources() != null) {
344            for (int i = 0; i < instance.getDatasources().size(); i++) {
345                table resultSet = instance.getDatasources().get(i);
346                if (resultSet.getId().equalsIgnoreCase(targetParent)) {
347                    return true;
348                }
349            }
350        }
351        return false;
352    }
353
354    private boolean isDatabase(dataflow instance, String targetParent) {
355        if (instance.getDatabases() != null) {
356            for (int i = 0; i < instance.getDatabases().size(); i++) {
357                table resultSet = instance.getDatabases().get(i);
358                if (resultSet.getId().equalsIgnoreCase(targetParent)) {
359                    return true;
360                }
361            }
362        }
363        return false;
364    }
365
366    private boolean isSchema(dataflow instance, String targetParent) {
367        if (instance.getSchemas() != null) {
368            for (int i = 0; i < instance.getSchemas().size(); i++) {
369                table resultSet = instance.getSchemas().get(i);
370                if (resultSet.getId().equalsIgnoreCase(targetParent)) {
371                    return true;
372                }
373            }
374        }
375        return false;
376    }
377
378    private boolean isStream(dataflow instance, String targetParent) {
379        if (instance.getStreams() != null) {
380            for (int i = 0; i < instance.getStreams().size(); i++) {
381                table resultSet = instance.getStreams().get(i);
382                if (resultSet.getId().equalsIgnoreCase(targetParent)) {
383                    return true;
384                }
385            }
386        }
387        return false;
388    }
389
390    private boolean isFile(dataflow instance, String targetParent) {
391        if (instance.getPaths() != null) {
392            for (int i = 0; i < instance.getPaths().size(); i++) {
393                table resultSet = instance.getPaths().get(i);
394                if (resultSet.getId().equalsIgnoreCase(targetParent)) {
395                    return true;
396                }
397            }
398        }
399        return false;
400    }
401
402    private boolean isVariable(dataflow instance, String targetParent) {
403        if (instance.getVariables() != null) {
404            for (int i = 0; i < instance.getVariables().size(); i++) {
405                table resultSet = instance.getVariables().get(i);
406                if (resultSet.getId().equalsIgnoreCase(targetParent)) {
407                    return true;
408                }
409            }
410        }
411        return false;
412    }
413
414    private boolean isTargetResultSet(dataflow instance, String targetParent) {
415        if (instance.getResultsets() != null) {
416            for (int i = 0; i < instance.getResultsets().size(); i++) {
417                table resultSet = instance.getResultsets().get(i);
418                if (resultSet.getId().equalsIgnoreCase(targetParent)) {
419                    return !isFunction(resultSet);
420                }
421            }
422        }
423        return false;
424    }
425
426    private String mergeRelationType(List<Pair<sourceColumn, List<String>>> typePaths) {
427        RelationshipType relationType = RelationshipType.join;
428        for (int i = 0; i < typePaths.size(); i++) {
429            List<String> path = typePaths.get(i).second;
430            RelationshipType type = RelationshipType.valueOf(getRelationType(path));
431            if (type.ordinal() < relationType.ordinal()) {
432                relationType = type;
433            }
434        }
435        return relationType.name();
436    }
437
438    private String getRelationType(List<String> typePaths) {
439        if (typePaths.contains("join"))
440            return "join";
441        if (typePaths.contains("fdr"))
442            return "fdr";
443        if (typePaths.contains("frd"))
444            return "frd";
445        if (typePaths.contains("fddi"))
446            return "fddi";
447        return "fdd";
448    }
449
450
451    /** Two column names of the output model, compared as column paths by the vendor's rules. */
452    private static boolean sameColumn(EDbVendor dbVendor, String a, String b) {
453        if (a == null || b == null) {
454            return false;
455        }
456        return DlineageUtil.sameColumnPath(dbVendor, a, b);
457    }
458}