Source code

001package gudusoft.gsqlparser.dlineage.util;
002
003import java.io.IOException;
004import java.io.StringWriter;
005import java.util.ArrayList;
006import java.util.Arrays;
007import java.util.Collections;
008import java.util.Comparator;
009import java.util.HashMap;
010import java.util.HashSet;
011import java.util.Iterator;
012import java.util.List;
013import java.util.Map;
014import java.util.Objects;
015import java.util.Set;
016import java.util.TreeMap;
017import java.util.TreeSet;
018
019import gudusoft.gsqlparser.EDbVendor;
020import gudusoft.gsqlparser.dlineage.DataFlowAnalyzer;
021import gudusoft.gsqlparser.dlineage.dataflow.model.ModelBindingManager;
022import gudusoft.gsqlparser.dlineage.dataflow.model.xml.column;
023import gudusoft.gsqlparser.dlineage.dataflow.model.xml.dataflow;
024import gudusoft.gsqlparser.dlineage.dataflow.model.xml.relationship;
025import gudusoft.gsqlparser.dlineage.dataflow.model.xml.sourceColumn;
026import gudusoft.gsqlparser.dlineage.dataflow.model.xml.table;
027import gudusoft.gsqlparser.dlineage.dataflow.model.xml.targetColumn;
028import gudusoft.gsqlparser.sqlenv.ESQLDataObjectType;
029import gudusoft.gsqlparser.sqlenv.IdentifierService;
030import gudusoft.gsqlparser.util.Logger;
031import gudusoft.gsqlparser.util.LoggerFactory;
032import gudusoft.gsqlparser.util.SQLUtil;
033import gudusoft.gsqlparser.util.csv.CsvWriter;
034import gudusoft.gsqlparser.util.json.JSON;
035
036public class TableFlowUtility {
037
038        private static final Logger logger = LoggerFactory.getLogger(TableFlowUtility.class);
039
040        public static class LeftMostSourceTable {
041                private Table[] targetTables;
042                private TableColumn[] attributes;
043
044                public TableColumn[] getAttributes() {
045                        return attributes;
046                }
047
048                public void setAttributes(TableColumn[] attributes) {
049                        this.attributes = attributes;
050                }
051
052                public Table[] getTargetTables() {
053                        return targetTables;
054                }
055
056                public void setTargetTables(Table[] targetTables) {
057                        this.targetTables = targetTables;
058                }
059
060                public void appendAttribute(TableColumn attribute) {
061                        if (attribute == null) {
062                                return;
063                        }
064                        if (attributes == null) {
065                                attributes = new TableColumn[] { attribute };
066                        } else {
067                                TableColumn[] newAttributes = new TableColumn[attributes.length + 1];
068                                System.arraycopy(attributes, 0, newAttributes, 0, attributes.length);
069                                newAttributes[attributes.length] = attribute;
070                                this.attributes = newAttributes;
071                        }
072                }
073
074                public void appendTargetTable(Table targetTable) {
075                        if (targetTable == null) {
076                                return;
077                        }
078                        if (targetTables == null) {
079                                targetTables = new Table[] { targetTable };
080                        } else {
081                                Table[] newTargetTables = new Table[targetTables.length + 1];
082                                System.arraycopy(targetTables, 0, newTargetTables, 0, targetTables.length);
083                                newTargetTables[targetTables.length] = targetTable;
084                                this.targetTables = newTargetTables;
085                        }
086                }
087
088        }
089
090        public static class ColumnUpstream {
091                private List<TableColumn> nodes;
092
093                public List<TableColumn> getNodes() {
094                        return nodes;
095                }
096
097                public void setNodes(List<TableColumn> nodes) {
098                        this.nodes = nodes;
099                }
100
101                public void appendNode(TableColumn node) {
102                        if (node == null) {
103                                return;
104                        }
105                        if (nodes == null) {
106                                nodes = new ArrayList<TableFlowUtility.TableColumn>();
107                                nodes.add(node);
108                        } else {
109                                if (!nodes.contains(node)) {
110                                        nodes.add(node);
111                                }
112                        }
113                }
114
115                public void appendNode(Table node) {
116                        if (node == null) {
117                                return;
118                        }
119                        TableColumn tableColumn = new TableColumn(node);
120                        if (nodes == null) {
121                                nodes = new ArrayList<TableColumn>();
122                                nodes.add(tableColumn);
123                        } else {
124                                if (!nodes.contains(tableColumn)) {
125                                        nodes.add(tableColumn);
126                                }
127                        }
128                }
129
130        }
131
132        public static class Table implements Comparable<Table> {
133                private String server;
134                private String userName;
135                private String databaseName;
136                private String schemaName;
137                private String relationName;
138                private ColumnUpstream upstreamAttributesIndirect;
139
140                public Table() {
141                }
142
143                public Table(String server, String userName, String databaseName, String schemaName, String relationName) {
144                        this.server = server;
145                        this.userName = userName;
146                        this.databaseName = databaseName;
147                        this.schemaName = schemaName;
148                        this.relationName = relationName;
149                }
150
151                public String getServer() {
152                        return server;
153                }
154
155                public void setServer(String server) {
156                        this.server = server;
157                }
158
159                public String getUserName() {
160                        return userName;
161                }
162
163                public void setUserName(String userName) {
164                        this.userName = userName;
165                }
166
167                public String getDatabaseName() {
168                        return databaseName;
169                }
170
171                public void setDatabaseName(String databaseName) {
172                        this.databaseName = databaseName;
173                }
174
175                public String getSchemaName() {
176                        return schemaName;
177                }
178
179                public void setSchemaName(String schemaName) {
180                        this.schemaName = schemaName;
181                }
182
183                public String getRelationName() {
184                        return relationName;
185                }
186
187                public void setRelationName(String relationName) {
188                        this.relationName = relationName;
189                }
190
191                public ColumnUpstream getUpstreamAttributesIndirect() {
192                        return upstreamAttributesIndirect;
193                }
194
195                public void setUpstreamAttributesIndirect(ColumnUpstream upstreamAttributesIndirect) {
196                        this.upstreamAttributesIndirect = upstreamAttributesIndirect;
197                }
198
199                @Override
200                public boolean equals(Object o) {
201                        if (this == o) return true;
202                        if (o == null || getClass() != o.getClass()) return false;
203                        Table table = (Table) o;
204                        return Objects.equals(server, table.server) &&
205                                        Objects.equals(userName, table.userName) &&
206                                        Objects.equals(databaseName, table.databaseName) &&
207                                        Objects.equals(schemaName, table.schemaName) &&
208                                        Objects.equals(relationName, table.relationName) &&
209                                        Objects.equals(upstreamAttributesIndirect, table.upstreamAttributesIndirect);
210                }
211
212                @Override
213                public int hashCode() {
214                        return Objects.hash(server, userName, databaseName, schemaName, relationName, upstreamAttributesIndirect);
215                }
216
217                @Override
218                public int compareTo(Table table) {
219                        return (getDatabaseName() + "." + getSchemaName() + "." + getRelationName())
220                                        .compareTo(table.getDatabaseName() + "." + table.getSchemaName() + "." + table.getRelationName());
221                }
222        }
223
224        public static class TableColumn {
225                private String server;
226                private String userName;
227                private String databaseName;
228                private String schemaName;
229                private String relationName;
230                private String attributeName;
231                private String timestampMin;
232                private String timestampMax;
233                private ColumnUpstream upstreamAttributesIndirect;
234
235                public TableColumn() {
236                }
237
238                public TableColumn(Table node) {
239                        this.server = node.getServer();
240                        this.userName = node.getUserName();
241                        this.databaseName = node.getDatabaseName();
242                        this.schemaName = node.getSchemaName();
243                        this.relationName = node.getRelationName();
244                }
245
246                public String getServer() {
247                        return server;
248                }
249
250                public void setServer(String server) {
251                        this.server = server;
252                }
253
254                public String getUserName() {
255                        return userName;
256                }
257
258                public void setUserName(String userName) {
259                        this.userName = userName;
260                }
261
262                public String getDatabaseName() {
263                        return databaseName;
264                }
265
266                public void setDatabaseName(String databaseName) {
267                        this.databaseName = databaseName;
268                }
269
270                public String getSchemaName() {
271                        return schemaName;
272                }
273
274                public void setSchemaName(String schemaName) {
275                        this.schemaName = schemaName;
276                }
277
278                public String getRelationName() {
279                        return relationName;
280                }
281
282                public void setRelationName(String relationName) {
283                        this.relationName = relationName;
284                }
285
286                public String getAttributeName() {
287                        return attributeName;
288                }
289
290                public void setAttributeName(String attributeName) {
291                        this.attributeName = attributeName;
292                }
293
294                public ColumnUpstream getUpstreamAttributesIndirect() {
295                        return upstreamAttributesIndirect;
296                }
297
298                public void setUpstreamAttributesIndirect(ColumnUpstream upstreamAttributesIndirect) {
299                        this.upstreamAttributesIndirect = upstreamAttributesIndirect;
300                }
301
302                public String getTimestampMin() {
303                        return timestampMin;
304                }
305
306                public void setTimestampMin(String timestampMin) {
307                        this.timestampMin = timestampMin;
308                }
309
310                public String getTimestampMax() {
311                        return timestampMax;
312                }
313
314                public void setTimestampMax(String timestampMax) {
315                        this.timestampMax = timestampMax;
316                }
317
318                @Override
319                public boolean equals(Object o) {
320                        if (this == o) return true;
321                        if (o == null || getClass() != o.getClass()) return false;
322                        TableColumn that = (TableColumn) o;
323                        return Objects.equals(server, that.server) &&
324                                        Objects.equals(userName, that.userName) &&
325                                        Objects.equals(databaseName, that.databaseName) &&
326                                        Objects.equals(schemaName, that.schemaName) &&
327                                        Objects.equals(relationName, that.relationName) &&
328                                        Objects.equals(attributeName, that.attributeName) &&
329                                        Objects.equals(timestampMin, that.timestampMin) &&
330                                        Objects.equals(timestampMax, that.timestampMax) &&
331                                        Objects.equals(upstreamAttributesIndirect, that.upstreamAttributesIndirect);
332                }
333
334                @Override
335                public int hashCode() {
336                        return Objects.hash(server, userName, databaseName, schemaName, relationName, attributeName, timestampMin, timestampMax, upstreamAttributesIndirect);
337                }
338
339                public Table parentTable() {
340                        return new Table(server, userName, databaseName, schemaName, relationName);
341                }
342        }
343
344        public static String generateLeftMostTable(DataFlowAnalyzer analyzer, dataflow instance, String tableNamePattern,
345                                                                                           String tableColumn, boolean isSimple) {
346                return generateLeftMostTable(analyzer, instance, tableNamePattern, tableColumn, isSimple, "json", null);
347        }
348
349        public static String generateLeftMostTable(DataFlowAnalyzer analyzer, dataflow instance, String tableNamePattern,
350                                                                                           String tableColumn, boolean isSimple, String outputType, String delimiter) {
351                dataflow dataflow = generateLeftMostTableFlow(analyzer, instance, tableNamePattern, tableColumn, isSimple);
352                LeftMostSourceTable leftMostSourceTable = exportToLeftMostTableModel(dataflow);
353
354                if ("json".equals(outputType)) {
355                        return JSON.toJSONString(leftMostSourceTable);
356                } else if ("csv".equals(outputType)) {
357                        return exportModelToCsv(leftMostSourceTable, delimiter);
358                } else {
359                        throw new UnsupportedOperationException("Unsupported output type: " + outputType);
360                }
361        }
362
363        static LeftMostSourceTable exportToLeftMostTableModel(dataflow dataflow) {
364                LeftMostSourceTable leftMostSourceTable = new LeftMostSourceTable();
365
366                Map<String, Set<relationship>> targetIdRelationMap = new HashMap<String, Set<relationship>>();
367                List<relationship> relations = dataflow.getRelationships();
368                for (relationship relation : relations) {
369                        if (relation.getTarget() != null) {
370                                String key = relation.getTarget().getParent_id() + ":" + relation.getTarget().getId();
371                                if (!targetIdRelationMap.containsKey(key)) {
372                                        targetIdRelationMap.put(key, new HashSet<relationship>());
373                                }
374                                targetIdRelationMap.get(key).add(relation);
375                        }
376                }
377
378                List<table> tables = new ArrayList<>();
379                if (dataflow.getTables() != null) {
380                        tables.addAll(dataflow.getTables());
381                }
382                if (dataflow.getPaths() != null) {
383                        tables.addAll(dataflow.getPaths());
384                }
385                if (dataflow.getStages() != null) {
386                        tables.addAll(dataflow.getStages());
387                }
388                if (dataflow.getSequences() != null) {
389                        tables.addAll(dataflow.getSequences());
390                }
391                if (dataflow.getDatasources() != null) {
392                        tables.addAll(dataflow.getDatasources());
393                }
394                if (dataflow.getDatabases() != null) {
395                        tables.addAll(dataflow.getDatabases());
396                }
397                if (dataflow.getSchemas() != null) {
398                        tables.addAll(dataflow.getSchemas());
399                }
400                if (dataflow.getVariables() != null) {
401                        tables.addAll(dataflow.getVariables());
402                }
403                if (dataflow.getViews() != null) {
404                        tables.addAll(dataflow.getViews());
405                }
406                Map<String, table> tableMap = new HashMap<String, table>();
407                Map<String, List<table>> duplicateTableMap = new HashMap<String, List<table>>();
408                Iterator<table> tableIter = tables.iterator();
409                while (tableIter.hasNext()) {
410                        table table = tableIter.next();
411                        tableMap.put(table.getId(), table);
412                        String tableName = table.getName();
413                        if(!duplicateTableMap.containsKey(tableName)) {
414                                duplicateTableMap.put(tableName, new ArrayList<table>());
415                        }
416                        duplicateTableMap.get(tableName).add(table);
417                }
418
419                Map<String, TableColumn> tableColumnMap = new TreeMap<String, TableColumn>();
420                Map<String, Table> targetTableMap = new TreeMap<String, Table>();
421                for (String key : targetIdRelationMap.keySet()) {
422                        String targetTableId = key.split(":")[0];
423                        String targetColumnId = key.split(":")[1];
424                        boolean matches = false;
425                        if (!tableColumnMap.containsKey(key)) {
426                                table targetTable = tableMap.get(targetTableId);
427                                for (column column : targetTable.getColumns()) {
428                                        if (column.getId().equals(targetColumnId)) {
429                                                TableColumn targetTableColumn = new TableColumn();
430                                                targetTableColumn.setServer(targetTable.getServer());
431                                                targetTableColumn.setUserName(targetTable.getUserName());
432                                                targetTableColumn.setDatabaseName(targetTable.getDatabase());
433                                                targetTableColumn.setSchemaName(targetTable.getSchema());
434                                                targetTableColumn.setRelationName(targetTable.getTableNameOnly());
435                                                targetTableColumn.setAttributeName(column.getName());
436                                                ColumnUpstream columnUpstream = new ColumnUpstream();
437                                                targetTableColumn.setUpstreamAttributesIndirect(columnUpstream);
438                                                tableColumnMap.put(key, targetTableColumn);
439                                                matches = true;
440                                                break;
441                                        }
442                                }
443                        }
444
445                        if (!matches) {
446                                continue;
447                        }
448                        ColumnUpstream columnUpstream = tableColumnMap.get(key).getUpstreamAttributesIndirect();
449                        Set<relationship> columnRelations = targetIdRelationMap.get(key);
450                        for (relationship columnRelation : columnRelations) {
451                                List<sourceColumn> sourceColumns = columnRelation.getSources();
452                                for (sourceColumn sourceColumn : sourceColumns) {
453                                        table sourceTable = tableMap.get(sourceColumn.getParent_id());
454                                        if (sourceTable == null)
455                                                continue;
456                                        TableColumn sourceTableColumn = new TableColumn();
457                                        sourceTableColumn.setServer(sourceTable.getServer());
458                                        sourceTableColumn.setUserName(sourceTable.getUserName());
459                                        sourceTableColumn.setDatabaseName(sourceTable.getDatabase());
460                                        sourceTableColumn.setSchemaName(sourceTable.getSchema());
461                                        sourceTableColumn.setRelationName(sourceTable.getTableNameOnly());
462                                        sourceTableColumn.setAttributeName(sourceColumn.getColumn());
463                                        sourceTableColumn.setTimestampMax(columnRelation.getTimestampMax());
464                                        sourceTableColumn.setTimestampMin(columnRelation.getTimestampMin());
465                                        columnUpstream.appendNode(sourceTableColumn);
466                                }
467                        }
468                        if (columnUpstream.getNodes() != null && !columnUpstream.getNodes().isEmpty()) {
469                                leftMostSourceTable.appendAttribute(tableColumnMap.get(key));
470                                if (!targetTableMap.containsKey(targetTableId)) {
471                                        Table targetTable = new Table();
472                                        table table = tableMap.get(targetTableId);
473                                        targetTable.setServer(table.getServer());
474                                        targetTable.setUserName(table.getUserName());
475                                        targetTable.setDatabaseName(table.getDatabase());
476                                        targetTable.setSchemaName(table.getSchema());
477                                        targetTable.setRelationName(table.getTableNameOnly());
478                                        leftMostSourceTable.appendTargetTable(targetTable);
479                                        targetTableMap.put(targetTableId, targetTable);
480                                }
481                        }
482                }
483
484                if (leftMostSourceTable.getAttributes() != null) {
485                        Arrays.sort(leftMostSourceTable.getAttributes(), new Comparator<TableColumn>() {
486
487                                @Override
488                                public int compare(TableColumn o1, TableColumn o2) {
489                                        int result = (o1.getDatabaseName() + "." + o1.getSchemaName() + "." + o1.getRelationName())
490                                                        .compareTo(o2.getDatabaseName() + "." + o2.getSchemaName() + "." + o2.getRelationName());
491                                        if (result == 0) {
492                                                return o1.getAttributeName().compareTo(o2.getAttributeName());
493                                        } else {
494                                                return result;
495                                        }
496                                }
497                        });
498
499                        Map<Table, TreeSet<Table>> targetSourceTableMap = new HashMap<TableFlowUtility.Table, TreeSet<Table>>();
500                        for (TableColumn attribute : leftMostSourceTable.getAttributes()) {
501                                if (attribute.getUpstreamAttributesIndirect() != null
502                                                && attribute.getUpstreamAttributesIndirect().getNodes() != null) {
503                                        Collections.sort(attribute.getUpstreamAttributesIndirect().getNodes(),
504                                                        new Comparator<TableColumn>() {
505                                                                @Override
506                                                                public int compare(TableColumn o1, TableColumn o2) {
507                                                                        int result = (o1.getDatabaseName() + "." + o1.getSchemaName() + "."
508                                                                                        + o1.getRelationName())
509                                                                                        .compareTo(o2.getDatabaseName() + "." + o2.getSchemaName() + "."
510                                                                                                        + o2.getRelationName());
511                                                                        if (result == 0) {
512                                                                                return o1.getAttributeName().compareTo(o2.getAttributeName());
513                                                                        } else {
514                                                                                return result;
515                                                                        }
516                                                                }
517                                                        });
518
519                                        if (!targetSourceTableMap.containsKey(attribute.parentTable())) {
520                                                targetSourceTableMap.put(attribute.parentTable(), new TreeSet<Table>());
521                                        }
522                                        for (TableColumn sourceTableColumn : attribute.getUpstreamAttributesIndirect().getNodes()) {
523                                                targetSourceTableMap.get(attribute.parentTable()).add(sourceTableColumn.parentTable());
524                                        }
525                                }
526                        }
527
528                        if (leftMostSourceTable.getTargetTables() != null) {
529                                for (Table targetTable : leftMostSourceTable.getTargetTables()) {
530                                        TreeSet<Table> sourceTableTree = targetSourceTableMap.get(targetTable);
531                                        if (sourceTableTree != null) {
532                                                ColumnUpstream columnUpstream = new ColumnUpstream();
533                                                targetTable.setUpstreamAttributesIndirect(columnUpstream);
534                                                for (Table table : sourceTableTree) {
535                                                        columnUpstream.appendNode(table);
536                                                }
537                                        }
538                                }
539                        }
540                }
541
542                if (leftMostSourceTable.getTargetTables() != null) {
543                        Arrays.sort(leftMostSourceTable.getTargetTables(), new Comparator<Table>() {
544                                @Override
545                                public int compare(Table o1, Table o2) {
546                                        int result = (o1.getDatabaseName() + "." + o1.getSchemaName() + "." + o1.getRelationName())
547                                                        .compareTo(o2.getDatabaseName() + "." + o2.getSchemaName() + "." + o2.getRelationName());
548                                        return result;
549                                }
550                        });
551                }
552                return leftMostSourceTable;
553        }
554
555        private static String exportModelToCsv(LeftMostSourceTable leftMostSourceTable, String delimiter) {
556                if (delimiter == null || delimiter.length() == 0) {
557                        delimiter = ",";
558                }
559                StringBuilder buffer = new StringBuilder();
560                buffer.append("SOURCE_TABLE"+delimiter+"SOURCE_COLUMN"+delimiter+"TARGET_TABLE"+delimiter+"TARGET_COLUMN").append(System.getProperty("line.separator"));
561                try {
562                        Set<String> lines = exportModelToLines(leftMostSourceTable, delimiter);
563                        for (String line : lines) {
564                                buffer.append(line).append(System.getProperty("line.separator"));
565                        }
566                } catch (Exception e) {
567                        logger.error("Generate column level csv failed.", e);
568                }
569                return buffer.toString();
570        }
571
572        public static Set<String> exportModelToLines(LeftMostSourceTable leftMostSourceTable, String delimiter){
573                TreeSet<String> lines = new TreeSet<String>(new Comparator<String>() {
574                        @Override
575                        public int compare(String o1, String o2) {
576                                return o1.toLowerCase(java.util.Locale.ROOT).compareTo(o2.toLowerCase(java.util.Locale.ROOT));
577                        }
578                });
579                if (leftMostSourceTable.getAttributes() != null) {
580                        StringWriter sw = new StringWriter();
581                        CsvWriter csvWriter = new CsvWriter(sw, delimiter.charAt(0));
582                        try {
583                                for (TableColumn targetColumn : leftMostSourceTable.getAttributes()) {
584                                        if (targetColumn.getUpstreamAttributesIndirect() != null
585                                                        && targetColumn.getUpstreamAttributesIndirect().getNodes() != null) {
586                                                String targetTableName = getTableFullName(targetColumn);
587                                                String targetColumnName = targetColumn.getAttributeName();
588                                                for (TableColumn sourceColumn : targetColumn.getUpstreamAttributesIndirect().getNodes()) {
589                                                        String sourceTableName = getTableFullName(sourceColumn);
590                                                        String sourceColumnName = sourceColumn.getAttributeName();
591                                                        csvWriter.writeRecord(new String[]{sourceTableName, sourceColumnName, targetTableName, targetColumnName});
592                                                }
593                                        }
594                                }
595                        } catch (Exception e) {
596                                logger.error("write csv failed.", e);
597                        }
598                        csvWriter.close();
599                        String systemRecordDelimiter = System.getProperty("line.separator");
600                        lines.addAll(Arrays.asList(sw.toString().split(systemRecordDelimiter)));
601                }
602                return lines;
603        }
604
605        private static String getTableFullName(TableColumn targetColumn) {
606                String tableName = targetColumn.getRelationName();
607                if (!SQLUtil.isEmpty(targetColumn.getSchemaName())) {
608                        tableName = targetColumn.getSchemaName() + "." + tableName;
609                }
610                if (!SQLUtil.isEmpty(targetColumn.getDatabaseName())) {
611                        tableName = targetColumn.getDatabaseName() + "." + tableName;
612                }
613                return tableName;
614        }
615
616        public static dataflow generateLeftMostTableFlow(DataFlowAnalyzer analyzer, dataflow instance, String tableNamePattern, String tableColumn, boolean isSimple) {
617                EDbVendor vendor = analyzer.getOption().getVendor();
618
619                if (tableNamePattern != null) {
620                        tableNamePattern = tableNamePattern.replace("[", "\\[").replace("]", "\\]");
621                }
622                try {
623                        dataflow simple = instance;
624                        if (!isSimple && instance.getResultsets() != null && instance.getResultsets().size() > 0) {
625                                simple = analyzer.getSimpleDataflow(instance, true);
626                        }
627                        if (simple.getTables() == null && simple.getViews() == null) {
628                                return simple;
629                        }
630                        List<relationship> relations = simple.getRelationships();
631                        if (relations == null || relations.size() == 0) {
632                                return simple;
633                        }
634                        long maxId = Long.parseLong(relations.get(relations.size() - 1).getId().split("\\-")[0].replace("_", ""))
635                                        * 100;
636                        
637                        List<table> tables = new ArrayList<>();
638                        if (simple.getTables() != null) {
639                                tables.addAll(simple.getTables());
640                        }
641                        if (simple.getPaths() != null) {
642                                tables.addAll(simple.getPaths());
643                        }
644                        if (simple.getStages() != null) {
645                                tables.addAll(simple.getStages());
646                        }
647                        if (simple.getSequences() != null) {
648                                tables.addAll(simple.getSequences());
649                        }
650                        if (simple.getDatasources() != null) {
651                                tables.addAll(simple.getDatasources());
652                        }
653                        if (simple.getDatabases() != null) {
654                                tables.addAll(simple.getDatabases());
655                        }
656                        if (simple.getSchemas() != null) {
657                                tables.addAll(simple.getSchemas());
658                        }
659                        if (simple.getVariables() != null) {
660                                tables.addAll(simple.getVariables());
661                        }
662                        if (simple.getViews() != null) {
663                                tables.addAll(simple.getViews());
664                        }
665                        Map<String, table> tableMap = new HashMap<String, table>();
666                        Iterator<table> tableIter = tables.iterator();
667                        while (tableIter.hasNext()) {
668                                table table = tableIter.next();
669                                tableMap.put(table.getId(), table);
670                        }
671                        
672                        Iterator<relationship> iter = simple.getRelationships().iterator();
673                        while (iter.hasNext()) {
674                                relationship relation = iter.next();
675                                if (!"fdd".equals(relation.getType())) {
676                                        iter.remove();
677                                }
678                                if(relation.getSources() != null && relation.getSources().size()>0) {
679                                        Iterator<sourceColumn> sourceIter = relation.getSources().iterator();
680                                        while(sourceIter.hasNext()) {
681                                                sourceColumn source = sourceIter.next();
682                                                if(source.getId().equals(relation.getTarget().getId())) {
683                                                        sourceIter.remove();
684                                                }
685                                                
686                                                table sourceTable = tableMap.get(source.getParent_id());
687                                                if(sourceTable.isFile()) {
688                                                        sourceIter.remove();
689                                                }
690                                        }
691                                        if(relation.getSources().isEmpty()) {
692                                                iter.remove();
693                                        }
694                                }
695                        }
696
697
698                        Map<String, Set<relationship>> targetIdRelationMap = new HashMap<String, Set<relationship>>();
699                        for (relationship relation : relations) {
700                                if (relation.getTarget() != null) {
701                                        String key = relation.getTarget().getParent_id() + "." + relation.getTarget().getId();
702                                        if (!targetIdRelationMap.containsKey(key)) {
703                                                targetIdRelationMap.put(key, new HashSet<relationship>());
704                                        }
705                                        targetIdRelationMap.get(key).add(relation);
706                                }
707                        }
708
709                        List<relationship> simpleRelations = new ArrayList<relationship>();
710                        Set<table> tableSet = new HashSet<table>();
711                        iter = simple.getRelationships().iterator();
712                        while (iter.hasNext()) {
713                                relationship relation = iter.next();
714                                targetColumn target = relation.getTarget();
715                                String targetParentId = target.getParent_id();
716                                table targetTable = tableMap.get(targetParentId);
717                                if (targetTable == null) {
718                                        continue;
719                                }
720                                if (!matchTableName(tableNamePattern, targetTable, SQLUtil.isEmpty(tableColumn), vendor)) {
721                                        continue;
722                                }
723                                if (!SQLUtil.isEmpty(tableColumn) && !storedName(vendor, ESQLDataObjectType.dotColumn, target.getColumn())
724                                                .equalsIgnoreCase(storedName(vendor, ESQLDataObjectType.dotColumn, tableColumn))) { // non-identifier-compare: lenient user filter
725                                        continue;
726                                }
727                                tableSet.add(targetTable);
728
729                                Set<sourceColumn> sourceColumnSet = new HashSet<sourceColumn>();
730                                Set<relationship> relationSet = new HashSet<relationship>();
731
732                                findSourceColumns(targetIdRelationMap, relation, sourceColumnSet, relationSet, tableMap);
733                                if (!sourceColumnSet.isEmpty()) {
734                                        relationship simpleRelation = (relationship) relation.clone();
735                                        simpleRelation.setSources(new ArrayList<sourceColumn>(sourceColumnSet));
736                                        simpleRelation.setId(String.valueOf(++maxId));
737                                        simpleRelations.add(simpleRelation);
738                                        for (sourceColumn sourceColumn : sourceColumnSet) {
739                                                table sourceTable = tableMap.get(sourceColumn.getParent_id());
740                                                if (sourceTable != null) {
741                                                        tableSet.add(sourceTable);
742                                                }
743                                        }
744                                }
745                        }
746                        simple.setRelationships(simpleRelations);
747
748                        if (simple.getTables() != null) {
749                                simple.getTables().retainAll(tableSet);
750                        }
751                        if (simple.getPaths() != null) {
752                                simple.getPaths().retainAll(tableSet);
753                        }
754                        if (simple.getStages() != null) {
755                                simple.getStages().retainAll(tableSet);
756                        }
757                        if (simple.getSequences() != null) {
758                                simple.getSequences().retainAll(tableSet);
759                        }
760                        if (simple.getDatasources() != null) {
761                                simple.getDatasources().retainAll(tableSet);
762                        }
763                        if (simple.getDatabases() != null) {
764                                simple.getDatabases().retainAll(tableSet);
765                        }
766                        if (simple.getSchemas() != null) {
767                                simple.getSchemas().retainAll(tableSet);
768                        }
769                        if (simple.getVariables() != null) {
770                                simple.getVariables().retainAll(tableSet);
771                        }
772                        if (simple.getViews() != null) {
773                                simple.getViews().retainAll(tableSet);
774                        }
775                        simple.getProcedures().clear();
776                        simple.getProcesses().clear();
777                        simple.getResultsets().clear();
778                        simple.getErrors().clear();
779                        return simple;
780                } catch (Exception e) {
781                        logger.error("Generate left most table flow failed.", e);
782                }
783                return new dataflow();
784        }
785
786        private static void findSourceColumns(Map<String, Set<relationship>> targetIdRelationMap, relationship relation,
787                                                                                  Set<sourceColumn> sourceColumnSet, Set<relationship> relationSet, Map<String, table> tableMap) {
788                if (relationSet.contains(relation)) {
789                        List<sourceColumn> sourceColumns = relation.getSources();
790                        for (sourceColumn sourceColumn : sourceColumns) {
791                                if (sourceColumn.getId().equals(relation.getTarget().getId())) {
792                                        sourceColumnSet.add(sourceColumn);
793                                        break;
794                                }
795                        }
796                        return;
797                } else {
798                        relationSet.add(relation);
799                }
800                List<sourceColumn> sourceColumns = relation.getSources();
801                for (sourceColumn sourceColumn : sourceColumns) {
802                        String columnId = sourceColumn.getParent_id() + "." + sourceColumn.getId();
803                        if (sourceColumnSet.contains(sourceColumn)) {
804                                continue;
805                        } else if (!targetIdRelationMap.containsKey(columnId)) {
806                                sourceColumnSet.add(sourceColumn);
807                        } else {
808                                Set<relationship> sourceRelations = targetIdRelationMap.get(columnId);
809                                for (relationship sourceRelation : sourceRelations) {
810                                        findSourceColumns(targetIdRelationMap, sourceRelation, sourceColumnSet, relationSet, tableMap);
811                                }
812                        }
813                }
814        }
815
816        public static boolean matchTableName(String tableNamePattern, table targetTable, boolean regex, EDbVendor vendor) {
817                if (SQLUtil.isEmpty(tableNamePattern))
818                        return false;
819                String tableNameOnly = storedName(vendor, ESQLDataObjectType.dotTable, targetTable.getTableNameOnly());
820                if (regex) {
821                        tableNamePattern = SQLUtil.trimColumnStringQuote(tableNamePattern);
822                        if (SQLUtil.trimColumnStringQuote(tableNameOnly).matches("(?i)" + tableNamePattern))
823                                return true;
824                        String tableName = targetTable.getName();
825                        if (SQLUtil.trimColumnStringQuote(tableName).matches("(?i)" + tableNamePattern))
826                                return true;
827                        if (vendor == EDbVendor.dbvmssql || vendor == EDbVendor.dbvazuresql) {
828                                if (tableName.indexOf("..") != -1) {
829                                        if (ModelBindingManager.getGlobalSchema() != null) {
830                                                tableName = tableName.replace("..", "." + ModelBindingManager.getGlobalSchema() + ".");
831                                        } else {
832                                                tableName = tableName.replace("..", ".dbo.");
833                                        }
834                                        if (SQLUtil.trimColumnStringQuote(tableName).matches("(?i)" + tableNamePattern))
835                                                return true;
836                                }
837                        }
838                } else {
839                        String pattern = storedName(vendor, ESQLDataObjectType.dotTable, tableNamePattern);
840                        return tableNameOnly.equalsIgnoreCase(pattern) // non-identifier-compare: lenient user filter
841                                        || storedName(vendor, ESQLDataObjectType.dotTable, targetTable.getName()).equalsIgnoreCase(pattern); // non-identifier-compare: lenient user filter
842                }
843                return false;
844        }
845
846        /**
847         * The name as the vendor stores it, segment by segment, with an omitted SQL Server schema filled
848         * in, upper-cased. The table-flow filters match a user's name against it ignoring case, as they
849         * always have: a lookup the user types is not an identifier comparison.
850         */
851        private static String storedName(EDbVendor vendor, ESQLDataObjectType objectType, String name) {
852                if (name == null || vendor == null) {
853                        return name;
854                }
855                // The legacy filter upper-cased with Locale.ROOT, so ß matched SS; a user's lookup keeps that.
856                return IdentifierService.defaultServiceFor(vendor)
857                                .normalizeQualifiedName(DlineageUtil.withSessionSchema(vendor, name), objectType)
858                                .toUpperCase(java.util.Locale.ROOT); // non-identifier-compare: lenient user filter
859        }
860}