Source code

001package gudusoft.gsqlparser.dlineage.util;
002
003import java.io.IOException;
004import java.io.StringWriter;
005import java.util.ArrayList;
006import java.util.Arrays;
007import java.util.Collections;
008import java.util.Comparator;
009import java.util.HashMap;
010import java.util.HashSet;
011import java.util.Iterator;
012import java.util.List;
013import java.util.Map;
014import java.util.Objects;
015import java.util.Set;
016import java.util.TreeMap;
017import java.util.TreeSet;
018
019import gudusoft.gsqlparser.EDbVendor;
020import gudusoft.gsqlparser.dlineage.DataFlowAnalyzer;
021import gudusoft.gsqlparser.dlineage.dataflow.model.ModelBindingManager;
022import gudusoft.gsqlparser.dlineage.dataflow.model.xml.column;
023import gudusoft.gsqlparser.dlineage.dataflow.model.xml.dataflow;
024import gudusoft.gsqlparser.dlineage.dataflow.model.xml.relationship;
025import gudusoft.gsqlparser.dlineage.dataflow.model.xml.sourceColumn;
026import gudusoft.gsqlparser.dlineage.dataflow.model.xml.table;
027import gudusoft.gsqlparser.dlineage.dataflow.model.xml.targetColumn;
028import gudusoft.gsqlparser.util.Logger;
029import gudusoft.gsqlparser.util.LoggerFactory;
030import gudusoft.gsqlparser.util.SQLUtil;
031import gudusoft.gsqlparser.util.csv.CsvWriter;
032import gudusoft.gsqlparser.util.json.JSON;
033
034public class TableFlowUtility {
035
036        private static final Logger logger = LoggerFactory.getLogger(TableFlowUtility.class);
037
038        public static class LeftMostSourceTable {
039                private Table[] targetTables;
040                private TableColumn[] attributes;
041
042                public TableColumn[] getAttributes() {
043                        return attributes;
044                }
045
046                public void setAttributes(TableColumn[] attributes) {
047                        this.attributes = attributes;
048                }
049
050                public Table[] getTargetTables() {
051                        return targetTables;
052                }
053
054                public void setTargetTables(Table[] targetTables) {
055                        this.targetTables = targetTables;
056                }
057
058                public void appendAttribute(TableColumn attribute) {
059                        if (attribute == null) {
060                                return;
061                        }
062                        if (attributes == null) {
063                                attributes = new TableColumn[] { attribute };
064                        } else {
065                                TableColumn[] newAttributes = new TableColumn[attributes.length + 1];
066                                System.arraycopy(attributes, 0, newAttributes, 0, attributes.length);
067                                newAttributes[attributes.length] = attribute;
068                                this.attributes = newAttributes;
069                        }
070                }
071
072                public void appendTargetTable(Table targetTable) {
073                        if (targetTable == null) {
074                                return;
075                        }
076                        if (targetTables == null) {
077                                targetTables = new Table[] { targetTable };
078                        } else {
079                                Table[] newTargetTables = new Table[targetTables.length + 1];
080                                System.arraycopy(targetTables, 0, newTargetTables, 0, targetTables.length);
081                                newTargetTables[targetTables.length] = targetTable;
082                                this.targetTables = newTargetTables;
083                        }
084                }
085
086        }
087
088        public static class ColumnUpstream {
089                private List<TableColumn> nodes;
090
091                public List<TableColumn> getNodes() {
092                        return nodes;
093                }
094
095                public void setNodes(List<TableColumn> nodes) {
096                        this.nodes = nodes;
097                }
098
099                public void appendNode(TableColumn node) {
100                        if (node == null) {
101                                return;
102                        }
103                        if (nodes == null) {
104                                nodes = new ArrayList<TableFlowUtility.TableColumn>();
105                                nodes.add(node);
106                        } else {
107                                if (!nodes.contains(node)) {
108                                        nodes.add(node);
109                                }
110                        }
111                }
112
113                public void appendNode(Table node) {
114                        if (node == null) {
115                                return;
116                        }
117                        TableColumn tableColumn = new TableColumn(node);
118                        if (nodes == null) {
119                                nodes = new ArrayList<TableColumn>();
120                                nodes.add(tableColumn);
121                        } else {
122                                if (!nodes.contains(tableColumn)) {
123                                        nodes.add(tableColumn);
124                                }
125                        }
126                }
127
128        }
129
130        public static class Table implements Comparable<Table> {
131                private String server;
132                private String userName;
133                private String databaseName;
134                private String schemaName;
135                private String relationName;
136                private ColumnUpstream upstreamAttributesIndirect;
137
138                public Table() {
139                }
140
141                public Table(String server, String userName, String databaseName, String schemaName, String relationName) {
142                        this.server = server;
143                        this.userName = userName;
144                        this.databaseName = databaseName;
145                        this.schemaName = schemaName;
146                        this.relationName = relationName;
147                }
148
149                public String getServer() {
150                        return server;
151                }
152
153                public void setServer(String server) {
154                        this.server = server;
155                }
156
157                public String getUserName() {
158                        return userName;
159                }
160
161                public void setUserName(String userName) {
162                        this.userName = userName;
163                }
164
165                public String getDatabaseName() {
166                        return databaseName;
167                }
168
169                public void setDatabaseName(String databaseName) {
170                        this.databaseName = databaseName;
171                }
172
173                public String getSchemaName() {
174                        return schemaName;
175                }
176
177                public void setSchemaName(String schemaName) {
178                        this.schemaName = schemaName;
179                }
180
181                public String getRelationName() {
182                        return relationName;
183                }
184
185                public void setRelationName(String relationName) {
186                        this.relationName = relationName;
187                }
188
189                public ColumnUpstream getUpstreamAttributesIndirect() {
190                        return upstreamAttributesIndirect;
191                }
192
193                public void setUpstreamAttributesIndirect(ColumnUpstream upstreamAttributesIndirect) {
194                        this.upstreamAttributesIndirect = upstreamAttributesIndirect;
195                }
196
197                @Override
198                public boolean equals(Object o) {
199                        if (this == o) return true;
200                        if (o == null || getClass() != o.getClass()) return false;
201                        Table table = (Table) o;
202                        return Objects.equals(server, table.server) &&
203                                        Objects.equals(userName, table.userName) &&
204                                        Objects.equals(databaseName, table.databaseName) &&
205                                        Objects.equals(schemaName, table.schemaName) &&
206                                        Objects.equals(relationName, table.relationName) &&
207                                        Objects.equals(upstreamAttributesIndirect, table.upstreamAttributesIndirect);
208                }
209
210                @Override
211                public int hashCode() {
212                        return Objects.hash(server, userName, databaseName, schemaName, relationName, upstreamAttributesIndirect);
213                }
214
215                @Override
216                public int compareTo(Table table) {
217                        return (getDatabaseName() + "." + getSchemaName() + "." + getRelationName())
218                                        .compareTo(table.getDatabaseName() + "." + table.getSchemaName() + "." + table.getRelationName());
219                }
220        }
221
222        public static class TableColumn {
223                private String server;
224                private String userName;
225                private String databaseName;
226                private String schemaName;
227                private String relationName;
228                private String attributeName;
229                private String timestampMin;
230                private String timestampMax;
231                private ColumnUpstream upstreamAttributesIndirect;
232
233                public TableColumn() {
234                }
235
236                public TableColumn(Table node) {
237                        this.server = node.getServer();
238                        this.userName = node.getUserName();
239                        this.databaseName = node.getDatabaseName();
240                        this.schemaName = node.getSchemaName();
241                        this.relationName = node.getRelationName();
242                }
243
244                public String getServer() {
245                        return server;
246                }
247
248                public void setServer(String server) {
249                        this.server = server;
250                }
251
252                public String getUserName() {
253                        return userName;
254                }
255
256                public void setUserName(String userName) {
257                        this.userName = userName;
258                }
259
260                public String getDatabaseName() {
261                        return databaseName;
262                }
263
264                public void setDatabaseName(String databaseName) {
265                        this.databaseName = databaseName;
266                }
267
268                public String getSchemaName() {
269                        return schemaName;
270                }
271
272                public void setSchemaName(String schemaName) {
273                        this.schemaName = schemaName;
274                }
275
276                public String getRelationName() {
277                        return relationName;
278                }
279
280                public void setRelationName(String relationName) {
281                        this.relationName = relationName;
282                }
283
284                public String getAttributeName() {
285                        return attributeName;
286                }
287
288                public void setAttributeName(String attributeName) {
289                        this.attributeName = attributeName;
290                }
291
292                public ColumnUpstream getUpstreamAttributesIndirect() {
293                        return upstreamAttributesIndirect;
294                }
295
296                public void setUpstreamAttributesIndirect(ColumnUpstream upstreamAttributesIndirect) {
297                        this.upstreamAttributesIndirect = upstreamAttributesIndirect;
298                }
299
300                public String getTimestampMin() {
301                        return timestampMin;
302                }
303
304                public void setTimestampMin(String timestampMin) {
305                        this.timestampMin = timestampMin;
306                }
307
308                public String getTimestampMax() {
309                        return timestampMax;
310                }
311
312                public void setTimestampMax(String timestampMax) {
313                        this.timestampMax = timestampMax;
314                }
315
316                @Override
317                public boolean equals(Object o) {
318                        if (this == o) return true;
319                        if (o == null || getClass() != o.getClass()) return false;
320                        TableColumn that = (TableColumn) o;
321                        return Objects.equals(server, that.server) &&
322                                        Objects.equals(userName, that.userName) &&
323                                        Objects.equals(databaseName, that.databaseName) &&
324                                        Objects.equals(schemaName, that.schemaName) &&
325                                        Objects.equals(relationName, that.relationName) &&
326                                        Objects.equals(attributeName, that.attributeName) &&
327                                        Objects.equals(timestampMin, that.timestampMin) &&
328                                        Objects.equals(timestampMax, that.timestampMax) &&
329                                        Objects.equals(upstreamAttributesIndirect, that.upstreamAttributesIndirect);
330                }
331
332                @Override
333                public int hashCode() {
334                        return Objects.hash(server, userName, databaseName, schemaName, relationName, attributeName, timestampMin, timestampMax, upstreamAttributesIndirect);
335                }
336
337                public Table parentTable() {
338                        return new Table(server, userName, databaseName, schemaName, relationName);
339                }
340        }
341
342        public static String generateLeftMostTable(DataFlowAnalyzer analyzer, dataflow instance, String tableNamePattern,
343                                                                                           String tableColumn, boolean isSimple) {
344                return generateLeftMostTable(analyzer, instance, tableNamePattern, tableColumn, isSimple, "json", null);
345        }
346
347        public static String generateLeftMostTable(DataFlowAnalyzer analyzer, dataflow instance, String tableNamePattern,
348                                                                                           String tableColumn, boolean isSimple, String outputType, String delimiter) {
349                dataflow dataflow = generateLeftMostTableFlow(analyzer, instance, tableNamePattern, tableColumn, isSimple);
350                LeftMostSourceTable leftMostSourceTable = exportToLeftMostTableModel(dataflow);
351
352                if ("json".equals(outputType)) {
353                        return JSON.toJSONString(leftMostSourceTable);
354                } else if ("csv".equals(outputType)) {
355                        return exportModelToCsv(leftMostSourceTable, delimiter);
356                } else {
357                        throw new UnsupportedOperationException("Unsupported output type: " + outputType);
358                }
359        }
360
361        static LeftMostSourceTable exportToLeftMostTableModel(dataflow dataflow) {
362                LeftMostSourceTable leftMostSourceTable = new LeftMostSourceTable();
363
364                Map<String, Set<relationship>> targetIdRelationMap = new HashMap<String, Set<relationship>>();
365                List<relationship> relations = dataflow.getRelationships();
366                for (relationship relation : relations) {
367                        if (relation.getTarget() != null) {
368                                String key = relation.getTarget().getParent_id() + ":" + relation.getTarget().getId();
369                                if (!targetIdRelationMap.containsKey(key)) {
370                                        targetIdRelationMap.put(key, new HashSet<relationship>());
371                                }
372                                targetIdRelationMap.get(key).add(relation);
373                        }
374                }
375
376                List<table> tables = new ArrayList<>();
377                if (dataflow.getTables() != null) {
378                        tables.addAll(dataflow.getTables());
379                }
380                if (dataflow.getPaths() != null) {
381                        tables.addAll(dataflow.getPaths());
382                }
383                if (dataflow.getStages() != null) {
384                        tables.addAll(dataflow.getStages());
385                }
386                if (dataflow.getSequences() != null) {
387                        tables.addAll(dataflow.getSequences());
388                }
389                if (dataflow.getDatasources() != null) {
390                        tables.addAll(dataflow.getDatasources());
391                }
392                if (dataflow.getDatabases() != null) {
393                        tables.addAll(dataflow.getDatabases());
394                }
395                if (dataflow.getSchemas() != null) {
396                        tables.addAll(dataflow.getSchemas());
397                }
398                if (dataflow.getVariables() != null) {
399                        tables.addAll(dataflow.getVariables());
400                }
401                if (dataflow.getViews() != null) {
402                        tables.addAll(dataflow.getViews());
403                }
404                Map<String, table> tableMap = new HashMap<String, table>();
405                Map<String, List<table>> duplicateTableMap = new HashMap<String, List<table>>();
406                Iterator<table> tableIter = tables.iterator();
407                while (tableIter.hasNext()) {
408                        table table = tableIter.next();
409                        tableMap.put(table.getId(), table);
410                        String tableName = table.getName();
411                        if(!duplicateTableMap.containsKey(tableName)) {
412                                duplicateTableMap.put(tableName, new ArrayList<table>());
413                        }
414                        duplicateTableMap.get(tableName).add(table);
415                }
416
417                Map<String, TableColumn> tableColumnMap = new TreeMap<String, TableColumn>();
418                Map<String, Table> targetTableMap = new TreeMap<String, Table>();
419                for (String key : targetIdRelationMap.keySet()) {
420                        String targetTableId = key.split(":")[0];
421                        String targetColumnId = key.split(":")[1];
422                        boolean matches = false;
423                        if (!tableColumnMap.containsKey(key)) {
424                                table targetTable = tableMap.get(targetTableId);
425                                for (column column : targetTable.getColumns()) {
426                                        if (column.getId().equals(targetColumnId)) {
427                                                TableColumn targetTableColumn = new TableColumn();
428                                                targetTableColumn.setServer(targetTable.getServer());
429                                                targetTableColumn.setUserName(targetTable.getUserName());
430                                                targetTableColumn.setDatabaseName(targetTable.getDatabase());
431                                                targetTableColumn.setSchemaName(targetTable.getSchema());
432                                                targetTableColumn.setRelationName(targetTable.getTableNameOnly());
433                                                targetTableColumn.setAttributeName(column.getName());
434                                                ColumnUpstream columnUpstream = new ColumnUpstream();
435                                                targetTableColumn.setUpstreamAttributesIndirect(columnUpstream);
436                                                tableColumnMap.put(key, targetTableColumn);
437                                                matches = true;
438                                                break;
439                                        }
440                                }
441                        }
442
443                        if (!matches) {
444                                continue;
445                        }
446                        ColumnUpstream columnUpstream = tableColumnMap.get(key).getUpstreamAttributesIndirect();
447                        Set<relationship> columnRelations = targetIdRelationMap.get(key);
448                        for (relationship columnRelation : columnRelations) {
449                                List<sourceColumn> sourceColumns = columnRelation.getSources();
450                                for (sourceColumn sourceColumn : sourceColumns) {
451                                        table sourceTable = tableMap.get(sourceColumn.getParent_id());
452                                        if (sourceTable == null)
453                                                continue;
454                                        TableColumn sourceTableColumn = new TableColumn();
455                                        sourceTableColumn.setServer(sourceTable.getServer());
456                                        sourceTableColumn.setUserName(sourceTable.getUserName());
457                                        sourceTableColumn.setDatabaseName(sourceTable.getDatabase());
458                                        sourceTableColumn.setSchemaName(sourceTable.getSchema());
459                                        sourceTableColumn.setRelationName(sourceTable.getTableNameOnly());
460                                        sourceTableColumn.setAttributeName(sourceColumn.getColumn());
461                                        sourceTableColumn.setTimestampMax(columnRelation.getTimestampMax());
462                                        sourceTableColumn.setTimestampMin(columnRelation.getTimestampMin());
463                                        columnUpstream.appendNode(sourceTableColumn);
464                                }
465                        }
466                        if (columnUpstream.getNodes() != null && !columnUpstream.getNodes().isEmpty()) {
467                                leftMostSourceTable.appendAttribute(tableColumnMap.get(key));
468                                if (!targetTableMap.containsKey(targetTableId)) {
469                                        Table targetTable = new Table();
470                                        table table = tableMap.get(targetTableId);
471                                        targetTable.setServer(table.getServer());
472                                        targetTable.setUserName(table.getUserName());
473                                        targetTable.setDatabaseName(table.getDatabase());
474                                        targetTable.setSchemaName(table.getSchema());
475                                        targetTable.setRelationName(table.getTableNameOnly());
476                                        leftMostSourceTable.appendTargetTable(targetTable);
477                                        targetTableMap.put(targetTableId, targetTable);
478                                }
479                        }
480                }
481
482                if (leftMostSourceTable.getAttributes() != null) {
483                        Arrays.sort(leftMostSourceTable.getAttributes(), new Comparator<TableColumn>() {
484
485                                @Override
486                                public int compare(TableColumn o1, TableColumn o2) {
487                                        int result = (o1.getDatabaseName() + "." + o1.getSchemaName() + "." + o1.getRelationName())
488                                                        .compareTo(o2.getDatabaseName() + "." + o2.getSchemaName() + "." + o2.getRelationName());
489                                        if (result == 0) {
490                                                return o1.getAttributeName().compareTo(o2.getAttributeName());
491                                        } else {
492                                                return result;
493                                        }
494                                }
495                        });
496
497                        Map<Table, TreeSet<Table>> targetSourceTableMap = new HashMap<TableFlowUtility.Table, TreeSet<Table>>();
498                        for (TableColumn attribute : leftMostSourceTable.getAttributes()) {
499                                if (attribute.getUpstreamAttributesIndirect() != null
500                                                && attribute.getUpstreamAttributesIndirect().getNodes() != null) {
501                                        Collections.sort(attribute.getUpstreamAttributesIndirect().getNodes(),
502                                                        new Comparator<TableColumn>() {
503                                                                @Override
504                                                                public int compare(TableColumn o1, TableColumn o2) {
505                                                                        int result = (o1.getDatabaseName() + "." + o1.getSchemaName() + "."
506                                                                                        + o1.getRelationName())
507                                                                                        .compareTo(o2.getDatabaseName() + "." + o2.getSchemaName() + "."
508                                                                                                        + o2.getRelationName());
509                                                                        if (result == 0) {
510                                                                                return o1.getAttributeName().compareTo(o2.getAttributeName());
511                                                                        } else {
512                                                                                return result;
513                                                                        }
514                                                                }
515                                                        });
516
517                                        if (!targetSourceTableMap.containsKey(attribute.parentTable())) {
518                                                targetSourceTableMap.put(attribute.parentTable(), new TreeSet<Table>());
519                                        }
520                                        for (TableColumn sourceTableColumn : attribute.getUpstreamAttributesIndirect().getNodes()) {
521                                                targetSourceTableMap.get(attribute.parentTable()).add(sourceTableColumn.parentTable());
522                                        }
523                                }
524                        }
525
526                        if (leftMostSourceTable.getTargetTables() != null) {
527                                for (Table targetTable : leftMostSourceTable.getTargetTables()) {
528                                        TreeSet<Table> sourceTableTree = targetSourceTableMap.get(targetTable);
529                                        if (sourceTableTree != null) {
530                                                ColumnUpstream columnUpstream = new ColumnUpstream();
531                                                targetTable.setUpstreamAttributesIndirect(columnUpstream);
532                                                for (Table table : sourceTableTree) {
533                                                        columnUpstream.appendNode(table);
534                                                }
535                                        }
536                                }
537                        }
538                }
539
540                if (leftMostSourceTable.getTargetTables() != null) {
541                        Arrays.sort(leftMostSourceTable.getTargetTables(), new Comparator<Table>() {
542                                @Override
543                                public int compare(Table o1, Table o2) {
544                                        int result = (o1.getDatabaseName() + "." + o1.getSchemaName() + "." + o1.getRelationName())
545                                                        .compareTo(o2.getDatabaseName() + "." + o2.getSchemaName() + "." + o2.getRelationName());
546                                        return result;
547                                }
548                        });
549                }
550                return leftMostSourceTable;
551        }
552
553        private static String exportModelToCsv(LeftMostSourceTable leftMostSourceTable, String delimiter) {
554                if (delimiter == null || delimiter.length() == 0) {
555                        delimiter = ",";
556                }
557                StringBuilder buffer = new StringBuilder();
558                buffer.append("SOURCE_TABLE"+delimiter+"SOURCE_COLUMN"+delimiter+"TARGET_TABLE"+delimiter+"TARGET_COLUMN").append(System.getProperty("line.separator"));
559                try {
560                        Set<String> lines = exportModelToLines(leftMostSourceTable, delimiter);
561                        for (String line : lines) {
562                                buffer.append(line).append(System.getProperty("line.separator"));
563                        }
564                } catch (Exception e) {
565                        logger.error("Generate column level csv failed.", e);
566                }
567                return buffer.toString();
568        }
569
570        public static Set<String> exportModelToLines(LeftMostSourceTable leftMostSourceTable, String delimiter){
571                TreeSet<String> lines = new TreeSet<String>(new Comparator<String>() {
572                        @Override
573                        public int compare(String o1, String o2) {
574                                return o1.toLowerCase().compareTo(o2.toLowerCase());
575                        }
576                });
577                if (leftMostSourceTable.getAttributes() != null) {
578                        StringWriter sw = new StringWriter();
579                        CsvWriter csvWriter = new CsvWriter(sw, delimiter.charAt(0));
580                        try {
581                                for (TableColumn targetColumn : leftMostSourceTable.getAttributes()) {
582                                        if (targetColumn.getUpstreamAttributesIndirect() != null
583                                                        && targetColumn.getUpstreamAttributesIndirect().getNodes() != null) {
584                                                String targetTableName = getTableFullName(targetColumn);
585                                                String targetColumnName = targetColumn.getAttributeName();
586                                                for (TableColumn sourceColumn : targetColumn.getUpstreamAttributesIndirect().getNodes()) {
587                                                        String sourceTableName = getTableFullName(sourceColumn);
588                                                        String sourceColumnName = sourceColumn.getAttributeName();
589                                                        csvWriter.writeRecord(new String[]{sourceTableName, sourceColumnName, targetTableName, targetColumnName});
590                                                }
591                                        }
592                                }
593                        } catch (Exception e) {
594                                logger.error("write csv failed.", e);
595                        }
596                        csvWriter.close();
597                        String systemRecordDelimiter = System.getProperty("line.separator");
598                        lines.addAll(Arrays.asList(sw.toString().split(systemRecordDelimiter)));
599                }
600                return lines;
601        }
602
603        private static String getTableFullName(TableColumn targetColumn) {
604                String tableName = targetColumn.getRelationName();
605                if (!SQLUtil.isEmpty(targetColumn.getSchemaName())) {
606                        tableName = targetColumn.getSchemaName() + "." + tableName;
607                }
608                if (!SQLUtil.isEmpty(targetColumn.getDatabaseName())) {
609                        tableName = targetColumn.getDatabaseName() + "." + tableName;
610                }
611                return tableName;
612        }
613
614        public static dataflow generateLeftMostTableFlow(DataFlowAnalyzer analyzer, dataflow instance, String tableNamePattern, String tableColumn, boolean isSimple) {
615                EDbVendor vendor = analyzer.getOption().getVendor();
616
617                if (tableNamePattern != null) {
618                        tableNamePattern = tableNamePattern.replace("[", "\\[").replace("]", "\\]");
619                }
620                try {
621                        dataflow simple = instance;
622                        if (!isSimple && instance.getResultsets() != null && instance.getResultsets().size() > 0) {
623                                simple = analyzer.getSimpleDataflow(instance, true);
624                        }
625                        if (simple.getTables() == null && simple.getViews() == null) {
626                                return simple;
627                        }
628                        List<relationship> relations = simple.getRelationships();
629                        if (relations == null || relations.size() == 0) {
630                                return simple;
631                        }
632                        long maxId = Long.parseLong(relations.get(relations.size() - 1).getId().split("\\-")[0].replace("_", ""))
633                                        * 100;
634                        
635                        List<table> tables = new ArrayList<>();
636                        if (simple.getTables() != null) {
637                                tables.addAll(simple.getTables());
638                        }
639                        if (simple.getPaths() != null) {
640                                tables.addAll(simple.getPaths());
641                        }
642                        if (simple.getStages() != null) {
643                                tables.addAll(simple.getStages());
644                        }
645                        if (simple.getSequences() != null) {
646                                tables.addAll(simple.getSequences());
647                        }
648                        if (simple.getDatasources() != null) {
649                                tables.addAll(simple.getDatasources());
650                        }
651                        if (simple.getDatabases() != null) {
652                                tables.addAll(simple.getDatabases());
653                        }
654                        if (simple.getSchemas() != null) {
655                                tables.addAll(simple.getSchemas());
656                        }
657                        if (simple.getVariables() != null) {
658                                tables.addAll(simple.getVariables());
659                        }
660                        if (simple.getViews() != null) {
661                                tables.addAll(simple.getViews());
662                        }
663                        Map<String, table> tableMap = new HashMap<String, table>();
664                        Iterator<table> tableIter = tables.iterator();
665                        while (tableIter.hasNext()) {
666                                table table = tableIter.next();
667                                tableMap.put(table.getId(), table);
668                        }
669                        
670                        Iterator<relationship> iter = simple.getRelationships().iterator();
671                        while (iter.hasNext()) {
672                                relationship relation = iter.next();
673                                if (!"fdd".equals(relation.getType())) {
674                                        iter.remove();
675                                }
676                                if(relation.getSources() != null && relation.getSources().size()>0) {
677                                        Iterator<sourceColumn> sourceIter = relation.getSources().iterator();
678                                        while(sourceIter.hasNext()) {
679                                                sourceColumn source = sourceIter.next();
680                                                if(source.getId().equals(relation.getTarget().getId())) {
681                                                        sourceIter.remove();
682                                                }
683                                                
684                                                table sourceTable = tableMap.get(source.getParent_id());
685                                                if(sourceTable.isFile()) {
686                                                        sourceIter.remove();
687                                                }
688                                        }
689                                        if(relation.getSources().isEmpty()) {
690                                                iter.remove();
691                                        }
692                                }
693                        }
694
695
696                        Map<String, Set<relationship>> targetIdRelationMap = new HashMap<String, Set<relationship>>();
697                        for (relationship relation : relations) {
698                                if (relation.getTarget() != null) {
699                                        String key = relation.getTarget().getParent_id() + "." + relation.getTarget().getId();
700                                        if (!targetIdRelationMap.containsKey(key)) {
701                                                targetIdRelationMap.put(key, new HashSet<relationship>());
702                                        }
703                                        targetIdRelationMap.get(key).add(relation);
704                                }
705                        }
706
707                        List<relationship> simpleRelations = new ArrayList<relationship>();
708                        Set<table> tableSet = new HashSet<table>();
709                        iter = simple.getRelationships().iterator();
710                        while (iter.hasNext()) {
711                                relationship relation = iter.next();
712                                targetColumn target = relation.getTarget();
713                                String targetParentId = target.getParent_id();
714                                table targetTable = tableMap.get(targetParentId);
715                                if (targetTable == null) {
716                                        continue;
717                                }
718                                if (!matchTableName(tableNamePattern, targetTable, SQLUtil.isEmpty(tableColumn), vendor)) {
719                                        continue;
720                                }
721                                if (!SQLUtil.isEmpty(tableColumn) && !SQLUtil.getIdentifierNormalColumnName(vendor, target.getColumn()).equalsIgnoreCase(SQLUtil.getIdentifierNormalColumnName(vendor, tableColumn))) {
722                                        continue;
723                                }
724                                tableSet.add(targetTable);
725
726                                Set<sourceColumn> sourceColumnSet = new HashSet<sourceColumn>();
727                                Set<relationship> relationSet = new HashSet<relationship>();
728
729                                findSourceColumns(targetIdRelationMap, relation, sourceColumnSet, relationSet, tableMap);
730                                if (!sourceColumnSet.isEmpty()) {
731                                        relationship simpleRelation = (relationship) relation.clone();
732                                        simpleRelation.setSources(new ArrayList<sourceColumn>(sourceColumnSet));
733                                        simpleRelation.setId(String.valueOf(++maxId));
734                                        simpleRelations.add(simpleRelation);
735                                        for (sourceColumn sourceColumn : sourceColumnSet) {
736                                                table sourceTable = tableMap.get(sourceColumn.getParent_id());
737                                                if (sourceTable != null) {
738                                                        tableSet.add(sourceTable);
739                                                }
740                                        }
741                                }
742                        }
743                        simple.setRelationships(simpleRelations);
744
745                        if (simple.getTables() != null) {
746                                simple.getTables().retainAll(tableSet);
747                        }
748                        if (simple.getPaths() != null) {
749                                simple.getPaths().retainAll(tableSet);
750                        }
751                        if (simple.getStages() != null) {
752                                simple.getStages().retainAll(tableSet);
753                        }
754                        if (simple.getSequences() != null) {
755                                simple.getSequences().retainAll(tableSet);
756                        }
757                        if (simple.getDatasources() != null) {
758                                simple.getDatasources().retainAll(tableSet);
759                        }
760                        if (simple.getDatabases() != null) {
761                                simple.getDatabases().retainAll(tableSet);
762                        }
763                        if (simple.getSchemas() != null) {
764                                simple.getSchemas().retainAll(tableSet);
765                        }
766                        if (simple.getVariables() != null) {
767                                simple.getVariables().retainAll(tableSet);
768                        }
769                        if (simple.getViews() != null) {
770                                simple.getViews().retainAll(tableSet);
771                        }
772                        simple.getProcedures().clear();
773                        simple.getProcesses().clear();
774                        simple.getResultsets().clear();
775                        simple.getErrors().clear();
776                        return simple;
777                } catch (Exception e) {
778                        logger.error("Generate left most table flow failed.", e);
779                }
780                return new dataflow();
781        }
782
783        private static void findSourceColumns(Map<String, Set<relationship>> targetIdRelationMap, relationship relation,
784                                                                                  Set<sourceColumn> sourceColumnSet, Set<relationship> relationSet, Map<String, table> tableMap) {
785                if (relationSet.contains(relation)) {
786                        List<sourceColumn> sourceColumns = relation.getSources();
787                        for (sourceColumn sourceColumn : sourceColumns) {
788                                if (sourceColumn.getId().equals(relation.getTarget().getId())) {
789                                        sourceColumnSet.add(sourceColumn);
790                                        break;
791                                }
792                        }
793                        return;
794                } else {
795                        relationSet.add(relation);
796                }
797                List<sourceColumn> sourceColumns = relation.getSources();
798                for (sourceColumn sourceColumn : sourceColumns) {
799                        String columnId = sourceColumn.getParent_id() + "." + sourceColumn.getId();
800                        if (sourceColumnSet.contains(sourceColumn)) {
801                                continue;
802                        } else if (!targetIdRelationMap.containsKey(columnId)) {
803                                sourceColumnSet.add(sourceColumn);
804                        } else {
805                                Set<relationship> sourceRelations = targetIdRelationMap.get(columnId);
806                                for (relationship sourceRelation : sourceRelations) {
807                                        findSourceColumns(targetIdRelationMap, sourceRelation, sourceColumnSet, relationSet, tableMap);
808                                }
809                        }
810                }
811        }
812
813        public static boolean matchTableName(String tableNamePattern, table targetTable, boolean regex, EDbVendor vendor) {
814                if (SQLUtil.isEmpty(tableNamePattern))
815                        return false;
816                String tableNameOnly = SQLUtil.getIdentifierNormalTableName(vendor, targetTable.getTableNameOnly());
817                if (regex) {
818                        tableNamePattern = SQLUtil.trimColumnStringQuote(tableNamePattern);
819                        if (SQLUtil.trimColumnStringQuote(tableNameOnly).matches("(?i)" + tableNamePattern))
820                                return true;
821                        String tableName = targetTable.getName();
822                        if (SQLUtil.trimColumnStringQuote(tableName).matches("(?i)" + tableNamePattern))
823                                return true;
824                        if (vendor == EDbVendor.dbvmssql || vendor == EDbVendor.dbvazuresql) {
825                                if (tableName.indexOf("..") != -1) {
826                                        if (ModelBindingManager.getGlobalSchema() != null) {
827                                                tableName = tableName.replace("..", "." + ModelBindingManager.getGlobalSchema() + ".");
828                                        } else {
829                                                tableName = tableName.replace("..", ".dbo.");
830                                        }
831                                        if (SQLUtil.trimColumnStringQuote(tableName).matches("(?i)" + tableNamePattern))
832                                                return true;
833                                }
834                        }
835                } else {
836                        return tableNameOnly.equalsIgnoreCase(SQLUtil.getIdentifierNormalTableName(vendor, tableNamePattern))
837                                        ||  SQLUtil.getIdentifierNormalTableName(vendor, targetTable.getName()).equalsIgnoreCase(SQLUtil.getIdentifierNormalTableName(vendor, tableNamePattern));
838                }
839                return false;
840        }
841}