Source code

001package gudusoft.gsqlparser.dlineage.util;
002
003import java.io.IOException;
004import java.io.StringWriter;
005import java.util.ArrayList;
006import java.util.Arrays;
007import java.util.Collections;
008import java.util.Comparator;
009import java.util.HashMap;
010import java.util.HashSet;
011import java.util.Iterator;
012import java.util.List;
013import java.util.Map;
014import java.util.Objects;
015import java.util.Set;
016import java.util.TreeMap;
017import java.util.TreeSet;
018
019import gudusoft.gsqlparser.EDbVendor;
020import gudusoft.gsqlparser.dlineage.DataFlowAnalyzer;
021import gudusoft.gsqlparser.dlineage.dataflow.model.ModelBindingManager;
022import gudusoft.gsqlparser.dlineage.dataflow.model.xml.column;
023import gudusoft.gsqlparser.dlineage.dataflow.model.xml.dataflow;
024import gudusoft.gsqlparser.dlineage.dataflow.model.xml.relationship;
025import gudusoft.gsqlparser.dlineage.dataflow.model.xml.sourceColumn;
026import gudusoft.gsqlparser.dlineage.dataflow.model.xml.table;
027import gudusoft.gsqlparser.dlineage.dataflow.model.xml.targetColumn;
028import gudusoft.gsqlparser.util.Logger;
029import gudusoft.gsqlparser.util.LoggerFactory;
030import gudusoft.gsqlparser.util.SQLUtil;
031import gudusoft.gsqlparser.util.csv.CsvWriter;
032import gudusoft.gsqlparser.util.json.JSON;
033
034public class TableFlowUtility {
035
036        private static final Logger logger = LoggerFactory.getLogger(TableFlowUtility.class);
037
038        public static class LeftMostSourceTable {
039                private Table[] targetTables;
040                private TableColumn[] attributes;
041
042                public TableColumn[] getAttributes() {
043                        return attributes;
044                }
045
046                public void setAttributes(TableColumn[] attributes) {
047                        this.attributes = attributes;
048                }
049
050                public Table[] getTargetTables() {
051                        return targetTables;
052                }
053
054                public void setTargetTables(Table[] targetTables) {
055                        this.targetTables = targetTables;
056                }
057
058                public void appendAttribute(TableColumn attribute) {
059                        if (attribute == null) {
060                                return;
061                        }
062                        if (attributes == null) {
063                                attributes = new TableColumn[] { attribute };
064                        } else {
065                                TableColumn[] newAttributes = new TableColumn[attributes.length + 1];
066                                System.arraycopy(attributes, 0, newAttributes, 0, attributes.length);
067                                newAttributes[attributes.length] = attribute;
068                                this.attributes = newAttributes;
069                        }
070                }
071
072                public void appendTargetTable(Table targetTable) {
073                        if (targetTable == null) {
074                                return;
075                        }
076                        if (targetTables == null) {
077                                targetTables = new Table[] { targetTable };
078                        } else {
079                                Table[] newTargetTables = new Table[targetTables.length + 1];
080                                System.arraycopy(targetTables, 0, newTargetTables, 0, targetTables.length);
081                                newTargetTables[targetTables.length] = targetTable;
082                                this.targetTables = newTargetTables;
083                        }
084                }
085
086        }
087
088        public static class ColumnUpstream {
089                private List<TableColumn> nodes;
090
091                public List<TableColumn> getNodes() {
092                        return nodes;
093                }
094
095                public void setNodes(List<TableColumn> nodes) {
096                        this.nodes = nodes;
097                }
098
099                public void appendNode(TableColumn node) {
100                        if (node == null) {
101                                return;
102                        }
103                        if (nodes == null) {
104                                nodes = new ArrayList<TableFlowUtility.TableColumn>();
105                                nodes.add(node);
106                        } else {
107                                if (!nodes.contains(node)) {
108                                        nodes.add(node);
109                                }
110                        }
111                }
112
113                public void appendNode(Table node) {
114                        if (node == null) {
115                                return;
116                        }
117                        TableColumn tableColumn = new TableColumn(node);
118                        if (nodes == null) {
119                                nodes = new ArrayList<TableColumn>();
120                                nodes.add(tableColumn);
121                        } else {
122                                if (!nodes.contains(tableColumn)) {
123                                        nodes.add(tableColumn);
124                                }
125                        }
126                }
127
128        }
129
130        public static class Table implements Comparable<Table> {
131                private String server;
132                private String userName;
133                private String databaseName;
134                private String schemaName;
135                private String relationName;
136                private ColumnUpstream upstreamAttributesIndirect;
137
138                public Table() {
139                }
140
141                public Table(String server, String userName, String databaseName, String schemaName, String relationName) {
142                        this.server = server;
143                        this.userName = userName;
144                        this.databaseName = databaseName;
145                        this.schemaName = schemaName;
146                        this.relationName = relationName;
147                }
148
149                public String getServer() {
150                        return server;
151                }
152
153                public void setServer(String server) {
154                        this.server = server;
155                }
156
157                public String getUserName() {
158                        return userName;
159                }
160
161                public void setUserName(String userName) {
162                        this.userName = userName;
163                }
164
165                public String getDatabaseName() {
166                        return databaseName;
167                }
168
169                public void setDatabaseName(String databaseName) {
170                        this.databaseName = databaseName;
171                }
172
173                public String getSchemaName() {
174                        return schemaName;
175                }
176
177                public void setSchemaName(String schemaName) {
178                        this.schemaName = schemaName;
179                }
180
181                public String getRelationName() {
182                        return relationName;
183                }
184
185                public void setRelationName(String relationName) {
186                        this.relationName = relationName;
187                }
188
189                public ColumnUpstream getUpstreamAttributesIndirect() {
190                        return upstreamAttributesIndirect;
191                }
192
193                public void setUpstreamAttributesIndirect(ColumnUpstream upstreamAttributesIndirect) {
194                        this.upstreamAttributesIndirect = upstreamAttributesIndirect;
195                }
196
197                @Override
198                public boolean equals(Object o) {
199                        if (this == o) return true;
200                        if (o == null || getClass() != o.getClass()) return false;
201                        Table table = (Table) o;
202                        return Objects.equals(server, table.server) &&
203                                        Objects.equals(userName, table.userName) &&
204                                        Objects.equals(databaseName, table.databaseName) &&
205                                        Objects.equals(schemaName, table.schemaName) &&
206                                        Objects.equals(relationName, table.relationName) &&
207                                        Objects.equals(upstreamAttributesIndirect, table.upstreamAttributesIndirect);
208                }
209
210                @Override
211                public int hashCode() {
212                        return Objects.hash(server, userName, databaseName, schemaName, relationName, upstreamAttributesIndirect);
213                }
214
215                @Override
216                public int compareTo(Table table) {
217                        return (getDatabaseName() + "." + getSchemaName() + "." + getRelationName())
218                                        .compareTo(table.getDatabaseName() + "." + table.getSchemaName() + "." + table.getRelationName());
219                }
220        }
221
222        public static class TableColumn {
223                private String server;
224                private String userName;
225                private String databaseName;
226                private String schemaName;
227                private String relationName;
228                private String attributeName;
229                private String timestampMin;
230                private String timestampMax;
231                private ColumnUpstream upstreamAttributesIndirect;
232
233                public TableColumn() {
234                }
235
236                public TableColumn(Table node) {
237                        this.server = node.getServer();
238                        this.userName = node.getUserName();
239                        this.databaseName = node.getDatabaseName();
240                        this.schemaName = node.getSchemaName();
241                        this.relationName = node.getRelationName();
242                }
243
244                public String getServer() {
245                        return server;
246                }
247
248                public void setServer(String server) {
249                        this.server = server;
250                }
251
252                public String getUserName() {
253                        return userName;
254                }
255
256                public void setUserName(String userName) {
257                        this.userName = userName;
258                }
259
260                public String getDatabaseName() {
261                        return databaseName;
262                }
263
264                public void setDatabaseName(String databaseName) {
265                        this.databaseName = databaseName;
266                }
267
268                public String getSchemaName() {
269                        return schemaName;
270                }
271
272                public void setSchemaName(String schemaName) {
273                        this.schemaName = schemaName;
274                }
275
276                public String getRelationName() {
277                        return relationName;
278                }
279
280                public void setRelationName(String relationName) {
281                        this.relationName = relationName;
282                }
283
284                public String getAttributeName() {
285                        return attributeName;
286                }
287
288                public void setAttributeName(String attributeName) {
289                        this.attributeName = attributeName;
290                }
291
292                public ColumnUpstream getUpstreamAttributesIndirect() {
293                        return upstreamAttributesIndirect;
294                }
295
296                public void setUpstreamAttributesIndirect(ColumnUpstream upstreamAttributesIndirect) {
297                        this.upstreamAttributesIndirect = upstreamAttributesIndirect;
298                }
299
300                public String getTimestampMin() {
301                        return timestampMin;
302                }
303
304                public void setTimestampMin(String timestampMin) {
305                        this.timestampMin = timestampMin;
306                }
307
308                public String getTimestampMax() {
309                        return timestampMax;
310                }
311
312                public void setTimestampMax(String timestampMax) {
313                        this.timestampMax = timestampMax;
314                }
315
316                @Override
317                public boolean equals(Object o) {
318                        if (this == o) return true;
319                        if (o == null || getClass() != o.getClass()) return false;
320                        TableColumn that = (TableColumn) o;
321                        return Objects.equals(server, that.server) &&
322                                        Objects.equals(userName, that.userName) &&
323                                        Objects.equals(databaseName, that.databaseName) &&
324                                        Objects.equals(schemaName, that.schemaName) &&
325                                        Objects.equals(relationName, that.relationName) &&
326                                        Objects.equals(attributeName, that.attributeName) &&
327                                        Objects.equals(timestampMin, that.timestampMin) &&
328                                        Objects.equals(timestampMax, that.timestampMax) &&
329                                        Objects.equals(upstreamAttributesIndirect, that.upstreamAttributesIndirect);
330                }
331
332                @Override
333                public int hashCode() {
334                        return Objects.hash(server, userName, databaseName, schemaName, relationName, attributeName, timestampMin, timestampMax, upstreamAttributesIndirect);
335                }
336
337                public Table parentTable() {
338                        return new Table(server, userName, databaseName, schemaName, relationName);
339                }
340        }
341
342        public static String generateLeftMostTable(DataFlowAnalyzer analyzer, dataflow instance, String tableNamePattern,
343                                                                                           String tableColumn, boolean isSimple) {
344                return generateLeftMostTable(analyzer, instance, tableNamePattern, tableColumn, isSimple, "json", null);
345        }
346
347        public static String generateLeftMostTable(DataFlowAnalyzer analyzer, dataflow instance, String tableNamePattern,
348                                                                                           String tableColumn, boolean isSimple, String outputType, String delimiter) {
349                dataflow dataflow = generateLeftMostTableFlow(analyzer, instance, tableNamePattern, tableColumn, isSimple);
350                LeftMostSourceTable leftMostSourceTable = exportToLeftMostTableModel(dataflow);
351
352                if ("json".equals(outputType)) {
353                        return JSON.toJSONString(leftMostSourceTable);
354                } else if ("csv".equals(outputType)) {
355                        return exportModelToCsv(leftMostSourceTable, delimiter);
356                } else {
357                        throw new UnsupportedOperationException("Unsupported output type: " + outputType);
358                }
359        }
360
361        static LeftMostSourceTable exportToLeftMostTableModel(dataflow dataflow) {
362                LeftMostSourceTable leftMostSourceTable = new LeftMostSourceTable();
363
364                Map<String, Set<relationship>> targetIdRelationMap = new HashMap<String, Set<relationship>>();
365                List<relationship> relations = dataflow.getRelationships();
366                for (relationship relation : relations) {
367                        if (relation.getTarget() != null) {
368                                String key = relation.getTarget().getParent_id() + ":" + relation.getTarget().getId();
369                                if (!targetIdRelationMap.containsKey(key)) {
370                                        targetIdRelationMap.put(key, new HashSet<relationship>());
371                                }
372                                targetIdRelationMap.get(key).add(relation);
373                        }
374                }
375
376                List<table> tables = new ArrayList<>();
377                if (dataflow.getTables() != null) {
378                        tables.addAll(dataflow.getTables());
379                }
380                if (dataflow.getPaths() != null) {
381                        tables.addAll(dataflow.getPaths());
382                }
383                if (dataflow.getStages() != null) {
384                        tables.addAll(dataflow.getStages());
385                }
386                if (dataflow.getSequences() != null) {
387                        tables.addAll(dataflow.getSequences());
388                }
389                if (dataflow.getDatasources() != null) {
390                        tables.addAll(dataflow.getDatasources());
391                }
392                if (dataflow.getDatabases() != null) {
393                        tables.addAll(dataflow.getDatabases());
394                }
395                if (dataflow.getSchemas() != null) {
396                        tables.addAll(dataflow.getSchemas());
397                }
398                if (dataflow.getVariables() != null) {
399                        tables.addAll(dataflow.getVariables());
400                }
401                if (dataflow.getViews() != null) {
402                        tables.addAll(dataflow.getViews());
403                }
404                Map<String, table> tableMap = new HashMap<String, table>();
405                Map<String, List<table>> duplicateTableMap = new HashMap<String, List<table>>();
406                Iterator<table> tableIter = tables.iterator();
407                while (tableIter.hasNext()) {
408                        table table = tableIter.next();
409                        tableMap.put(table.getId(), table);
410                        String tableName = table.getName();
411                        if(!duplicateTableMap.containsKey(tableName)) {
412                                duplicateTableMap.put(tableName, new ArrayList<table>());
413                        }
414                        duplicateTableMap.get(tableName).add(table);
415                }
416
417                Map<String, TableColumn> tableColumnMap = new TreeMap<String, TableColumn>();
418                Map<String, Table> targetTableMap = new TreeMap<String, Table>();
419                for (String key : targetIdRelationMap.keySet()) {
420                        String targetTableId = key.split(":")[0];
421                        String targetColumnId = key.split(":")[1];
422                        boolean matches = false;
423                        if (!tableColumnMap.containsKey(key)) {
424                                table targetTable = tableMap.get(targetTableId);
425                                for (column column : targetTable.getColumns()) {
426                                        if (column.getId().equals(targetColumnId)) {
427                                                TableColumn targetTableColumn = new TableColumn();
428                                                targetTableColumn.setServer(targetTable.getServer());
429                                                targetTableColumn.setUserName(targetTable.getUserName());
430                                                targetTableColumn.setDatabaseName(targetTable.getDatabase());
431                                                targetTableColumn.setSchemaName(targetTable.getSchema());
432                                                targetTableColumn.setRelationName(targetTable.getTableNameOnly());
433                                                targetTableColumn.setAttributeName(column.getName());
434                                                ColumnUpstream columnUpstream = new ColumnUpstream();
435                                                targetTableColumn.setUpstreamAttributesIndirect(columnUpstream);
436                                                tableColumnMap.put(key, targetTableColumn);
437                                                matches = true;
438                                                break;
439                                        }
440                                }
441                        }
442
443                        if (!matches) {
444                                continue;
445                        }
446                        ColumnUpstream columnUpstream = tableColumnMap.get(key).getUpstreamAttributesIndirect();
447                        Set<relationship> columnRelations = targetIdRelationMap.get(key);
448                        for (relationship columnRelation : columnRelations) {
449                                List<sourceColumn> sourceColumns = columnRelation.getSources();
450                                for (sourceColumn sourceColumn : sourceColumns) {
451                                        table sourceTable = tableMap.get(sourceColumn.getParent_id());
452                                        if (sourceTable == null)
453                                                continue;
454                                        TableColumn sourceTableColumn = new TableColumn();
455                                        sourceTableColumn.setServer(sourceTable.getServer());
456                                        sourceTableColumn.setUserName(sourceTable.getUserName());
457                                        sourceTableColumn.setDatabaseName(sourceTable.getDatabase());
458                                        sourceTableColumn.setSchemaName(sourceTable.getSchema());
459                                        sourceTableColumn.setRelationName(sourceTable.getTableNameOnly());
460                                        sourceTableColumn.setAttributeName(sourceColumn.getColumn());
461                                        sourceTableColumn.setTimestampMax(columnRelation.getTimestampMax());
462                                        sourceTableColumn.setTimestampMin(columnRelation.getTimestampMin());
463                                        columnUpstream.appendNode(sourceTableColumn);
464                                }
465                        }
466                        if (columnUpstream.getNodes() != null && !columnUpstream.getNodes().isEmpty()) {
467                                leftMostSourceTable.appendAttribute(tableColumnMap.get(key));
468                                if (!targetTableMap.containsKey(targetTableId)) {
469                                        Table targetTable = new Table();
470                                        table table = tableMap.get(targetTableId);
471                                        targetTable.setServer(table.getServer());
472                                        targetTable.setUserName(table.getUserName());
473                                        targetTable.setDatabaseName(table.getDatabase());
474                                        targetTable.setSchemaName(table.getSchema());
475                                        targetTable.setRelationName(table.getTableNameOnly());
476                                        leftMostSourceTable.appendTargetTable(targetTable);
477                                        targetTableMap.put(targetTableId, targetTable);
478                                }
479                        }
480                }
481
482                if (leftMostSourceTable.getAttributes() != null) {
483                        Arrays.sort(leftMostSourceTable.getAttributes(), new Comparator<TableColumn>() {
484
485                                @Override
486                                public int compare(TableColumn o1, TableColumn o2) {
487                                        int result = (o1.getDatabaseName() + "." + o1.getSchemaName() + "." + o1.getRelationName())
488                                                        .compareTo(o2.getDatabaseName() + "." + o2.getSchemaName() + "." + o2.getRelationName());
489                                        if (result == 0) {
490                                                return o1.getAttributeName().compareTo(o2.getAttributeName());
491                                        } else {
492                                                return result;
493                                        }
494                                }
495                        });
496
497                        Map<Table, TreeSet<Table>> targetSourceTableMap = new HashMap<TableFlowUtility.Table, TreeSet<Table>>();
498                        for (TableColumn attribute : leftMostSourceTable.getAttributes()) {
499                                if (attribute.getUpstreamAttributesIndirect() != null
500                                                && attribute.getUpstreamAttributesIndirect().getNodes() != null) {
501                                        Collections.sort(attribute.getUpstreamAttributesIndirect().getNodes(),
502                                                        new Comparator<TableColumn>() {
503                                                                @Override
504                                                                public int compare(TableColumn o1, TableColumn o2) {
505                                                                        int result = (o1.getDatabaseName() + "." + o1.getSchemaName() + "."
506                                                                                        + o1.getRelationName())
507                                                                                        .compareTo(o2.getDatabaseName() + "." + o2.getSchemaName() + "."
508                                                                                                        + o2.getRelationName());
509                                                                        if (result == 0) {
510                                                                                return o1.getAttributeName().compareTo(o2.getAttributeName());
511                                                                        } else {
512                                                                                return result;
513                                                                        }
514                                                                }
515                                                        });
516
517                                        if (!targetSourceTableMap.containsKey(attribute.parentTable())) {
518                                                targetSourceTableMap.put(attribute.parentTable(), new TreeSet<Table>());
519                                        }
520                                        for (TableColumn sourceTableColumn : attribute.getUpstreamAttributesIndirect().getNodes()) {
521                                                targetSourceTableMap.get(attribute.parentTable()).add(sourceTableColumn.parentTable());
522                                        }
523                                }
524                        }
525
526                        if (leftMostSourceTable.getTargetTables() != null) {
527                                for (Table targetTable : leftMostSourceTable.getTargetTables()) {
528                                        TreeSet<Table> sourceTableTree = targetSourceTableMap.get(targetTable);
529                                        if (sourceTableTree != null) {
530                                                ColumnUpstream columnUpstream = new ColumnUpstream();
531                                                targetTable.setUpstreamAttributesIndirect(columnUpstream);
532                                                for (Table table : sourceTableTree) {
533                                                        columnUpstream.appendNode(table);
534                                                }
535                                        }
536                                }
537                        }
538                }
539
540                if (leftMostSourceTable.getTargetTables() != null) {
541                        Arrays.sort(leftMostSourceTable.getTargetTables(), new Comparator<Table>() {
542                                @Override
543                                public int compare(Table o1, Table o2) {
544                                        int result = (o1.getDatabaseName() + "." + o1.getSchemaName() + "." + o1.getRelationName())
545                                                        .compareTo(o2.getDatabaseName() + "." + o2.getSchemaName() + "." + o2.getRelationName());
546                                        return result;
547                                }
548                        });
549                }
550                return leftMostSourceTable;
551        }
552
553        private static String exportModelToCsv(LeftMostSourceTable leftMostSourceTable, String delimiter) {
554                if (delimiter == null || delimiter.length() == 0) {
555                        delimiter = ",";
556                }
557                StringBuilder buffer = new StringBuilder();
558                buffer.append("SOURCE_TABLE"+delimiter+"SOURCE_COLUMN"+delimiter+"TARGET_TABLE"+delimiter+"TARGET_COLUMN").append(System.getProperty("line.separator"));
559                try {
560                        Set<String> lines = exportModelToLines(leftMostSourceTable, delimiter);
561                        for (String line : lines) {
562                                buffer.append(line).append(System.getProperty("line.separator"));
563                        }
564                } catch (Exception e) {
565                        logger.error("Generate column level csv failed.", e);
566                }
567                return buffer.toString();
568        }
569
570        public static Set<String> exportModelToLines(LeftMostSourceTable leftMostSourceTable, String delimiter){
571                TreeSet<String> lines = new TreeSet<String>(new Comparator<String>() {
572                        @Override
573                        public int compare(String o1, String o2) {
574                                return o1.toLowerCase().compareTo(o2.toLowerCase());
575                        }
576                });
577                if (leftMostSourceTable.getAttributes() != null) {
578                        StringWriter sw = new StringWriter();
579                        CsvWriter csvWriter = new CsvWriter(sw, delimiter.charAt(0));
580                        try {
581                                for (TableColumn targetColumn : leftMostSourceTable.getAttributes()) {
582                                        if (targetColumn.getUpstreamAttributesIndirect() != null
583                                                        && targetColumn.getUpstreamAttributesIndirect().getNodes() != null) {
584                                                String targetTableName = getTableFullName(targetColumn);
585                                                String targetColumnName = targetColumn.getAttributeName();
586                                                for (TableColumn sourceColumn : targetColumn.getUpstreamAttributesIndirect().getNodes()) {
587                                                        String sourceTableName = getTableFullName(sourceColumn);
588                                                        String sourceColumnName = sourceColumn.getAttributeName();
589                                                        csvWriter.writeRecord(new String[]{sourceTableName, sourceColumnName, targetTableName, targetColumnName});
590                                                }
591                                        }
592                                }
593                        } catch (Exception e) {
594                                logger.error("write csv failed.", e);
595                        }
596                        csvWriter.close();
597                        String systemRecordDelimiter = System.getProperty("line.separator");
598                        lines.addAll(Arrays.asList(sw.toString().split(systemRecordDelimiter)));
599                }
600                return lines;
601        }
602
603        private static String getTableFullName(TableColumn targetColumn) {
604                String tableName = targetColumn.getRelationName();
605                if (!SQLUtil.isEmpty(targetColumn.getSchemaName())) {
606                        tableName = targetColumn.getSchemaName() + "." + tableName;
607                }
608                if (!SQLUtil.isEmpty(targetColumn.getDatabaseName())) {
609                        tableName = targetColumn.getDatabaseName() + "." + tableName;
610                }
611                return tableName;
612        }
613
614        public static dataflow generateLeftMostTableFlow(DataFlowAnalyzer analyzer, dataflow instance,
615                                                                                                         String tableNamePattern, String tableColumn, boolean isSimple) {
616                if (tableNamePattern != null) {
617                        tableNamePattern = tableNamePattern.replace("[", "\\[").replace("]", "\\]");
618                }
619                try {
620                        dataflow simple = instance;
621                        if (!isSimple && instance.getResultsets() != null && instance.getResultsets().size() > 0) {
622                                simple = analyzer.getSimpleDataflow(instance, true);
623                        }
624                        if (simple.getTables() == null && simple.getViews() == null) {
625                                return simple;
626                        }
627                        List<relationship> relations = simple.getRelationships();
628                        if (relations == null || relations.size() == 0) {
629                                return simple;
630                        }
631                        long maxId = Long.parseLong(relations.get(relations.size() - 1).getId().split("\\-")[0].replace("_", ""))
632                                        * 100;
633                        
634                        List<table> tables = new ArrayList<>();
635                        if (simple.getTables() != null) {
636                                tables.addAll(simple.getTables());
637                        }
638                        if (simple.getPaths() != null) {
639                                tables.addAll(simple.getPaths());
640                        }
641                        if (simple.getStages() != null) {
642                                tables.addAll(simple.getStages());
643                        }
644                        if (simple.getSequences() != null) {
645                                tables.addAll(simple.getSequences());
646                        }
647                        if (simple.getDatasources() != null) {
648                                tables.addAll(simple.getDatasources());
649                        }
650                        if (simple.getDatabases() != null) {
651                                tables.addAll(simple.getDatabases());
652                        }
653                        if (simple.getSchemas() != null) {
654                                tables.addAll(simple.getSchemas());
655                        }
656                        if (simple.getVariables() != null) {
657                                tables.addAll(simple.getVariables());
658                        }
659                        if (simple.getViews() != null) {
660                                tables.addAll(simple.getViews());
661                        }
662                        Map<String, table> tableMap = new HashMap<String, table>();
663                        Iterator<table> tableIter = tables.iterator();
664                        while (tableIter.hasNext()) {
665                                table table = tableIter.next();
666                                tableMap.put(table.getId(), table);
667                        }
668                        
669                        Iterator<relationship> iter = simple.getRelationships().iterator();
670                        while (iter.hasNext()) {
671                                relationship relation = iter.next();
672                                if (!"fdd".equals(relation.getType())) {
673                                        iter.remove();
674                                }
675                                if(relation.getSources() != null && relation.getSources().size()>0) {
676                                        Iterator<sourceColumn> sourceIter = relation.getSources().iterator();
677                                        while(sourceIter.hasNext()) {
678                                                sourceColumn source = sourceIter.next();
679                                                if(source.getId().equals(relation.getTarget().getId())) {
680                                                        sourceIter.remove();
681                                                }
682                                                
683                                                table sourceTable = tableMap.get(source.getParent_id());
684                                                if(sourceTable.isFile()) {
685                                                        sourceIter.remove();
686                                                }
687                                        }
688                                        if(relation.getSources().isEmpty()) {
689                                                iter.remove();
690                                        }
691                                }
692                        }
693
694
695                        Map<String, Set<relationship>> targetIdRelationMap = new HashMap<String, Set<relationship>>();
696                        for (relationship relation : relations) {
697                                if (relation.getTarget() != null) {
698                                        String key = relation.getTarget().getParent_id() + "." + relation.getTarget().getId();
699                                        if (!targetIdRelationMap.containsKey(key)) {
700                                                targetIdRelationMap.put(key, new HashSet<relationship>());
701                                        }
702                                        targetIdRelationMap.get(key).add(relation);
703                                }
704                        }
705
706                        List<relationship> simpleRelations = new ArrayList<relationship>();
707                        Set<table> tableSet = new HashSet<table>();
708                        iter = simple.getRelationships().iterator();
709                        while (iter.hasNext()) {
710                                relationship relation = iter.next();
711                                targetColumn target = relation.getTarget();
712                                String targetParentId = target.getParent_id();
713                                table targetTable = tableMap.get(targetParentId);
714                                if (targetTable == null) {
715                                        continue;
716                                }
717                                if (!matchTableName(tableNamePattern, targetTable, SQLUtil.isEmpty(tableColumn), analyzer.getOption().getVendor())) {
718                                        continue;
719                                }
720                                if (!SQLUtil.isEmpty(tableColumn) && !target.getColumn().equalsIgnoreCase(tableColumn)) {
721                                        continue;
722                                }
723                                tableSet.add(targetTable);
724
725                                Set<sourceColumn> sourceColumnSet = new HashSet<sourceColumn>();
726                                Set<relationship> relationSet = new HashSet<relationship>();
727
728                                findSourceColumns(targetIdRelationMap, relation, sourceColumnSet, relationSet, tableMap);
729                                if (!sourceColumnSet.isEmpty()) {
730                                        relationship simpleRelation = (relationship) relation.clone();
731                                        simpleRelation.setSources(new ArrayList<sourceColumn>(sourceColumnSet));
732                                        simpleRelation.setId(String.valueOf(++maxId));
733                                        simpleRelations.add(simpleRelation);
734                                        for (sourceColumn sourceColumn : sourceColumnSet) {
735                                                table sourceTable = tableMap.get(sourceColumn.getParent_id());
736                                                if (sourceTable != null) {
737                                                        tableSet.add(sourceTable);
738                                                }
739                                        }
740                                }
741                        }
742                        simple.setRelationships(simpleRelations);
743
744                        if (simple.getTables() != null) {
745                                simple.getTables().retainAll(tableSet);
746                        }
747                        if (simple.getPaths() != null) {
748                                simple.getPaths().retainAll(tableSet);
749                        }
750                        if (simple.getStages() != null) {
751                                simple.getStages().retainAll(tableSet);
752                        }
753                        if (simple.getSequences() != null) {
754                                simple.getSequences().retainAll(tableSet);
755                        }
756                        if (simple.getDatasources() != null) {
757                                simple.getDatasources().retainAll(tableSet);
758                        }
759                        if (simple.getDatabases() != null) {
760                                simple.getDatabases().retainAll(tableSet);
761                        }
762                        if (simple.getSchemas() != null) {
763                                simple.getSchemas().retainAll(tableSet);
764                        }
765                        if (simple.getVariables() != null) {
766                                simple.getVariables().retainAll(tableSet);
767                        }
768                        if (simple.getViews() != null) {
769                                simple.getViews().retainAll(tableSet);
770                        }
771                        simple.getProcedures().clear();
772                        simple.getProcesses().clear();
773                        simple.getResultsets().clear();
774                        simple.getErrors().clear();
775                        return simple;
776                } catch (Exception e) {
777                        logger.error("Generate left most table flow failed.", e);
778                }
779                return new dataflow();
780        }
781
782        private static void findSourceColumns(Map<String, Set<relationship>> targetIdRelationMap, relationship relation,
783                                                                                  Set<sourceColumn> sourceColumnSet, Set<relationship> relationSet, Map<String, table> tableMap) {
784                if (relationSet.contains(relation)) {
785                        List<sourceColumn> sourceColumns = relation.getSources();
786                        for (sourceColumn sourceColumn : sourceColumns) {
787                                if (sourceColumn.getId().equals(relation.getTarget().getId())) {
788                                        sourceColumnSet.add(sourceColumn);
789                                        break;
790                                }
791                        }
792                        return;
793                } else {
794                        relationSet.add(relation);
795                }
796                List<sourceColumn> sourceColumns = relation.getSources();
797                for (sourceColumn sourceColumn : sourceColumns) {
798                        String columnId = sourceColumn.getParent_id() + "." + sourceColumn.getId();
799                        if (sourceColumnSet.contains(sourceColumn)) {
800                                continue;
801                        } else if (!targetIdRelationMap.containsKey(columnId)) {
802                                sourceColumnSet.add(sourceColumn);
803                        } else {
804                                Set<relationship> sourceRelations = targetIdRelationMap.get(columnId);
805                                for (relationship sourceRelation : sourceRelations) {
806                                        findSourceColumns(targetIdRelationMap, sourceRelation, sourceColumnSet, relationSet, tableMap);
807                                }
808                        }
809                }
810        }
811
812        private static boolean matchTableName(String tableNamePattern, table targetTable, boolean regex, EDbVendor vendor) {
813                if (SQLUtil.isEmpty(tableNamePattern))
814                        return false;
815                String tableNameOnly = SQLUtil.getIdentifierNormalTableName(vendor, targetTable.getTableNameOnly());
816                if (regex) {
817                        tableNamePattern = SQLUtil.trimColumnStringQuote(tableNamePattern);
818                        if (SQLUtil.trimColumnStringQuote(tableNameOnly).matches("(?i)" + tableNamePattern))
819                                return true;
820                        String tableName = targetTable.getName();
821                        if (SQLUtil.trimColumnStringQuote(tableName).matches("(?i)" + tableNamePattern))
822                                return true;
823                        if (vendor == EDbVendor.dbvmssql || vendor == EDbVendor.dbvazuresql) {
824                                if (tableName.indexOf("..") != -1) {
825                                        if (ModelBindingManager.getGlobalSchema() != null) {
826                                                tableName = tableName.replace("..", "." + ModelBindingManager.getGlobalSchema() + ".");
827                                        } else {
828                                                tableName = tableName.replace("..", ".dbo.");
829                                        }
830                                        if (SQLUtil.trimColumnStringQuote(tableName).matches("(?i)" + tableNamePattern))
831                                                return true;
832                                }
833                        }
834                } else {
835                        return tableNameOnly.equalsIgnoreCase(SQLUtil.getIdentifierNormalTableName(vendor, tableNamePattern))
836                                        ||  SQLUtil.getIdentifierNormalTableName(vendor, targetTable.getName()).equalsIgnoreCase(SQLUtil.getIdentifierNormalTableName(vendor, tableNamePattern));
837                }
838                return false;
839        }
840}