001package gudusoft.gsqlparser.dlineage.graph; 002 003import gudusoft.gsqlparser.dlineage.dataflow.model.RelationshipType; 004import gudusoft.gsqlparser.dlineage.dataflow.model.xml.*; 005import gudusoft.gsqlparser.dlineage.graph.utils.StringEscapeUtils; 006import gudusoft.gsqlparser.dlineage.util.Pair3; 007import gudusoft.gsqlparser.util.SQLUtil; 008import java.io.IOException; 009import java.util.*; 010import java.util.Map.Entry; 011import java.util.stream.Collectors; 012 013public class DataFlowGraph { 014 015 public Pair3<String, Boolean, Set<String>> generateDataFlowGraph(dataflow dataflow, 016 boolean showLinkOnly, boolean hideColumn, List<RelationshipType> relationshipTypes, 017 Map<String, String> tooltipMap) throws IOException { 018 return generateDataFlowGraph(dataflow, showLinkOnly, hideColumn, relationshipTypes, tooltipMap, false); 019 } 020 021 /** 022 * @param includeIsolatedNodes keep the tables and resultsets that take part in no relationship (a table a 023 * CREATE TABLE declares that nothing reads, the resultset of SELECT 1); the 024 * overload without it drops them from every graph but the ER graph (Mantis 4812). 025 * Use it with {@code showLinkOnly = false}, which keeps unlinked nodes too. 026 */ 027 public Pair3<String, Boolean, Set<String>> generateDataFlowGraph(dataflow dataflow, 028 boolean showLinkOnly, boolean hideColumn, List<RelationshipType> relationshipTypes, 029 Map<String, String> tooltipMap, boolean includeIsolatedNodes) throws IOException { 030 return convertMetaInfoToGraphXML(dataflow, 031 showLinkOnly, hideColumn, 032 relationshipTypes, 033 tooltipMap, includeIsolatedNodes); 034 } 035 036 private List<Object> nodes = new ArrayList<>(); 037 038 private Set<relationship> linkRelations = new HashSet<relationship>(); 039 040 private List<Object> linkIds = new ArrayList<Object>(); 041 042 public List<Object> getNodes() { 043 return nodes; 044 } 045 046 public List<relationship> getRelationships() { 047 return new ArrayList<relationship>(linkRelations); 048 } 049 050 private Pair3<String, Boolean, Set<String>> convertMetaInfoToGraphXML(dataflow dataflow, 051 boolean showLinkOnly, boolean hideColumn, List<RelationshipType> relationshipTypes, 052 Map<String, String> tooltipMap, boolean includeIsolatedNodes) throws IOException { 053 nodes.clear(); 054 linkIds.clear(); 055 linkRelations.clear(); 056 057 boolean containGroup = false; 058 String columnContent = SQLUtil.getInputStreamContent(DataFlowGraph.class.getResourceAsStream("/gudusoft/gsqlparser/graph/column.template"), 059 false); 060 String tableContent = SQLUtil.getInputStreamContent(DataFlowGraph.class.getResourceAsStream("/gudusoft/gsqlparser/graph/table.template"), 061 false); 062 String linkContent = SQLUtil.getInputStreamContent(DataFlowGraph.class.getResourceAsStream("/gudusoft/gsqlparser/graph/link.template"), 063 false); 064 String graphContent = SQLUtil.getInputStreamContent(DataFlowGraph.class.getResourceAsStream("/gudusoft/gsqlparser/graph/graph.template"), 065 false); 066 067 StringBuffer tableFullBuffer = new StringBuffer(); 068 069 List<table> tables = new ArrayList<table>(); 070 if (dataflow.getTables() != null) 071 tables.addAll(dataflow.getTables()); 072 073 if(!relationshipTypes.contains(RelationshipType.er)){ 074 if (dataflow.getViews() != null) 075 tables.addAll(dataflow.getViews()); 076 if (dataflow.getStages() != null) 077 tables.addAll(dataflow.getStages()); 078 if (dataflow.getSequences() != null) 079 tables.addAll(dataflow.getSequences()); 080 if (dataflow.getDatasources() != null) 081 tables.addAll(dataflow.getDatasources()); 082 if (dataflow.getDatabases() != null) 083 tables.addAll(dataflow.getDatabases()); 084 if (dataflow.getSchemas() != null) 085 tables.addAll(dataflow.getSchemas()); 086 if (dataflow.getStreams() != null) 087 tables.addAll(dataflow.getStreams()); 088 if (dataflow.getPaths() != null) 089 tables.addAll(dataflow.getPaths()); 090 if (dataflow.getVariables() != null) 091 tables.addAll(dataflow.getVariables()); 092 if (dataflow.getResultsets() != null) 093 tables.addAll(dataflow.getResultsets()); 094 } 095 Map<String, table> tableMap = new HashMap<>(); 096 for (table table: tables) { 097 tableMap.put(table.getId(), table); 098 } 099 100 Set<String> relationshipTableIds = new HashSet<>(); 101 for (relationship relation : dataflow.getRelationships()) { 102 if( "call".equals(relation.getType())) { 103 relationshipTableIds.add(relation.getCaller().getId()); 104 if(relation.getCallees()!=null){ 105 for (sourceColumn sourceColumn: relation.getCallees()){ 106 relationshipTableIds.add(sourceColumn.getId()); 107 } 108 } 109 continue; 110 } 111 if (relation.getTarget().getParent_id() != null) { 112 relationshipTableIds.add(relation.getTarget().getParent_id()); 113 } else { 114 relationshipTableIds.add(relation.getTarget().getId()); 115 } 116 117 if (relation.getSources() != null) { 118 for (sourceColumn sourceColumn : relation.getSources()) { 119 if (sourceColumn.getParent_id() != null) { 120 relationshipTableIds.add(sourceColumn.getParent_id()); 121 } else { 122 relationshipTableIds.add(sourceColumn.getId()); 123 } 124 } 125 } 126 } 127 Iterator<String> tableIter = tableMap.keySet().iterator(); 128 if(relationshipTypes.contains(RelationshipType.er)){ 129// while(tableIter.hasNext()){ 130// String tableId = tableIter.next(); 131// if(!relationshipTableIds.contains(tableId)){ 132// table tb = tableMap.get(tableId); 133// if((!SQLUtil.isEmpty(tableName) && !tableName.equalsIgnoreCase(tb.getName())) || 134// (!SQLUtil.isEmpty(databaseName) && !databaseName.equalsIgnoreCase(tb.getDatabase())) || 135// (!SQLUtil.isEmpty(schemaName) && !schemaName.equalsIgnoreCase(tb.getSchema())) || 136// (!SQLUtil.isEmpty(serverName) && !serverName.equalsIgnoreCase(tb.getServer()))){ 137// tableIter.remove(); 138// } 139// } 140// } 141 } 142 else if (!includeIsolatedNodes) { 143 while(tableIter.hasNext()){ 144 if(!relationshipTableIds.contains(tableIter.next())){ 145 tableIter.remove(); 146 } 147 } 148 } 149 Map<String, Map<String, List<column>>> tableColumns = new HashMap<String, Map<String, List<column>>>(); 150 Set<String> columnSet = new HashSet<String>(); 151 Set<String> linkColumnSet = new HashSet<String>(); 152 153 boolean showCallRelation = relationshipTypes.contains(RelationshipType.call); 154 155 Map<String, procedure> procedureMap = new HashMap<>(); 156 if (relationshipTypes.contains(RelationshipType.call)) { 157 List<procedure> procedures = new ArrayList<>(dataflow.getProcedures()); 158 if (dataflow.getPackages() != null) { 159 for (oraclePackage pkg : dataflow.getPackages()) { 160 procedures.addAll(pkg.getProcedures()); 161 } 162 } 163 procedureMap = procedures.stream().collect(Collectors.toMap( 164 t -> t.getId(), 165 t -> t, 166 (existingValue, newValue) -> newValue 167 )); 168 169 170 Iterator<String> procedureIter = procedureMap.keySet().iterator(); 171 172 while (procedureIter.hasNext()) { 173 if (!relationshipTableIds.contains(procedureIter.next())) { 174 procedureIter.remove(); 175 } 176 } 177 178 procedureIter = procedureMap.keySet().iterator(); 179 180 while (procedureIter.hasNext()) { 181 procedure procedure = procedureMap.get(procedureIter.next()); 182 String tableId = "table-" + procedure.getId(); 183 nodes.add(procedure); 184 String content = new String(tableContent); 185 content = content.replace("{tableId}", "table-" + procedure.getId()); 186 187 String tableLabel = procedure.getName(); 188 if (tableLabel.length() > 25) { 189 String shortLabel = getShortLabel(tableLabel, 25); 190 if (tooltipMap != null) { 191 tooltipMap.put(shortLabel.replace("\r\n", "\n").replace("\n", " "), tableLabel); 192 } 193 tableLabel = shortLabel; 194 195 } 196 197 content = content.replace("{tableLabel}", StringEscapeUtils.escapeXml(tableLabel)); 198 199 StringBuffer columnBuffer = new StringBuffer(); 200 String columnId = "column-" + procedure.getId(); 201 columnSet.add(columnId); 202 String content1 = new String(columnContent); 203 content1 = content1.replace("{columnId}", columnId); 204 content1 = content1.replace("{columnPosY}", "0"); 205 content1 = content1.replace("{columnLabel}", getProcedureType(procedure)); 206// content1 = content1.replace("height=\"16\"", "height=\"0\""); 207 columnBuffer.append(content1).append("\n"); 208 209 content = content.replace("{columns}", columnBuffer.toString()); 210 { 211 content = content.replace("{contentColor}", "#9999ff"); 212 content = content.replace("{labelColor}", "#DBDADA"); 213 } 214 tableFullBuffer.append(content).append("\n"); 215 } 216 217 218 Iterator<String> dbObjIter = tableMap.keySet().iterator(); 219 while (dbObjIter.hasNext()) { 220 table table = tableMap.get(dbObjIter.next()); 221 String tableId = "table-" + table.getId(); 222 nodes.add(table); 223 String content = new String(tableContent); 224 content = content.replace("{tableId}", "table-" + table.getId()); 225 226 String tableLabel = table.getName(); 227 if (tableLabel.length() > 25) { 228 String shortLabel = getShortLabel(tableLabel, 25); 229 if (tooltipMap != null) { 230 tooltipMap.put(shortLabel.replace("\r\n", "\n").replace("\n", " "), tableLabel); 231 } 232 tableLabel = shortLabel; 233 234 } 235 236 content = content.replace("{tableLabel}", StringEscapeUtils.escapeXml(tableLabel)); 237 238 StringBuffer columnBuffer = new StringBuffer(); 239 String columnId = "column-" + table.getId(); 240 columnSet.add(columnId); 241 String content1 = new String(columnContent); 242 content1 = content1.replace("{columnId}", columnId); 243 content1 = content1.replace("{columnPosY}", "0"); 244 content1 = content1.replace("{columnLabel}", table.getType()); 245// content1 = content1.replace("height=\"16\"", "height=\"0\""); 246 columnBuffer.append(content1).append("\n"); 247 248 content = content.replace("{columns}", columnBuffer.toString()); 249 { 250 content = content.replace("{contentColor}", "#9999ff"); 251 content = content.replace("{labelColor}", "#DBDADA"); 252 } 253 tableFullBuffer.append(content).append("\n"); 254 } 255 } 256 257 Map<String, process> processMap = new HashMap<>(); 258 for (process process : dataflow.getProcesses()) { 259 processMap.put(process.getId(), process); 260 } 261 if(dataflow.getProcesses()!=null){ 262 Iterator<String> processIter = processMap.keySet().iterator(); 263 while (processIter.hasNext()) { 264 if (!relationshipTableIds.contains(processIter.next())) { 265 processIter.remove(); 266 } 267 } 268 269 processIter = processMap.keySet().iterator(); 270 while (processIter.hasNext()) { 271 process process = processMap.get(processIter.next()); 272 String tableId = "table-"+process.getId(); 273 columnSet.add(tableId); 274 nodes.add(process); 275 String content = new String(tableContent); 276 content = content.replace("{tableId}", "table-" + process.getId()); 277 278 String tableLabel = process.getName(); 279 if (tableLabel.length() > 25) { 280 String shortLabel = getShortLabel(tableLabel, 25); 281 if (tooltipMap != null) { 282 tooltipMap.put(shortLabel.replace("\r\n", "\n").replace("\n", " "), tableLabel); 283 } 284 tableLabel = shortLabel; 285 286 } 287 288 content = content.replace("{tableLabel}", StringEscapeUtils.escapeXml(tableLabel)); 289 290 StringBuffer columnBuffer = new StringBuffer(); 291 String columnId = "column-" + process.getId(); 292 String content1 = new String(columnContent); 293 content1 = content1.replace("{columnId}", columnId); 294 content1 = content1.replace("{columnPosY}", "0"); 295 content1 = content1.replace("{columnLabel}", "hide-table-column"); 296 content1 = content1.replace("height=\"16\"", "height=\"0\""); 297 columnBuffer.append(content1).append("\n"); 298 299 300 content = content.replace("{columns}", columnBuffer.toString()); 301 { 302 content = content.replace("{contentColor}", "#9999ff"); 303 content = content.replace("{labelColor}", "#DBDADA"); 304 } 305 tableFullBuffer.append(content).append("\n"); 306 } 307 } 308 tableIter = tableMap.keySet().iterator(); 309 while (tableIter.hasNext()) { 310 table currentTable = tableMap.get(tableIter.next()); 311// if (currentTable.getColumns() == null 312// || currentTable.getColumns().isEmpty()) 313// continue; 314 if(showCallRelation){ 315 continue; 316 } 317 nodes.add(currentTable); 318 String tableId = "table-"+currentTable.getId(); 319 columnSet.add(tableId); 320 321 StringBuffer columnBuffer = new StringBuffer(); 322 323 if (currentTable.isTable() || currentTable.isView() || currentTable.isCursor() || currentTable.isDatabaseType() || currentTable.isSchemaType() || currentTable.isStage() || currentTable.isSequence() || currentTable.isDataSource() || currentTable.isStream() || currentTable.isFile()|| currentTable.isVariable()) { 324 Map<String, List<column>> columnMap = new LinkedHashMap<String, List<column>>(); 325 tableColumns.put(currentTable.getId(), columnMap); 326 for (int k = 0; k < currentTable.getColumns().size(); k++) { 327 column column = currentTable.getColumns().get(k); 328 String columnLabel = column.getName(); 329 //add by grq 2023.01.10 issue=I69NOU 330 if(relationshipTypes.contains(RelationshipType.er)){ 331 if(column.standsForStar() || columnLabel.equalsIgnoreCase("relationRows")){ 332 continue; 333 } 334 } 335 //end by grq 336 List<column> columns; 337 if (!columnMap.containsKey(columnLabel)) { 338 columns = new ArrayList<column>(); 339 columnMap.put(columnLabel, columns); 340 } else { 341 columns = columnMap.get(columnLabel); 342 } 343 columns.add(column); 344 } 345 346 Iterator<Entry<String, List<column>>> iterator = columnMap 347 .entrySet().iterator(); 348 for (int k = 0; k < columnMap.size(); k++) { 349 Entry<String, List<column>> entry = iterator.next(); 350 nodes.add(entry.getValue()); 351 String content = columnContent; 352 353 String columnId = "column-" 354 + currentTable.getId() 355 + "-index-" 356 + k; 357 if(columnSet.contains(columnId)){ 358 continue; 359 } 360 columnSet.add(columnId); 361 362 content = content.replace("{columnId}", columnId); 363 content = content.replace("{columnPosY}", 364 String.valueOf(35 * k)); 365 String columnLabel = entry.getKey(); 366 if (columnLabel.length() > 25) { 367 String shortLabel = getShortLabel(columnLabel, 25); 368 if (tooltipMap != null) { 369 tooltipMap.put(shortLabel.replace("\r\n", "\n") 370 .replace("\n", " "), columnLabel); 371 } 372 columnLabel = shortLabel; 373 374 } 375 content = content.replace("{columnLabel}", 376 StringEscapeUtils.escapeXml(columnLabel)); 377 columnBuffer.append(content).append("\n"); 378 } 379 } else { 380 for (int k = 0; k < currentTable.getColumns().size(); k++) { 381 column column = currentTable.getColumns().get(k); 382 nodes.add(column); 383 String content = new String(columnContent); 384 385 String columnParentId = currentTable.getId(); 386 String columnId = column.getId(); 387 columnId = convertColumnId(tableColumns, 388 columnId, 389 columnParentId); 390 391 columnId = "column-" + columnId; 392 if(columnSet.contains(columnId)){ 393 continue; 394 } 395 columnSet.add(columnId); 396 397 content = content.replace("{columnId}", columnId); 398 content = content.replace("{columnPosY}", 399 String.valueOf(35 * k)); 400 String columnLabel = column.getName(); 401 if (columnLabel.length() > 25) { 402 String shortLabel = getShortLabel(columnLabel, 25); 403 if (tooltipMap != null) { 404 tooltipMap.put(shortLabel.replace("\r\n", "\n") 405 .replace("\n", " "), columnLabel); 406 } 407 columnLabel = shortLabel; 408 409 } 410 content = content.replace("{columnLabel}", 411 StringEscapeUtils.escapeXml(columnLabel)); 412 columnBuffer.append(content).append("\n"); 413 } 414 } 415 String content = new String(tableContent); 416 content = content.replace("{tableId}", 417 "table-" + currentTable.getId()); 418 419 String tableLabel = currentTable.getName(); 420 if (tableLabel.length() > 25) { 421 String shortLabel = getShortLabel(tableLabel, 25); 422 if (tooltipMap != null) { 423 tooltipMap.put(shortLabel.replace("\r\n", "\n") 424 .replace("\n", " "), 425 tableLabel); 426 } 427 tableLabel = shortLabel; 428 429 } 430 431 if(columnBuffer.length() == 0){ 432 String columnId = "column-" + currentTable.getId(); 433 String content1 = new String(columnContent); 434 content1 = content1.replace("{columnId}", columnId); 435 content1 = content1.replace("{columnPosY}", "0"); 436 content1 = content1.replace("{columnLabel}", "hide-table-column"); 437 content1 = content1.replace("height=\"16\"", "height=\"0\""); 438 columnBuffer.append(content1).append("\n"); 439 } 440 441 content = content.replace("{tableLabel}", 442 StringEscapeUtils.escapeXml(tableLabel)); 443 444 String columnString = columnBuffer.toString(); 445 if (hideColumn) { 446 columnString = columnString.replace("height=\"16\"", "height=\"0\""); 447 } 448 content = content.replace("{columns}", columnString); 449 if (currentTable.isView()) { 450 content = content.replace("{contentColor}", "#ff99cc"); 451 content = content.replace("{labelColor}", "#ffccff"); 452 } else if (currentTable.isStage()) { 453 content = content.replace("{contentColor}", "#87CEFA"); 454 content = content.replace("{labelColor}", "#87CEFA"); 455 } else if (currentTable.isSequence()) { 456 content = content.replace("{contentColor}", "#87CEFA"); 457 content = content.replace("{labelColor}", "#87CEFA"); 458 } else if (currentTable.isDataSource()) { 459 content = content.replace("{contentColor}", "#87CEFA"); 460 content = content.replace("{labelColor}", "#87CEFA"); 461 } else if (currentTable.isDatabaseType()) { 462 content = content.replace("{contentColor}", "#87CEFA"); 463 content = content.replace("{labelColor}", "#87CEFA"); 464 } else if (currentTable.isSchemaType()) { 465 content = content.replace("{contentColor}", "#87CEFA"); 466 content = content.replace("{labelColor}", "#87CEFA"); 467 } else if (currentTable.isStream()) { 468 content = content.replace("{contentColor}", "#87CEFA"); 469 content = content.replace("{labelColor}", "#87CEFA"); 470 } else if (currentTable.isFile()) { 471 content = content.replace("{contentColor}", "#87CEFA"); 472 content = content.replace("{labelColor}", "#87CEFA"); 473 } else if (currentTable.isCursor()) { 474 content = content.replace("{contentColor}", "#87CEFA"); 475 content = content.replace("{labelColor}", "#87CEFA"); 476 } else if (currentTable.isVariable()) { 477 content = content.replace("{contentColor}", "#87CEFA"); 478 content = content.replace("{labelColor}", "#87CEFA"); 479 } else if (currentTable.isResultSet()) { 480 content = content.replace("{contentColor}", "#009944"); 481 content = content.replace("{labelColor}", "#aaffcc"); 482 } else { 483 content = content.replace("{contentColor}", "#9999ff"); 484 content = content.replace("{labelColor}", "#DBDADA"); 485 } 486 tableFullBuffer.append(content).append("\n"); 487 488 containGroup = true; 489 } 490 491 StringBuffer linkBuffer = new StringBuffer(); 492 List<relationship> relationships = dataflow.getRelationships(); 493 if (relationshipTypes.contains(RelationshipType.join)) { 494 relationships = relationships.stream().filter(t -> t.getType().equals("join") || t.getType().equals("fdd")) 495 .collect(Collectors.toList()); 496 } 497 else if (relationshipTypes.contains(RelationshipType.call)) { 498 relationships = relationships.stream().filter(t -> t.getType().equals("call")) 499 .collect(Collectors.toList()); 500 } 501 502 if (relationships != null) { 503 List<String> links = new ArrayList<String>(); 504 for (int i = 0; i < relationships.size(); i++) { 505 relationship relationship = relationships.get(i); 506 507 if (relationship.getTarget() == null || relationship.getSources() == null || relationship.getSources().isEmpty()) 508 continue; 509 if (!relationshipTypes.contains(RelationshipType.valueOf(relationship.getType()))) { 510 if (relationshipTypes.contains(RelationshipType.join) 511 && relationship.getType() 512 .equals(RelationshipType.fdd.name())) { 513 if (!traceJoin(relationship, relationships, tableColumns, 0)) { 514 continue; 515 } 516 } else { 517 continue; 518 } 519 } 520 521 if (RelationshipType.call.equals(RelationshipType.valueOf(relationship.getType()))) { 522 targetColumn target = relationship.getCaller(); 523 String targetColumnId = target.getId(); 524 if (relationship.getCallees() != null) { 525 List<sourceColumn> linkTables = relationship.getCallees(); 526 for (int k = 0; k < linkTables.size(); k++) { 527 sourceColumn source = linkTables.get(k); 528 String sourceColumnId = source.getId(); 529 String content = linkContent; 530 String sourceId = "column-" + sourceColumnId; 531// if (sourceColumnId != null && sourceColumnId.indexOf("-") == -1) { 532// sourceId = "table-" + sourceColumnId; 533// } 534 String targetId = "column-" + targetColumnId; 535// if (targetColumnId != null && targetColumnId.indexOf("-") == -1) { 536// targetId = "table-" + targetColumnId; 537// } 538 539 //call relation的箭头方向和fdd是相反的 540 content = content.replace("{sourceId}", targetId); 541 content = content.replace("{targetId}", sourceId); 542 543 if (columnSet.contains(sourceId) && columnSet.contains(targetId)) { 544 String temp = content; 545 if (!links.contains(temp)) { 546 links.add(temp); 547 548 linkColumnSet.add(sourceId); 549 linkColumnSet.add(targetId); 550 551 String linkId = "link_" + relationship.getId() + "_" + sourceColumnId + "_" 552 + targetColumnId; 553 content = content.replace("{linkId}", linkId); 554 linkIds.add(linkId); 555 linkRelations.add(relationship); 556 linkBuffer.append(content).append("\n"); 557 } 558 } else { 559 System.err.println("Can't get " + "link_" + relationship.getId() + "_" + sourceColumnId + "_" 560 + targetColumnId); 561 } 562 } 563 } 564 } else { 565 targetColumn target = relationship.getTarget(); 566 String targetColumnId = target.getId(); 567 String targetParentId = target.getParent_id(); 568 569 targetColumnId = convertColumnId(tableColumns, 570 targetColumnId, 571 targetParentId); 572 if (targetParentId == null) { 573 targetColumnId = target.getTarget_id(); 574 } 575 576 if (relationship.getSources() != null) { 577 List<sourceColumn> linkTables = relationship.getSources(); 578 for (int k = 0; k < linkTables.size(); k++) { 579 sourceColumn source = linkTables.get(k); 580 581 String sourceColumnId = source.getId(); 582 String sourceParentId = source.getParent_id(); 583 sourceColumnId = convertColumnId(tableColumns, 584 sourceColumnId, 585 sourceParentId); 586 if (sourceParentId == null) { 587 sourceColumnId = source.getSource_id(); 588 } 589 590 String content = linkContent; 591 592 if (relationship.getType() 593 .equals(RelationshipType.join.name())) { 594 content = content.replace("type=\"line\"", 595 "type=\"dashed\""); 596 } 597 598 String sourceId = "column-" + sourceColumnId; 599 if (sourceColumnId != null && sourceColumnId.indexOf("-") == -1) { 600 sourceId = "table-" + sourceColumnId; 601 } 602 String targetId = "column-" + targetColumnId; 603 if (targetColumnId != null && targetColumnId.indexOf("-") == -1) { 604 targetId = "table-" + targetColumnId; 605 } 606 content = content.replace("{sourceId}", sourceId); 607 content = content.replace("{targetId}", targetId); 608 609 if (columnSet.contains(sourceId) 610 && columnSet.contains(targetId)) { 611 String temp = content; 612 if (!links.contains(temp)) { 613 links.add(temp); 614 615 linkColumnSet.add(sourceId); 616 linkColumnSet.add(targetId); 617 618 String linkId = "link_" 619 + relationship.getId() 620 + "_" 621 + sourceColumnId 622 + "_" 623 + targetColumnId; 624 content = content.replace("{linkId}", linkId); 625 linkIds.add(linkId); 626 linkRelations.add(relationship); 627 linkBuffer.append(content).append("\n"); 628 } else { 629 linkRelations.add(relationship); 630 } 631 } else { 632 System.err.println("Can't get " 633 + "link_" 634 + relationship.getId() 635 + "_" 636 + sourceColumnId 637 + "_" 638 + targetColumnId); 639 } 640 } 641 } 642 } 643 } 644 } 645 646 if (showLinkOnly) { 647 nodes.clear(); 648 StringBuffer tableLinkBuffer = new StringBuffer(); 649 int showColumnCount = 0; 650 651 if (relationshipTypes.contains(RelationshipType.call)) { 652 Iterator<String> procedureIter = procedureMap.keySet().iterator(); 653 while (procedureIter.hasNext()) { 654 procedure procedure = procedureMap.get(procedureIter.next()); 655 String tableId = "table-" + procedure.getId(); 656 columnSet.add(tableId); 657 nodes.add(procedure); 658 String content = new String(tableContent); 659 content = content.replace("{tableId}", "table-" + procedure.getId()); 660 661 String tableLabel = procedure.getName(); 662 if (tableLabel.length() > 25) { 663 String shortLabel = getShortLabel(tableLabel, 25); 664 if (tooltipMap != null) { 665 tooltipMap.put(shortLabel.replace("\r\n", "\n").replace("\n", " "), tableLabel); 666 } 667 tableLabel = shortLabel; 668 669 } 670 671 content = content.replace("{tableLabel}", StringEscapeUtils.escapeXml(tableLabel)); 672 673 StringBuffer columnBuffer = new StringBuffer(); 674 String columnId = "column-" + procedure.getId(); 675 String content1 = new String(columnContent); 676 content1 = content1.replace("{columnId}", columnId); 677 content1 = content1.replace("{columnPosY}", "0"); 678 content1 = content1.replace("{columnLabel}", getProcedureType(procedure)); 679// content1 = content1.replace("height=\"16\"", "height=\"0\""); 680 columnBuffer.append(content1).append("\n"); 681 682 content = content.replace("{columns}", columnBuffer.toString()); 683 { 684 content = content.replace("{contentColor}", "#9999ff"); 685 content = content.replace("{labelColor}", "#DBDADA"); 686 } 687 tableFullBuffer.append(content).append("\n"); 688 tableLinkBuffer.append(content).append("\n"); 689 } 690 691 Iterator<String> dbObjIter = tableMap.keySet().iterator(); 692 while (dbObjIter.hasNext()) { 693 table table = tableMap.get(dbObjIter.next()); 694 String tableId = "table-" + table.getId(); 695 nodes.add(table); 696 String content = new String(tableContent); 697 content = content.replace("{tableId}", "table-" + table.getId()); 698 699 String tableLabel = table.getName(); 700 if (tableLabel.length() > 25) { 701 String shortLabel = getShortLabel(tableLabel, 25); 702 if (tooltipMap != null) { 703 tooltipMap.put(shortLabel.replace("\r\n", "\n").replace("\n", " "), tableLabel); 704 } 705 tableLabel = shortLabel; 706 707 } 708 709 content = content.replace("{tableLabel}", StringEscapeUtils.escapeXml(tableLabel)); 710 711 StringBuffer columnBuffer = new StringBuffer(); 712 String columnId = "column-" + table.getId(); 713 columnSet.add(columnId); 714 String content1 = new String(columnContent); 715 content1 = content1.replace("{columnId}", columnId); 716 content1 = content1.replace("{columnPosY}", "0"); 717 content1 = content1.replace("{columnLabel}", table.getType()); 718 columnBuffer.append(content1).append("\n"); 719 720 content = content.replace("{columns}", columnBuffer.toString()); 721 { 722 content = content.replace("{contentColor}", "#9999ff"); 723 content = content.replace("{labelColor}", "#DBDADA"); 724 } 725 tableFullBuffer.append(content).append("\n"); 726 tableLinkBuffer.append(content).append("\n"); 727 } 728 } 729 730 if(dataflow.getProcesses()!=null){ 731 Iterator<String> processIter = processMap.keySet().iterator(); 732 while (processIter.hasNext()) { 733 process process = processMap.get(processIter.next()); 734 if(!linkColumnSet.contains("table-" + process.getId())) { 735 continue; 736 } 737 nodes.add(process); 738 String tableId = "table-"+process.getId(); 739 columnSet.add(tableId); 740 String content = new String(tableContent); 741 content = content.replace("{tableId}", "table-" + process.getId()); 742 743 String tableLabel = process.getName(); 744 if (tableLabel.length() > 25) { 745 String shortLabel = getShortLabel(tableLabel, 25); 746 if (tooltipMap != null) { 747 tooltipMap.put(shortLabel.replace("\r\n", "\n").replace("\n", " "), tableLabel); 748 } 749 tableLabel = shortLabel; 750 751 } 752 753 content = content.replace("{tableLabel}", StringEscapeUtils.escapeXml(tableLabel)); 754 755 StringBuffer columnBuffer = new StringBuffer(); 756 String columnId = "column-" + process.getId(); 757 String content1 = new String(columnContent); 758 content1 = content1.replace("{columnId}", columnId); 759 content1 = content1.replace("{columnPosY}", "0"); 760 content1 = content1.replace("{columnLabel}", "hide-table-column"); 761 content1 = content1.replace("height=\"16\"", "height=\"0\""); 762 columnBuffer.append(content1).append("\n"); 763 764 765 content = content.replace("{columns}", columnBuffer.toString()); 766 { 767 content = content.replace("{contentColor}", "#9999ff"); 768 content = content.replace("{labelColor}", "#DBDADA"); 769 } 770 tableFullBuffer.append(content).append("\n"); 771 tableLinkBuffer.append(content).append("\n"); 772 } 773 } 774 775 tableIter = tableMap.keySet().iterator(); 776 while (tableIter.hasNext()) { 777 table currentTable = tableMap.get(tableIter.next()); 778 if(showCallRelation){ 779 continue; 780 } 781 if (currentTable.getColumns() == null || currentTable.getColumns().isEmpty()) { 782 if(!linkColumnSet.contains("table-" + currentTable.getId())) { 783 continue; 784 } 785 } 786 787 //包含在TableLevelLineage中则不删除，不包含如无link则删除 788 boolean removeTable = !linkColumnSet.contains("table-" + currentTable.getId()); 789 790 nodes.add(currentTable); 791 StringBuffer columnBuffer = new StringBuffer(); 792 if (currentTable.isTable() || currentTable.isView() || currentTable.isCursor() || currentTable.isDatabaseType() || currentTable.isSchemaType() || currentTable.isStage() || currentTable.isSequence() || currentTable.isDataSource() || currentTable.isStream() || currentTable.isFile()|| currentTable.isVariable()) { 793 Map<String, List<column>> columnMap = new LinkedHashMap<String, List<column>>(); 794 for (int k = 0; k < currentTable.getColumns().size(); k++) { 795 column column = currentTable.getColumns().get(k); 796 String columnLabel = column.getName(); 797 List<column> columns; 798 if (!columnMap.containsKey(columnLabel)) { 799 columns = new ArrayList<column>(); 800 columnMap.put(columnLabel, columns); 801 } else { 802 columns = columnMap.get(columnLabel); 803 } 804 columns.add(column); 805 } 806 807 Iterator<Entry<String, List<column>>> iterator = columnMap.entrySet() 808 .iterator(); 809 for (int k = 0; k < columnMap.size(); k++) { 810 Entry<String, List<column>> entry = iterator.next(); 811 String columnId = "column-" 812 + currentTable.getId() 813 + "-index-" 814 + k; 815 if ((currentTable.isTable() || currentTable.isView() || currentTable.isCursor() || currentTable.isDatabaseType() || currentTable.isSchemaType() || currentTable.isStage() || currentTable.isSequence() || currentTable.isDataSource() || currentTable.isStream() || currentTable.isFile()|| currentTable.isVariable()) 816 && !linkColumnSet.contains(columnId)) 817 continue; 818 819 removeTable = false; 820 nodes.add(entry.getValue()); 821 822 String content = new String(columnContent); 823 content = content.replace("{columnId}", columnId); 824 content = content.replace("{columnPosY}", 825 String.valueOf(35 * showColumnCount)); 826 String columnLabel = entry.getKey(); 827 if (columnLabel.length() > 25) { 828 String shortLabel = getShortLabel(columnLabel, 25); 829 if (tooltipMap != null) { 830 tooltipMap.put(shortLabel.replace("\r\n", 831 "\n").replace("\n", " "), 832 columnLabel); 833 } 834 columnLabel = shortLabel; 835 836 } 837 content = content.replace("{columnLabel}", 838 StringEscapeUtils.escapeXml(columnLabel)); 839 columnBuffer.append(content).append("\n"); 840 showColumnCount++; 841 } 842 } else { 843 for (int k = 0; k < currentTable.getColumns().size(); k++) { 844 column column = currentTable.getColumns().get(k); 845 846 String columnParentId = currentTable.getId(); 847 String columnId = column.getId(); 848 columnId = convertColumnId(tableColumns, 849 columnId, 850 columnParentId); 851 852 columnId = "column-" + columnId; 853 if (!(currentTable.isTable() || currentTable.isView() || currentTable.isCursor() || currentTable.isDatabaseType() || currentTable.isSchemaType() || currentTable.isStage() || currentTable.isSequence() || currentTable.isDataSource() || currentTable.isStream() || currentTable.isFile()|| currentTable.isVariable()) 854 && !linkColumnSet.contains(columnId)) 855 continue; 856 857 removeTable = false; 858 nodes.add(column); 859 860 String content = new String(columnContent); 861 content = content.replace("{columnId}", columnId); 862 content = content.replace("{columnPosY}", 863 String.valueOf(35 * showColumnCount)); 864 String columnLabel = column.getName(); 865 if (columnLabel.length() > 25) { 866 String shortLabel = getShortLabel(columnLabel, 25); 867 if (tooltipMap != null) { 868 tooltipMap.put(shortLabel.replace("\r\n", 869 "\n").replace("\n", " "), 870 columnLabel); 871 } 872 columnLabel = shortLabel; 873 874 } 875 content = content.replace("{columnLabel}", 876 StringEscapeUtils.escapeXml(columnLabel)); 877 columnBuffer.append(content).append("\n"); 878 showColumnCount++; 879 } 880 } 881 882 if (removeTable) { 883 nodes.remove(currentTable); 884 continue; 885 } else { 886 if (columnBuffer.toString().trim().length() == 0) { 887 String columnId = "column-" + currentTable.getId(); 888 String content1 = new String(columnContent); 889 content1 = content1.replace("{columnId}", columnId); 890 content1 = content1.replace("{columnPosY}", "0"); 891 content1 = content1.replace("{columnLabel}", "hide-table-column"); 892 content1 = content1.replace("height=\"16\"", "height=\"0\""); 893 columnBuffer.append(content1).append("\n"); 894 } 895 } 896 897 String content = new String(tableContent); 898 content = content.replace("{tableId}", 899 "table-" + currentTable.getId()); 900 901 String tableLabel = currentTable.getName(); 902 if (tableLabel.length() > 25) { 903 String shortLabel = getShortLabel(tableLabel, 25); 904 if (tooltipMap != null) { 905 tooltipMap.put(shortLabel.replace("\r\n", "\n") 906 .replace("\n", " "), tableLabel); 907 } 908 tableLabel = shortLabel; 909 } 910 911 content = content.replace("{tableLabel}", 912 StringEscapeUtils.escapeXml(tableLabel)); 913 String columnString = columnBuffer.toString(); 914 if (hideColumn) { 915 columnString = columnString.replace("height=\"16\"", "height=\"0\""); 916 } 917 content = content.replace("{columns}", columnString); 918 919 if (currentTable.isView()) { 920 content = content.replace("{contentColor}", "#ff99cc"); 921 content = content.replace("{labelColor}", "#ffccff"); 922 } else if (currentTable.isStage()) { 923 content = content.replace("{contentColor}", "#87CEFA"); 924 content = content.replace("{labelColor}", "#87CEFA"); 925 } else if (currentTable.isSequence()) { 926 content = content.replace("{contentColor}", "#87CEFA"); 927 content = content.replace("{labelColor}", "#87CEFA"); 928 } else if (currentTable.isDataSource()) { 929 content = content.replace("{contentColor}", "#87CEFA"); 930 content = content.replace("{labelColor}", "#87CEFA"); 931 } else if (currentTable.isDatabaseType()) { 932 content = content.replace("{contentColor}", "#87CEFA"); 933 content = content.replace("{labelColor}", "#87CEFA"); 934 } else if (currentTable.isSchemaType()) { 935 content = content.replace("{contentColor}", "#87CEFA"); 936 content = content.replace("{labelColor}", "#87CEFA"); 937 } else if (currentTable.isStream()) { 938 content = content.replace("{contentColor}", "#87CEFA"); 939 content = content.replace("{labelColor}", "#87CEFA"); 940 } else if (currentTable.isFile()) { 941 content = content.replace("{contentColor}", "#87CEFA"); 942 content = content.replace("{labelColor}", "#87CEFA"); 943 } else if (currentTable.isVariable()) { 944 content = content.replace("{contentColor}", "#87CEFA"); 945 content = content.replace("{labelColor}", "#87CEFA"); 946 } else if (currentTable.isCursor()) { 947 content = content.replace("{contentColor}", "#87CEFA"); 948 content = content.replace("{labelColor}", "#87CEFA"); 949 } else if (currentTable.isResultSet()) { 950 content = content.replace("{contentColor}", "#009944"); 951 content = content.replace("{labelColor}", "#aaffcc"); 952 } else { 953 content = content.replace("{contentColor}", "#9999ff"); 954 content = content.replace("{labelColor}", "#DBDADA"); 955 } 956 tableLinkBuffer.append(content).append("\n"); 957 containGroup = true; 958 } 959 960 graphContent = graphContent.replace("{nodes}", 961 tableLinkBuffer.toString()); 962 } else { 963 graphContent = graphContent.replace("{nodes}", 964 tableFullBuffer.toString()); 965 } 966 967 nodes.addAll(linkIds); 968 graphContent = graphContent.replace("{links}", linkBuffer.toString()); 969 Set<String> dbObjIds = new LinkedHashSet<>(); 970 dbObjIds.addAll(tableMap.keySet()); 971 dbObjIds.addAll(processMap.keySet()); 972 dbObjIds.addAll(procedureMap.keySet()); 973 return new Pair3<String, Boolean, Set<String>>(graphContent, containGroup, dbObjIds); 974 } 975 976 private CharSequence getProcedureType(procedure procedure) { 977 String type = procedure.getType().toLowerCase(Locale.ROOT); 978 if (type.indexOf("procedure") != -1) { 979 return "procedure"; 980 } else if (type.indexOf("function") != -1) { 981 return "function"; 982 } else if (type.indexOf("trigger") != -1) { 983 return "trigger"; 984 } else if (type.indexOf("package") != -1) { 985 return "package"; 986 } else if (type.indexOf("macro") != -1) { 987 return "macro"; 988 } else { 989 return type; 990 } 991 } 992 993 private boolean traceJoin(relationship relationship, List<relationship> relationships, 994 Map<String, Map<String, List<column>>> tableColumns, int level) { 995 targetColumn target = relationship.getTarget(); 996 String targetColumnId = target.getId(); 997 String targetParentId = target.getParent_id(); 998 999 targetColumnId = convertColumnId(tableColumns, 1000 targetColumnId, 1001 targetParentId); 1002 1003 for (int i = 0; i < relationships.size(); i++) { 1004 relationship tempRelation = relationships.get(i); 1005 1006 if (relationship == tempRelation) 1007 continue; 1008 1009 List<sourceColumn> sources = tempRelation.getSources(); 1010 if (sources != null) { 1011 if (relationships.size() * sources.size() > 5000) { 1012 System.out.println("The relationship is too huge, skip the join trace."); 1013 continue; 1014 } 1015 } 1016 1017 String tempColumnId = tempRelation.getTarget().getId(); 1018 String tempParentId = tempRelation.getTarget().getParent_id(); 1019 tempColumnId = convertColumnId(tableColumns, 1020 tempColumnId, 1021 tempParentId); 1022 if (tempColumnId.equals(targetColumnId)) { 1023 if (tempRelation.getType().equals(RelationshipType.join.name())) { 1024 return true; 1025 } else if (tempRelation.getType() 1026 .equals(RelationshipType.fdd.name())) { 1027 if (level < 2 1028 && traceJoin(tempRelation, 1029 relationships, 1030 tableColumns, 1031 level + 1)) { 1032 return true; 1033 } 1034 } 1035 } 1036 1037 if (sources != null) { 1038 for (int j = 0; j < sources.size(); j++) { 1039 if ("constant".equals(sources.get(j).getColumn_type())) { 1040 continue; 1041 } 1042 tempColumnId = sources.get(j).getId(); 1043 tempParentId = sources.get(j).getParent_id(); 1044 tempColumnId = convertColumnId(tableColumns, 1045 tempColumnId, 1046 tempParentId); 1047 if (tempColumnId.equals(targetColumnId)) { 1048 if (tempRelation.getType() 1049 .equals(RelationshipType.join.name())) { 1050 return true; 1051 } else if (tempRelation.getType() 1052 .equals(RelationshipType.fdd.name())) { 1053 if (level < 2 1054 && traceJoin(tempRelation, 1055 relationships, 1056 tableColumns, 1057 level + 1)) { 1058 return true; 1059 } 1060 } 1061 } 1062 } 1063 } 1064 } 1065 1066 return false; 1067 } 1068 1069 private String convertColumnId( 1070 Map<String, Map<String, List<column>>> tableColumns, 1071 String columnId, String parentId) { 1072 if (tableColumns.containsKey(parentId)) { 1073 Map<String, List<column>> columnMap = tableColumns.get(parentId); 1074 Iterator<List<column>> iter = columnMap.values().iterator(); 1075 for (int i = 0; i < columnMap.size(); i++) { 1076 List<column> columns = iter.next(); 1077 for (int j = 0; j < columns.size(); j++) { 1078 column column = columns.get(j); 1079 if (column.getId().equals(columnId)) { 1080 return parentId + "-index-" + i; 1081 } 1082 } 1083 } 1084 } 1085 if (parentId != null) { 1086 return parentId + "-id-" + columnId; 1087 } else 1088 return columnId; 1089 } 1090 1091 private String getShortLabel(String label, int length) { 1092 int index = length / 2 - 1; 1093 return (label.substring(0, index - 1) + "..." + label.substring(label.length() 1094 - (index + 1))); 1095 } 1096 1097}