001package gudusoft.gsqlparser.dlineage.dataflow.metadata.sqlflow; 002 003import gudusoft.gsqlparser.EDbVendor; 004import gudusoft.gsqlparser.sqlenv.ESQLDataObjectType; 005import gudusoft.gsqlparser.dlineage.dataflow.metadata.MetadataAnalyzer; 006import gudusoft.gsqlparser.dlineage.dataflow.model.EffectType; 007import gudusoft.gsqlparser.dlineage.dataflow.model.ModelBindingManager; 008import gudusoft.gsqlparser.dlineage.dataflow.model.RelationshipType; 009import gudusoft.gsqlparser.dlineage.dataflow.model.SubType; 010import gudusoft.gsqlparser.dlineage.dataflow.model.TableColumnRelationshipElement; 011import gudusoft.gsqlparser.dlineage.dataflow.model.xml.*; 012import gudusoft.gsqlparser.dlineage.util.DlineageUtil; 013import gudusoft.gsqlparser.dlineage.util.Pair3; 014import gudusoft.gsqlparser.dlineage.util.XML2Model; 015import gudusoft.gsqlparser.sqlenv.TSQLEnv; 016import gudusoft.gsqlparser.sqlenv.parser.sqlflow.SqlflowNameForm; 017import gudusoft.gsqlparser.util.SQLUtil; 018import gudusoft.gsqlparser.util.json.JSON; 019 020import java.io.File; 021import java.util.*; 022 023public class SqlflowMetadataAnalyzer implements MetadataAnalyzer<String> { 024 025 private Map<Object, procedure> procedureMap = new LinkedHashMap<Object, procedure>(); 026 private Map<Object, table> tableMap = new LinkedHashMap<Object, table>(); 027 private Map<Object, oraclePackage> oraclePackageMap = new LinkedHashMap<Object, oraclePackage>(); 028 private Map<Object, column> columnMap = new LinkedHashMap<Object, column>(); 029 private EDbVendor vendor; 030 private TSQLEnv sqlenv; 031 032 public SqlflowMetadataAnalyzer() { 033 } 034 035 public SqlflowMetadataAnalyzer(TSQLEnv sqlenv) { 036 this.sqlenv = sqlenv; 037 } 038 039 @Override 040 public synchronized dataflow analyzeMetadata(EDbVendor metadataVendor, String metadata) { 041 init(metadataVendor); 042 dataflow dataflow = new dataflow(); 043 Map json = (Map) JSON.parseObject(metadata.trim()); 044 // A file may declare its names as catalog-stored (root "nameForm"); everything below 045 // reads names as SQL text, so convert once here. No-op for every existing file. 046 SqlflowNameForm.toLexicalInPlace(metadataVendor, json); 047 if (json.containsKey("createdBy")) { 048 String createdBy = (String) json.get("createdBy"); 049 if (createdBy.toLowerCase(Locale.ROOT).indexOf("sqlflow") != -1) { 050 List servers = (List) json.get("servers"); 051 for (int x = 0; x < servers.size(); x++) { 052 Map server = (Map) servers.get(x); 053 String serverName = (String) server.get("name"); 054 String dbVendor = (String) server.get("dbVendor"); 055 056 EDbVendor vendor = metadataVendor; 057 if (dbVendor != null) { 058 vendor = EDbVendor.valueOf(dbVendor); 059 } 060 061 boolean supportsCatalogs = TSQLEnv.supportCatalog(vendor); 062 boolean supportsSchemas = TSQLEnv.supportSchema(vendor); 063 064 if (supportsCatalogs && supportsSchemas) { 065 List databases = (List) server.get("databases"); 066 if (databases == null) { 067 continue; 068 } 069 for (int i = 0; i < databases.size(); i++) { 070 Map jsonDatabase = (Map) databases.get(i); 071 String databaseName = (String) jsonDatabase.get("name"); 072 if (SQLUtil.parseNames(databaseName).size() > 1) { 073 databaseName = "\"" + databaseName + "\""; 074 } 075 List schemas = (List) jsonDatabase.get("schemas"); 076 if (schemas == null) { 077 continue; 078 } 079 for (int j = 0; j < schemas.size(); j++) { 080 Map jsonSchema = (Map) schemas.get(j); 081 String schemaName = (String) jsonSchema.get("name"); 082 if (SQLUtil.parseNames(schemaName).size() > 1) { 083 schemaName = "\"" + schemaName + "\""; 084 } 085 List tables = (List) jsonSchema.get("tables"); 086 List views = (List) jsonSchema.get("views"); 087 088 List dbObjs = new ArrayList(); 089 if (tables != null) { 090 dbObjs.addAll(tables); 091 } 092 if (views != null) { 093 dbObjs.addAll(views); 094 } 095 for (int k = 0; k < dbObjs.size(); k++) { 096 Map jsonTable = (Map) dbObjs.get(k); 097 String tableName = (String) jsonTable.get("name"); 098 String type = (String) jsonTable.get("type"); 099 String fromDDL = (String) jsonTable.get("fromDDL"); 100 boolean isView = false; 101 if (type != null && type.toLowerCase(Locale.ROOT).indexOf("view") != -1) { 102 isView = true; 103 } 104 List columns = (List) jsonTable.get("columns"); 105 for (int l = 0; l < columns.size(); l++) { 106 Map jsonColumn = (Map) columns.get(l); 107 String columnName = (String) jsonColumn.get("name"); 108 String dataType = null; 109 if (jsonColumn.containsKey("dataType")){ 110 dataType = (String) jsonColumn.get("dataType"); 111 } 112 Boolean primaryKey = null; 113 if (jsonColumn.containsKey("primaryKey")){ 114 primaryKey = (Boolean) jsonColumn.get("primaryKey"); 115 } 116 Boolean unqiueKey = null; 117 if (jsonColumn.containsKey("unqiueKey")){ 118 unqiueKey = (Boolean) jsonColumn.get("unqiueKey"); 119 } 120 Boolean indexKey = null; 121 if (jsonColumn.containsKey("indexKey")){ 122 indexKey = (Boolean) jsonColumn.get("indexKey"); 123 } 124 Boolean foreignKey = null; 125 if (jsonColumn.containsKey("foreignKey")){ 126 foreignKey = (Boolean) jsonColumn.get("foreignKey"); 127 } 128 appendTable(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, databaseName, 129 schemaName, tableName, isView, false, columnName, dataType, primaryKey, unqiueKey, indexKey, foreignKey, fromDDL, readColumnMeta(jsonColumn)); 130 } 131 } 132 133 List synonyms = (List) jsonSchema.get("synonyms"); 134 if (synonyms != null) { 135 for (int k = 0; k < synonyms.size(); k++) { 136 Map jsonTable = (Map) synonyms.get(k); 137 String synonymsName = (String) jsonTable.get("name"); 138 String synonymsDatabaseName = (String) jsonTable.get("database"); 139 String synonymsSchemaName = (String) jsonTable.get("schema"); 140 141 String synonymsSourceName = (String) jsonTable.get("sourceName"); 142 String synonymsSourceDatabaseName = (String) jsonTable.get("sourceDbLinkName"); 143 String synonymsSourceSchemaName = (String) jsonTable.get("sourceSchema"); 144 145 if (SQLUtil.isEmpty(synonymsSourceDatabaseName)) { 146 synonymsSourceDatabaseName = synonymsDatabaseName; 147 } 148 149 if (SQLUtil.isEmpty(synonymsSourceSchemaName)) { 150 synonymsSourceSchemaName = synonymsSchemaName; 151 } 152 153 Object tableKey = getFullTableName(vendor, serverName, synonymsSourceDatabaseName, 154 synonymsSourceSchemaName, synonymsSourceName); 155 table sourceTable = tableMap.get(tableKey); 156 if (sourceTable == null) { 157 // the synonym of a synonym 158 sourceTable = tableMap.get(getFullName(vendor, ESQLDataObjectType.dotSynonyms, serverName, synonymsSourceDatabaseName, 159 synonymsSourceSchemaName, synonymsSourceName)); 160 } 161 if (sourceTable != null) { 162 List columns = (List) sourceTable.getColumns(); 163 String fromDDL = sourceTable.getFromDDL(); 164 dbObjs.add(jsonTable); 165 for (int l = 0; l < columns.size(); l++) { 166 column jsonColumn = (column) columns.get(l); 167 String columnName = (String) jsonColumn.getName(); 168 String dataType = jsonColumn.getDataType(); 169 appendTable(vendor, supportsCatalogs, supportsSchemas, dataflow, 170 serverName, databaseName, schemaName, synonymsName, false, true, 171 columnName, dataType, false, false, false, false, fromDDL); 172 } 173 174 Object synonymsTableKey = getFullName(vendor, ESQLDataObjectType.dotSynonyms, serverName, 175 synonymsDatabaseName, synonymsSchemaName, synonymsName); 176 table synonymsTable = tableMap.get(synonymsTableKey); 177 if (synonymsTable != null) { 178 List synonymsColumns = (List) synonymsTable.getColumns(); 179 180 for (int l = 0; l < synonymsColumns.size(); l++) { 181 relationship relationElement = new relationship(); 182 relationElement.setType(RelationshipType.fdd.name()); 183 relationElement.setEffectType(EffectType.synonym.name()); 184 185 long id = ++ModelBindingManager.get().RELATION_ID; 186 relationElement.setId(String.valueOf(id)); 187 188 column targetColumn = (column)synonymsColumns.get(l); 189 targetColumn target = new targetColumn(); 190 target.setId(String.valueOf(targetColumn.getId())); 191 target.setColumn(targetColumn.getName()); 192 target.setParent_id(String.valueOf(synonymsTable.getId())); 193 target.setParent_name(synonymsTable.getName()); 194 relationElement.setTarget(target); 195 196 column sourceColumn = (column)columns.get(l); 197 sourceColumn source = new sourceColumn(); 198 source.setId(String.valueOf(sourceColumn.getId())); 199 source.setColumn(sourceColumn.getName()); 200 source.setParent_id(String.valueOf(sourceTable.getId())); 201 source.setParent_name(sourceTable.getName()); 202 relationElement.addSource(source); 203 204 dataflow.getRelationships().add(relationElement); 205 } 206 } 207 } 208 } 209 } 210 211 List<Map> procedures = (List<Map>) jsonSchema.get("procedures"); 212 if (procedures != null) { 213 for (Map procedure : procedures) { 214 appendProcedure(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, databaseName, schemaName, procedure, null, ESQLDataObjectType.dotProcedure); 215 } 216 } 217 List<Map> functions = (List<Map>) jsonSchema.get("functions"); 218 if (functions != null) { 219 for (Map function : functions) { 220 appendProcedure(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, databaseName, schemaName, function, null, ESQLDataObjectType.dotFunction); 221 } 222 } 223 List<Map> triggers = (List<Map>) jsonSchema.get("triggers"); 224 if (triggers != null) { 225 for (Map trigger : triggers) { 226 appendProcedure(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, databaseName, schemaName, trigger, null, ESQLDataObjectType.dotTrigger); 227 } 228 } 229 } 230 } 231 } else if (supportsCatalogs) { 232 List databases = (List) server.get("databases"); 233 if (databases == null) { 234 continue; 235 } 236 for (int i = 0; i < databases.size(); i++) { 237 Map jsonDatabase = (Map) databases.get(i); 238 String databaseName = (String) jsonDatabase.get("name"); 239 if (SQLUtil.parseNames(databaseName).size() > 1) { 240 databaseName = "\"" + databaseName + "\""; 241 } 242 List tables = (List) jsonDatabase.get("tables"); 243 List views = (List) jsonDatabase.get("views"); 244 List dbObjs = new ArrayList(); 245 if (tables != null) { 246 dbObjs.addAll(tables); 247 } 248 if (views != null) { 249 dbObjs.addAll(views); 250 } 251 for (int k = 0; k < dbObjs.size(); k++) { 252 Map jsonTable = (Map) dbObjs.get(k); 253 String tableName = (String) jsonTable.get("name"); 254 String type = (String) jsonTable.get("type"); 255 String fromDDL = (String) jsonTable.get("fromDDL"); 256 boolean isView = false; 257 if (type != null && type.toLowerCase(Locale.ROOT).indexOf("view") != -1) { 258 isView = true; 259 } 260 List columns = (List) jsonTable.get("columns"); 261 for (int l = 0; l < columns.size(); l++) { 262 Map jsonColumn = (Map) columns.get(l); 263 String columnName = (String) jsonColumn.get("name"); 264 String dataType = null; 265 if (jsonColumn.containsKey("dataType")){ 266 dataType = (String) jsonColumn.get("dataType"); 267 } 268 Boolean primaryKey = null; 269 if (jsonColumn.containsKey("primaryKey")){ 270 primaryKey = (Boolean) jsonColumn.get("primaryKey"); 271 } 272 Boolean unqiueKey = null; 273 if (jsonColumn.containsKey("unqiueKey")){ 274 unqiueKey = (Boolean) jsonColumn.get("unqiueKey"); 275 } 276 Boolean indexKey = null; 277 if (jsonColumn.containsKey("indexKey")){ 278 indexKey = (Boolean) jsonColumn.get("indexKey"); 279 } 280 Boolean foreignKey = null; 281 if (jsonColumn.containsKey("foreignKey")){ 282 foreignKey = (Boolean) jsonColumn.get("foreignKey"); 283 } 284 appendTable(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, databaseName, 285 null, tableName, isView, false, columnName, dataType, primaryKey, unqiueKey, indexKey, foreignKey, fromDDL, readColumnMeta(jsonColumn)); 286 } 287 } 288 289 List synonyms = (List) jsonDatabase.get("synonyms"); 290 if (synonyms != null) { 291 for (int k = 0; k < synonyms.size(); k++) { 292 Map jsonTable = (Map) synonyms.get(k); 293 String synonymsName = (String) jsonTable.get("name"); 294 String synonymsDatabaseName = (String) jsonTable.get("database"); 295 296 String synonymsSourceName = (String) jsonTable.get("sourceName"); 297 String synonymsSourceDatabaseName = (String) jsonTable.get("sourceDbLinkName"); 298 299 if (SQLUtil.isEmpty(synonymsSourceDatabaseName)) { 300 synonymsSourceDatabaseName = synonymsDatabaseName; 301 } 302 303 Object tableKey = getFullTableName(vendor, serverName, synonymsSourceDatabaseName, 304 null, synonymsSourceName); 305 table sourceTable = tableMap.get(tableKey); 306 if (sourceTable == null) { 307 // the synonym of a synonym 308 sourceTable = tableMap.get(getFullName(vendor, ESQLDataObjectType.dotSynonyms, serverName, synonymsSourceDatabaseName, 309 null, synonymsSourceName)); 310 } 311 if (sourceTable != null) { 312 List columns = (List) sourceTable.getColumns(); 313 String fromDDL = sourceTable.getFromDDL(); 314 dbObjs.add(jsonTable); 315 for (int l = 0; l < columns.size(); l++) { 316 column jsonColumn = (column) columns.get(l); 317 String columnName = (String) jsonColumn.getName(); 318 String dataType = jsonColumn.getDataType(); 319 appendTable(vendor, supportsCatalogs, supportsSchemas, dataflow, 320 serverName, databaseName, null, synonymsName, false, true, 321 columnName, dataType, false, false, false, false, fromDDL); 322 } 323 324 Object synonymsTableKey = getFullName(vendor, ESQLDataObjectType.dotSynonyms, serverName, 325 synonymsDatabaseName, null, synonymsName); 326 table synonymsTable = tableMap.get(synonymsTableKey); 327 if (synonymsTable != null) { 328 List synonymsColumns = (List) synonymsTable.getColumns(); 329 330 for (int l = 0; l < synonymsColumns.size(); l++) { 331 relationship relationElement = new relationship(); 332 relationElement.setType(RelationshipType.fdd.name()); 333 relationElement.setEffectType(EffectType.synonym.name()); 334 335 long id = ++ModelBindingManager.get().RELATION_ID; 336 relationElement.setId(String.valueOf(id)); 337 338 column targetColumn = (column)synonymsColumns.get(l); 339 targetColumn target = new targetColumn(); 340 target.setId(String.valueOf(targetColumn.getId())); 341 target.setColumn(targetColumn.getName()); 342 target.setParent_id(String.valueOf(synonymsTable.getId())); 343 target.setParent_name(synonymsTable.getName()); 344 relationElement.setTarget(target); 345 346 column sourceColumn = (column)columns.get(l); 347 sourceColumn source = new sourceColumn(); 348 source.setId(String.valueOf(sourceColumn.getId())); 349 source.setColumn(sourceColumn.getName()); 350 source.setParent_id(String.valueOf(sourceTable.getId())); 351 source.setParent_name(sourceTable.getName()); 352 relationElement.addSource(source); 353 354 dataflow.getRelationships().add(relationElement); 355 } 356 } 357 } 358 } 359 } 360 361 List<Map> procedures = (List<Map>) jsonDatabase.get("procedures"); 362 if (procedures != null) { 363 for (Map procedure : procedures) { 364 appendProcedure(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, databaseName, null, procedure, null, ESQLDataObjectType.dotProcedure); 365 } 366 } 367 List<Map> functions = (List<Map>) jsonDatabase.get("functions"); 368 if (functions != null) { 369 for (Map function : functions) { 370 appendProcedure(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, databaseName, null, function, null, ESQLDataObjectType.dotFunction); 371 } 372 } 373 List<Map> triggers = (List<Map>) jsonDatabase.get("triggers"); 374 if (triggers != null) { 375 for (Map trigger : triggers) { 376 appendProcedure(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, databaseName, null, trigger, null, ESQLDataObjectType.dotTrigger); 377 } 378 } 379 } 380 } else if (supportsSchemas) { 381 List schemas = (List) server.get("schemas"); 382 if (schemas == null) { 383 continue; 384 } 385 for (int i = 0; i < schemas.size(); i++) { 386 Map jsonSchema = (Map) schemas.get(i); 387 String schemaName = (String) jsonSchema.get("name"); 388 if (SQLUtil.parseNames(schemaName).size() > 1) { 389 schemaName = "\"" + schemaName + "\""; 390 } 391 List tables = (List) jsonSchema.get("tables"); 392 List views = (List) jsonSchema.get("views"); 393 List dbObjs = new ArrayList(); 394 if (tables != null) { 395 dbObjs.addAll(tables); 396 } 397 if (views != null) { 398 dbObjs.addAll(views); 399 } 400 for (int k = 0; k < dbObjs.size(); k++) { 401 Map jsonTable = (Map) dbObjs.get(k); 402 String tableName = (String) jsonTable.get("name"); 403 String type = (String) jsonTable.get("type"); 404 String fromDDL = (String) jsonTable.get("fromDDL"); 405 boolean isView = false; 406 if (type != null && type.toLowerCase(Locale.ROOT).indexOf("view") != -1) { 407 isView = true; 408 } 409 List columns = (List) jsonTable.get("columns"); 410 for (int l = 0; l < columns.size(); l++) { 411 Map jsonColumn = (Map) columns.get(l); 412 String columnName = (String) jsonColumn.get("name"); 413 String dataType = null; 414 if (jsonColumn.containsKey("dataType")){ 415 dataType = (String) jsonColumn.get("dataType"); 416 } 417 Boolean primaryKey = null; 418 if (jsonColumn.containsKey("primaryKey")){ 419 primaryKey = (Boolean) jsonColumn.get("primaryKey"); 420 } 421 Boolean unqiueKey = null; 422 if (jsonColumn.containsKey("unqiueKey")){ 423 unqiueKey = (Boolean) jsonColumn.get("unqiueKey"); 424 } 425 Boolean indexKey = null; 426 if (jsonColumn.containsKey("indexKey")){ 427 indexKey = (Boolean) jsonColumn.get("indexKey"); 428 } 429 Boolean foreignKey = null; 430 if (jsonColumn.containsKey("foreignKey")){ 431 foreignKey = (Boolean) jsonColumn.get("foreignKey"); 432 } 433 appendTable(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, null, 434 schemaName, tableName, isView, false, columnName, dataType, primaryKey, unqiueKey, indexKey, foreignKey, fromDDL, readColumnMeta(jsonColumn)); 435 } 436 } 437 438 List synonyms = (List) jsonSchema.get("synonyms"); 439 if (synonyms != null) { 440 for (int k = 0; k < synonyms.size(); k++) { 441 Map jsonTable = (Map) synonyms.get(k); 442 String synonymsName = (String) jsonTable.get("name"); 443 String synonymsSchemaName = (String) jsonTable.get("schema"); 444 445 String synonymsSourceName = (String) jsonTable.get("sourceName"); 446 String synonymsSourceSchemaName = (String) jsonTable.get("sourceSchema"); 447 448 if (SQLUtil.isEmpty(synonymsSourceSchemaName)) { 449 synonymsSourceSchemaName = synonymsSchemaName; 450 } 451 452 Object tableKey = getFullTableName(vendor, serverName, null, 453 synonymsSourceSchemaName, synonymsSourceName); 454 table sourceTable = tableMap.get(tableKey); 455 if (sourceTable == null) { 456 // the synonym of a synonym 457 sourceTable = tableMap.get(getFullName(vendor, ESQLDataObjectType.dotSynonyms, serverName, null, 458 synonymsSourceSchemaName, synonymsSourceName)); 459 } 460 if (sourceTable != null) { 461 String fromDDL = sourceTable.getFromDDL(); 462 List columns = (List) sourceTable.getColumns(); 463 dbObjs.add(jsonTable); 464 for (int l = 0; l < columns.size(); l++) { 465 column jsonColumn = (column) columns.get(l); 466 String columnName = (String) jsonColumn.getName(); 467 String dataType = jsonColumn.getDataType(); 468 appendTable(vendor, supportsCatalogs, supportsSchemas, dataflow, 469 serverName, null, schemaName, synonymsName, false, true, 470 columnName, dataType, false, false, false, false, fromDDL); 471 } 472 473 Object synonymsTableKey = getFullName(vendor, ESQLDataObjectType.dotSynonyms, serverName, 474 null, synonymsSchemaName, synonymsName); 475 table synonymsTable = tableMap.get(synonymsTableKey); 476 if (synonymsTable != null) { 477 List synonymsColumns = (List) synonymsTable.getColumns(); 478 479 for (int l = 0; l < synonymsColumns.size(); l++) { 480 relationship relationElement = new relationship(); 481 relationElement.setType(RelationshipType.fdd.name()); 482 relationElement.setEffectType(EffectType.synonym.name()); 483 484 long id = ++ModelBindingManager.get().RELATION_ID; 485 relationElement.setId(String.valueOf(id)); 486 487 column targetColumn = (column)synonymsColumns.get(l); 488 targetColumn target = new targetColumn(); 489 target.setId(String.valueOf(targetColumn.getId())); 490 target.setColumn(targetColumn.getName()); 491 target.setParent_id(String.valueOf(synonymsTable.getId())); 492 target.setParent_name(synonymsTable.getName()); 493 relationElement.setTarget(target); 494 495 column sourceColumn = (column)columns.get(l); 496 sourceColumn source = new sourceColumn(); 497 source.setId(String.valueOf(sourceColumn.getId())); 498 source.setColumn(sourceColumn.getName()); 499 source.setParent_id(String.valueOf(sourceTable.getId())); 500 source.setParent_name(sourceTable.getName()); 501 relationElement.addSource(source); 502 503 dataflow.getRelationships().add(relationElement); 504 } 505 } 506 } 507 } 508 } 509 510 List<Map> oraclePackages = (List<Map>) jsonSchema.get("packages"); 511 if (oraclePackages != null) { 512 for (Map oraclePackageItem : oraclePackages) { 513 String oraclePackageName = (String) oraclePackageItem.get("name"); 514 oraclePackage oraclePackage = appendOraclePackage(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, null, schemaName, oraclePackageName); 515 List<Map> procedures = (List<Map>) oraclePackageItem.get("procedures"); 516 if (procedures != null) { 517 for (Map procedure : procedures) { 518 appendProcedure(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, null, schemaName, procedure, oraclePackage, ESQLDataObjectType.dotProcedure); 519 } 520 } 521 List<Map> functions = (List<Map>) oraclePackageItem.get("functions"); 522 if (functions != null) { 523 for (Map function : functions) { 524 appendProcedure(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, null, schemaName, function, oraclePackage, ESQLDataObjectType.dotFunction); 525 } 526 } 527 List<Map> triggers = (List<Map>) oraclePackageItem.get("triggers"); 528 if (triggers != null) { 529 for (Map trigger : triggers) { 530 appendProcedure(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, null, schemaName, trigger, oraclePackage, ESQLDataObjectType.dotTrigger); 531 } 532 } 533 } 534 } 535 List<Map> procedures = (List<Map>) jsonSchema.get("procedures"); 536 if (procedures != null) { 537 for (Map procedure : procedures) { 538 appendProcedure(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, null, schemaName, procedure, null, ESQLDataObjectType.dotProcedure); 539 } 540 } 541 List<Map> functions = (List<Map>) jsonSchema.get("functions"); 542 if (functions != null) { 543 for (Map function : functions) { 544 appendProcedure(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, null, schemaName, function, null, ESQLDataObjectType.dotFunction); 545 } 546 } 547 List<Map> triggers = (List<Map>) jsonSchema.get("triggers"); 548 if (triggers != null) { 549 for (Map trigger : triggers) { 550 appendProcedure(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, null, schemaName, trigger, null, ESQLDataObjectType.dotTrigger); 551 } 552 } 553 } 554 } 555 } 556 } 557 } 558 sortTableColumns(dataflow); 559 return dataflow; 560 } 561 562 private void init(EDbVendor vendor) { 563 this.vendor = vendor; 564 procedureMap.clear(); 565 tableMap.clear(); 566 columnMap.clear(); 567 if (ModelBindingManager.get() == null) { 568 ModelBindingManager.set(new ModelBindingManager()); 569 } 570 } 571 572 private void sortTableColumns(dataflow dataflow) { 573 if (dataflow.getTables() != null) { 574 for (table table : dataflow.getTables()) { 575 Collections.sort(table.getColumns(), new Comparator<column>() { 576 public int compare(column t1, column t2) { 577 if (t1.getName().equalsIgnoreCase("RelationRows")) 578 return 1; 579 if (t2.getName().equalsIgnoreCase("RelationRows")) 580 return -1; 581 return 0; 582 } 583 }); 584 } 585 } 586 587 if (dataflow.getResultsets() != null) { 588 for (table table : dataflow.getResultsets()) { 589 Collections.sort(table.getColumns(), new Comparator<column>() { 590 public int compare(column t1, column t2) { 591 if (t1.getName().equalsIgnoreCase("RelationRows")) 592 return 1; 593 if (t2.getName().equalsIgnoreCase("RelationRows")) 594 return -1; 595 return 0; 596 } 597 }); 598 } 599 } 600 } 601 602 private oraclePackage appendOraclePackage(EDbVendor vendor, boolean supportsCatalogs, boolean supportsSchemas, dataflow dataflow, String serverName, String databaseName, String schemaName, String oraclePackageName) { 603 Object oraclePackageKey = getFullName(vendor, ESQLDataObjectType.dotOraclePackage, serverName, databaseName, schemaName, oraclePackageName); 604 if (!oraclePackageMap.containsKey(oraclePackageKey)) { 605 oraclePackage oraclePackage = new oraclePackage(); 606 if (serverName != null) { 607 oraclePackage.setServer(serverName); 608 } else if (sqlenv != null && sqlenv.getDefaultServerName() != null 609 && !TSQLEnv.DEFAULT_SERVER_NAME.equalsIgnoreCase(sqlenv.getDefaultServerName())) { 610 oraclePackage.setServer(sqlenv.getDefaultServerName()); 611 } 612 if (supportsCatalogs && databaseName != null) { 613 oraclePackage.setDatabase(databaseName); 614 } else if (supportsCatalogs && sqlenv != null && sqlenv.getDefaultCatalogName() != null 615 && !TSQLEnv.DEFAULT_DB_NAME.equalsIgnoreCase(sqlenv.getDefaultCatalogName())) { 616 oraclePackage.setDatabase(sqlenv.getDefaultCatalogName()); 617 } 618 619 if (supportsSchemas && schemaName != null) { 620 oraclePackage.setSchema(schemaName); 621 } else if (supportsSchemas && sqlenv != null && sqlenv.getDefaultSchemaName() != null 622 && !TSQLEnv.DEFAULT_SCHEMA_NAME.equalsIgnoreCase(sqlenv.getDefaultSchemaName())) { 623 oraclePackage.setSchema(sqlenv.getDefaultSchemaName()); 624 } 625 626 oraclePackage.setName(oraclePackageName); 627 if (supportsSchemas && !SQLUtil.isEmpty(oraclePackage.getSchema())) { 628 oraclePackage.setName(oraclePackage.getSchema() + "." + oraclePackage.getName()); 629 } 630 if (supportsCatalogs && !SQLUtil.isEmpty(oraclePackage.getDatabase())) { 631 oraclePackage.setName(oraclePackage.getDatabase() + "." + oraclePackage.getName()); 632 } 633 oraclePackage.setId(String.valueOf(++ModelBindingManager.get().TABLE_COLUMN_ID)); 634 if (ModelBindingManager.getGlobalSqlInfo() != null) { 635 oraclePackage.setCoordinate(new Pair3<Long, Long, Integer>(-1L, -1L, 636 ModelBindingManager.getGlobalSqlInfo().getIndexOf(ModelBindingManager.getGlobalHash())) + "," 637 + new Pair3<Long, Long, Integer>(-1L, -1L, 638 ModelBindingManager.getGlobalSqlInfo().getIndexOf(ModelBindingManager.getGlobalHash()))); 639 } 640 dataflow.getPackages().add(oraclePackage); 641 oraclePackageMap.put(oraclePackageKey, oraclePackage); 642 } 643 return oraclePackageMap.get(oraclePackageKey); 644 } 645 646 private void appendProcedure(EDbVendor vendor, boolean supportsCatalogs, boolean supportsSchemas, dataflow dataflow, String serverName, String databaseName, String schemaName, Map procedureObj, oraclePackage oraclePackage, ESQLDataObjectType role) { 647 String procedureName = (String) procedureObj.get("name"); 648 Object procedureKey = getFullName(vendor, role, serverName, databaseName, schemaName, procedureName); 649 if (!procedureMap.containsKey(procedureKey)) { 650 procedure procedure = new procedure(); 651 if (serverName != null) { 652 procedure.setServer(serverName); 653 } else if (sqlenv != null && sqlenv.getDefaultServerName() != null 654 && !TSQLEnv.DEFAULT_SERVER_NAME.equalsIgnoreCase(sqlenv.getDefaultServerName())) { 655 procedure.setServer(sqlenv.getDefaultServerName()); 656 } 657 if (supportsCatalogs && databaseName != null) { 658 procedure.setDatabase(databaseName); 659 } else if (supportsCatalogs && sqlenv != null && sqlenv.getDefaultCatalogName() != null 660 && !TSQLEnv.DEFAULT_DB_NAME.equalsIgnoreCase(sqlenv.getDefaultCatalogName())) { 661 procedure.setDatabase(sqlenv.getDefaultCatalogName()); 662 } 663 664 if (supportsSchemas && schemaName != null) { 665 procedure.setSchema(schemaName); 666 } else if (supportsSchemas && sqlenv != null && sqlenv.getDefaultSchemaName() != null 667 && !TSQLEnv.DEFAULT_SCHEMA_NAME.equalsIgnoreCase(sqlenv.getDefaultSchemaName())) { 668 procedure.setSchema(sqlenv.getDefaultSchemaName()); 669 } 670 671 procedure.setName(procedureName); 672 if (supportsSchemas && !SQLUtil.isEmpty(procedure.getSchema())) { 673 procedure.setName(procedure.getSchema() + "." + procedure.getName()); 674 } 675 if (supportsCatalogs && !SQLUtil.isEmpty(procedure.getDatabase())) { 676 procedure.setName(procedure.getDatabase() + "." + procedure.getName()); 677 } 678 procedure.setId(String.valueOf(++ModelBindingManager.get().TABLE_COLUMN_ID)); 679 if (ModelBindingManager.getGlobalSqlInfo() != null) { 680 procedure.setCoordinate(new Pair3<Long, Long, Integer>(-1L, -1L, 681 ModelBindingManager.getGlobalSqlInfo().getIndexOf(ModelBindingManager.getGlobalHash())) + "," 682 + new Pair3<Long, Long, Integer>(-1L, -1L, 683 ModelBindingManager.getGlobalSqlInfo().getIndexOf(ModelBindingManager.getGlobalHash()))); 684 } 685 procedure.setType((String) procedureObj.get("type")); 686 if (oraclePackage != null) { 687 oraclePackage.getProcedures().add(procedure); 688 } else { 689 dataflow.getProcedures().add(procedure); 690 } 691 procedureMap.put(procedureKey, procedure); 692 } 693 } 694 695 /** 696 * Optional, additive column attributes (catalog completeness only; not consumed by lineage). 697 * Carried from the metadata JSON through to the in-memory column model. Any field may be null 698 * (unknown); a Boolean false is meaningful (e.g. nullable=false means NOT NULL) and preserved. 699 */ 700 private static class ColumnMeta { 701 Boolean nullable; 702 Boolean identity; 703 Boolean computed; 704 Boolean persisted; 705 String computedDefinition; 706 String defaultValue; 707 String defaultName; 708 } 709 710 private static ColumnMeta readColumnMeta(Map jsonColumn) { 711 ColumnMeta meta = new ColumnMeta(); 712 if (jsonColumn.containsKey("nullable")) { 713 meta.nullable = (Boolean) jsonColumn.get("nullable"); 714 } 715 if (jsonColumn.containsKey("identity")) { 716 meta.identity = (Boolean) jsonColumn.get("identity"); 717 } 718 if (jsonColumn.containsKey("computed")) { 719 meta.computed = (Boolean) jsonColumn.get("computed"); 720 } 721 if (jsonColumn.containsKey("persisted")) { 722 meta.persisted = (Boolean) jsonColumn.get("persisted"); 723 } 724 if (jsonColumn.containsKey("computedDefinition")) { 725 meta.computedDefinition = (String) jsonColumn.get("computedDefinition"); 726 } 727 if (jsonColumn.containsKey("defaultValue")) { 728 meta.defaultValue = (String) jsonColumn.get("defaultValue"); 729 } 730 if (jsonColumn.containsKey("defaultName")) { 731 meta.defaultName = (String) jsonColumn.get("defaultName"); 732 } 733 return meta; 734 } 735 736 private void appendTable(EDbVendor vendor, boolean supportsCatalogs, boolean supportsSchemas, dataflow dataflow, String serverName, String databaseName, String schemaName, String tableName, 737 boolean isView, boolean isSynonym, String columnName, String dataType, Boolean primaryKey, Boolean unqiueKey,Boolean indexKey, Boolean foreignKey, String fromDDL) { 738 appendTable(vendor, supportsCatalogs, supportsSchemas, dataflow, serverName, databaseName, schemaName, tableName, 739 isView, isSynonym, columnName, dataType, primaryKey, unqiueKey, indexKey, foreignKey, fromDDL, null); 740 } 741 742 private void appendTable(EDbVendor vendor, boolean supportsCatalogs, boolean supportsSchemas, dataflow dataflow, String serverName, String databaseName, String schemaName, String tableName, 743 boolean isView, boolean isSynonym, String columnName, String dataType, Boolean primaryKey, Boolean unqiueKey,Boolean indexKey, Boolean foreignKey, String fromDDL, ColumnMeta columnMeta) { 744 // A synonym is keyed in its own role: a profile may give synonyms rules of their own. 745 Object tableKey = getFullName(vendor, isSynonym ? ESQLDataObjectType.dotSynonyms : ESQLDataObjectType.dotTable, 746 serverName, databaseName, schemaName, tableName); 747 if (!tableMap.containsKey(tableKey)) { 748 table table = new table(); 749 // This reader publishes SQL spellings (its input names, converted by their declared form). 750 table.setPublishedNameSqlSpelling(true); 751 if (serverName != null) { 752 table.setServer(serverName); 753 } else if (sqlenv != null && sqlenv.getDefaultServerName() != null 754 && !TSQLEnv.DEFAULT_SERVER_NAME.equalsIgnoreCase(sqlenv.getDefaultServerName())) { 755 table.setServer(sqlenv.getDefaultServerName()); 756 } 757 if (supportsCatalogs && databaseName != null) { 758 table.setDatabase(databaseName); 759 } else if (supportsCatalogs && sqlenv != null && sqlenv.getDefaultCatalogName() != null 760 && !TSQLEnv.DEFAULT_DB_NAME.equalsIgnoreCase(sqlenv.getDefaultCatalogName())) { 761 table.setDatabase(sqlenv.getDefaultCatalogName()); 762 } 763 764 if (supportsSchemas && schemaName != null) { 765 table.setSchema(schemaName); 766 } else if (supportsSchemas && sqlenv != null && sqlenv.getDefaultSchemaName() != null 767 && !TSQLEnv.DEFAULT_SCHEMA_NAME.equalsIgnoreCase(sqlenv.getDefaultSchemaName())) { 768 table.setSchema(sqlenv.getDefaultSchemaName()); 769 } 770 771 table.setName(tableName); 772 if (supportsSchemas && !SQLUtil.isEmpty(table.getSchema())) { 773 table.setName(table.getSchema() + "." + table.getName()); 774 } 775 if (supportsCatalogs && !SQLUtil.isEmpty(table.getDatabase())) { 776 table.setName(table.getDatabase() + "." + table.getName()); 777 } 778 table.setId(String.valueOf(++ModelBindingManager.get().TABLE_COLUMN_ID)); 779 if (ModelBindingManager.getGlobalSqlInfo() != null) { 780 table.setCoordinate(new Pair3<Long, Long, Integer>(-1L, -1L, 781 ModelBindingManager.getGlobalSqlInfo().getIndexOf(ModelBindingManager.getGlobalHash())) + "," 782 + new Pair3<Long, Long, Integer>(-1L, -1L, 783 ModelBindingManager.getGlobalSqlInfo().getIndexOf(ModelBindingManager.getGlobalHash()))); 784 } 785 786 if(!SQLUtil.isEmpty(fromDDL)){ 787 table.setFromDDL(fromDDL); 788 } 789 790 if (isView) { 791 table.setType("view"); 792 dataflow.getViews().add(table); 793 } else { 794 table.setType("table"); 795 dataflow.getTables().add(table); 796 } 797 if (isSynonym) { 798 table.setSubType(SubType.synonym.name()); 799 } 800 801 tableMap.put(tableKey, table); 802 } 803 804 table table = tableMap.get(tableKey); 805 // A column belongs to its table or synonym under that object's own key, and is named under column rules. 806 Object sourceColumnKey = java.util.Arrays.asList(tableKey, 807 DlineageUtil.lexicalNameKey(vendor, columnName, ESQLDataObjectType.dotColumn)); 808 if (!columnMap.containsKey(sourceColumnKey)) { 809 column column = new column(); 810 column.setId(String.valueOf(++ModelBindingManager.get().TABLE_COLUMN_ID)); 811 column.setName(columnName); 812 if ("*".equals(columnName)) { 813 // a metadata column named *, not a star 814 column.setStar(Boolean.FALSE); 815 } 816 // Input names have been converted to SQL lexical form at the reader boundary. 817 column.setIdentifierKey(DlineageUtil.lexicalColumnKey(vendor, columnName)); 818 column.setDataType(dataType); 819 column.setPrimaryKey(primaryKey); 820 column.setForeignKey(foreignKey); 821 column.setIndexKey(indexKey); 822 column.setUnqiueKey(unqiueKey); 823 if (columnMeta != null) { 824 column.setNullable(columnMeta.nullable); 825 column.setIdentity(columnMeta.identity); 826 column.setComputed(columnMeta.computed); 827 column.setPersisted(columnMeta.persisted); 828 column.setComputedDefinition(columnMeta.computedDefinition); 829 column.setDefaultValue(columnMeta.defaultValue); 830 column.setDefaultName(columnMeta.defaultName); 831 } 832 if ("RelationRows".equalsIgnoreCase(columnName)) { 833 column.setSource("system"); 834 } 835 if (ModelBindingManager.getGlobalSqlInfo() != null) { 836 column.setCoordinate(new Pair3<Long, Long, Integer>(-1L, -1L, 837 ModelBindingManager.getGlobalSqlInfo().getIndexOf(ModelBindingManager.getGlobalHash())) + "," 838 + new Pair3<Long, Long, Integer>(-1L, -1L, 839 ModelBindingManager.getGlobalSqlInfo().getIndexOf(ModelBindingManager.getGlobalHash()))); 840 } 841 table.getColumns().add(column); 842 columnMap.put(sourceColumnKey, column); 843 } 844 } 845 846 private Object getFullTableName(EDbVendor vendor, String... segments) { 847 return getFullName(vendor, ESQLDataObjectType.dotTable, segments); 848 } 849 850 /** 851 * The reader's key of an object: its server as given, and its name joined from the other segments under the 852 * analysis's identifier rules for {@code objectType} (the names are SQL spellings at this boundary). 853 */ 854 private Object getFullName(EDbVendor vendor, ESQLDataObjectType objectType, String... segments) { 855 StringBuilder builder = new StringBuilder(); 856 for (int i = 1; i < segments.length; i++) { 857 if (segments[i] == null) { 858 continue; 859 } 860 builder.append(segments[i]); 861 if (i < segments.length - 1) { 862 builder.append("."); 863 } 864 } 865 866 return java.util.Arrays.asList(segments[0], DlineageUtil.lexicalNameKey(vendor, builder.toString(), objectType)); 867 } 868 869 public static void main(String[] args) throws Exception { 870 dataflow dataflow = new SqlflowMetadataAnalyzer().analyzeMetadata(EDbVendor.dbvoracle, SQLUtil.getFileContent(new File("C:\\Users\\KK\\Downloads\\metadata (9).json"))); 871 System.out.println(XML2Model.saveXML(dataflow)); 872 } 873}