cartography 0.104.0rc2__py3-none-any.whl → 0.123.0__py3-none-any.whl

cartography/intel/azure/cosmosdb.py

@@ -12,7 +12,39 @@ from azure.core.exceptions import HttpResponseError
 from azure.core.exceptions import ResourceNotFoundError
 from azure.mgmt.cosmosdb import CosmosDBManagementClient
 
-from cartography.util import run_cleanup_job
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.azure.cosmosdb.account import AzureCosmosDBAccountSchema
+from cartography.models.azure.cosmosdb.accountfailoverpolicy import (
+    AzureCosmosDBAccountFailoverPolicySchema,
+)
+from cartography.models.azure.cosmosdb.cassandrakeyspace import (
+    AzureCosmosDBCassandraKeyspaceSchema,
+)
+from cartography.models.azure.cosmosdb.cassandratable import (
+    AzureCosmosDBCassandraTableSchema,
+)
+from cartography.models.azure.cosmosdb.corspolicy import AzureCosmosDBCorsPolicySchema
+from cartography.models.azure.cosmosdb.dblocation import AzureCosmosDBLocationSchema
+from cartography.models.azure.cosmosdb.mongodbcollection import (
+    AzureCosmosDBMongoDBCollectionSchema,
+)
+from cartography.models.azure.cosmosdb.mongodbdatabase import (
+    AzureCosmosDBMongoDBDatabaseSchema,
+)
+from cartography.models.azure.cosmosdb.privateendpointconnection import (
+    AzureCDBPrivateEndpointConnectionSchema,
+)
+from cartography.models.azure.cosmosdb.sqlcontainer import (
+    AzureCosmosDBSqlContainerSchema,
+)
+from cartography.models.azure.cosmosdb.sqldatabase import AzureCosmosDBSqlDatabaseSchema
+from cartography.models.azure.cosmosdb.tableresource import (
+    AzureCosmosDBTableResourceSchema,
+)
+from cartography.models.azure.cosmosdb.virtualnetworkrule import (
+    AzureCosmosDBVirtualNetworkRuleSchema,
+)
 from cartography.util import timeit
 
 from .util.credentials import Credentials
@@ -83,7 +115,7 @@ def transform_database_account_data(database_account_list: List[Dict]) -> List[D
         if "ip_rules" in database_account and len(database_account["ip_rules"]) > 0:
             iprules = [x["ip_address_or_range"] for x in database_account["ip_rules"]]
         database_account["ipruleslist"] = iprules
-        database_account["list_of_capabilities"] = capabilities
+        database_account["capabilities"] = capabilities
 
     return database_account_list
 
@@ -98,45 +130,12 @@ def load_database_account_data(
     """
     Ingest data of all database accounts into neo4j.
     """
-    ingest_database_account = """
-    UNWIND $database_accounts_list AS da
-    MERGE (d:AzureCosmosDBAccount{id: da.id})
-    ON CREATE SET d.firstseen = timestamp(),
-    d.type = da.type, d.resourcegroup = da.resourceGroup,
-    d.location = da.location
-    SET d.lastupdated = $azure_update_tag,
-    d.kind = da.kind,
-    d.name = da.name,
-    d.ipranges = da.ipruleslist,
-    d.capabilities = da.list_of_capabilities,
-    d.documentendpoint = da.document_endpoint,
-    d.virtualnetworkfilterenabled = da.is_virtual_network_filter_enabled,
-    d.enableautomaticfailover = da.enable_automatic_failover,
-    d.provisioningstate = da.provisioning_state,
-    d.multiplewritelocations = da.enable_multiple_write_locations,
-    d.accountoffertype = da.database_account_offer_type,
-    d.publicnetworkaccess = da.public_network_access,
-    d.enablecassandraconnector = da.enable_cassandra_connector,
-    d.connectoroffer = da.connector_offer,
-    d.disablekeybasedmetadatawriteaccess = da.disable_key_based_metadata_write_access,
-    d.keyvaulturi = da.key_vault_key_uri,
-    d.enablefreetier = da.enable_free_tier,
-    d.enableanalyticalstorage = da.enable_analytical_storage,
-    d.defaultconsistencylevel = da.consistency_policy.default_consistency_level,
-    d.maxstalenessprefix = da.consistency_policy.max_staleness_prefix,
-    d.maxintervalinseconds = da.consistency_policy.max_interval_in_seconds
-    WITH d
-    MATCH (owner:AzureSubscription{id: $AZURE_SUBSCRIPTION_ID})
-    MERGE (owner)-[r:RESOURCE]->(d)
-    ON CREATE SET r.firstseen = timestamp()
-    SET r.lastupdated = $azure_update_tag
-    """
-
-    neo4j_session.run(
-        ingest_database_account,
-        database_accounts_list=database_account_list,
+    load(
+        neo4j_session,
+        AzureCosmosDBAccountSchema(),
+        database_account_list,
+        lastupdated=azure_update_tag,
         AZURE_SUBSCRIPTION_ID=subscription_id,
-        azure_update_tag=azure_update_tag,
     )
 
 
@@ -152,35 +151,46 @@ def sync_database_account_data_resources(
     response (like cors policy, failover policy, private endpoint connections, virtual network rules and locations).
     """
     for database_account in database_account_list:
-        _load_cosmosdb_cors_policy(neo4j_session, database_account, azure_update_tag)
+        _load_cosmosdb_cors_policy(
+            neo4j_session,
+            database_account,
+            subscription_id,
+            azure_update_tag,
+        )
         _load_cosmosdb_failover_policies(
             neo4j_session,
             database_account,
+            subscription_id,
             azure_update_tag,
         )
         _load_cosmosdb_private_endpoint_connections(
             neo4j_session,
             database_account,
+            subscription_id,
             azure_update_tag,
         )
         _load_cosmosdb_virtual_network_rules(
             neo4j_session,
             database_account,
+            subscription_id,
             azure_update_tag,
         )
         _load_database_account_write_locations(
             neo4j_session,
             database_account,
+            subscription_id,
             azure_update_tag,
         )
         _load_database_account_read_locations(
             neo4j_session,
             database_account,
+            subscription_id,
             azure_update_tag,
         )
         _load_database_account_associated_locations(
             neo4j_session,
             database_account,
+            subscription_id,
             azure_update_tag,
         )
 
@@ -189,6 +199,7 @@ def sync_database_account_data_resources(
 def _load_database_account_write_locations(
     neo4j_session: neo4j.Session,
     database_account: Dict,
+    subscription_id: str,
     azure_update_tag: int,
 ) -> None:
     """
@@ -198,31 +209,16 @@ def _load_database_account_write_locations(
         "write_locations" in database_account
         and len(database_account["write_locations"]) > 0
     ):
-        database_account_id = database_account["id"]
         write_locations = database_account["write_locations"]
+        for wl in write_locations:
+            wl["db_write_account_id"] = database_account["id"]
 
-        ingest_write_location = """
-        UNWIND $write_locations_list as wl
-        MERGE (loc:AzureCosmosDBLocation{id: wl.id})
-        ON CREATE SET loc.firstseen = timestamp()
-        SET loc.lastupdated = $azure_update_tag,
-        loc.locationname = wl.location_name,
-        loc.documentendpoint = wl.document_endpoint,
-        loc.provisioningstate = wl.provisioning_state,
-        loc.failoverpriority = wl.failover_priority,
-        loc.iszoneredundant = wl.is_zone_redundant
-        WITH loc
-        MATCH (d:AzureCosmosDBAccount{id: $DatabaseAccountId})
-        MERGE (d)-[r:CAN_WRITE_FROM]->(loc)
-        ON CREATE SET r.firstseen = timestamp()
-        SET r.lastupdated = $azure_update_tag
-        """
-
-        neo4j_session.run(
-            ingest_write_location,
-            write_locations_list=write_locations,
-            DatabaseAccountId=database_account_id,
-            azure_update_tag=azure_update_tag,
+        load(
+            neo4j_session,
+            AzureCosmosDBLocationSchema(),
+            write_locations,
+            lastupdated=azure_update_tag,
+            AZURE_SUBSCRIPTION_ID=subscription_id,
         )
 
 
@@ -230,6 +226,7 @@ def _load_database_account_write_locations(
 def _load_database_account_read_locations(
     neo4j_session: neo4j.Session,
     database_account: Dict,
+    subscription_id: str,
     azure_update_tag: int,
 ) -> None:
     """
@@ -239,31 +236,16 @@ def _load_database_account_read_locations(
         "read_locations" in database_account
         and len(database_account["read_locations"]) > 0
     ):
-        database_account_id = database_account["id"]
         read_locations = database_account["read_locations"]
+        for rl in read_locations:
+            rl["db_read_account_id"] = database_account["id"]
 
-        ingest_read_location = """
-        UNWIND $read_locations_list as rl
-        MERGE (loc:AzureCosmosDBLocation{id: rl.id})
-        ON CREATE SET loc.firstseen = timestamp()
-        SET loc.lastupdated = $azure_update_tag,
-        loc.locationname = rl.location_name,
-        loc.documentendpoint = rl.document_endpoint,
-        loc.provisioningstate = rl.provisioning_state,
-        loc.failoverpriority = rl.failover_priority,
-        loc.iszoneredundant = rl.is_zone_redundant
-        WITH loc
-        MATCH (d:AzureCosmosDBAccount{id: $DatabaseAccountId})
-        MERGE (d)-[r:CAN_READ_FROM]->(loc)
-        ON CREATE SET r.firstseen = timestamp()
-        SET r.lastupdated = $azure_update_tag
-        """
-
-        neo4j_session.run(
-            ingest_read_location,
-            read_locations_list=read_locations,
-            DatabaseAccountId=database_account_id,
-            azure_update_tag=azure_update_tag,
+        load(
+            neo4j_session,
+            AzureCosmosDBLocationSchema(),
+            read_locations,
+            lastupdated=azure_update_tag,
+            AZURE_SUBSCRIPTION_ID=subscription_id,
         )
 
 
@@ -271,37 +253,23 @@ def _load_database_account_read_locations(
 def _load_database_account_associated_locations(
     neo4j_session: neo4j.Session,
     database_account: Dict,
+    subscription_id: str,
     azure_update_tag: int,
 ) -> None:
     """
     Ingest the details of enabled location for the database account.
     """
     if "locations" in database_account and len(database_account["locations"]) > 0:
-        database_account_id = database_account["id"]
         associated_locations = database_account["locations"]
+        for al in associated_locations:
+            al["db_associated_account_id"] = database_account["id"]
 
-        ingest_associated_location = """
-        UNWIND $associated_locations_list as al
-        MERGE (loc:AzureCosmosDBLocation{id: al.id})
-        ON CREATE SET loc.firstseen = timestamp()
-        SET loc.lastupdated = $azure_update_tag,
-        loc.locationname = al.location_name,
-        loc.documentendpoint = al.document_endpoint,
-        loc.provisioningstate = al.provisioning_state,
-        loc.failoverpriority = al.failover_priority,
-        loc.iszoneredundant = al.is_zone_redundant
-        WITH loc
-        MATCH (d:AzureCosmosDBAccount{id: $DatabaseAccountId})
-        MERGE (d)-[r:ASSOCIATED_WITH]->(loc)
-        ON CREATE SET r.firstseen = timestamp()
-        SET r.lastupdated = $azure_update_tag
-        """
-
-        neo4j_session.run(
-            ingest_associated_location,
-            associated_locations_list=associated_locations,
-            DatabaseAccountId=database_account_id,
-            azure_update_tag=azure_update_tag,
+        load(
+            neo4j_session,
+            AzureCosmosDBLocationSchema(),
+            associated_locations,
+            lastupdated=azure_update_tag,
+            AZURE_SUBSCRIPTION_ID=subscription_id,
         )
 
 
@@ -321,6 +289,7 @@ def transform_cosmosdb_cors_policy(database_account: Dict) -> Dict:
 def _load_cosmosdb_cors_policy(
     neo4j_session: neo4j.Session,
     database_account: Dict,
+    subscription_id: str,
     azure_update_tag: int,
 ) -> None:
     """
@@ -331,28 +300,13 @@ def _load_cosmosdb_cors_policy(
         database_account_id = database_account["id"]
         cors_policies = database_account["cors"]
 
-        ingest_cors_policy = """
-        UNWIND $cors_policies_list AS cp
-        MERGE (corspolicy:AzureCosmosDBCorsPolicy{id: cp.cors_policy_unique_id})
-        ON CREATE SET corspolicy.firstseen = timestamp(),
-        corspolicy.allowedorigins = cp.allowed_origins
-        SET corspolicy.lastupdated = $azure_update_tag,
-        corspolicy.allowedmethods = cp.allowed_methods,
-        corspolicy.allowedheaders = cp.allowed_headers,
-        corspolicy.exposedheaders = cp.exposed_headers,
-        corspolicy.maxageinseconds = cp.max_age_in_seconds
-        WITH corspolicy
-        MATCH (d:AzureCosmosDBAccount{id: $DatabaseAccountId})
-        MERGE (d)-[r:CONTAINS]->(corspolicy)
-        ON CREATE SET r.firstseen = timestamp()
-        SET r.lastupdated = $azure_update_tag
-        """
-
-        neo4j_session.run(
-            ingest_cors_policy,
-            cors_policies_list=cors_policies,
+        load(
+            neo4j_session,
+            AzureCosmosDBCorsPolicySchema(),
+            cors_policies,
+            lastupdated=azure_update_tag,
+            AZURE_SUBSCRIPTION_ID=subscription_id,
             DatabaseAccountId=database_account_id,
-            azure_update_tag=azure_update_tag,
         )
 
 
@@ -360,6 +314,7 @@ def _load_cosmosdb_cors_policy(
 def _load_cosmosdb_failover_policies(
     neo4j_session: neo4j.Session,
     database_account: Dict,
+    subscription_id: str,
     azure_update_tag: int,
 ) -> None:
     """
@@ -372,25 +327,13 @@ def _load_cosmosdb_failover_policies(
         database_account_id = database_account["id"]
         failover_policies = database_account["failover_policies"]
 
-        ingest_failover_policies = """
-        UNWIND $failover_policies_list AS fp
-        MERGE (fpolicy:AzureCosmosDBAccountFailoverPolicy{id: fp.id})
-        ON CREATE SET fpolicy.firstseen = timestamp()
-        SET fpolicy.lastupdated = $azure_update_tag,
-        fpolicy.locationname = fp.location_name,
-        fpolicy.failoverpriority = fp.failover_priority
-        WITH fpolicy
-        MATCH (d:AzureCosmosDBAccount{id: $DatabaseAccountId})
-        MERGE (d)-[r:CONTAINS]->(fpolicy)
-        ON CREATE SET r.firstseen = timestamp()
-        SET r.lastupdated = $azure_update_tag
-        """
-
-        neo4j_session.run(
-            ingest_failover_policies,
-            failover_policies_list=failover_policies,
+        load(
+            neo4j_session,
+            AzureCosmosDBAccountFailoverPolicySchema(),
+            failover_policies,
+            lastupdated=azure_update_tag,
+            AZURE_SUBSCRIPTION_ID=subscription_id,
             DatabaseAccountId=database_account_id,
-            azure_update_tag=azure_update_tag,
         )
 
 
@@ -398,6 +341,7 @@ def _load_cosmosdb_failover_policies(
 def _load_cosmosdb_private_endpoint_connections(
     neo4j_session: neo4j.Session,
     database_account: Dict,
+    subscription_id: str,
     azure_update_tag: int,
 ) -> None:
     """
@@ -413,27 +357,13 @@ def _load_cosmosdb_private_endpoint_connections(
         database_account_id = database_account["id"]
         private_endpoint_connections = database_account["private_endpoint_connections"]
 
-        ingest_private_endpoint_connections = """
-        UNWIND $private_endpoint_connections_list AS connection
-        MERGE (pec:AzureCDBPrivateEndpointConnection{id: connection.id})
-        ON CREATE SET pec.firstseen = timestamp()
-        SET pec.lastupdated = $azure_update_tag,
-        pec.name = connection.name,
-        pec.privateendpointid = connection.private_endpoint.id,
-        pec.status = connection.private_link_service_connection_state.status,
-        pec.actionrequired = connection.private_link_service_connection_state.actions_required
-        WITH pec
-        MATCH (d:AzureCosmosDBAccount{id: $DatabaseAccountId})
-        MERGE (d)-[r:CONFIGURED_WITH]->(pec)
-        ON CREATE SET r.firstseen = timestamp()
-        SET r.lastupdated = $azure_update_tag
-        """
-
-        neo4j_session.run(
-            ingest_private_endpoint_connections,
-            private_endpoint_connections_list=private_endpoint_connections,
+        load(
+            neo4j_session,
+            AzureCDBPrivateEndpointConnectionSchema(),
+            private_endpoint_connections,
+            lastupdated=azure_update_tag,
+            AZURE_SUBSCRIPTION_ID=subscription_id,
             DatabaseAccountId=database_account_id,
-            azure_update_tag=azure_update_tag,
         )
 
 
@@ -441,6 +371,7 @@ def _load_cosmosdb_private_endpoint_connections(
 def _load_cosmosdb_virtual_network_rules(
     neo4j_session: neo4j.Session,
     database_account: Dict,
+    subscription_id: str,
     azure_update_tag: int,
 ) -> None:
     """
@@ -453,24 +384,13 @@ def _load_cosmosdb_virtual_network_rules(
         database_account_id = database_account["id"]
         virtual_network_rules = database_account["virtual_network_rules"]
 
-        ingest_virtual_network_rules = """
-        UNWIND $virtual_network_rules_list AS vnr
-        MERGE (rules:AzureCosmosDBVirtualNetworkRule{id: vnr.id})
-        ON CREATE SET rules.firstseen = timestamp()
-        SET rules.lastupdated = $azure_update_tag,
-        rules.ignoremissingvnetserviceendpoint = vnr.ignore_missing_v_net_service_endpoint
-        WITH rules
-        MATCH (d:AzureCosmosDBAccount{id: $DatabaseAccountId})
-        MERGE (d)-[r:CONFIGURED_WITH]->(rules)
-        ON CREATE SET r.firstseen = timestamp()
-        SET r.lastupdated = $azure_update_tag
-        """
-
-        neo4j_session.run(
-            ingest_virtual_network_rules,
-            virtual_network_rules_list=virtual_network_rules,
+        load(
+            neo4j_session,
+            AzureCosmosDBVirtualNetworkRuleSchema(),
+            virtual_network_rules,
+            lastupdated=azure_update_tag,
+            AZURE_SUBSCRIPTION_ID=subscription_id,
             DatabaseAccountId=database_account_id,
-            azure_update_tag=azure_update_tag,
         )
 
 
@@ -762,14 +682,18 @@ def load_database_account_details(
             table_resources.extend(t)
 
     # Loading the table resources
-    _load_table_resources(neo4j_session, table_resources, update_tag)
+    _load_table_resources(neo4j_session, table_resources, subscription_id, update_tag)
     # Cleanup of table resources (done here because table resource doesn't have any other child resources in it)
     cleanup_table_resources(neo4j_session, common_job_parameters)
 
     # Loading SQL databases, Cassandra Keyspaces and MongoDB databases
-    _load_sql_databases(neo4j_session, sql_databases, update_tag)
-    _load_cassandra_keyspaces(neo4j_session, cassandra_keyspaces, update_tag)
-    _load_mongodb_databases(neo4j_session, mongodb_databases, update_tag)
+    _load_sql_databases(neo4j_session, sql_databases, subscription_id, update_tag)
+    _load_cassandra_keyspaces(
+        neo4j_session, cassandra_keyspaces, subscription_id, update_tag
+    )
+    _load_mongodb_databases(
+        neo4j_session, mongodb_databases, subscription_id, update_tag
+    )
 
     sync_sql_database_details(
         neo4j_session,
@@ -801,31 +725,18 @@ def load_database_account_details(
 def _load_sql_databases(
     neo4j_session: neo4j.Session,
     sql_databases: List[Dict],
+    subscription_id: str,
     update_tag: int,
 ) -> None:
     """
     Ingest SQL Databases into neo4j.
     """
-    ingest_sql_databases = """
-    UNWIND $sql_databases_list AS database
-    MERGE (sdb:AzureCosmosDBSqlDatabase{id: database.id})
-    ON CREATE SET sdb.firstseen = timestamp(), sdb.type = database.type,
-    sdb.location = database.location
-    SET sdb.name = database.name,
-    sdb.throughput = database.options.throughput,
-    sdb.maxthroughput = database.options.autoscale_setting.max_throughput,
-    sdb.lastupdated = $azure_update_tag
-    WITH sdb, database
-    MATCH (d:AzureCosmosDBAccount{id: database.database_account_id})
-    MERGE (d)-[r:CONTAINS]->(sdb)
-    ON CREATE SET r.firstseen = timestamp()
-    SET r.lastupdated = $azure_update_tag
-    """
-
-    neo4j_session.run(
-        ingest_sql_databases,
-        sql_databases_list=sql_databases,
-        azure_update_tag=update_tag,
+    load(
+        neo4j_session,
+        AzureCosmosDBSqlDatabaseSchema(),
+        sql_databases,
+        lastupdated=update_tag,
+        AZURE_SUBSCRIPTION_ID=subscription_id,
     )
 
 
@@ -833,31 +744,18 @@ def _load_sql_databases(
 def _load_cassandra_keyspaces(
     neo4j_session: neo4j.Session,
     cassandra_keyspaces: List[Dict],
+    subscription_id: str,
     update_tag: int,
 ) -> None:
     """
     Ingest Cassandra keyspaces into neo4j.
     """
-    ingest_cassandra_keyspaces = """
-    UNWIND $cassandra_keyspaces_list AS keyspace
-    MERGE (ck:AzureCosmosDBCassandraKeyspace{id: keyspace.id})
-    ON CREATE SET ck.firstseen = timestamp(), ck.type = keyspace.type,
-    ck.location = keyspace.location
-    SET ck.name = keyspace.name,
-    ck.lastupdated = $azure_update_tag,
-    ck.throughput = keyspace.options.throughput,
-    ck.maxthroughput = keyspace.options.autoscale_setting.max_throughput
-    WITH ck, keyspace
-    MATCH (d:AzureCosmosDBAccount{id: keyspace.database_account_id})
-    MERGE (d)-[r:CONTAINS]->(ck)
-    ON CREATE SET r.firstseen = timestamp()
-    SET r.lastupdated = $azure_update_tag
-    """
-
-    neo4j_session.run(
-        ingest_cassandra_keyspaces,
-        cassandra_keyspaces_list=cassandra_keyspaces,
-        azure_update_tag=update_tag,
+    load(
+        neo4j_session,
+        AzureCosmosDBCassandraKeyspaceSchema(),
+        cassandra_keyspaces,
+        lastupdated=update_tag,
+        AZURE_SUBSCRIPTION_ID=subscription_id,
     )
 
 
@@ -865,31 +763,18 @@ def _load_cassandra_keyspaces(
 def _load_mongodb_databases(
     neo4j_session: neo4j.Session,
     mongodb_databases: List[Dict],
+    subscription_id: str,
     update_tag: int,
 ) -> None:
     """
     Ingest MongoDB databases into neo4j.
     """
-    ingest_mongodb_databases = """
-    UNWIND $mongodb_databases_list AS database
-    MERGE (mdb:AzureCosmosDBMongoDBDatabase{id: database.id})
-    ON CREATE SET mdb.firstseen = timestamp(), mdb.type = database.type,
-    mdb.location = database.location
-    SET mdb.name = database.name,
-    mdb.throughput = database.options.throughput,
-    mdb.maxthroughput = database.options.autoscale_setting.max_throughput,
-    mdb.lastupdated = $azure_update_tag
-    WITH mdb, database
-    MATCH (d:AzureCosmosDBAccount{id: database.database_account_id})
-    MERGE (d)-[r:CONTAINS]->(mdb)
-    ON CREATE SET r.firstseen = timestamp()
-    SET r.lastupdated = $azure_update_tag
-    """
-
-    neo4j_session.run(
-        ingest_mongodb_databases,
-        mongodb_databases_list=mongodb_databases,
-        azure_update_tag=update_tag,
+    load(
+        neo4j_session,
+        AzureCosmosDBMongoDBDatabaseSchema(),
+        mongodb_databases,
+        lastupdated=update_tag,
+        AZURE_SUBSCRIPTION_ID=subscription_id,
     )
 
 
@@ -897,31 +782,18 @@ def _load_mongodb_databases(
 def _load_table_resources(
     neo4j_session: neo4j.Session,
     table_resources: List[Dict],
+    subscription_id: str,
     update_tag: int,
 ) -> None:
     """
     Ingest Table resources into neo4j.
     """
-    ingest_tables = """
-    UNWIND $table_resources_list AS table
-    MERGE (tr:AzureCosmosDBTableResource{id: table.id})
-    ON CREATE SET tr.firstseen = timestamp(), tr.type = table.type,
-    tr.location = table.location
-    SET tr.name = table.name,
-    tr.lastupdated = $azure_update_tag,
-    tr.throughput = table.options.throughput,
-    tr.maxthroughput = table.options.autoscale_setting.max_throughput
-    WITH tr, table
-    MATCH (d:AzureCosmosDBAccount{id: table.database_account_id})
-    MERGE (d)-[r:CONTAINS]->(tr)
-    ON CREATE SET r.firstseen = timestamp()
-    SET r.lastupdated = $azure_update_tag
-    """
-
-    neo4j_session.run(
-        ingest_tables,
-        table_resources_list=table_resources,
-        azure_update_tag=update_tag,
+    load(
+        neo4j_session,
+        AzureCosmosDBTableResourceSchema(),
+        table_resources,
+        lastupdated=update_tag,
+        AZURE_SUBSCRIPTION_ID=subscription_id,
     )
 
 
@@ -939,7 +811,9 @@ def sync_sql_database_details(
         subscription_id,
         sql_databases,
     )
-    load_sql_database_details(neo4j_session, sql_database_details, update_tag)
+    load_sql_database_details(
+        neo4j_session, sql_database_details, subscription_id, update_tag
+    )
     cleanup_sql_database_details(neo4j_session, common_job_parameters)
 
 
@@ -999,6 +873,7 @@ def get_sql_containers(
 def load_sql_database_details(
     neo4j_session: neo4j.Session,
     details: List[Tuple[Any, Any]],
+    subscription_id: str,
     update_tag: int,
 ) -> None:
     """
@@ -1012,44 +887,25 @@ def load_sql_database_details(
                 c["database_id"] = database_id
             containers.extend(container)
 
-    _load_sql_containers(neo4j_session, containers, update_tag)
+    _load_sql_containers(neo4j_session, containers, subscription_id, update_tag)
 
 
 @timeit
 def _load_sql_containers(
     neo4j_session: neo4j.Session,
     containers: List[Dict],
+    subscription_id: str,
     update_tag: int,
 ) -> None:
     """
     Ingest SQL Container details into neo4j.
     """
-    ingest_containers = """
-    UNWIND $sql_containers_list AS container
-    MERGE (c:AzureCosmosDBSqlContainer{id: container.id})
-    ON CREATE SET c.firstseen = timestamp(), c.type = container.type,
-    c.location = container.location
-    SET c.name = container.name,
-    c.lastupdated = $azure_update_tag,
-    c.throughput = container.options.throughput,
-    c.maxthroughput = container.options.autoscale_setting.max_throughput,
-    c.container = container.resource.id,
-    c.defaultttl = container.resource.default_ttl,
-    c.analyticalttl = container.resource.analytical_storage_ttl,
-    c.isautomaticindexingpolicy = container.resource.indexing_policy.automatic,
-    c.indexingmode = container.resource.indexing_policy.indexing_mode,
-    c.conflictresolutionpolicymode = container.resource.conflict_resolution_policy.mode
-    WITH c, container
-    MATCH (sdb:AzureCosmosDBSqlDatabase{id: container.database_id})
-    MERGE (sdb)-[r:CONTAINS]->(c)
-    ON CREATE SET r.firstseen = timestamp()
-    SET r.lastupdated = $azure_update_tag
-    """
-
-    neo4j_session.run(
-        ingest_containers,
-        sql_containers_list=containers,
-        azure_update_tag=update_tag,
+    load(
+        neo4j_session,
+        AzureCosmosDBSqlContainerSchema(),
+        containers,
+        lastupdated=update_tag,
+        AZURE_SUBSCRIPTION_ID=subscription_id,
     )
 
 
@@ -1070,6 +926,7 @@ def sync_cassandra_keyspace_details(
     load_cassandra_keyspace_details(
         neo4j_session,
         cassandra_keyspace_details,
+        subscription_id,
         update_tag,
     )
     cleanup_cassandra_keyspace_details(neo4j_session, common_job_parameters)
@@ -1131,6 +988,7 @@ def get_cassandra_tables(
 def load_cassandra_keyspace_details(
     neo4j_session: neo4j.Session,
     details: List[Tuple[Any, Any]],
+    subscription_id: str,
     update_tag: int,
 ) -> None:
     """
@@ -1144,41 +1002,25 @@ def load_cassandra_keyspace_details(
                 t["keyspace_id"] = keyspace_id
             cassandra_tables.extend(cassandra_table)
 
-    _load_cassandra_tables(neo4j_session, cassandra_tables, update_tag)
+    _load_cassandra_tables(neo4j_session, cassandra_tables, subscription_id, update_tag)
 
 
 @timeit
 def _load_cassandra_tables(
     neo4j_session: neo4j.Session,
     cassandra_tables: List[Dict],
+    subscription_id: str,
     update_tag: int,
 ) -> None:
     """
     Ingest Cassandra Tables into neo4j.
     """
-    ingest_cassandra_tables = """
-    UNWIND $cassandra_tables_list AS table
-    MERGE (ct:AzureCosmosDBCassandraTable{id: table.id})
-    ON CREATE SET ct.firstseen = timestamp(), ct.type = table.type,
-    ct.location = table.location
-    SET ct.name = table.name,
-    ct.lastupdated = $azure_update_tag,
-    ct.throughput = table.options.throughput,
-    ct.maxthroughput = table.options.autoscale_setting.max_throughput,
-    ct.container = table.resource.id,
-    ct.defaultttl = table.resource.default_ttl,
-    ct.analyticalttl = table.resource.analytical_storage_ttl
-    WITH ct, table
-    MATCH (ck:AzureCosmosDBCassandraKeyspace{id: table.keyspace_id})
-    MERGE (ck)-[r:CONTAINS]->(ct)
-    ON CREATE SET r.firstseen = timestamp()
-    SET r.lastupdated = $azure_update_tag
-    """
-
-    neo4j_session.run(
-        ingest_cassandra_tables,
-        cassandra_tables_list=cassandra_tables,
-        azure_update_tag=update_tag,
+    load(
+        neo4j_session,
+        AzureCosmosDBCassandraTableSchema(),
+        cassandra_tables,
+        lastupdated=update_tag,
+        AZURE_SUBSCRIPTION_ID=subscription_id,
     )
 
 
@@ -1196,7 +1038,9 @@ def sync_mongodb_database_details(
         subscription_id,
         mongodb_databases,
     )
-    load_mongodb_databases_details(neo4j_session, mongodb_databases_details, update_tag)
+    load_mongodb_databases_details(
+        neo4j_session, mongodb_databases_details, subscription_id, update_tag
+    )
     cleanup_mongodb_database_details(neo4j_session, common_job_parameters)
 
 
@@ -1256,6 +1100,7 @@ def get_mongodb_collections(
 def load_mongodb_databases_details(
     neo4j_session: neo4j.Session,
     details: List[Tuple[Any, Any]],
+    subscription_id: str,
     update_tag: int,
 ) -> None:
     """
@@ -1269,40 +1114,25 @@ def load_mongodb_databases_details(
                 c["database_id"] = database_id
             collections.extend(collection)
 
-    _load_collections(neo4j_session, collections, update_tag)
+    _load_collections(neo4j_session, collections, subscription_id, update_tag)
 
 
 @timeit
 def _load_collections(
     neo4j_session: neo4j.Session,
     collections: List[Dict],
+    subscription_id: str,
     update_tag: int,
 ) -> None:
     """
     Ingest MongoDB Collections into neo4j.
     """
-    ingest_collections = """
-    UNWIND $mongodb_collections_list AS collection
-    MERGE (col:AzureCosmosDBMongoDBCollection{id: collection.id})
-    ON CREATE SET col.firstseen = timestamp(), col.type = collection.type,
-    col.location = collection.location
-    SET col.name = collection.name,
-    col.lastupdated = $azure_update_tag,
-    col.throughput = collection.options.throughput,
-    col.maxthroughput = collection.options.autoscale_setting.max_throughput,
-    col.collectionname = collection.resource.id,
-    col.analyticalttl = collection.resource.analytical_storage_ttl
-    WITH col, collection
-    MATCH (mdb:AzureCosmosDBMongoDBDatabase{id: collection.database_id})
-    MERGE (mdb)-[r:CONTAINS]->(col)
-    ON CREATE SET r.firstseen = timestamp()
-    SET r.lastupdated = $azure_update_tag
-    """
-
-    neo4j_session.run(
-        ingest_collections,
-        mongodb_collections_list=collections,
-        azure_update_tag=update_tag,
+    load(
+        neo4j_session,
+        AzureCosmosDBMongoDBCollectionSchema(),
+        collections,
+        lastupdated=update_tag,
+        AZURE_SUBSCRIPTION_ID=subscription_id,
     )
 
 
@@ -1311,10 +1141,26 @@ def cleanup_azure_database_accounts(
     neo4j_session: neo4j.Session,
     common_job_parameters: Dict,
 ) -> None:
-    run_cleanup_job(
-        "azure_database_account_cleanup.json",
+    GraphJob.from_node_schema(AzureCosmosDBAccountSchema(), common_job_parameters).run(
+        neo4j_session,
+    )
+    GraphJob.from_node_schema(AzureCosmosDBLocationSchema(), common_job_parameters).run(
+        neo4j_session,
+    )
+    GraphJob.from_node_schema(
+        AzureCosmosDBCorsPolicySchema(), common_job_parameters
+    ).run(
+        neo4j_session,
+    )
+    GraphJob.from_node_schema(
+        AzureCosmosDBVirtualNetworkRuleSchema(), common_job_parameters
+    ).run(
+        neo4j_session,
+    )
+    GraphJob.from_node_schema(
+        AzureCDBPrivateEndpointConnectionSchema(), common_job_parameters
+    ).run(
         neo4j_session,
-        common_job_parameters,
     )
 
 
@@ -1323,10 +1169,15 @@ def cleanup_sql_database_details(
     neo4j_session: neo4j.Session,
     common_job_parameters: Dict,
 ) -> None:
-    run_cleanup_job(
-        "azure_cosmosdb_sql_database_cleanup.json",
+    GraphJob.from_node_schema(
+        AzureCosmosDBSqlContainerSchema(), common_job_parameters
+    ).run(
+        neo4j_session,
+    )
+    GraphJob.from_node_schema(
+        AzureCosmosDBSqlDatabaseSchema(), common_job_parameters
+    ).run(
         neo4j_session,
-        common_job_parameters,
     )
 
 
@@ -1335,10 +1186,15 @@ def cleanup_cassandra_keyspace_details(
     neo4j_session: neo4j.Session,
     common_job_parameters: Dict,
 ) -> None:
-    run_cleanup_job(
-        "azure_cosmosdb_cassandra_keyspace_cleanup.json",
+    GraphJob.from_node_schema(
+        AzureCosmosDBCassandraTableSchema(), common_job_parameters
+    ).run(
+        neo4j_session,
+    )
+    GraphJob.from_node_schema(
+        AzureCosmosDBCassandraKeyspaceSchema(), common_job_parameters
+    ).run(
         neo4j_session,
-        common_job_parameters,
     )
 
 
@@ -1347,10 +1203,15 @@ def cleanup_mongodb_database_details(
     neo4j_session: neo4j.Session,
     common_job_parameters: Dict,
 ) -> None:
-    run_cleanup_job(
-        "azure_cosmosdb_mongodb_database_cleanup.json",
+    GraphJob.from_node_schema(
+        AzureCosmosDBMongoDBCollectionSchema(), common_job_parameters
+    ).run(
+        neo4j_session,
+    )
+    GraphJob.from_node_schema(
+        AzureCosmosDBMongoDBDatabaseSchema(), common_job_parameters
+    ).run(
         neo4j_session,
-        common_job_parameters,
     )
 
 
@@ -1359,10 +1220,10 @@ def cleanup_table_resources(
     neo4j_session: neo4j.Session,
     common_job_parameters: Dict,
 ) -> None:
-    run_cleanup_job(
-        "azure_cosmosdb_table_resources_cleanup.json",
+    GraphJob.from_node_schema(
+        AzureCosmosDBTableResourceSchema(), common_job_parameters
+    ).run(
         neo4j_session,
-        common_job_parameters,
     )