cartography 0.104.0rc2__py3-none-any.whl → 0.123.0__py3-none-any.whl

cartography/intel/gcp/bigtable_app_profile.py

@@ -0,0 +1,101 @@
+import logging
+
+import neo4j
+from googleapiclient.discovery import Resource
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.gcp.bigtable.app_profile import GCPBigtableAppProfileSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def get_bigtable_app_profiles(client: Resource, instance_id: str) -> list[dict]:
+    app_profiles: list[dict] = []
+    request = client.projects().instances().appProfiles().list(parent=instance_id)
+    while request is not None:
+        response = request.execute()
+        app_profiles.extend(response.get("appProfiles", []))
+        request = (
+            client.projects()
+            .instances()
+            .appProfiles()
+            .list_next(
+                previous_request=request,
+                previous_response=response,
+            )
+        )
+    return app_profiles
+
+
+def transform_app_profiles(
+    app_profiles_data: list[dict],
+    instance_id: str,
+) -> list[dict]:
+    transformed: list[dict] = []
+    for app_profile in app_profiles_data:
+        app_profile["instance_id"] = instance_id
+        routing = app_profile.get("singleClusterRouting")
+        if routing:
+            short_cluster_id = routing.get("clusterId")
+            if short_cluster_id:
+                app_profile["single_cluster_routing_cluster_id"] = (
+                    f"{instance_id}/clusters/{short_cluster_id}"
+                )
+        transformed.append(app_profile)
+    return transformed
+
+
+@timeit
+def load_bigtable_app_profiles(
+    neo4j_session: neo4j.Session,
+    data: list[dict],
+    project_id: str,
+    update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        GCPBigtableAppProfileSchema(),
+        data,
+        lastupdated=update_tag,
+        PROJECT_ID=project_id,
+    )
+
+
+@timeit
+def cleanup_bigtable_app_profiles(
+    neo4j_session: neo4j.Session, common_job_parameters: dict
+) -> None:
+    GraphJob.from_node_schema(GCPBigtableAppProfileSchema(), common_job_parameters).run(
+        neo4j_session,
+    )
+
+
+@timeit
+def sync_bigtable_app_profiles(
+    neo4j_session: neo4j.Session,
+    client: Resource,
+    instances: list[dict],
+    project_id: str,
+    update_tag: int,
+    common_job_parameters: dict,
+) -> None:
+    logger.info(f"Syncing Bigtable App Profiles for project {project_id}.")
+    all_app_profiles_transformed: list[dict] = []
+
+    for inst in instances:
+        instance_id = inst["name"]
+        app_profiles_raw = get_bigtable_app_profiles(client, instance_id)
+        all_app_profiles_transformed.extend(
+            transform_app_profiles(app_profiles_raw, instance_id),
+        )
+
+    load_bigtable_app_profiles(
+        neo4j_session, all_app_profiles_transformed, project_id, update_tag
+    )
+
+    cleanup_job_params = common_job_parameters.copy()
+    cleanup_job_params["PROJECT_ID"] = project_id
+    cleanup_bigtable_app_profiles(neo4j_session, cleanup_job_params)

cartography/intel/gcp/bigtable_backup.py

@@ -0,0 +1,91 @@
+import logging
+
+import neo4j
+from googleapiclient.discovery import Resource
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.gcp.bigtable.backup import GCPBigtableBackupSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def get_bigtable_backups(client: Resource, cluster_id: str) -> list[dict]:
+    backups: list[dict] = []
+    request = client.projects().instances().clusters().backups().list(parent=cluster_id)
+    while request is not None:
+        response = request.execute()
+        backups.extend(response.get("backups", []))
+        request = (
+            client.projects()
+            .instances()
+            .clusters()
+            .backups()
+            .list_next(
+                previous_request=request,
+                previous_response=response,
+            )
+        )
+    return backups
+
+
+def transform_backups(backups_data: list[dict], cluster_id: str) -> list[dict]:
+    transformed: list[dict] = []
+    for backup in backups_data:
+        backup["cluster_id"] = cluster_id
+        backup["source_table"] = backup.get("sourceTable")
+        transformed.append(backup)
+    return transformed
+
+
+@timeit
+def load_bigtable_backups(
+    neo4j_session: neo4j.Session,
+    data: list[dict],
+    project_id: str,
+    update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        GCPBigtableBackupSchema(),
+        data,
+        lastupdated=update_tag,
+        PROJECT_ID=project_id,
+    )
+
+
+@timeit
+def cleanup_bigtable_backups(
+    neo4j_session: neo4j.Session, common_job_parameters: dict
+) -> None:
+    GraphJob.from_node_schema(GCPBigtableBackupSchema(), common_job_parameters).run(
+        neo4j_session,
+    )
+
+
+@timeit
+def sync_bigtable_backups(
+    neo4j_session: neo4j.Session,
+    client: Resource,
+    clusters: list[dict],
+    project_id: str,
+    update_tag: int,
+    common_job_parameters: dict,
+) -> None:
+    logger.info(f"Syncing Bigtable Backups for project {project_id}.")
+    all_backups_transformed: list[dict] = []
+
+    for cluster in clusters:
+        cluster_id = cluster["name"]
+        backups_raw = get_bigtable_backups(client, cluster_id)
+        all_backups_transformed.extend(transform_backups(backups_raw, cluster_id))
+
+    load_bigtable_backups(
+        neo4j_session, all_backups_transformed, project_id, update_tag
+    )
+
+    cleanup_job_params = common_job_parameters.copy()
+    cleanup_job_params["PROJECT_ID"] = project_id
+    cleanup_bigtable_backups(neo4j_session, cleanup_job_params)

cartography/intel/gcp/bigtable_cluster.py

@@ -0,0 +1,93 @@
+import logging
+
+import neo4j
+from googleapiclient.discovery import Resource
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.gcp.bigtable.cluster import GCPBigtableClusterSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def get_bigtable_clusters(client: Resource, instance_id: str) -> list[dict]:
+    clusters: list[dict] = []
+    request = client.projects().instances().clusters().list(parent=instance_id)
+    while request is not None:
+        response = request.execute()
+        clusters.extend(response.get("clusters", []))
+        request = (
+            client.projects()
+            .instances()
+            .clusters()
+            .list_next(
+                previous_request=request,
+                previous_response=response,
+            )
+        )
+    return clusters
+
+
+def transform_clusters(clusters_data: list[dict], instance_id: str) -> list[dict]:
+    transformed: list[dict] = []
+    for cluster in clusters_data:
+        cluster["instance_id"] = instance_id
+        transformed.append(cluster)
+    return transformed
+
+
+@timeit
+def load_bigtable_clusters(
+    neo4j_session: neo4j.Session,
+    data: list[dict],
+    project_id: str,
+    update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        GCPBigtableClusterSchema(),
+        data,
+        lastupdated=update_tag,
+        PROJECT_ID=project_id,
+    )
+
+
+@timeit
+def cleanup_bigtable_clusters(
+    neo4j_session: neo4j.Session, common_job_parameters: dict
+) -> None:
+    GraphJob.from_node_schema(GCPBigtableClusterSchema(), common_job_parameters).run(
+        neo4j_session,
+    )
+
+
+@timeit
+def sync_bigtable_clusters(
+    neo4j_session: neo4j.Session,
+    client: Resource,
+    instances: list[dict],
+    project_id: str,
+    update_tag: int,
+    common_job_parameters: dict,
+) -> list[dict]:
+    logger.info(f"Syncing Bigtable Clusters for project {project_id}.")
+    all_clusters_raw: list[dict] = []
+    all_clusters_transformed: list[dict] = []
+
+    for inst in instances:
+        instance_id = inst["name"]
+        clusters_raw = get_bigtable_clusters(client, instance_id)
+        all_clusters_raw.extend(clusters_raw)
+        all_clusters_transformed.extend(transform_clusters(clusters_raw, instance_id))
+
+    load_bigtable_clusters(
+        neo4j_session, all_clusters_transformed, project_id, update_tag
+    )
+
+    cleanup_job_params = common_job_parameters.copy()
+    cleanup_job_params["PROJECT_ID"] = project_id
+    cleanup_bigtable_clusters(neo4j_session, cleanup_job_params)
+
+    return all_clusters_raw

cartography/intel/gcp/bigtable_instance.py

@@ -0,0 +1,86 @@
+import logging
+
+import neo4j
+from googleapiclient.discovery import Resource
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.gcp.bigtable.instance import GCPBigtableInstanceSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def get_bigtable_instances(client: Resource, project_id: str) -> list[dict]:
+    instances: list[dict] = []
+    request = client.projects().instances().list(parent=f"projects/{project_id}")
+    while request is not None:
+        response = request.execute()
+        instances.extend(response.get("instances", []))
+        request = (
+            client.projects()
+            .instances()
+            .list_next(
+                previous_request=request,
+                previous_response=response,
+            )
+        )
+    return instances
+
+
+def transform_instances(instances_data: list[dict], project_id: str) -> list[dict]:
+    transformed: list[dict] = []
+    for inst in instances_data:
+        inst["project_id"] = project_id
+        transformed.append(inst)
+    return transformed
+
+
+@timeit
+def load_bigtable_instances(
+    neo4j_session: neo4j.Session,
+    data: list[dict],
+    project_id: str,
+    update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        GCPBigtableInstanceSchema(),
+        data,
+        lastupdated=update_tag,
+        PROJECT_ID=project_id,
+    )
+
+
+@timeit
+def cleanup_bigtable_instances(
+    neo4j_session: neo4j.Session, common_job_parameters: dict
+) -> None:
+    GraphJob.from_node_schema(GCPBigtableInstanceSchema(), common_job_parameters).run(
+        neo4j_session,
+    )
+
+
+@timeit
+def sync_bigtable_instances(
+    neo4j_session: neo4j.Session,
+    client: Resource,
+    project_id: str,
+    update_tag: int,
+    common_job_parameters: dict,
+) -> list[dict]:
+    logger.info(f"Syncing Bigtable Instances for project {project_id}.")
+    instances_raw = get_bigtable_instances(client, project_id)
+    if not instances_raw:
+        logger.info(f"No Bigtable instances found for project {project_id}.")
+        return []
+
+    instances = transform_instances(instances_raw, project_id)
+    load_bigtable_instances(neo4j_session, instances, project_id, update_tag)
+
+    cleanup_job_params = common_job_parameters.copy()
+    cleanup_job_params["PROJECT_ID"] = project_id
+    cleanup_bigtable_instances(neo4j_session, cleanup_job_params)
+
+    return instances_raw

cartography/intel/gcp/bigtable_table.py

@@ -0,0 +1,87 @@
+import logging
+
+import neo4j
+from googleapiclient.discovery import Resource
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.gcp.bigtable.table import GCPBigtableTableSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def get_bigtable_tables(client: Resource, instance_id: str) -> list[dict]:
+    tables: list[dict] = []
+    request = client.projects().instances().tables().list(parent=instance_id)
+    while request is not None:
+        response = request.execute()
+        tables.extend(response.get("tables", []))
+        request = (
+            client.projects()
+            .instances()
+            .tables()
+            .list_next(
+                previous_request=request,
+                previous_response=response,
+            )
+        )
+    return tables
+
+
+def transform_tables(tables_data: list[dict], instance_id: str) -> list[dict]:
+    transformed: list[dict] = []
+    for table in tables_data:
+        table["instance_id"] = instance_id
+        transformed.append(table)
+    return transformed
+
+
+@timeit
+def load_bigtable_tables(
+    neo4j_session: neo4j.Session,
+    data: list[dict],
+    project_id: str,
+    update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        GCPBigtableTableSchema(),
+        data,
+        lastupdated=update_tag,
+        PROJECT_ID=project_id,
+    )
+
+
+@timeit
+def cleanup_bigtable_tables(
+    neo4j_session: neo4j.Session, common_job_parameters: dict
+) -> None:
+    GraphJob.from_node_schema(GCPBigtableTableSchema(), common_job_parameters).run(
+        neo4j_session,
+    )
+
+
+@timeit
+def sync_bigtable_tables(
+    neo4j_session: neo4j.Session,
+    client: Resource,
+    instances: list[dict],
+    project_id: str,
+    update_tag: int,
+    common_job_parameters: dict,
+) -> None:
+    logger.info(f"Syncing Bigtable Tables for project {project_id}.")
+    all_tables_transformed: list[dict] = []
+
+    for inst in instances:
+        instance_id = inst["name"]
+        tables_raw = get_bigtable_tables(client, instance_id)
+        all_tables_transformed.extend(transform_tables(tables_raw, instance_id))
+
+    load_bigtable_tables(neo4j_session, all_tables_transformed, project_id, update_tag)
+
+    cleanup_job_params = common_job_parameters.copy()
+    cleanup_job_params["PROJECT_ID"] = project_id
+    cleanup_bigtable_tables(neo4j_session, cleanup_job_params)