cartography 0.93.0rc1__py3-none-any.whl → 0.123.0__py3-none-any.whl

cartography/intel/pagerduty/services.py

@@ -7,6 +7,7 @@ import dateutil.parser
 import neo4j
 from pdpyras import APISession
 
+from cartography.client.core.tx import run_write_query
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -34,7 +35,8 @@ def get_services(pd_session: APISession) -> List[Dict[str, Any]]:
 
 @timeit
 def get_integrations(
-    pd_session: APISession, services: List[Dict[str, Any]],
+    pd_session: APISession,
+    services: List[Dict[str, Any]],
 ) -> List[Dict[str, Any]]:
     """
     Get integrations from services.
@@ -51,7 +53,9 @@ def get_integrations(
 
 
 def load_service_data(
-    neo4j_session: neo4j.Session, data: List[Dict], update_tag: int,
+    neo4j_session: neo4j.Session,
+    data: List[Dict],
+    update_tag: int,
 ) -> None:
     """
     Transform and load service information
@@ -93,7 +97,8 @@ def load_service_data(
             for team in service["teams"]:
                 team_relations.append({"service": service["id"], "team": team["id"]})
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Services=data,
         update_tag=update_tag,
@@ -104,7 +109,9 @@ def load_service_data(
 
 
 def _attach_teams(
-    neo4j_session: neo4j.Session, data: List[Dict], update_tag: int,
+    neo4j_session: neo4j.Session,
+    data: List[Dict],
+    update_tag: int,
 ) -> None:
     """
     Add relationship between teams and services.
@@ -115,7 +122,8 @@ def _attach_teams(
         MERGE (t)-[r:ASSOCIATED_WITH]->(s)
         ON CREATE SET r.firstseen = timestamp()
     """
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Relations=data,
         update_tag=update_tag,
@@ -123,7 +131,9 @@ def _attach_teams(
 
 
 def load_integration_data(
-    neo4j_session: neo4j.Session, data: List[Dict], update_tag: int,
+    neo4j_session: neo4j.Session,
+    data: List[Dict],
+    update_tag: int,
 ) -> None:
     """
     Transform and load integration information
@@ -155,7 +165,8 @@ def load_integration_data(
         created_at = dateutil.parser.parse(integration["created_at"])
         integration["created_at"] = int(created_at.timestamp())
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Integrations=data,
         update_tag=update_tag,

cartography/intel/pagerduty/teams.py

@@ -6,6 +6,7 @@ from typing import List
 import neo4j
 from pdpyras import APISession
 
+from cartography.client.core.tx import run_write_query
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -33,7 +34,8 @@ def get_teams(pd_session: APISession) -> List[Dict[str, Any]]:
 
 @timeit
 def get_team_members(
-    pd_session: APISession, teams: List[Dict[str, Any]],
+    pd_session: APISession,
+    teams: List[Dict[str, Any]],
 ) -> List[Dict[str, str]]:
     relations: List[Dict[str, str]] = []
     for team in teams:
@@ -46,7 +48,9 @@ def get_team_members(
 
 
 def load_team_data(
-    neo4j_session: neo4j.Session, data: List[Dict], update_tag: int,
+    neo4j_session: neo4j.Session,
+    data: List[Dict],
+    update_tag: int,
 ) -> None:
     """
     Transform and load teamuser information
@@ -65,7 +69,8 @@ def load_team_data(
     """
     logger.info(f"Loading {len(data)} pagerduty teams.")
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Teams=data,
         update_tag=update_tag,
@@ -73,7 +78,9 @@ def load_team_data(
 
 
 def load_team_relations(
-    neo4j_session: neo4j.Session, data: List[Dict], update_tag: int,
+    neo4j_session: neo4j.Session,
+    data: List[Dict],
+    update_tag: int,
 ) -> None:
     """
     Attach users to their teams
@@ -85,7 +92,8 @@ def load_team_relations(
         ON CREATE SET r.firstseen = timestamp()
         SET r.role = relation.role
     """
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Relations=data,
         update_tag=update_tag,

cartography/intel/pagerduty/users.py

@@ -6,6 +6,7 @@ from typing import List
 import neo4j
 from pdpyras import APISession
 
+from cartography.client.core.tx import run_write_query
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -30,7 +31,9 @@ def get_users(pd_session: APISession) -> List[Dict[str, Any]]:
 
 
 def load_user_data(
-    neo4j_session: neo4j.Session, data: List[Dict], update_tag: int,
+    neo4j_session: neo4j.Session,
+    data: List[Dict],
+    update_tag: int,
 ) -> None:
     """
     Transform and load user information
@@ -55,7 +58,8 @@ def load_user_data(
     """
     logger.info(f"Loading {len(data)} pagerduty users.")
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Users=data,
         update_tag=update_tag,

cartography/intel/pagerduty/vendors.py

@@ -6,6 +6,7 @@ from typing import List
 import neo4j
 from pdpyras import APISession
 
+from cartography.client.core.tx import run_write_query
 from cartography.util import timeit
 
 logger = logging.getLogger(__name__)
@@ -30,7 +31,9 @@ def get_vendors(pd_session: APISession) -> List[Dict[str, Any]]:
 
 
 def load_vendor_data(
-    neo4j_session: neo4j.Session, data: List[Dict], update_tag: int,
+    neo4j_session: neo4j.Session,
+    data: List[Dict],
+    update_tag: int,
 ) -> None:
     """
     Transform and load vendor information
@@ -51,7 +54,8 @@ def load_vendor_data(
     """
     logger.info(f"Loading {len(data)} pagerduty vendors.")
 
-    neo4j_session.run(
+    run_write_query(
+        neo4j_session,
         ingestion_cypher_query,
         Vendors=data,
         update_tag=update_tag,

cartography/intel/scaleway/__init__.py

@@ -0,0 +1,127 @@
+import logging
+
+import neo4j
+import scaleway
+
+import cartography.intel.scaleway.iam.apikeys
+import cartography.intel.scaleway.iam.applications
+import cartography.intel.scaleway.iam.groups
+import cartography.intel.scaleway.iam.users
+import cartography.intel.scaleway.instances.flexibleips
+import cartography.intel.scaleway.instances.instances
+import cartography.intel.scaleway.projects
+import cartography.intel.scaleway.storage.snapshots
+import cartography.intel.scaleway.storage.volumes
+from cartography.config import Config
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def start_scaleway_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
+    """
+    If this module is configured, perform ingestion of Scaleway data. Otherwise warn and exit
+    :param neo4j_session: Neo4J session for database interface
+    :param config: A cartography.config object
+    :return: None
+    """
+
+    if (
+        not config.scaleway_access_key
+        or not config.scaleway_secret_key
+        or not config.scaleway_org
+    ):
+        logger.info(
+            "Tailscale import is not configured - skipping this module. "
+            "See docs to configure.",
+        )
+        return
+
+    # Create client
+    client = scaleway.Client(
+        access_key=config.scaleway_access_key,
+        secret_key=config.scaleway_secret_key,
+    )
+
+    common_job_parameters = {
+        "UPDATE_TAG": config.update_tag,
+        "ORG_ID": config.scaleway_org,
+    }
+
+    # Organization level
+    projects = cartography.intel.scaleway.projects.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+        org_id=config.scaleway_org,
+        update_tag=config.update_tag,
+    )
+    projects_id = [project["id"] for project in projects]
+    cartography.intel.scaleway.iam.users.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+        org_id=config.scaleway_org,
+        update_tag=config.update_tag,
+    )
+    cartography.intel.scaleway.iam.applications.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+        org_id=config.scaleway_org,
+        update_tag=config.update_tag,
+    )
+    cartography.intel.scaleway.iam.groups.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+        org_id=config.scaleway_org,
+        update_tag=config.update_tag,
+    )
+    cartography.intel.scaleway.iam.apikeys.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+        org_id=config.scaleway_org,
+        update_tag=config.update_tag,
+    )
+
+    # Storage
+    cartography.intel.scaleway.storage.volumes.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+        org_id=config.scaleway_org,
+        projects_id=projects_id,
+        update_tag=config.update_tag,
+    )
+    cartography.intel.scaleway.storage.snapshots.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+        org_id=config.scaleway_org,
+        projects_id=projects_id,
+        update_tag=config.update_tag,
+    )
+
+    # Instances
+    # DISABLED due to https://github.com/scaleway/scaleway-sdk-python/issues/1040
+    """
+    cartography.intel.scaleway.instances.flexibleips.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+        org_id=config.scaleway_org,
+        projects_id=projects_id,
+        update_tag=config.update_tag,
+    )
+    """
+    cartography.intel.scaleway.instances.instances.sync(
+        neo4j_session,
+        client,
+        common_job_parameters,
+        org_id=config.scaleway_org,
+        projects_id=projects_id,
+        update_tag=config.update_tag,
+    )

cartography/intel/scaleway/iam/apikeys.py

@@ -0,0 +1,71 @@
+import logging
+from typing import Any
+
+import neo4j
+import scaleway
+from scaleway.iam.v1alpha1 import APIKey
+from scaleway.iam.v1alpha1 import IamV1Alpha1API
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.scaleway.utils import scaleway_obj_to_dict
+from cartography.models.scaleway.iam.apikey import ScalewayApiKeySchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: scaleway.Client,
+    common_job_parameters: dict[str, Any],
+    org_id: str,
+    update_tag: int,
+) -> None:
+    apikeys = get(client, org_id)
+    formatted_apikeys = transform_apikeys(apikeys)
+    load_apikeys(neo4j_session, formatted_apikeys, org_id, update_tag)
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    client: scaleway.Client,
+    org_id: str,
+) -> list[APIKey]:
+    api = IamV1Alpha1API(client)
+    return api.list_api_keys_all(organization_id=org_id)
+
+
+def transform_apikeys(apikeys: list[APIKey]) -> list[dict[str, Any]]:
+    formatted_apikeys = []
+    for apikey in apikeys:
+        formatted_apikeys.append(scaleway_obj_to_dict(apikey))
+    return formatted_apikeys
+
+
+@timeit
+def load_apikeys(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    org_id: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Scaleway ApiKeys into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        ScalewayApiKeySchema(),
+        data,
+        lastupdated=update_tag,
+        ORG_ID=org_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(ScalewayApiKeySchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/scaleway/iam/applications.py

@@ -0,0 +1,71 @@
+import logging
+from typing import Any
+
+import neo4j
+import scaleway
+from scaleway.iam.v1alpha1 import Application
+from scaleway.iam.v1alpha1 import IamV1Alpha1API
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.scaleway.utils import scaleway_obj_to_dict
+from cartography.models.scaleway.iam.application import ScalewayApplicationSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: scaleway.Client,
+    common_job_parameters: dict[str, Any],
+    org_id: str,
+    update_tag: int,
+) -> None:
+    applications = get(client, org_id)
+    formatted_applications = transform_applications(applications)
+    load_applications(neo4j_session, formatted_applications, org_id, update_tag)
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    client: scaleway.Client,
+    org_id: str,
+) -> list[Application]:
+    api = IamV1Alpha1API(client)
+    return api.list_applications_all(organization_id=org_id)
+
+
+def transform_applications(applications: list[Application]) -> list[dict[str, Any]]:
+    formatted_applications = []
+    for application in applications:
+        formatted_applications.append(scaleway_obj_to_dict(application))
+    return formatted_applications
+
+
+@timeit
+def load_applications(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    org_id: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Scaleway Applications into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        ScalewayApplicationSchema(),
+        data,
+        lastupdated=update_tag,
+        ORG_ID=org_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(ScalewayApplicationSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/scaleway/iam/groups.py

@@ -0,0 +1,71 @@
+import logging
+from typing import Any
+
+import neo4j
+import scaleway
+from scaleway.iam.v1alpha1 import Group
+from scaleway.iam.v1alpha1 import IamV1Alpha1API
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.scaleway.utils import scaleway_obj_to_dict
+from cartography.models.scaleway.iam.group import ScalewayGroupSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: scaleway.Client,
+    common_job_parameters: dict[str, Any],
+    org_id: str,
+    update_tag: int,
+) -> None:
+    groups = get(client, org_id)
+    formatted_groups = transform_groups(groups)
+    load_groups(neo4j_session, formatted_groups, org_id, update_tag)
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    client: scaleway.Client,
+    org_id: str,
+) -> list[Group]:
+    api = IamV1Alpha1API(client)
+    return api.list_groups_all(organization_id=org_id)
+
+
+def transform_groups(groups: list[Group]) -> list[dict[str, Any]]:
+    formatted_groups = []
+    for group in groups:
+        formatted_groups.append(scaleway_obj_to_dict(group))
+    return formatted_groups
+
+
+@timeit
+def load_groups(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    org_id: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Scaleway Groups into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        ScalewayGroupSchema(),
+        data,
+        lastupdated=update_tag,
+        ORG_ID=org_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(ScalewayGroupSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/scaleway/iam/users.py

@@ -0,0 +1,71 @@
+import logging
+from typing import Any
+
+import neo4j
+import scaleway
+from scaleway.iam.v1alpha1 import IamV1Alpha1API
+from scaleway.iam.v1alpha1 import User
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.scaleway.utils import scaleway_obj_to_dict
+from cartography.models.scaleway.iam.user import ScalewayUserSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: scaleway.Client,
+    common_job_parameters: dict[str, Any],
+    org_id: str,
+    update_tag: int,
+) -> None:
+    users = get(client, org_id)
+    formatted_users = transform_users(users)
+    load_users(neo4j_session, formatted_users, org_id, update_tag)
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    client: scaleway.Client,
+    org_id: str,
+) -> list[User]:
+    api = IamV1Alpha1API(client)
+    return api.list_users_all(organization_id=org_id)
+
+
+def transform_users(users: list[User]) -> list[dict[str, Any]]:
+    formatted_users = []
+    for user in users:
+        formatted_users.append(scaleway_obj_to_dict(user))
+    return formatted_users
+
+
+@timeit
+def load_users(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    org_id: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Scaleway Users into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        ScalewayUserSchema(),
+        data,
+        lastupdated=update_tag,
+        ORG_ID=org_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(ScalewayUserSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/scaleway/instances/flexibleips.py

@@ -0,0 +1,86 @@
+import logging
+from typing import Any
+
+import neo4j
+import scaleway
+from scaleway.instance.v1 import InstanceV1API
+from scaleway.instance.v1 import Ip
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.scaleway.utils import DEFAULT_ZONE
+from cartography.intel.scaleway.utils import scaleway_obj_to_dict
+from cartography.models.scaleway.instance.flexibleip import ScalewayFlexibleIpSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: scaleway.Client,
+    common_job_parameters: dict[str, Any],
+    org_id: str,
+    projects_id: list[str],
+    update_tag: int,
+) -> None:
+    flexibleips = get(client, org_id)
+    flexibleips_by_project = transform_flexibleips(flexibleips)
+    load_flexibleips(neo4j_session, flexibleips_by_project, update_tag)
+    cleanup(neo4j_session, projects_id, common_job_parameters)
+
+
+@timeit
+def get(
+    client: scaleway.Client,
+    org_id: str,
+) -> list[Ip]:
+    api = InstanceV1API(client)
+    return api.list_ips_all(organization=org_id, zone=DEFAULT_ZONE)
+
+
+def transform_flexibleips(
+    flexibleips: list[Ip],
+) -> dict[str, list[dict[str, Any]]]:
+    result: dict[str, list[dict[str, Any]]] = {}
+    for flexibleip in flexibleips:
+        project_id = flexibleip.project
+        formatted_flexibleip = scaleway_obj_to_dict(flexibleip)
+        result.setdefault(project_id, []).append(formatted_flexibleip)
+    return result
+
+
+@timeit
+def load_flexibleips(
+    neo4j_session: neo4j.Session,
+    data: dict[str, list[dict[str, Any]]],
+    update_tag: int,
+) -> None:
+    for project_id, flexibleips in data.items():
+        logger.info(
+            "Loading %d Scaleway Flexible IPs in project '%s' into Neo4j.",
+            len(flexibleips),
+            project_id,
+        )
+        load(
+            neo4j_session,
+            ScalewayFlexibleIpSchema(),
+            flexibleips,
+            lastupdated=update_tag,
+            PROJECT_ID=project_id,
+        )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session,
+    projects_id: list[str],
+    common_job_parameters: dict[str, Any],
+) -> None:
+    for project_id in projects_id:
+        scopped_job_parameters = common_job_parameters.copy()
+        scopped_job_parameters["PROJECT_ID"] = project_id
+        GraphJob.from_node_schema(
+            ScalewayFlexibleIpSchema(), scopped_job_parameters
+        ).run(neo4j_session)