cartography 0.103.0rc1__py3-none-any.whl → 0.104.0rc2__py3-none-any.whl

cartography/intel/cloudflare/roles.py

@@ -0,0 +1,65 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+from cloudflare import Cloudflare
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.cloudflare.role import CloudflareRoleSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: Cloudflare,
+    common_job_parameters: Dict[str, Any],
+    account_id: str,
+) -> None:
+    roles = get(client, account_id)
+    load_roles(
+        neo4j_session,
+        roles,
+        account_id,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    client: Cloudflare,
+    account_id: str,
+) -> List[Dict[str, Any]]:
+    return [
+        role.to_dict() for role in client.accounts.roles.list(account_id=account_id)
+    ]
+
+
+def load_roles(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    account_id: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Cloudflare roles into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        CloudflareRoleSchema(),
+        data,
+        lastupdated=update_tag,
+        account_id=account_id,
+    )
+
+
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(CloudflareRoleSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/cloudflare/zones.py

@@ -0,0 +1,64 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+from cloudflare import Cloudflare
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.cloudflare.zone import CloudflareZoneSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: Cloudflare,
+    common_job_parameters: Dict[str, Any],
+    account_id: str,
+) -> List[Dict]:
+    zones = get(client, account_id)
+    load_zones(
+        neo4j_session,
+        zones,
+        account_id,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+    return zones
+
+
+@timeit
+def get(
+    client: Cloudflare,
+    account_id: str,
+) -> List[Dict[str, Any]]:
+    return [zone.to_dict() for zone in client.zones.list(account=account_id)]
+
+
+def load_zones(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    account_id: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Cloudflare zones into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        CloudflareZoneSchema(),
+        data,
+        lastupdated=update_tag,
+        account_id=account_id,
+    )
+
+
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(CloudflareZoneSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/openai/__init__.py

@@ -0,0 +1,86 @@
+import logging
+
+import neo4j
+import requests
+
+import cartography.intel.openai.adminapikeys
+import cartography.intel.openai.apikeys
+import cartography.intel.openai.projects
+import cartography.intel.openai.serviceaccounts
+import cartography.intel.openai.users
+from cartography.config import Config
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def start_openai_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
+    """
+    If this module is configured, perform ingestion of OpenAI data. Otherwise warn and exit
+    :param neo4j_session: Neo4J session for database interface
+    :param config: A cartography.config object
+    :return: None
+    """
+
+    if not config.openai_apikey or not config.openai_org_id:
+        logger.info(
+            "OpenAI import is not configured - skipping this module. "
+            "See docs to configure.",
+        )
+        return
+
+    # Create requests sessions
+    api_session = requests.session()
+    api_session.headers.update(
+        {
+            "Authorization": f"Bearer {config.openai_apikey}",
+            "OpenAI-Organization": config.openai_org_id,
+        }
+    )
+
+    common_job_parameters = {
+        "UPDATE_TAG": config.update_tag,
+        "BASE_URL": "https://api.openai.com/v1",
+        "ORG_ID": config.openai_org_id,
+    }
+
+    # Organization node is created during the users sync
+    cartography.intel.openai.users.sync(
+        neo4j_session,
+        api_session,
+        common_job_parameters,
+        ORG_ID=config.openai_org_id,
+    )
+
+    for project in cartography.intel.openai.projects.sync(
+        neo4j_session,
+        api_session,
+        common_job_parameters,
+        ORG_ID=config.openai_org_id,
+    ):
+        project_job_parameters = {
+            "UPDATE_TAG": config.update_tag,
+            "BASE_URL": "https://api.openai.com/v1",
+            "ORG_ID": config.openai_org_id,
+            "project_id": project["id"],
+        }
+        cartography.intel.openai.serviceaccounts.sync(
+            neo4j_session,
+            api_session,
+            project_job_parameters,
+            project_id=project["id"],
+        )
+        cartography.intel.openai.apikeys.sync(
+            neo4j_session,
+            api_session,
+            project_job_parameters,
+            project_id=project["id"],
+        )
+
+    cartography.intel.openai.adminapikeys.sync(
+        neo4j_session,
+        api_session,
+        common_job_parameters,
+        ORG_ID=config.openai_org_id,
+    )

cartography/intel/openai/adminapikeys.py

@@ -0,0 +1,89 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.openai.util import paginated_get
+from cartography.models.openai.adminapikey import OpenAIAdminApiKeySchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: requests.Session,
+    common_job_parameters: Dict[str, Any],
+    ORG_ID: str,
+) -> None:
+    adminapikeys = get(
+        api_session,
+        common_job_parameters["BASE_URL"],
+    )
+    transformed_adminapikeys = transform(adminapikeys)
+    load_adminapikeys(
+        neo4j_session,
+        transformed_adminapikeys,
+        ORG_ID,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    api_session: requests.Session,
+    base_url: str,
+) -> List[Dict[str, Any]]:
+    return list(
+        paginated_get(
+            api_session, f"{base_url}/organization/admin_api_keys", timeout=_TIMEOUT
+        )
+    )
+
+
+def transform(
+    adminapikeys: List[Dict[str, Any]],
+) -> List[Dict[str, Any]]:
+    result: List[Dict[str, Any]] = []
+    for adminapikey in adminapikeys:
+        if adminapikey["owner"]["type"] == "user":
+            adminapikey["owner_user_id"] = adminapikey["owner"]["id"]
+        else:
+            adminapikey["owner_sa_id"] = adminapikey["owner"]["id"]
+        result.append(adminapikey)
+    return result
+
+
+@timeit
+def load_adminapikeys(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    ORG_ID: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d OpenAI AdminApiKey into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        OpenAIAdminApiKeySchema(),
+        data,
+        lastupdated=update_tag,
+        ORG_ID=ORG_ID,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(OpenAIAdminApiKeySchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/openai/apikeys.py

@@ -0,0 +1,96 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.openai.util import paginated_get
+from cartography.models.openai.apikey import OpenAIApiKeySchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: requests.Session,
+    common_job_parameters: Dict[str, Any],
+    project_id: str,
+) -> None:
+    apikeys = get(
+        api_session,
+        common_job_parameters["BASE_URL"],
+        project_id,
+    )
+    transformed_apikeys = transform(apikeys)
+    load_apikeys(
+        neo4j_session,
+        transformed_apikeys,
+        project_id,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    api_session: requests.Session,
+    base_url: str,
+    project_id: str,
+) -> List[Dict[str, Any]]:
+    return list(
+        paginated_get(
+            api_session,
+            "{base_url}/organization/projects/{project_id}/api_keys".format(
+                base_url=base_url,
+                project_id=project_id,
+            ),
+            timeout=_TIMEOUT,
+        )
+    )
+
+
+def transform(
+    apikeys: List[Dict[str, Any]],
+) -> List[Dict[str, Any]]:
+    result: List[Dict[str, Any]] = []
+    for apikey in apikeys:
+        if apikey["owner"]["type"] == "user":
+            apikey["owner_user_id"] = apikey["owner"]["user"]["id"]
+        else:
+            apikey["owner_sa_id"] = apikey["owner"]["service_account"]["id"]
+        result.append(apikey)
+    return result
+
+
+@timeit
+def load_apikeys(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    project_id: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d OpenAI APIKey into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        OpenAIApiKeySchema(),
+        data,
+        lastupdated=update_tag,
+        project_id=project_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(OpenAIApiKeySchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/openai/projects.py

@@ -0,0 +1,97 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.openai.util import paginated_get
+from cartography.models.openai.project import OpenAIProjectSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: requests.Session,
+    common_job_parameters: Dict[str, Any],
+    ORG_ID: str,
+) -> List[Dict]:
+    projects = get(
+        api_session,
+        common_job_parameters["BASE_URL"],
+    )
+    for project in projects:
+        project["users"] = []
+        project["admins"] = []
+        for user in get_project_users(
+            api_session,
+            common_job_parameters["BASE_URL"],
+            project["id"],
+        ):
+            project["users"].append(user["id"])
+            if user["role"] == "owner":
+                project["admins"].append(user["id"])
+    load_projects(neo4j_session, projects, ORG_ID, common_job_parameters["UPDATE_TAG"])
+    cleanup(neo4j_session, common_job_parameters)
+    return projects
+
+
+@timeit
+def get(
+    api_session: requests.Session,
+    base_url: str,
+) -> List[Dict[str, Any]]:
+    return list(
+        paginated_get(
+            api_session, f"{base_url}/organization/projects", timeout=_TIMEOUT
+        )
+    )
+
+
+@timeit
+def get_project_users(
+    api_session: requests.Session,
+    base_url: str,
+    project_id: str,
+) -> List[Dict[str, Any]]:
+    return list(
+        paginated_get(
+            api_session,
+            f"{base_url}/organization/projects/{project_id}/users",
+            timeout=_TIMEOUT,
+        )
+    )
+
+
+@timeit
+def load_projects(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    ORG_ID: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d OpenAI Projects into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        OpenAIProjectSchema(),
+        data,
+        lastupdated=update_tag,
+        ORG_ID=ORG_ID,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(OpenAIProjectSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/openai/serviceaccounts.py

@@ -0,0 +1,82 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.openai.util import paginated_get
+from cartography.models.openai.serviceaccount import OpenAIServiceAccountSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: requests.Session,
+    common_job_parameters: Dict[str, Any],
+    project_id: str,
+) -> None:
+    serviceaccountss = get(
+        api_session,
+        common_job_parameters["BASE_URL"],
+        project_id,
+    )
+    load_serviceaccounts(
+        neo4j_session,
+        serviceaccountss,
+        project_id,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    api_session: requests.Session,
+    base_url: str,
+    project_id: str,
+) -> List[Dict[str, Any]]:
+    return list(
+        paginated_get(
+            api_session,
+            "{base_url}/organization/projects/{project_id}/service_accounts".format(
+                base_url=base_url,
+                project_id=project_id,
+            ),
+            timeout=_TIMEOUT,
+        )
+    )
+
+
+@timeit
+def load_serviceaccounts(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    project_id: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d OpenAI ServiceAccount into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        OpenAIServiceAccountSchema(),
+        data,
+        lastupdated=update_tag,
+        project_id=project_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(OpenAIServiceAccountSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/openai/users.py

@@ -0,0 +1,75 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.openai.util import paginated_get
+from cartography.models.openai.organization import OpenAIOrganizationSchema
+from cartography.models.openai.user import OpenAIUserSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: requests.Session,
+    common_job_parameters: Dict[str, Any],
+    ORG_ID: str,
+) -> None:
+    users = get(
+        api_session,
+        common_job_parameters["BASE_URL"],
+    )
+    load_users(neo4j_session, users, ORG_ID, common_job_parameters["UPDATE_TAG"])
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    api_session: requests.Session,
+    base_url: str,
+) -> List[Dict[str, Any]]:
+    return list(
+        paginated_get(api_session, f"{base_url}/organization/users", timeout=_TIMEOUT)
+    )
+
+
+@timeit
+def load_users(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    ORG_ID: str,
+    update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        OpenAIOrganizationSchema(),
+        [{"id": ORG_ID}],
+        lastupdated=update_tag,
+    )
+    logger.info("Loading %d OpenAI User into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        OpenAIUserSchema(),
+        data,
+        lastupdated=update_tag,
+        ORG_ID=ORG_ID,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(OpenAIUserSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/openai/util.py

@@ -0,0 +1,45 @@
+from typing import Any
+from typing import Generator
+
+import requests
+
+
+def paginated_get(
+    api_session: requests.Session,
+    url: str,
+    timeout: tuple[int, int],
+    after: str | None = None,
+) -> Generator[dict[str, Any], None, None]:
+    """Helper function to get paginated data from the OpenAI API.
+
+    This function handles the pagination of the API requests and returns
+    the results as a generator. It will continue to make requests until
+    all pages of data have been retrieved. The results are returned as a
+    list of dictionaries, where each dictionary represents a single
+    entity.
+
+    Args:
+        api_session (requests.Session): The requests session to use for making API calls.
+        url (str): The URL to make the API call to.
+        timeout (tuple[int, int]): The timeout for the API call.
+        after (str | None): The ID of the last item retrieved in the previous request.
+            If None, the first page of results will be retrieved.
+    Returns:
+        Generator[dict[str, Any], None, None]: A generator yielding dictionaries representing the results.
+    """
+    params = {"after": after} if after else {}
+    req = api_session.get(
+        url,
+        params=params,
+        timeout=timeout,
+    )
+    req.raise_for_status()
+    result = req.json()
+    yield from result.get("data", [])
+    if result.get("has_more"):
+        yield from paginated_get(
+            api_session,
+            url,
+            timeout=timeout,
+            after=result.get("last_id"),
+        )