cartography 0.103.0rc1__py3-none-any.whl → 0.104.0__py3-none-any.whl

cartography/intel/cloudflare/dnsrecords.py

@@ -0,0 +1,64 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+from cloudflare import Cloudflare
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.cloudflare.dnsrecord import CloudflareDNSRecordSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: Cloudflare,
+    common_job_parameters: Dict[str, Any],
+    zone_id: str,
+) -> None:
+    dnsrecords = get(client, zone_id)
+    load_dnsrecords(
+        neo4j_session,
+        dnsrecords,
+        zone_id,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(client: Cloudflare, zone_id: str) -> List[Dict[str, Any]]:
+    return [
+        dnsrecord.to_dict() for dnsrecord in client.dns.records.list(zone_id=zone_id)
+    ]
+
+
+def load_dnsrecords(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    zone_id: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Cloudflare DNSRecords into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        CloudflareDNSRecordSchema(),
+        data,
+        lastupdated=update_tag,
+        zone_id=zone_id,
+    )
+
+
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(CloudflareDNSRecordSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/cloudflare/members.py

@@ -0,0 +1,75 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+from cloudflare import Cloudflare
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.cloudflare.member import CloudflareMemberSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: Cloudflare,
+    common_job_parameters: Dict[str, Any],
+    account_id: str,
+) -> None:
+    members = get(client, account_id)
+    transformed_members = transform_members(members)
+    load_members(
+        neo4j_session,
+        transformed_members,
+        account_id,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(client: Cloudflare, account_id: str) -> List[Dict[str, Any]]:
+    return [
+        member.to_dict()
+        for member in client.accounts.members.list(account_id=account_id)
+    ]
+
+
+def load_members(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    account_id: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Cloudflare members into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        CloudflareMemberSchema(),
+        data,
+        lastupdated=update_tag,
+        account_id=account_id,
+    )
+
+
+def transform_members(data: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    result: List[Dict[str, Any]] = []
+    for member in data:
+        member["roles_ids"] = [role["id"] for role in member.get("roles", [])]
+        member["policies_ids"] = [policy["id"] for policy in member.get("policies", [])]
+        result.append(member)
+    return result
+
+
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(CloudflareMemberSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/cloudflare/roles.py

@@ -0,0 +1,65 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+from cloudflare import Cloudflare
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.cloudflare.role import CloudflareRoleSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: Cloudflare,
+    common_job_parameters: Dict[str, Any],
+    account_id: str,
+) -> None:
+    roles = get(client, account_id)
+    load_roles(
+        neo4j_session,
+        roles,
+        account_id,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    client: Cloudflare,
+    account_id: str,
+) -> List[Dict[str, Any]]:
+    return [
+        role.to_dict() for role in client.accounts.roles.list(account_id=account_id)
+    ]
+
+
+def load_roles(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    account_id: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Cloudflare roles into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        CloudflareRoleSchema(),
+        data,
+        lastupdated=update_tag,
+        account_id=account_id,
+    )
+
+
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(CloudflareRoleSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/cloudflare/zones.py

@@ -0,0 +1,64 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+from cloudflare import Cloudflare
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.cloudflare.zone import CloudflareZoneSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    client: Cloudflare,
+    common_job_parameters: Dict[str, Any],
+    account_id: str,
+) -> List[Dict]:
+    zones = get(client, account_id)
+    load_zones(
+        neo4j_session,
+        zones,
+        account_id,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+    return zones
+
+
+@timeit
+def get(
+    client: Cloudflare,
+    account_id: str,
+) -> List[Dict[str, Any]]:
+    return [zone.to_dict() for zone in client.zones.list(account=account_id)]
+
+
+def load_zones(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    account_id: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Cloudflare zones into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        CloudflareZoneSchema(),
+        data,
+        lastupdated=update_tag,
+        account_id=account_id,
+    )
+
+
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(CloudflareZoneSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/entra/ou.py

@@ -22,12 +22,28 @@ async def get_entra_ous(client: GraphServiceClient) -> list[AdministrativeUnit]:
     Get all OUs from Microsoft Graph API with pagination support
     """
     all_units: list[AdministrativeUnit] = []
-    request = client.directory.administrative_units.request()
 
-    while request:
-        response = await request.get()
-        all_units.extend(response.value)
-        request = response.odata_next_link if response.odata_next_link else None
+    # Initialize first page request
+    current_request = client.directory.administrative_units
+
+    while current_request:
+        try:
+            response = await current_request.get()
+            if response and response.value:
+                all_units.extend(response.value)
+
+                # Handle next page using OData link
+                if response.odata_next_link:
+                    current_request = client.directory.administrative_units.with_url(
+                        response.odata_next_link
+                    )
+                else:
+                    current_request = None
+            else:
+                current_request = None
+        except Exception as e:
+            logger.error(f"Failed to retrieve administrative units: {str(e)}")
+            current_request = None
 
     return all_units
 

cartography/intel/openai/__init__.py

@@ -0,0 +1,86 @@
+import logging
+
+import neo4j
+import requests
+
+import cartography.intel.openai.adminapikeys
+import cartography.intel.openai.apikeys
+import cartography.intel.openai.projects
+import cartography.intel.openai.serviceaccounts
+import cartography.intel.openai.users
+from cartography.config import Config
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def start_openai_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
+    """
+    If this module is configured, perform ingestion of OpenAI data. Otherwise warn and exit
+    :param neo4j_session: Neo4J session for database interface
+    :param config: A cartography.config object
+    :return: None
+    """
+
+    if not config.openai_apikey or not config.openai_org_id:
+        logger.info(
+            "OpenAI import is not configured - skipping this module. "
+            "See docs to configure.",
+        )
+        return
+
+    # Create requests sessions
+    api_session = requests.session()
+    api_session.headers.update(
+        {
+            "Authorization": f"Bearer {config.openai_apikey}",
+            "OpenAI-Organization": config.openai_org_id,
+        }
+    )
+
+    common_job_parameters = {
+        "UPDATE_TAG": config.update_tag,
+        "BASE_URL": "https://api.openai.com/v1",
+        "ORG_ID": config.openai_org_id,
+    }
+
+    # Organization node is created during the users sync
+    cartography.intel.openai.users.sync(
+        neo4j_session,
+        api_session,
+        common_job_parameters,
+        ORG_ID=config.openai_org_id,
+    )
+
+    for project in cartography.intel.openai.projects.sync(
+        neo4j_session,
+        api_session,
+        common_job_parameters,
+        ORG_ID=config.openai_org_id,
+    ):
+        project_job_parameters = {
+            "UPDATE_TAG": config.update_tag,
+            "BASE_URL": "https://api.openai.com/v1",
+            "ORG_ID": config.openai_org_id,
+            "project_id": project["id"],
+        }
+        cartography.intel.openai.serviceaccounts.sync(
+            neo4j_session,
+            api_session,
+            project_job_parameters,
+            project_id=project["id"],
+        )
+        cartography.intel.openai.apikeys.sync(
+            neo4j_session,
+            api_session,
+            project_job_parameters,
+            project_id=project["id"],
+        )
+
+    cartography.intel.openai.adminapikeys.sync(
+        neo4j_session,
+        api_session,
+        common_job_parameters,
+        ORG_ID=config.openai_org_id,
+    )

cartography/intel/openai/adminapikeys.py

@@ -0,0 +1,89 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.openai.util import paginated_get
+from cartography.models.openai.adminapikey import OpenAIAdminApiKeySchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: requests.Session,
+    common_job_parameters: Dict[str, Any],
+    ORG_ID: str,
+) -> None:
+    adminapikeys = get(
+        api_session,
+        common_job_parameters["BASE_URL"],
+    )
+    transformed_adminapikeys = transform(adminapikeys)
+    load_adminapikeys(
+        neo4j_session,
+        transformed_adminapikeys,
+        ORG_ID,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    api_session: requests.Session,
+    base_url: str,
+) -> List[Dict[str, Any]]:
+    return list(
+        paginated_get(
+            api_session, f"{base_url}/organization/admin_api_keys", timeout=_TIMEOUT
+        )
+    )
+
+
+def transform(
+    adminapikeys: List[Dict[str, Any]],
+) -> List[Dict[str, Any]]:
+    result: List[Dict[str, Any]] = []
+    for adminapikey in adminapikeys:
+        if adminapikey["owner"]["type"] == "user":
+            adminapikey["owner_user_id"] = adminapikey["owner"]["id"]
+        else:
+            adminapikey["owner_sa_id"] = adminapikey["owner"]["id"]
+        result.append(adminapikey)
+    return result
+
+
+@timeit
+def load_adminapikeys(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    ORG_ID: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d OpenAI AdminApiKey into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        OpenAIAdminApiKeySchema(),
+        data,
+        lastupdated=update_tag,
+        ORG_ID=ORG_ID,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(OpenAIAdminApiKeySchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/openai/apikeys.py

@@ -0,0 +1,96 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.openai.util import paginated_get
+from cartography.models.openai.apikey import OpenAIApiKeySchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: requests.Session,
+    common_job_parameters: Dict[str, Any],
+    project_id: str,
+) -> None:
+    apikeys = get(
+        api_session,
+        common_job_parameters["BASE_URL"],
+        project_id,
+    )
+    transformed_apikeys = transform(apikeys)
+    load_apikeys(
+        neo4j_session,
+        transformed_apikeys,
+        project_id,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    api_session: requests.Session,
+    base_url: str,
+    project_id: str,
+) -> List[Dict[str, Any]]:
+    return list(
+        paginated_get(
+            api_session,
+            "{base_url}/organization/projects/{project_id}/api_keys".format(
+                base_url=base_url,
+                project_id=project_id,
+            ),
+            timeout=_TIMEOUT,
+        )
+    )
+
+
+def transform(
+    apikeys: List[Dict[str, Any]],
+) -> List[Dict[str, Any]]:
+    result: List[Dict[str, Any]] = []
+    for apikey in apikeys:
+        if apikey["owner"]["type"] == "user":
+            apikey["owner_user_id"] = apikey["owner"]["user"]["id"]
+        else:
+            apikey["owner_sa_id"] = apikey["owner"]["service_account"]["id"]
+        result.append(apikey)
+    return result
+
+
+@timeit
+def load_apikeys(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    project_id: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d OpenAI APIKey into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        OpenAIApiKeySchema(),
+        data,
+        lastupdated=update_tag,
+        project_id=project_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(OpenAIApiKeySchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/openai/projects.py

@@ -0,0 +1,97 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.openai.util import paginated_get
+from cartography.models.openai.project import OpenAIProjectSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: requests.Session,
+    common_job_parameters: Dict[str, Any],
+    ORG_ID: str,
+) -> List[Dict]:
+    projects = get(
+        api_session,
+        common_job_parameters["BASE_URL"],
+    )
+    for project in projects:
+        project["users"] = []
+        project["admins"] = []
+        for user in get_project_users(
+            api_session,
+            common_job_parameters["BASE_URL"],
+            project["id"],
+        ):
+            project["users"].append(user["id"])
+            if user["role"] == "owner":
+                project["admins"].append(user["id"])
+    load_projects(neo4j_session, projects, ORG_ID, common_job_parameters["UPDATE_TAG"])
+    cleanup(neo4j_session, common_job_parameters)
+    return projects
+
+
+@timeit
+def get(
+    api_session: requests.Session,
+    base_url: str,
+) -> List[Dict[str, Any]]:
+    return list(
+        paginated_get(
+            api_session, f"{base_url}/organization/projects", timeout=_TIMEOUT
+        )
+    )
+
+
+@timeit
+def get_project_users(
+    api_session: requests.Session,
+    base_url: str,
+    project_id: str,
+) -> List[Dict[str, Any]]:
+    return list(
+        paginated_get(
+            api_session,
+            f"{base_url}/organization/projects/{project_id}/users",
+            timeout=_TIMEOUT,
+        )
+    )
+
+
+@timeit
+def load_projects(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    ORG_ID: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d OpenAI Projects into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        OpenAIProjectSchema(),
+        data,
+        lastupdated=update_tag,
+        ORG_ID=ORG_ID,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(OpenAIProjectSchema(), common_job_parameters).run(
+        neo4j_session
+    )