cartography 0.106.0rc2__py3-none-any.whl → 0.107.0rc2__py3-none-any.whl

cartography/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.106.0rc2'
-__version_tuple__ = version_tuple = (0, 106, 0, 'rc2')
+__version__ = version = '0.107.0rc2'
+__version_tuple__ = version_tuple = (0, 107, 0, 'rc2')

cartography/cli.py

@@ -71,8 +71,8 @@ class CLI:
             default="bolt://localhost:7687",
             help=(
                 "A valid Neo4j URI to sync against. See "
-                "https://neo4j.com/docs/api/python-driver/current/driver.html#uri for complete documentation on the "
-                "structure of a Neo4j URI."
+                "https://neo4j.com/docs/browser-manual/current/operations/dbms-connection/#uri-scheme for complete "
+                "documentation on the structure of a Neo4j URI."
             ),
         )
         parser.add_argument(
@@ -182,6 +182,14 @@ class CLI:
                 "syncing other accounts and delay raising an exception until the very end."
             ),
         )
+        parser.add_argument(
+            "--aws-cloudtrail-management-events-lookback-hours",
+            type=int,
+            default=None,
+            help=(
+                "Number of hours back to retrieve CloudTrail management events from. If not specified, CloudTrail management events will not be retrieved."
+            ),
+        )
         parser.add_argument(
             "--oci-sync-all-profiles",
             action="store_true",
@@ -637,6 +645,33 @@ class CLI:
                 "Required if you are using the Anthropic intel module. Ignored otherwise."
             ),
         )
+        parser.add_argument(
+            "--airbyte-client-id",
+            type=str,
+            default=None,
+            help=(
+                "The Airbyte client ID to use for authentication. "
+                "Required if you are using the Airbyte intel module. Ignored otherwise."
+            ),
+        )
+        parser.add_argument(
+            "--airbyte-client-secret-env-var",
+            type=str,
+            default=None,
+            help=(
+                "The name of an environment variable containing the Airbyte client secret for authentication. "
+                "Required if you are using the Airbyte intel module. Ignored otherwise."
+            ),
+        )
+        parser.add_argument(
+            "--airbyte-api-url",
+            type=str,
+            default="https://api.airbyte.com/v1",
+            help=(
+                "The base URL for the Airbyte API (default is the public Airbyte Cloud API). "
+                "Required if you are using the Airbyte intel module. Ignored otherwise."
+            ),
+        )
         parser.add_argument(
             "--trivy-s3-bucket",
             type=str,
@@ -655,6 +690,59 @@ class CLI:
                 "Required if you are using the Trivy module. Ignored otherwise."
             ),
         )
+        parser.add_argument(
+            "--scaleway-org",
+            type=str,
+            default=None,
+            help=(
+                "The Scaleway organization ID to sync. "
+                "Required if you are using the Scaleway intel module. Ignored otherwise."
+            ),
+        )
+        parser.add_argument(
+            "--scaleway-access-key",
+            type=str,
+            default=None,
+            help=(
+                "The Scaleway access key to use for authentication. "
+                "Required if you are using the Scaleway intel module. Ignored otherwise."
+            ),
+        )
+        parser.add_argument(
+            "--scaleway-secret-key-env-var",
+            type=str,
+            default=None,
+            help=(
+                "The name of an environment variable containing the Scaleway secret key for authentication. "
+                "Required if you are using the Scaleway intel module. Ignored otherwise."
+            ),
+        )
+        parser.add_argument(
+            "--sentinelone-account-ids",
+            type=str,
+            default=None,
+            help=(
+                "Comma-separated list of SentinelOne account IDs to sync. "
+                "If not specified, all accessible accounts will be synced."
+            ),
+        )
+        parser.add_argument(
+            "--sentinelone-api-url",
+            type=str,
+            default=None,
+            help=(
+                "SentinelOne API URL. Required if you are using the SentinelOne intel module. Ignored otherwise."
+            ),
+        )
+        parser.add_argument(
+            "--sentinelone-api-token-env-var",
+            type=str,
+            default="SENTINELONE_API_TOKEN",
+            help=(
+                "The name of an environment variable containing the SentinelOne API token. "
+                "Required if you are using the SentinelOne intel module. Ignored otherwise."
+            ),
+        )
 
         return parser
 
@@ -973,6 +1061,17 @@ class CLI:
         else:
             config.anthropic_apikey = None
 
+        # Airbyte config
+        if config.airbyte_client_id and config.airbyte_client_secret_env_var:
+            logger.debug(
+                f"Reading Airbyte client secret from environment variable {config.airbyte_client_secret_env_var}",
+            )
+            config.airbyte_client_secret = os.environ.get(
+                config.airbyte_client_secret_env_var,
+            )
+        else:
+            config.airbyte_client_secret = None
+
         # Trivy config
         if config.trivy_s3_bucket:
             logger.debug(f"Trivy S3 bucket: {config.trivy_s3_bucket}")
@@ -980,6 +1079,36 @@ class CLI:
         if config.trivy_s3_prefix:
             logger.debug(f"Trivy S3 prefix: {config.trivy_s3_prefix}")
 
+        # Scaleway config
+        if config.scaleway_secret_key_env_var:
+            logger.debug(
+                f"Reading Scaleway secret key from environment variable {config.scaleway_secret_key_env_var}",
+            )
+            config.scaleway_secret_key = os.environ.get(
+                config.scaleway_secret_key_env_var,
+            )
+        else:
+            config.scaleway_secret_key = None
+
+        # SentinelOne config
+        if config.sentinelone_account_ids:
+            config.sentinelone_account_ids = [
+                id.strip() for id in config.sentinelone_account_ids.split(",")
+            ]
+            logger.debug(
+                f"Parsed {len(config.sentinelone_account_ids)} SentinelOne account IDs to sync"
+            )
+        else:
+            config.sentinelone_account_ids = None
+
+        if config.sentinelone_api_url and config.sentinelone_api_token_env_var:
+            logger.debug(
+                f"Reading API token for SentinelOne from environment variable {config.sentinelone_api_token_env_var}",
+            )
+            config.sentinelone_api_token = os.environ.get(
+                config.sentinelone_api_token_env_var
+            )
+
         # Run cartography
         try:
             return cartography.sync.run_with_config(self.sync, config)

cartography/config.py

@@ -31,6 +31,8 @@ class Config:
     :type aws_best_effort_mode: bool
     :param aws_best_effort_mode: If True, AWS sync will not raise any exceptions, just log. If False (default),
         exceptions will be raised.
+    :type aws_cloudtrail_management_events_lookback_hours: int
+    :param aws_cloudtrail_management_events_lookback_hours: Number of hours back to retrieve CloudTrail management events from. Optional.
     :type azure_sync_all_subscriptions: bool
     :param azure_sync_all_subscriptions: If True, Azure sync will run for all profiles in azureProfile.json. If
         False (default), Azure sync will run using current user session via CLI credentials. Optional.
@@ -137,10 +139,28 @@ class Config:
     :param openai_org_id: OpenAI organization id. Optional.
     :type anthropic_apikey: string
     :param anthropic_apikey: Anthropic API key. Optional.
+    :type airbyte_client_id: str
+    :param airbyte_client_id: Airbyte client ID for API authentication. Optional.
+    :type airbyte_client_secret: str
+    :param airbyte_client_secret: Airbyte client secret for API authentication. Optional.
+    :type airbyte_api_url: str
+    :param airbyte_api_url: Airbyte API base URL, e.g. https://api.airbyte.com/v1. Optional.
     :type trivy_s3_bucket: str
     :param trivy_s3_bucket: The S3 bucket name containing Trivy scan results. Optional.
     :type trivy_s3_prefix: str
     :param trivy_s3_prefix: The S3 prefix path containing Trivy scan results. Optional.
+    :type scaleway_access_key: str
+    :param scaleway_access_key: Scaleway access key. Optional.
+    :type scaleway_secret_key: str
+    :param scaleway_secret_key: Scaleway secret key. Optional.
+    :type scaleway_org: str
+    :param scaleway_org: Scaleway organization id. Optional.
+    :type sentinelone_api_url: string
+    :param sentinelone_api_url: SentinelOne API URL. Optional.
+    :type sentinelone_api_token: string
+    :param sentinelone_api_token: SentinelOne API token for authentication. Optional.
+    :type sentinelone_account_ids: list[str]
+    :param sentinelone_account_ids: List of SentinelOne account IDs to sync. Optional.
     """
 
     def __init__(
@@ -155,6 +175,7 @@ class Config:
         aws_sync_all_profiles=False,
         aws_regions=None,
         aws_best_effort_mode=False,
+        aws_cloudtrail_management_events_lookback_hours=None,
         azure_sync_all_subscriptions=False,
         azure_sp_auth=None,
         azure_tenant_id=None,
@@ -213,8 +234,17 @@ class Config:
         openai_apikey=None,
         openai_org_id=None,
         anthropic_apikey=None,
+        airbyte_client_id=None,
+        airbyte_client_secret=None,
+        airbyte_api_url=None,
         trivy_s3_bucket=None,
         trivy_s3_prefix=None,
+        scaleway_access_key=None,
+        scaleway_secret_key=None,
+        scaleway_org=None,
+        sentinelone_api_url=None,
+        sentinelone_api_token=None,
+        sentinelone_account_ids=None,
     ):
         self.neo4j_uri = neo4j_uri
         self.neo4j_user = neo4j_user
@@ -226,6 +256,9 @@ class Config:
         self.aws_sync_all_profiles = aws_sync_all_profiles
         self.aws_regions = aws_regions
         self.aws_best_effort_mode = aws_best_effort_mode
+        self.aws_cloudtrail_management_events_lookback_hours = (
+            aws_cloudtrail_management_events_lookback_hours
+        )
         self.azure_sync_all_subscriptions = azure_sync_all_subscriptions
         self.azure_sp_auth = azure_sp_auth
         self.azure_tenant_id = azure_tenant_id
@@ -284,5 +317,14 @@ class Config:
         self.openai_apikey = openai_apikey
         self.openai_org_id = openai_org_id
         self.anthropic_apikey = anthropic_apikey
+        self.airbyte_client_id = airbyte_client_id
+        self.airbyte_client_secret = airbyte_client_secret
+        self.airbyte_api_url = airbyte_api_url
         self.trivy_s3_bucket = trivy_s3_bucket
         self.trivy_s3_prefix = trivy_s3_prefix
+        self.scaleway_access_key = scaleway_access_key
+        self.scaleway_secret_key = scaleway_secret_key
+        self.scaleway_org = scaleway_org
+        self.sentinelone_api_url = sentinelone_api_url
+        self.sentinelone_api_token = sentinelone_api_token
+        self.sentinelone_account_ids = sentinelone_account_ids

cartography/driftdetect/cli.py

@@ -63,8 +63,9 @@ class CLI:
             default="bolt://localhost:7687",
             help=(
                 "A valid Neo4j URI to sync against. See "
-                "https://neo4j.com/docs/api/python-driver/current/driver.html#uri for complete documentation on the "
-                "structure of a Neo4j URI."
+                "https://neo4j.com/docs/browser-manual/current/operations/dbms-connection/#uri-scheme for "
+                "documentation on the structure of a Neo4j URI, and "
+                "https://neo4j.com/docs/api/python-driver/current/ for complete documentation on the Python driver."
             ),
         )
         parser_get_state.add_argument(

cartography/intel/airbyte/__init__.py

@@ -0,0 +1,105 @@
+import logging
+
+import neo4j
+
+import cartography.intel.airbyte.connections
+import cartography.intel.airbyte.destinations
+import cartography.intel.airbyte.organizations
+import cartography.intel.airbyte.sources
+import cartography.intel.airbyte.tags
+import cartography.intel.airbyte.users
+import cartography.intel.airbyte.workspaces
+from cartography.config import Config
+from cartography.intel.airbyte.util import AirbyteClient
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def start_airbyte_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
+    """
+    If this module is configured, perform ingestion of Airbyte data. Otherwise warn and exit
+    :param neo4j_session: Neo4J session for database interface
+    :param config: A cartography.config object
+    :return: None
+    """
+    if (
+        not config.airbyte_api_url
+        or not config.airbyte_client_id
+        or not config.airbyte_client_secret
+    ):
+        logger.info(
+            "Airbyte import is not configured - skipping this module. "
+            "See docs to configure.",
+        )
+        return
+
+    # Create api session
+    api_client = AirbyteClient(
+        base_url=config.airbyte_api_url,
+        client_id=config.airbyte_client_id,
+        client_secret=config.airbyte_client_secret,
+    )
+
+    common_job_parameters = {
+        "UPDATE_TAG": config.update_tag,
+    }
+
+    organizations = cartography.intel.airbyte.organizations.sync(
+        neo4j_session,
+        api_client,
+        common_job_parameters,
+    )
+
+    for organization in organizations:
+        org_common_job_parameters = {
+            "UPDATE_TAG": config.update_tag,
+            "ORG_ID": organization["organizationId"],
+        }
+        workspaces = cartography.intel.airbyte.workspaces.sync(
+            neo4j_session,
+            api_client,
+            organization["organizationId"],
+            org_common_job_parameters,
+        )
+        workspace_ids = [workspace["workspaceId"] for workspace in workspaces]
+
+        cartography.intel.airbyte.users.sync(
+            neo4j_session,
+            api_client,
+            organization["organizationId"],
+            org_common_job_parameters,
+        )
+
+        cartography.intel.airbyte.sources.sync(
+            neo4j_session,
+            api_client,
+            organization["organizationId"],
+            workspace_ids,
+            org_common_job_parameters,
+        )
+
+        cartography.intel.airbyte.destinations.sync(
+            neo4j_session,
+            api_client,
+            organization["organizationId"],
+            workspace_ids,
+            org_common_job_parameters,
+        )
+
+        cartography.intel.airbyte.tags.sync(
+            neo4j_session,
+            api_client,
+            organization["organizationId"],
+            workspace_ids,
+            org_common_job_parameters,
+        )
+
+        cartography.intel.airbyte.connections.sync(
+            neo4j_session,
+            api_client,
+            organization["organizationId"],
+            workspace_ids,
+            org_common_job_parameters,
+        )

cartography/intel/airbyte/connections.py

@@ -0,0 +1,120 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+from typing import Tuple
+
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.airbyte.util import AirbyteClient
+from cartography.intel.airbyte.util import list_to_string
+from cartography.models.airbyte.connection import AirbyteConnectionSchema
+from cartography.models.airbyte.stream import AirbyteStreamSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: AirbyteClient,
+    org_id: str,
+    workspace_ids: List[str],
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    connections = get(api_session, workspace_ids)
+    transformed_connections, transformed_streams = transform(connections)
+    load_connections(
+        neo4j_session,
+        transformed_connections,
+        org_id,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    load_streams(
+        neo4j_session, transformed_streams, org_id, common_job_parameters["UPDATE_TAG"]
+    )
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    api_session: AirbyteClient,
+    workspace_ids: List[str],
+) -> List[Dict[str, Any]]:
+    return api_session.get(
+        "/connections",
+        {"workspaceIds": ",".join(workspace_ids)} if workspace_ids else None,
+    )
+
+
+def transform(
+    connections: List[Dict[str, Any]],
+) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
+    transformed_connections = []
+    transformed_streams = []
+    for connection in connections:
+        connection["tags_ids"] = [tag["tagId"] for tag in connection.get("tags", [])]
+        transformed_connections.append(connection)
+        for stream in connection.get("configurations", {}).get("streams", []):
+            formated_stream = {
+                "connectionId": connection["connectionId"],
+                "streamId": f"{connection['connectionId']}_{stream['name']}",
+                "name": stream["name"],
+                "syncMode": stream["syncMode"],
+                "cursorField": list_to_string(stream.get("cursorField", [])),
+                "primaryKey": list_to_string(stream.get("primaryKey", [])),
+                "includeFiles": stream.get("includeFiles", False),
+                "selectedFields": list_to_string(stream.get("selectedFields", [])),
+                "mappers": list_to_string(stream.get("mappers", [])),
+            }
+            transformed_streams.append(formated_stream)
+    return transformed_connections, transformed_streams
+
+
+@timeit
+def load_connections(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    org_id: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Airbyte Connections into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        AirbyteConnectionSchema(),
+        data,
+        lastupdated=update_tag,
+        ORG_ID=org_id,
+    )
+
+
+@timeit
+def load_streams(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    org_id: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Airbyte Streams into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        AirbyteStreamSchema(),
+        data,
+        lastupdated=update_tag,
+        ORG_ID=org_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(AirbyteStreamSchema(), common_job_parameters).run(
+        neo4j_session
+    )
+    GraphJob.from_node_schema(AirbyteConnectionSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/airbyte/destinations.py

@@ -0,0 +1,81 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.airbyte.util import AirbyteClient
+from cartography.intel.airbyte.util import normalize_airbyte_config
+from cartography.models.airbyte.destination import AirbyteDestinationSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: AirbyteClient,
+    org_id: str,
+    workspace_ids: List[str],
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    destinations = get(api_session, workspace_ids)
+    transformed_destinations = transform(destinations)
+    load_destinations(
+        neo4j_session,
+        transformed_destinations,
+        org_id,
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    api_session: AirbyteClient,
+    workspace_ids: List[str],
+) -> List[Dict[str, Any]]:
+    return api_session.get(
+        "/destinations",
+        params={"workspaceIds": ",".join(workspace_ids)} if workspace_ids else None,
+    )
+
+
+def transform(destinations: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    transformed_destinations = []
+    for destination in destinations:
+        destination["configuration"] = normalize_airbyte_config(
+            destination.get("configuration", {})
+        )
+        transformed_destinations.append(destination)
+    return transformed_destinations
+
+
+@timeit
+def load_destinations(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    org_id: str,
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Airbyte Destinations into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        AirbyteDestinationSchema(),
+        data,
+        lastupdated=update_tag,
+        ORG_ID=org_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(AirbyteDestinationSchema(), common_job_parameters).run(
+        neo4j_session
+    )

cartography/intel/airbyte/organizations.py

@@ -0,0 +1,59 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.airbyte.util import AirbyteClient
+from cartography.models.airbyte.organization import AirbyteOrganizationSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: AirbyteClient,
+    common_job_parameters: Dict[str, Any],
+) -> List[Dict]:
+    organizations = get(api_session)
+    load_organizations(
+        neo4j_session, organizations, common_job_parameters["UPDATE_TAG"]
+    )
+    cleanup(neo4j_session, common_job_parameters)
+    return organizations
+
+
+@timeit
+def get(
+    api_session: AirbyteClient,
+) -> List[Dict[str, Any]]:
+    return api_session.get("/organizations")
+
+
+@timeit
+def load_organizations(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    update_tag: int,
+) -> None:
+    logger.info("Loading %d Airbyte Organizations into Neo4j.", len(data))
+    load(
+        neo4j_session,
+        AirbyteOrganizationSchema(),
+        data,
+        lastupdated=update_tag,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(AirbyteOrganizationSchema(), common_job_parameters).run(
+        neo4j_session
+    )