cartography 0.90.0rc2__tar.gz → 0.92.0__tar.gz

{cartography-0.90.0rc2/cartography.egg-info → cartography-0.92.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cartography
-Version: 0.90.0rc2
+Version: 0.92.0
 Summary: Explore assets and their relationships across your technical infrastructure.
 Home-page: https://www.github.com/lyft/cartography
 Maintainer: Lyft

{cartography-0.90.0rc2 → cartography-0.92.0}/cartography/cli.py

@@ -325,6 +325,30 @@ class CLI:
             default=None,
             help='The name of an environment variable containing a password with which to authenticate to Jamf.',
         )
+        parser.add_argument(
+            '--kandji-base-uri',
+            type=str,
+            default=None,
+            help=(
+                'Your Kandji base URI, e.g. https://company.api.kandji.io.'
+                'Required if you are using the Kandji intel module. Ignored otherwise.'
+            ),
+        )
+        parser.add_argument(
+            '--kandji-tenant-id',
+            type=str,
+            default=None,
+            help=(
+                'Your Kandji tenant id e.g. company.'
+                'Required using the Kandji intel module. Ignored otherwise.'
+            ),
+        )
+        parser.add_argument(
+            '--kandji-token-env-var',
+            type=str,
+            default=None,
+            help='The name of an environment variable containing token with which to authenticate to Kandji.',
+        )
         parser.add_argument(
             '--k8s-kubeconfig',
             default=None,
@@ -620,6 +644,26 @@ class CLI:
             config.jamf_user = None
             config.jamf_password = None
 
+        # Kandji config
+        if config.kandji_base_uri:
+            if config.kandji_token_env_var:
+                logger.debug(
+                    "Reading Kandji API token from environment variable '%s'.",
+                    config.kandji_token_env_var,
+                )
+                config.kandji_token = os.environ.get(config.kandji_token_env_var)
+            elif os.environ.get('KANDJI_TOKEN'):
+                logger.debug(
+                    "Reading Kandji API token from environment variable 'KANDJI_TOKEN'.",
+                )
+                config.kandji_token = os.environ.get('KANDJI_TOKEN')
+            else:
+                logger.warning("A Kandji base URI was provided but a token was not.")
+                config.kandji_token = None
+        else:
+            logger.warning("A Kandji base URI was not provided.")
+            config.kandji_base_uri = None
+
         if config.statsd_enabled:
             logger.debug(
                 f'statsd enabled. Sending metrics to server {config.statsd_host}:{config.statsd_port}. '

{cartography-0.90.0rc2 → cartography-0.92.0}/cartography/config.py

@@ -69,6 +69,12 @@ class Config:
     :param jamf_user: User name used to authenticate to the Jamf data provider. Optional.
     :type jamf_password: string
     :param jamf_password: Password used to authenticate to the Jamf data provider. Optional.
+    :type kandji_base_uri: string
+    :param kandji_base_uri: Kandji data provider base URI, e.g. https://company.api.kandji.io. Optional.
+    :type kandji_tenant_id: string
+    :param kandji_tenant_id: Kandji tenant id. e.g. company Optional.
+    :type kandji_token: string
+    :param kandji_token: Token used to authenticate to the Kandji data provider. Optional.
     :type statsd_enabled: bool
     :param statsd_enabled: Whether to collect statsd metrics such as sync execution times. Optional.
     :type statsd_host: str
@@ -137,6 +143,9 @@ class Config:
         jamf_base_uri=None,
         jamf_user=None,
         jamf_password=None,
+        kandji_base_uri=None,
+        kandji_tenant_id=None,
+        kandji_token=None,
         k8s_kubeconfig=None,
         statsd_enabled=False,
         statsd_prefix=None,
@@ -190,6 +199,9 @@ class Config:
         self.jamf_base_uri = jamf_base_uri
         self.jamf_user = jamf_user
         self.jamf_password = jamf_password
+        self.kandji_base_uri = kandji_base_uri
+        self.kandji_tenant_id = kandji_tenant_id
+        self.kandji_token = kandji_token
         self.k8s_kubeconfig = k8s_kubeconfig
         self.statsd_enabled = statsd_enabled
         self.statsd_prefix = statsd_prefix

{cartography-0.90.0rc2 → cartography-0.92.0}/cartography/data/indexes.cypher

@@ -200,12 +200,6 @@ CREATE INDEX IF NOT EXISTS FOR (n:KMSGrant) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:LaunchConfiguration) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:LaunchConfiguration) ON (n.name);
 CREATE INDEX IF NOT EXISTS FOR (n:LaunchConfiguration) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:LaunchTemplate) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:LaunchTemplate) ON (n.name);
-CREATE INDEX IF NOT EXISTS FOR (n:LaunchTemplate) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:LaunchTemplateVersion) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:LaunchTemplateVersion) ON (n.name);
-CREATE INDEX IF NOT EXISTS FOR (n:LaunchTemplateVersion) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:LoadBalancer) ON (n.dnsname);
 CREATE INDEX IF NOT EXISTS FOR (n:LoadBalancer) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:LoadBalancer) ON (n.lastupdated);

{cartography-0.90.0rc2 → cartography-0.92.0}/cartography/intel/aws/ec2/images.py

@@ -19,23 +19,23 @@ logger = logging.getLogger(__name__)
 
 @timeit
 def get_images_in_use(neo4j_session: neo4j.Session, region: str, current_aws_account_id: str) -> List[str]:
-    # We use OPTIONAL here to allow query chaining with queries that may not match.
     get_images_query = """
-    OPTIONAL MATCH (:AWSAccount{id: $AWS_ACCOUNT_ID})-[:RESOURCE]->(i:EC2Instance)
+    MATCH (:AWSAccount{id: $AWS_ACCOUNT_ID})-[:RESOURCE]->(i:EC2Instance)
     WHERE i.region = $Region
-    WITH collect(DISTINCT i.imageid) AS images
-    OPTIONAL MATCH (:AWSAccount{id: $AWS_ACCOUNT_ID})-[:RESOURCE]->(lc:LaunchConfiguration)
+    RETURN DISTINCT(i.imageid) as image
+    UNION
+    MATCH (:AWSAccount{id: $AWS_ACCOUNT_ID})-[:RESOURCE]->(lc:LaunchConfiguration)
     WHERE lc.region = $Region
-    WITH collect(DISTINCT lc.image_id)+images AS images
-    OPTIONAL MATCH (:AWSAccount{id: $AWS_ACCOUNT_ID})-[:RESOURCE]->(ltv:LaunchTemplateVersion)
+    RETURN DISTINCT(lc.image_id) as image
+    UNION
+    MATCH (:AWSAccount{id: $AWS_ACCOUNT_ID})-[:RESOURCE]->(ltv:LaunchTemplateVersion)
     WHERE ltv.region = $Region
-    WITH collect(DISTINCT ltv.image_id)+images AS images
-    RETURN images
+    RETURN DISTINCT(ltv.image_id) as image
     """
     results = neo4j_session.run(get_images_query, AWS_ACCOUNT_ID=current_aws_account_id, Region=region)
     images = []
     for r in results:
-        images.extend(r['images'])
+        images.append(r['image'])
     return images
 
 
@@ -51,6 +51,7 @@ def get_images(boto3_session: boto3.session.Session, region: str, image_ids: Lis
         logger.warning(f"Failed retrieve images for region - {region}. Error - {e}")
     try:
         if image_ids:
+            image_ids = [image_id for image_id in image_ids if image_id is not None]
             images_in_use = client.describe_images(ImageIds=image_ids)['Images']
             # Ensure we're not adding duplicates
             _ids = [image["ImageId"] for image in images]

cartography-0.92.0/cartography/intel/aws/ec2/launch_templates.py

@@ -0,0 +1,147 @@
+import logging
+from typing import Any
+
+import boto3
+import neo4j
+
+from .util import get_botocore_config
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.aws.ec2.launch_template_versions import LaunchTemplateVersionSchema
+from cartography.models.aws.ec2.launch_templates import LaunchTemplateSchema
+from cartography.util import aws_handle_regions
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+@aws_handle_regions
+def get_launch_templates(boto3_session: boto3.session.Session, region: str) -> list[dict[str, Any]]:
+    client = boto3_session.client('ec2', region_name=region, config=get_botocore_config())
+    paginator = client.get_paginator('describe_launch_templates')
+    templates: list[dict[str, Any]] = []
+    for page in paginator.paginate():
+        templates.extend(page['LaunchTemplates'])
+    return templates
+
+
+def transform_launch_templates(templates: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    result: list[dict[str, Any]] = []
+    for template in templates:
+        current = template.copy()
+        current['CreateTime'] = str(int(current['CreateTime'].timestamp()))
+        result.append(current)
+    return result
+
+
+@timeit
+def load_launch_templates(
+        neo4j_session: neo4j.Session,
+        data: list[dict[str, Any]],
+        region: str,
+        current_aws_account_id: str,
+        update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        LaunchTemplateSchema(),
+        data,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+        lastupdated=update_tag,
+    )
+
+
+@timeit
+@aws_handle_regions
+def get_launch_template_versions(
+        boto3_session: boto3.session.Session,
+        templates: list[dict[str, Any]],
+        region: str,
+) -> list[dict[str, Any]]:
+    client = boto3_session.client('ec2', region_name=region, config=get_botocore_config())
+    v_paginator = client.get_paginator('describe_launch_template_versions')
+    template_versions = []
+    for template in templates:
+        for versions in v_paginator.paginate(LaunchTemplateId=template['LaunchTemplateId']):
+            template_versions.extend(versions['LaunchTemplateVersions'])
+    return template_versions
+
+
+def transform_launch_template_versions(versions: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    result: list[dict[str, Any]] = []
+    for version in versions:
+        current = version.copy()
+
+        # Reformat some fields
+        current['Id'] = f"{version['LaunchTemplateId']}-{version['VersionNumber']}"
+        current['CreateTime'] = str(int(version['CreateTime'].timestamp()))
+
+        # Handle the nested object returned from boto
+        ltd = version['LaunchTemplateData']
+        current['KernelId'] = ltd.get('KernelId')
+        current['EbsOptimized'] = ltd.get('EbsOptimized')
+        current['IamInstanceProfileArn'] = ltd.get('IamInstanceProfileArn')
+        current['IamInstanceProfileName'] = ltd.get('IamInstanceProfileName')
+        current['ImageId'] = ltd.get('ImageId')
+        current['InstanceType'] = ltd.get('InstanceType')
+        current['KeyName'] = ltd.get('KeyName')
+        current['MonitoringEnabled'] = ltd.get('MonitoringEnabled')
+        current['RamdiskId'] = ltd.get('RamdiskId')
+        current['DisableApiTermination'] = ltd.get('DisableApiTermination')
+        current['InstanceInitiatedShutDownBehavior'] = ltd.get('InstanceInitiatedShutDownBehavior')
+        current['SecurityGroupIds'] = ltd.get('SecurityGroupIds')
+        current['SecurityGroups'] = ltd.get('SecurityGroups')
+        result.append(current)
+    return result
+
+
+@timeit
+def load_launch_template_versions(
+        neo4j_session: neo4j.Session,
+        data: list[dict[str, Any]],
+        region: str,
+        current_aws_account_id: str,
+        update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        LaunchTemplateVersionSchema(),
+        data,
+        Region=region,
+        AWS_ID=current_aws_account_id,
+        lastupdated=update_tag,
+    )
+
+
+@timeit
+def cleanup(neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]) -> None:
+    logger.info("Running launch template cleanup job.")
+    cleanup_job = GraphJob.from_node_schema(LaunchTemplateSchema(), common_job_parameters)
+    cleanup_job.run(neo4j_session)
+
+    cleanup_job = GraphJob.from_node_schema(LaunchTemplateVersionSchema(), common_job_parameters)
+    cleanup_job.run(neo4j_session)
+
+
+@timeit
+def sync_ec2_launch_templates(
+        neo4j_session: neo4j.Session,
+        boto3_session: boto3.session.Session,
+        regions: list[str],
+        current_aws_account_id: str,
+        update_tag: int,
+        common_job_parameters: dict[str, Any],
+) -> None:
+    for region in regions:
+        logger.info(f"Syncing launch templates for region '{region}' in account '{current_aws_account_id}'.")
+        templates = get_launch_templates(boto3_session, region)
+        templates = transform_launch_templates(templates)
+        load_launch_templates(neo4j_session, templates, region, current_aws_account_id, update_tag)
+
+        versions = get_launch_template_versions(boto3_session, templates, region)
+        versions = transform_launch_template_versions(versions)
+        load_launch_template_versions(neo4j_session, versions, region, current_aws_account_id, update_tag)
+
+    cleanup(neo4j_session, common_job_parameters)

{cartography-0.90.0rc2 → cartography-0.92.0}/cartography/intel/aws/ec2/snapshots.py

@@ -42,8 +42,10 @@ def get_snapshots(boto3_session: boto3.session.Session, region: str, in_use_snap
                 snapshots.extend(page['Snapshots'])
         except ClientError as e:
             if e.response['Error']['Code'] == 'InvalidSnapshot.NotFound':
-                logger.warning(f"Failed to retrieve page of in-use, \
-                    not owned snapshots. Continuing anyway. Error - {e}")
+                logger.warning(
+                    f"Failed to retrieve page of in-use, \
+                    not owned snapshots. Continuing anyway. Error - {e}",
+                )
             else:
                 raise
 

{cartography-0.90.0rc2 → cartography-0.92.0}/cartography/intel/aws/iam.py

@@ -227,6 +227,7 @@ def get_account_access_key_data(boto3_session: boto3.session.Session, username:
         logger.warning(
             f"Could not get access key for user {username} due to NoSuchEntityException; skipping.",
         )
+        return access_keys
     for access_key in access_keys['AccessKeyMetadata']:
         access_key_id = access_key['AccessKeyId']
         last_used_info = client.get_access_key_last_used(

{cartography-0.90.0rc2 → cartography-0.92.0}/cartography/intel/github/teams.py

@@ -57,10 +57,13 @@ def _get_team_repos_for_multiple_teams(
 
         team_repos = _get_team_repos(org, api_url, token, team_name) if repo_count > 0 else None
 
-        # Shape = [(repo_url, 'WRITE'), ...]]
-        repo_urls = [t['url'] for t in team_repos.nodes] if team_repos else []
-        repo_permissions = [t['permission'] for t in team_repos.edges] if team_repos else []
+        repo_urls = []
+        repo_permissions = []
+        if team_repos:
+            repo_urls = [t['url'] for t in team_repos.nodes] if team_repos.nodes else []
+            repo_permissions = [t['permission'] for t in team_repos.edges] if team_repos.edges else []
 
+        # Shape = [(repo_url, 'WRITE'), ...]]
         result[team_name] = list(zip(repo_urls, repo_permissions))
     return result
 

{cartography-0.90.0rc2 → cartography-0.92.0}/cartography/intel/github/util.py

@@ -81,12 +81,12 @@ def call_github_api(query: str, variables: str, token: str, api_url: str) -> Dic
 
 
 def fetch_page(
-        token: str,
-        api_url: str,
-        organization: str,
-        query: str,
-        cursor: Optional[str] = None,
-        **kwargs: Any,
+    token: str,
+    api_url: str,
+    organization: str,
+    query: str,
+    cursor: Optional[str] = None,
+    **kwargs: Any,
 ) -> Dict[str, Any]:
     """
     Return a single page of max size 100 elements from the Github api_url using the given `query` and `cursor` params.
@@ -139,6 +139,7 @@ def fetch_all(
     """
     cursor = None
     has_next_page = True
+    org_data: Dict[str, Any] = {}
     data: PaginatedGraphqlData = PaginatedGraphqlData(nodes=[], edges=[])
     retry = 0
 
@@ -170,6 +171,15 @@ def fetch_all(
             time.sleep(2 ** retry)
             continue
 
+        if 'data' not in resp:
+            logger.warning(
+                f'Got no "data" attribute in response: {resp}. '
+                f'Stopping requests for organization: {organization} and '
+                f'resource_type: {resource_type}',
+            )
+            has_next_page = False
+            continue
+
         resource = resp['data']['organization'][resource_type]
         if resource_inner_type:
             resource = resp['data']['organization'][resource_type][resource_inner_type]
@@ -180,6 +190,14 @@ def fetch_all(
 
         cursor = resource['pageInfo']['endCursor']
         has_next_page = resource['pageInfo']['hasNextPage']
-
-    org_data = {'url': resp['data']['organization']['url'], 'login': resp['data']['organization']['login']}
+        if not org_data:
+            org_data = {
+                'url': resp['data']['organization']['url'],
+                'login': resp['data']['organization']['login'],
+            }
+
+    if not org_data:
+        raise ValueError(
+            f"Didn't get any organization data for organization: {organization} and resource_type: {resource_type}",
+        )
     return data, org_data

cartography-0.92.0/cartography/intel/kandji/__init__.py

@@ -0,0 +1,39 @@
+import logging
+
+import neo4j
+
+import cartography.intel.kandji.devices
+from cartography.config import Config
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def start_kandji_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
+    """
+    If this module is configured, perform ingestion of Kandji devices. Otherwise warn and exit
+
+    :param neo4j_session: Neo4J session for database interface
+    :param config: A cartography.config object
+
+    :return: None
+    """
+    if config.kandji_base_uri is None or config.kandji_token is None or config.kandji_tenant_id is None:
+        logger.warning(
+            'Required parameter(s) missing. Skipping sync.',
+            'See docs to configure.',
+        )
+        return
+
+    common_job_parameters = {
+        "UPDATE_TAG": config.update_tag,
+        "TENANT_ID": config.kandji_tenant_id,
+    }
+
+    cartography.intel.kandji.devices.sync(
+        neo4j_session,
+        config.kandji_base_uri,
+        config.kandji_token,
+        common_job_parameters=common_job_parameters,
+    )

cartography-0.92.0/cartography/intel/kandji/devices.py

@@ -0,0 +1,84 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+from requests import Session
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.kandji.device import KandjiDeviceSchema
+from cartography.models.kandji.tenant import KandjiTenantSchema
+from cartography.util import timeit
+
+
+logger = logging.getLogger(__name__)
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def get(kandji_base_uri: str, kandji_token: str) -> List[Dict[str, Any]]:
+    api_endpoint = f"{kandji_base_uri}/api/v1/devices"
+    headers = {
+        'Accept': 'application/json',
+        'Authorization': f'Bearer {kandji_token}',
+    }
+
+    session = Session()
+    req = session.get(api_endpoint, headers=headers, timeout=_TIMEOUT)
+    req.raise_for_status()
+    return req.json()
+
+
+@timeit
+def transform(api_result: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    result: List[Dict[str, Any]] = []
+    for device in api_result:
+        n_device = device
+        n_device['id'] = device['device_id']
+        result.append(n_device)
+    return result
+
+
+@timeit
+def load_devices(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+    data: List[Dict[str, Any]],
+) -> None:
+
+    tenant_id = common_job_parameters["TENANT_ID"]
+    update_tag = common_job_parameters["UPDATE_TAG"]
+
+    load(
+        neo4j_session,
+        KandjiTenantSchema(),
+        [{'id': tenant_id}],
+        lastupdated=update_tag,
+    )
+
+    load(
+        neo4j_session,
+        KandjiDeviceSchema(),
+        data,
+        lastupdated=update_tag,
+        TENANT_ID=tenant_id,
+    )
+
+
+def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None:
+    GraphJob.from_node_schema(KandjiDeviceSchema(), common_job_parameters).run(neo4j_session)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    kandji_base_uri: str,
+    kandji_token: str,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    devices = get(kandji_base_uri=kandji_base_uri, kandji_token=kandji_token)
+    formatted_devices = transform(devices)
+    load_devices(neo4j_session, common_job_parameters, formatted_devices)
+    cleanup(neo4j_session, common_job_parameters)

{cartography-0.90.0rc2 → cartography-0.92.0}/cartography/intel/okta/__init__.py

@@ -68,7 +68,7 @@ def start_okta_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
     applications.sync_okta_applications(neo4j_session, config.okta_org_id, config.update_tag, config.okta_api_key)
     factors.sync_users_factors(neo4j_session, config.okta_org_id, config.update_tag, config.okta_api_key, state)
     origins.sync_trusted_origins(neo4j_session, config.okta_org_id, config.update_tag, config.okta_api_key)
-    awssaml.sync_okta_aws_saml(neo4j_session, config.okta_saml_role_regex, config.update_tag)
+    awssaml.sync_okta_aws_saml(neo4j_session, config.okta_saml_role_regex, config.update_tag, config.okta_org_id)
 
     # need creds with permission
     # soft fail as some won't be able to get such high priv token