cartography 0.102.0rc1__py3-none-any.whl → 0.103.0rc1__py3-none-any.whl

cartography/intel/okta/roles.py

@@ -26,7 +26,7 @@ def _get_user_roles(api_client: ApiClient, user_id: str, okta_org_id: str) -> st
     """
 
     # https://developer.okta.com/docs/reference/api/roles/#list-roles
-    response = api_client.get_path(f'/{user_id}/roles')
+    response = api_client.get_path(f"/{user_id}/roles")
     check_rate_limit(response)
     return response.text
 
@@ -42,7 +42,7 @@ def _get_group_roles(api_client: ApiClient, group_id: str, okta_org_id: str) ->
     """
 
     # https://developer.okta.com/docs/reference/api/roles/#list-roles-assigned-to-group
-    response = api_client.get_path(f'/{group_id}/roles')
+    response = api_client.get_path(f"/{group_id}/roles")
     check_rate_limit(response)
     return response.text
 
@@ -94,7 +94,12 @@ def transform_group_roles_data(data: str, okta_org_id: str) -> List[Dict]:
 
 
 @timeit
-def _load_user_role(neo4j_session: neo4j.Session, user_id: str, roles_data: List[Dict], okta_update_tag: int) -> None:
+def _load_user_role(
+    neo4j_session: neo4j.Session,
+    user_id: str,
+    roles_data: List[Dict],
+    okta_update_tag: int,
+) -> None:
     ingest = """
     MATCH (user:OktaUser{id: $USER_ID})<-[:RESOURCE]-(org:OktaOrganization)
     WITH user,org
@@ -122,7 +127,9 @@ def _load_user_role(neo4j_session: neo4j.Session, user_id: str, roles_data: List
 
 @timeit
 def _load_group_role(
-    neo4j_session: neo4j.Session, group_id: str, roles_data: List[Dict],
+    neo4j_session: neo4j.Session,
+    group_id: str,
+    roles_data: List[Dict],
     okta_update_tag: int,
 ) -> None:
     ingest = """
@@ -152,7 +159,10 @@ def _load_group_role(
 
 @timeit
 def sync_roles(
-    neo4j_session: str, okta_org_id: str, okta_update_tag: int, okta_api_key: str,
+    neo4j_session: str,
+    okta_org_id: str,
+    okta_update_tag: int,
+    okta_api_key: str,
     sync_state: OktaSyncState,
 ) -> None:
     """

cartography/intel/okta/users.py

@@ -12,7 +12,6 @@ from cartography.intel.okta.sync_state import OktaSyncState
 from cartography.intel.okta.utils import check_rate_limit
 from cartography.util import timeit
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -56,7 +55,9 @@ def _get_okta_users(user_client: UsersClient) -> List[Dict]:
 
 
 @timeit
-def transform_okta_user_list(okta_user_list: List[User]) -> Tuple[List[Dict], List[str]]:
+def transform_okta_user_list(
+    okta_user_list: List[User],
+) -> Tuple[List[Dict], List[str]]:
     users: List[Dict] = []
     user_ids: List[str] = []
 
@@ -91,7 +92,9 @@ def transform_okta_user(okta_user: User) -> Dict:
         user_props["activated"] = None
 
     if okta_user.statusChanged:
-        user_props["status_changed"] = okta_user.statusChanged.strftime("%m/%d/%Y, %H:%M:%S")
+        user_props["status_changed"] = okta_user.statusChanged.strftime(
+            "%m/%d/%Y, %H:%M:%S",
+        )
     else:
         user_props["status_changed"] = None
 
@@ -101,12 +104,16 @@ def transform_okta_user(okta_user: User) -> Dict:
         user_props["last_login"] = None
 
     if okta_user.lastUpdated:
-        user_props["okta_last_updated"] = okta_user.lastUpdated.strftime("%m/%d/%Y, %H:%M:%S")
+        user_props["okta_last_updated"] = okta_user.lastUpdated.strftime(
+            "%m/%d/%Y, %H:%M:%S",
+        )
     else:
         user_props["okta_last_updated"] = None
 
     if okta_user.passwordChanged:
-        user_props["password_changed"] = okta_user.passwordChanged.strftime("%m/%d/%Y, %H:%M:%S")
+        user_props["password_changed"] = okta_user.passwordChanged.strftime(
+            "%m/%d/%Y, %H:%M:%S",
+        )
     else:
         user_props["password_changed"] = None
 
@@ -120,7 +127,9 @@ def transform_okta_user(okta_user: User) -> Dict:
 
 @timeit
 def _load_okta_users(
-    neo4j_session: neo4j.Session, okta_org_id: str, user_list: List[Dict],
+    neo4j_session: neo4j.Session,
+    okta_org_id: str,
+    user_list: List[Dict],
     okta_update_tag: int,
 ) -> None:
     """
@@ -175,8 +184,11 @@ def _load_okta_users(
 
 @timeit
 def sync_okta_users(
-    neo4j_session: neo4j.Session, okta_org_id: str, okta_update_tag: int,
-    okta_api_key: str, sync_state: OktaSyncState,
+    neo4j_session: neo4j.Session,
+    okta_org_id: str,
+    okta_update_tag: int,
+    okta_api_key: str,
+    sync_state: OktaSyncState,
 ) -> None:
     """
     Sync okta users

cartography/intel/okta/utils.py

@@ -43,9 +43,9 @@ def check_rate_limit(response: Response) -> None:
     """
     rate_limit_threshold = 0.1
 
-    remaining = response.headers.get('x-rate-limit-remaining')
-    limit = response.headers.get('x-rate-limit-limit')
-    reset_time = response.headers.get('x-rate-limit-reset')
+    remaining = response.headers.get("x-rate-limit-remaining")
+    limit = response.headers.get("x-rate-limit-limit")
+    reset_time = response.headers.get("x-rate-limit-reset")
 
     if remaining and limit and reset_time:
         if (int(remaining) / int(limit)) < rate_limit_threshold:
@@ -58,5 +58,7 @@ def check_rate_limit(response: Response) -> None:
                     f"Okta API limit exceeded. Sleep time of {sleep_time_seconds} would exceed one minute. Crashing "
                     f"Okta sync to avoid blocking.",
                 )
-            logger.warning(f"Okta rate limit threshold reached. Waiting {sleep_time_seconds} seconds.")
+            logger.warning(
+                f"Okta rate limit threshold reached. Waiting {sleep_time_seconds} seconds.",
+            )
             time.sleep(sleep_time_seconds)

cartography/intel/pagerduty/__init__.py

@@ -4,9 +4,7 @@ import neo4j
 from pdpyras import APISession
 
 from cartography.config import Config
-from cartography.intel.pagerduty.escalation_policies import (
-    sync_escalation_policies,
-)
+from cartography.intel.pagerduty.escalation_policies import sync_escalation_policies
 from cartography.intel.pagerduty.schedules import sync_schedules
 from cartography.intel.pagerduty.services import sync_services
 from cartography.intel.pagerduty.teams import sync_teams
@@ -23,7 +21,8 @@ stat_handler = get_stats_client(__name__)
 
 @timeit
 def start_pagerduty_ingestion(
-    neo4j_session: neo4j.Session, config: Config,
+    neo4j_session: neo4j.Session,
+    config: Config,
 ) -> None:
     """
     Perform ingestion of pagerduty data.
@@ -35,7 +34,9 @@ def start_pagerduty_ingestion(
         "UPDATE_TAG": config.update_tag,
     }
     if not config.pagerduty_api_key:
-        logger.info('PagerDuty import is not configured - skipping this module. See docs to configure.')
+        logger.info(
+            "PagerDuty import is not configured - skipping this module. See docs to configure.",
+        )
         return
     session = APISession(config.pagerduty_api_key)
     if config.pagerduty_request_timeout is not None:
@@ -54,8 +55,8 @@ def start_pagerduty_ingestion(
 
     merge_module_sync_metadata(
         neo4j_session,
-        group_type='pagerduty',
-        group_id='module',
+        group_type="pagerduty",
+        group_id="module",
         synced_type="pagerduty",
         update_tag=config.update_tag,
         stat_handler=stat_handler,

cartography/intel/pagerduty/escalation_policies.py

@@ -31,7 +31,9 @@ def get_escalation_policies(pd_session: APISession) -> List[Dict[str, Any]]:
 
 
 def load_escalation_policy_data(
-    neo4j_session: neo4j.Session, data: List[Dict], update_tag: int,
+    neo4j_session: neo4j.Session,
+    data: List[Dict],
+    update_tag: int,
 ) -> None:
     """
     Transform and load escalation_policy information
@@ -82,7 +84,9 @@ def load_escalation_policy_data(
 
 
 def _attach_rules(
-    neo4j_session: neo4j.Session, data: List[Dict], update_tag: int,
+    neo4j_session: neo4j.Session,
+    data: List[Dict],
+    update_tag: int,
 ) -> None:
     """
     Add escalation policy rules, and attach them to targets.
@@ -122,7 +126,9 @@ def _attach_rules(
 
 
 def _attach_user_targets(
-    neo4j_session: neo4j.Session, data: List[Dict], update_tag: int,
+    neo4j_session: neo4j.Session,
+    data: List[Dict],
+    update_tag: int,
 ) -> None:
     """
     Add relationship between escalation policy and services.
@@ -142,7 +148,9 @@ def _attach_user_targets(
 
 
 def _attach_schedule_targets(
-    neo4j_session: neo4j.Session, data: List[Dict], update_tag: int,
+    neo4j_session: neo4j.Session,
+    data: List[Dict],
+    update_tag: int,
 ) -> None:
     """
     Add relationship between escalation policy and services.
@@ -162,7 +170,9 @@ def _attach_schedule_targets(
 
 
 def _attach_services(
-    neo4j_session: neo4j.Session, data: List[Dict], update_tag: int,
+    neo4j_session: neo4j.Session,
+    data: List[Dict],
+    update_tag: int,
 ) -> None:
     """
     Add relationship between escalation policy and services.
@@ -182,7 +192,9 @@ def _attach_services(
 
 
 def _attach_teams(
-    neo4j_session: neo4j.Session, data: List[Dict], update_tag: int,
+    neo4j_session: neo4j.Session,
+    data: List[Dict],
+    update_tag: int,
 ) -> None:
     """
     Add relationship between escalation policy and teams.

cartography/intel/pagerduty/schedules.py

@@ -32,7 +32,9 @@ def get_schedules(pd_session: APISession) -> List[Dict[str, Any]]:
 
 
 def load_schedule_data(
-    neo4j_session: neo4j.Session, data: List[Dict], update_tag: int,
+    neo4j_session: neo4j.Session,
+    data: List[Dict],
+    update_tag: int,
 ) -> None:
     """
     Transform and load schedule information
@@ -72,7 +74,9 @@ def load_schedule_data(
 
 
 def _attach_users(
-    neo4j_session: neo4j.Session, data: List[Dict], update_tag: int,
+    neo4j_session: neo4j.Session,
+    data: List[Dict],
+    update_tag: int,
 ) -> None:
     """
     Add relationship between schedule and users.
@@ -91,7 +95,9 @@ def _attach_users(
 
 
 def _attach_layers(
-    neo4j_session: neo4j.Session, data: List[Dict], update_tag: int,
+    neo4j_session: neo4j.Session,
+    data: List[Dict],
+    update_tag: int,
 ) -> None:
     """
     Create layers for a schedule and attach them together
@@ -133,7 +139,9 @@ def _attach_layers(
 
 
 def _attach_layer_users(
-    neo4j_session: neo4j.Session, data: List[Dict], update_tag: int,
+    neo4j_session: neo4j.Session,
+    data: List[Dict],
+    update_tag: int,
 ) -> None:
     """
     Add relationship between schedule layers and users.

cartography/intel/pagerduty/services.py

@@ -34,7 +34,8 @@ def get_services(pd_session: APISession) -> List[Dict[str, Any]]:
 
 @timeit
 def get_integrations(
-    pd_session: APISession, services: List[Dict[str, Any]],
+    pd_session: APISession,
+    services: List[Dict[str, Any]],
 ) -> List[Dict[str, Any]]:
     """
     Get integrations from services.
@@ -51,7 +52,9 @@ def get_integrations(
 
 
 def load_service_data(
-    neo4j_session: neo4j.Session, data: List[Dict], update_tag: int,
+    neo4j_session: neo4j.Session,
+    data: List[Dict],
+    update_tag: int,
 ) -> None:
     """
     Transform and load service information
@@ -104,7 +107,9 @@ def load_service_data(
 
 
 def _attach_teams(
-    neo4j_session: neo4j.Session, data: List[Dict], update_tag: int,
+    neo4j_session: neo4j.Session,
+    data: List[Dict],
+    update_tag: int,
 ) -> None:
     """
     Add relationship between teams and services.
@@ -123,7 +128,9 @@ def _attach_teams(
 
 
 def load_integration_data(
-    neo4j_session: neo4j.Session, data: List[Dict], update_tag: int,
+    neo4j_session: neo4j.Session,
+    data: List[Dict],
+    update_tag: int,
 ) -> None:
     """
     Transform and load integration information

cartography/intel/pagerduty/teams.py

@@ -33,7 +33,8 @@ def get_teams(pd_session: APISession) -> List[Dict[str, Any]]:
 
 @timeit
 def get_team_members(
-    pd_session: APISession, teams: List[Dict[str, Any]],
+    pd_session: APISession,
+    teams: List[Dict[str, Any]],
 ) -> List[Dict[str, str]]:
     relations: List[Dict[str, str]] = []
     for team in teams:
@@ -46,7 +47,9 @@ def get_team_members(
 
 
 def load_team_data(
-    neo4j_session: neo4j.Session, data: List[Dict], update_tag: int,
+    neo4j_session: neo4j.Session,
+    data: List[Dict],
+    update_tag: int,
 ) -> None:
     """
     Transform and load teamuser information
@@ -73,7 +76,9 @@ def load_team_data(
 
 
 def load_team_relations(
-    neo4j_session: neo4j.Session, data: List[Dict], update_tag: int,
+    neo4j_session: neo4j.Session,
+    data: List[Dict],
+    update_tag: int,
 ) -> None:
     """
     Attach users to their teams

cartography/intel/pagerduty/users.py

@@ -30,7 +30,9 @@ def get_users(pd_session: APISession) -> List[Dict[str, Any]]:
 
 
 def load_user_data(
-    neo4j_session: neo4j.Session, data: List[Dict], update_tag: int,
+    neo4j_session: neo4j.Session,
+    data: List[Dict],
+    update_tag: int,
 ) -> None:
     """
     Transform and load user information

cartography/intel/pagerduty/vendors.py

@@ -30,7 +30,9 @@ def get_vendors(pd_session: APISession) -> List[Dict[str, Any]]:
 
 
 def load_vendor_data(
-    neo4j_session: neo4j.Session, data: List[Dict], update_tag: int,
+    neo4j_session: neo4j.Session,
+    data: List[Dict],
+    update_tag: int,
 ) -> None:
     """
     Transform and load vendor information

cartography/intel/semgrep/__init__.py

@@ -8,23 +8,41 @@ from cartography.intel.semgrep.deployment import sync_deployment
 from cartography.intel.semgrep.findings import sync_findings
 from cartography.util import timeit
 
-
 logger = logging.getLogger(__name__)
 
 
 @timeit
 def start_semgrep_ingestion(
-    neo4j_session: neo4j.Session, config: Config,
+    neo4j_session: neo4j.Session,
+    config: Config,
 ) -> None:
     common_job_parameters = {
         "UPDATE_TAG": config.update_tag,
     }
     if not config.semgrep_app_token:
-        logger.info('Semgrep import is not configured - skipping this module. See docs to configure.')
+        logger.info(
+            "Semgrep import is not configured - skipping this module. See docs to configure.",
+        )
         return
 
     # sync_deployment must be called first since it populates common_job_parameters
     # with the deployment ID and slug, which are required by the other sync functions
-    sync_deployment(neo4j_session, config.semgrep_app_token, config.update_tag, common_job_parameters)
-    sync_dependencies(neo4j_session, config.semgrep_app_token, config.semgrep_dependency_ecosystems, config.update_tag, common_job_parameters)  # noqa: E501
-    sync_findings(neo4j_session, config.semgrep_app_token, config.update_tag, common_job_parameters)
+    sync_deployment(
+        neo4j_session,
+        config.semgrep_app_token,
+        config.update_tag,
+        common_job_parameters,
+    )
+    sync_dependencies(
+        neo4j_session,
+        config.semgrep_app_token,
+        config.semgrep_dependency_ecosystems,
+        config.update_tag,
+        common_job_parameters,
+    )  # noqa: E501
+    sync_findings(
+        neo4j_session,
+        config.semgrep_app_token,
+        config.update_tag,
+        common_job_parameters,
+    )

cartography/intel/semgrep/dependencies.py

@@ -26,30 +26,34 @@ _MAX_RETRIES = 3
 # The keys in this dictionary must be in Semgrep's list of supported ecosystems, defined here:
 # https://semgrep.dev/api/v1/docs/#tag/SupplyChainService/operation/semgrep_app.products.sca.handlers.dependency.list_dependencies_conexxion
 ECOSYSTEM_TO_SCHEMA: Dict = {
-    'gomod': SemgrepGoLibrarySchema,
-    'npm': SemgrepNpmLibrarySchema,
+    "gomod": SemgrepGoLibrarySchema,
+    "npm": SemgrepNpmLibrarySchema,
 }
 
 
 def parse_and_validate_semgrep_ecosystems(ecosystems: str) -> List[str]:
     validated_ecosystems: List[str] = []
-    for ecosystem in ecosystems.split(','):
+    for ecosystem in ecosystems.split(","):
         ecosystem = ecosystem.strip().lower()
 
         if ecosystem in ECOSYSTEM_TO_SCHEMA:
             validated_ecosystems.append(ecosystem)
         else:
-            valid_ecosystems: str = ','.join(ECOSYSTEM_TO_SCHEMA.keys())
+            valid_ecosystems: str = ",".join(ECOSYSTEM_TO_SCHEMA.keys())
             raise ValueError(
                 f'Error parsing `semgrep-dependency-ecosystems`. You specified "{ecosystems}". '
                 f'Please check that your input is formatted as comma-separated values, e.g. "gomod,npm". '
-                f'Full list of supported ecosystems: {valid_ecosystems}.',
+                f"Full list of supported ecosystems: {valid_ecosystems}.",
             )
     return validated_ecosystems
 
 
 @timeit
-def get_dependencies(semgrep_app_token: str, deployment_id: str, ecosystem: str) -> List[Dict[str, Any]]:
+def get_dependencies(
+    semgrep_app_token: str,
+    deployment_id: str,
+    ecosystem: str,
+) -> List[Dict[str, Any]]:
     """
     Gets all dependencies for the given ecosystem within the given Semgrep deployment ID.
     param: semgrep_app_token: The Semgrep App token to use for authentication.
@@ -73,14 +77,23 @@ def get_dependencies(semgrep_app_token: str, deployment_id: str, ecosystem: str)
         },
     }
 
-    logger.info(f"Retrieving Semgrep {ecosystem} dependencies for deployment '{deployment_id}'.")
+    logger.info(
+        f"Retrieving Semgrep {ecosystem} dependencies for deployment '{deployment_id}'.",
+    )
     while has_more:
         try:
-            response = requests.post(deps_url, json=request_data, headers=headers, timeout=_TIMEOUT)
+            response = requests.post(
+                deps_url,
+                json=request_data,
+                headers=headers,
+                timeout=_TIMEOUT,
+            )
             response.raise_for_status()
             data = response.json()
         except (ReadTimeout, HTTPError):
-            logger.warning(f"Failed to retrieve Semgrep {ecosystem} dependencies for page {page}. Retrying...")
+            logger.warning(
+                f"Failed to retrieve Semgrep {ecosystem} dependencies for page {page}. Retrying...",
+            )
             retries += 1
             if retries >= _MAX_RETRIES:
                 raise
@@ -93,7 +106,9 @@ def get_dependencies(semgrep_app_token: str, deployment_id: str, ecosystem: str)
         page += 1
         request_data["cursor"] = data.get("cursor")
 
-    logger.info(f"Retrieved {len(all_deps)} Semgrep {ecosystem} dependencies in {page} pages.")
+    logger.info(
+        f"Retrieved {len(all_deps)} Semgrep {ecosystem} dependencies in {page} pages.",
+    )
     return all_deps
 
 
@@ -142,19 +157,20 @@ def transform_dependencies(raw_deps: List[Dict[str, Any]]) -> List[Dict[str, Any
         # If Semgrep eventually supports version specifiers, update this line accordingly.
         specifier = f"=={version}"
 
-        deps.append({
-            # existing dependency properties:
-            "id": id,
-            "name": name,
-            "specifier": specifier,
-            "version": version,
-            "repo_url": repo_url,
-
-            # Semgrep-specific properties:
-            "ecosystem": raw_dep["ecosystem"],
-            "transitivity": raw_dep["transitivity"].lower(),
-            "url": raw_dep["definedAt"]["url"],
-        })
+        deps.append(
+            {
+                # existing dependency properties:
+                "id": id,
+                "name": name,
+                "specifier": specifier,
+                "version": version,
+                "repo_url": repo_url,
+                # Semgrep-specific properties:
+                "ecosystem": raw_dep["ecosystem"],
+                "transitivity": raw_dep["transitivity"].lower(),
+                "url": raw_dep["definedAt"]["url"],
+            },
+        )
 
     return deps
 
@@ -167,7 +183,9 @@ def load_dependencies(
     deployment_id: str,
     update_tag: int,
 ) -> None:
-    logger.info(f"Loading {len(dependencies)} {dependency_schema().label} objects into the graph.")
+    logger.info(
+        f"Loading {len(dependencies)} {dependency_schema().label} objects into the graph.",
+    )
     load(
         neo4j_session,
         dependency_schema(),
@@ -183,8 +201,12 @@ def cleanup(
     dependency_schema: Callable,
     common_job_parameters: Dict[str, Any],
 ) -> None:
-    logger.info(f"Running Semgrep Dependencies cleanup job for {dependency_schema().label}.")
-    GraphJob.from_node_schema(dependency_schema(), common_job_parameters).run(neo4j_session)
+    logger.info(
+        f"Running Semgrep Dependencies cleanup job for {dependency_schema().label}.",
+    )
+    GraphJob.from_node_schema(dependency_schema(), common_job_parameters).run(
+        neo4j_session,
+    )
 
 
 @timeit
@@ -225,9 +247,9 @@ def sync_dependencies(
 
     merge_module_sync_metadata(
         neo4j_session=neo4j_session,
-        group_type='Semgrep',
+        group_type="Semgrep",
         group_id=deployment_id,
-        synced_type='SemgrepDependency',
+        synced_type="SemgrepDependency",
         update_tag=update_tag,
         stat_handler=stat_handler,
     )

cartography/intel/semgrep/deployment.py

@@ -40,7 +40,9 @@ def get_deployment(semgrep_app_token: str) -> Dict[str, Any]:
 
 @timeit
 def load_semgrep_deployment(
-    neo4j_session: neo4j.Session, deployment: Dict[str, Any], update_tag: int,
+    neo4j_session: neo4j.Session,
+    deployment: Dict[str, Any],
+    update_tag: int,
 ) -> None:
     logger.info(f"Loading SemgrepDeployment {deployment} into the graph.")
     load(