cartography 0.110.0rc1__py3-none-any.whl → 0.111.0__py3-none-any.whl

cartography/intel/github/repos.py

@@ -41,12 +41,12 @@ UserAffiliationAndRepoPermission = namedtuple(
 
 
 GITHUB_ORG_REPOS_PAGINATED_GRAPHQL = """
-    query($login: String!, $cursor: String) {
+    query($login: String!, $cursor: String, $count: Int!) {
     organization(login: $login)
         {
             url
             login
-            repositories(first: 50, after: $cursor){
+            repositories(first: $count, after: $cursor){
                 pageInfo{
                     endCursor
                     hasNextPage
@@ -168,14 +168,22 @@ def _get_repo_collaborators_inner_func(
         repo_name = repo["name"]
         repo_url = repo["url"]
 
-        if (
-            affiliation == "OUTSIDE" and repo["outsideCollaborators"]["totalCount"] == 0
-        ) or (
-            affiliation == "DIRECT" and repo["directCollaborators"]["totalCount"] == 0
-        ):
-            # repo has no collabs of the affiliation type we're looking for, so don't waste time making an API call
-            result[repo_url] = []
-            continue
+        # Guard against None when collaborator fields are not accessible due to permissions.
+        direct_info = repo.get("directCollaborators")
+        outside_info = repo.get("outsideCollaborators")
+
+        if affiliation == "OUTSIDE":
+            total_outside = 0 if not outside_info else outside_info.get("totalCount", 0)
+            if total_outside == 0:
+                # No outside collaborators or not permitted to view; skip API calls for this repo.
+                result[repo_url] = []
+                continue
+        else:  # DIRECT
+            total_direct = 0 if not direct_info else direct_info.get("totalCount", 0)
+            if total_direct == 0:
+                # No direct collaborators or not permitted to view; skip API calls for this repo.
+                result[repo_url] = []
+                continue
 
         logger.info(f"Loading {affiliation} collaborators for repo {repo_name}.")
         collaborators = _get_repo_collaborators(
@@ -290,6 +298,7 @@ def get(token: str, api_url: str, organization: str) -> List[Dict]:
         organization,
         GITHUB_ORG_REPOS_PAGINATED_GRAPHQL,
         "repositories",
+        count=50,
     )
     return repos.nodes
 
@@ -405,9 +414,16 @@ def _create_default_branch_id(repo_url: str, default_branch_ref_id: str) -> str:
 
 def _create_git_url_from_ssh_url(ssh_url: str) -> str:
     """
-    Return a git:// URL from the given ssh_url
+    Convert SSH URL to git:// URL.
+    Example:
+        git@github.com:cartography-cncf/cartography.git
+        -> git://github.com/cartography-cncf/cartography.git
     """
-    return ssh_url.replace("/", ":").replace("git@", "git://")
+    # Remove the user part (e.g., "git@")
+    _, host_and_path = ssh_url.split("@", 1)
+    # Replace first ':' (separating host and repo) with '/'
+    host, path = host_and_path.split(":", 1)
+    return f"git://{host}/{path}"
 
 
 def _transform_repo_objects(input_repo_object: Dict, out_repo_list: List[Dict]) -> None:
@@ -647,9 +663,6 @@ def _transform_dependency_graph(
             requirements = dep.get("requirements", "")
             package_manager = dep.get("packageManager", "").upper()
 
-            # Extract version from requirements string if available
-            pinned_version = _extract_version_from_requirements(requirements)
-
             # Create ecosystem-specific canonical name
             canonical_name = _canonicalize_dependency_name(
                 package_name, package_manager
@@ -658,11 +671,12 @@ def _transform_dependency_graph(
             # Create ecosystem identifier
             ecosystem = package_manager.lower() if package_manager else "unknown"
 
-            # Create simple dependency ID using canonical name and version
+            # Create simple dependency ID using canonical name and requirements
             # This allows the same dependency to be shared across multiple repos
+            requirements_for_id = (requirements or "").strip()
             dependency_id = (
-                f"{canonical_name}|{pinned_version}"
-                if pinned_version
+                f"{canonical_name}|{requirements_for_id}"
+                if requirements_for_id
                 else canonical_name
             )
 
@@ -677,15 +691,12 @@ def _transform_dependency_graph(
                     "id": dependency_id,
                     "name": canonical_name,
                     "original_name": package_name,  # Keep original for reference
-                    "version": pinned_version,
                     "requirements": normalized_requirements,
                     "ecosystem": ecosystem,
                     "package_manager": package_manager,
                     "manifest_path": manifest_path,
                     "manifest_id": manifest_id,
                     "repo_url": repo_url,
-                    # Add separate fields for easier querying
-                    "repo_name": repo_url.split("/")[-1] if repo_url else "",
                     "manifest_file": (
                         manifest_path.split("/")[-1] if manifest_path else ""
                     ),
@@ -698,33 +709,6 @@ def _transform_dependency_graph(
         logger.info(f"Found {dependencies_added} dependencies in {repo_name}")
 
 
-def _extract_version_from_requirements(requirements: Optional[str]) -> Optional[str]:
-    """
-    Extract a pinned version from a requirements string if it exists.
-    Examples: "1.2.3" -> "1.2.3", "^1.2.3" -> None, ">=1.0,<2.0" -> None
-    """
-    if not requirements or not requirements.strip():
-        return None
-
-    # Handle exact version specifications (no operators)
-    if requirements and not any(
-        op in requirements for op in ["^", "~", ">", "<", "=", "*"]
-    ):
-        stripped = requirements.strip()
-        return stripped if stripped else None
-
-    # Handle == specifications
-    if "==" in requirements:
-        parts = requirements.split("==")
-        if len(parts) == 2:
-            version = parts[1].strip()
-            # Remove any trailing constraints
-            version = version.split(",")[0].split(" ")[0]
-            return version if version else None
-
-    return None
-
-
 def _canonicalize_dependency_name(name: str, package_manager: Optional[str]) -> str:
     """
     Canonicalize dependency names based on ecosystem conventions.

cartography/intel/github/util.py

@@ -157,6 +157,18 @@ def fetch_all(
             retry += 1
             exc = err
         except requests.exceptions.HTTPError as err:
+            if (
+                err.response is not None
+                and err.response.status_code == 502
+                and kwargs.get("count")
+                and kwargs["count"] > 1
+            ):
+                kwargs["count"] = max(1, kwargs["count"] // 2)
+                logger.warning(
+                    "GitHub: Received 502 response. Reducing page size to %s and retrying.",
+                    kwargs["count"],
+                )
+                continue
             retry += 1
             exc = err
         except requests.exceptions.ChunkedEncodingError as err:

cartography/intel/keycloak/__init__.py

@@ -0,0 +1,153 @@
+import logging
+
+import neo4j
+import requests
+
+import cartography.intel.keycloak.authenticationexecutions
+import cartography.intel.keycloak.authenticationflows
+import cartography.intel.keycloak.clients
+import cartography.intel.keycloak.groups
+import cartography.intel.keycloak.identityproviders
+import cartography.intel.keycloak.organizations
+import cartography.intel.keycloak.realms
+import cartography.intel.keycloak.roles
+import cartography.intel.keycloak.scopes
+import cartography.intel.keycloak.users
+from cartography.config import Config
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def start_keycloak_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
+    """
+    If this module is configured, perform ingestion of Keycloak data. Otherwise warn and exit
+    :param neo4j_session: Neo4J session for database interface
+    :param config: A cartography.config object
+    :return: None
+    """
+    if (
+        not config.keycloak_client_id
+        or not config.keycloak_client_secret
+        or not config.keycloak_url
+        or not config.keycloak_realm
+    ):
+        logger.info(
+            "Keycloak import is not configured - skipping this module. "
+            "See docs to configure.",
+        )
+        return
+
+    # Create requests sessions
+    with requests.session() as api_session:
+        payload = {
+            "grant_type": "client_credentials",
+            "client_id": config.keycloak_client_id,
+            "client_secret": config.keycloak_client_secret,
+        }
+        req = api_session.post(
+            f"{config.keycloak_url}/realms/{config.keycloak_realm}/protocol/openid-connect/token",
+            data=payload,
+            timeout=_TIMEOUT,
+        )
+        req.raise_for_status()
+        api_session.headers.update(
+            {"Authorization": f'Bearer {req.json()["access_token"]}'}
+        )
+
+        common_job_parameters = {
+            "UPDATE_TAG": config.update_tag,
+        }
+
+        for realm in cartography.intel.keycloak.realms.sync(
+            neo4j_session, api_session, config.keycloak_url, common_job_parameters
+        ):
+            realm_scopped_job_parameters = {
+                "UPDATE_TAG": config.update_tag,
+                "REALM": realm["realm"],
+                "REALM_ID": realm["id"],
+            }
+            cartography.intel.keycloak.users.sync(
+                neo4j_session,
+                api_session,
+                config.keycloak_url,
+                realm_scopped_job_parameters,
+            )
+            cartography.intel.keycloak.identityproviders.sync(
+                neo4j_session,
+                api_session,
+                config.keycloak_url,
+                realm_scopped_job_parameters,
+            )
+            scopes = cartography.intel.keycloak.scopes.sync(
+                neo4j_session,
+                api_session,
+                config.keycloak_url,
+                realm_scopped_job_parameters,
+            )
+            scope_ids = [s["id"] for s in scopes]
+            flows = cartography.intel.keycloak.authenticationflows.sync(
+                neo4j_session,
+                api_session,
+                config.keycloak_url,
+                realm_scopped_job_parameters,
+            )
+            flow_aliases_to_id = {f["alias"]: f["id"] for f in flows}
+            cartography.intel.keycloak.authenticationexecutions.sync(
+                neo4j_session,
+                api_session,
+                config.keycloak_url,
+                realm_scopped_job_parameters,
+                list(flow_aliases_to_id.keys()),
+            )
+            realm_default_flows = {
+                "browser": flow_aliases_to_id.get(realm.get("browserFlow")),
+                "registration": flow_aliases_to_id.get(realm.get("registrationFlow")),
+                "direct_grant": flow_aliases_to_id.get(realm.get("directGrantFlow")),
+                "reset_credentials": flow_aliases_to_id.get(
+                    realm.get("resetCredentialsFlow")
+                ),
+                "client_authentication": flow_aliases_to_id.get(
+                    realm.get("clientAuthenticationFlow")
+                ),
+                "docker_authentication": flow_aliases_to_id.get(
+                    realm.get("dockerAuthenticationFlow")
+                ),
+                "first_broker_login": flow_aliases_to_id.get(
+                    realm.get("firstBrokerLoginFlow")
+                ),
+            }
+
+            clients = cartography.intel.keycloak.clients.sync(
+                neo4j_session,
+                api_session,
+                config.keycloak_url,
+                realm_scopped_job_parameters,
+                realm_default_flows,
+            )
+            client_ids = [c["id"] for c in clients]
+            cartography.intel.keycloak.roles.sync(
+                neo4j_session,
+                api_session,
+                config.keycloak_url,
+                realm_scopped_job_parameters,
+                client_ids,
+                scope_ids,
+            )
+            cartography.intel.keycloak.groups.sync(
+                neo4j_session,
+                api_session,
+                config.keycloak_url,
+                realm_scopped_job_parameters,
+            )
+
+            # Organizations if they are enabled
+            if realm.get("organizationsEnabled", False):
+                cartography.intel.keycloak.organizations.sync(
+                    neo4j_session,
+                    api_session,
+                    config.keycloak_url,
+                    realm_scopped_job_parameters,
+                )

cartography/intel/keycloak/authenticationexecutions.py

@@ -0,0 +1,322 @@
+import logging
+from collections import OrderedDict
+from typing import Any
+from urllib.parse import quote
+
+import neo4j
+import requests
+
+from cartography.client.core.tx import load
+from cartography.client.core.tx import load_matchlinks
+from cartography.graph.job import GraphJob
+from cartography.models.keycloak.authenticationexecution import (
+    ExecutionToExecutionMatchLink,
+)
+from cartography.models.keycloak.authenticationexecution import ExecutionToFlowMatchLink
+from cartography.models.keycloak.authenticationexecution import (
+    KeycloakAuthenticationExecutionSchema,
+)
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    api_session: requests.Session,
+    base_url: str,
+    common_job_parameters: dict[str, Any],
+    flow_aliases: list[str],
+) -> None:
+    exec_by_flow = get(
+        api_session,
+        base_url,
+        common_job_parameters["REALM"],
+        flow_aliases,
+    )
+    transformed_exec, flow_steps, initial_flow_steps = transform(
+        exec_by_flow, common_job_parameters["REALM"]
+    )
+    load_authenticationexecutions(
+        neo4j_session,
+        transformed_exec,
+        common_job_parameters["REALM"],
+        common_job_parameters["UPDATE_TAG"],
+    )
+    load_execution_flow(
+        neo4j_session,
+        flow_steps,
+        initial_flow_steps,
+        common_job_parameters["REALM_ID"],
+        common_job_parameters["UPDATE_TAG"],
+    )
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(
+    api_session: requests.Session, base_url: str, realm: str, flow_aliases: list[str]
+) -> dict[str, list[dict[str, Any]]]:
+    """Fetch authentication execution data for each flow from Keycloak API.
+
+    Args:
+        api_session: Authenticated requests session
+        base_url: Keycloak base URL
+        realm: Target realm name
+        flow_aliases: List of authentication flow names to process
+
+    Returns:
+        Dictionary mapping flow names to their execution lists
+    """
+    results: dict[str, list[dict[str, Any]]] = {}
+    for flow_name in flow_aliases:
+        # URL-encode flow names to handle special characters safely
+        encoded_flow_name = quote(flow_name, safe="")
+        req = api_session.get(
+            f"{base_url}/admin/realms/{realm}/authentication/flows/{encoded_flow_name}/executions",
+            timeout=_TIMEOUT,
+        )
+        req.raise_for_status()
+        results[flow_name] = req.json()
+    return results
+
+
+def _recursive_transform_flow(
+    root_executions: list[dict[str, Any]],
+) -> tuple[list[str], list[tuple[str, str]], list[str]]:
+    """Recursively transforms Keycloak authentication executions into a flow graph structure.
+
+    This function processes authentication executions and builds a directed graph representation
+    suitable for Neo4j ingestion. It handles different execution requirements (REQUIRED,
+    ALTERNATIVE, CONDITIONAL, DISABLED) and nested subflows.
+
+    The function returns three components:
+        - entries: Execution IDs that serve as entry points to the flow
+        - links: Tuples representing directed edges between executions
+        - outs: Execution IDs that serve as exit points from the flow
+
+    Each execution dict must contain:
+        - id: Unique execution identifier
+        - requirement: Execution requirement type (REQUIRED/ALTERNATIVE/CONDITIONAL/DISABLED)
+        - _children: List of nested child executions (for subflows)
+
+    Args:
+        root_executions: List of execution dictionaries to process
+
+    Returns:
+        A tuple containing (entry_points, execution_links, exit_points)
+    """
+    entries: list[str] = []
+    links: list[tuple[str, str]] = []
+    outs: list[str] = []
+
+    for execution in root_executions:
+        # Skip disabled executions as they don't participate in the flow
+        if execution["requirement"] == "DISABLED":
+            continue
+
+        if execution["requirement"] == "REQUIRED":
+            # If no entry point exists, this required execution becomes the flow's starting point
+            if len(entries) == 0:
+                entries.append(execution["id"])
+
+            # Connect all current outputs to this required execution
+            for i in outs:
+                links.append((i, execution["id"]))
+
+            # Handle subflow execution: recursively process children and wire them up
+            if len(execution.get("_children", [])) > 0:
+                c_ins, c_links, c_outs = _recursive_transform_flow(
+                    execution["_children"]
+                )
+                for c_in in c_ins:
+                    links.append((execution["id"], c_in))
+                outs = c_outs
+                links.extend(c_links)
+            # For leaf executions, this becomes the sole output
+            else:
+                outs = [execution["id"]]  # Reset outs to the current execution
+
+            continue
+
+        if execution["requirement"] == "ALTERNATIVE":
+            # Alternative executions create branching paths (OR logic)
+            # This execution becomes an alternative entry point while preserving existing outputs
+            entries.append(execution["id"])
+
+            # Process subflow: wire up child inputs and aggregate child outputs
+            if len(execution.get("_children", [])) > 0:
+                c_ins, c_links, c_outs = _recursive_transform_flow(
+                    execution["_children"]
+                )
+                for c_in in c_ins:
+                    links.append((execution["id"], c_in))
+                for c_out in c_outs:
+                    outs.append(c_out)
+                links.extend(c_links)
+            else:
+                outs.append(execution["id"])
+
+            continue
+
+        if execution["requirement"] == "CONDITIONAL":
+            # Conditional executions only apply to subflows - skip if no children
+            if len(execution.get("_children", [])) == 0:
+                continue
+
+            # Conditional logic creates two possible paths:
+            # 1. Subflow evaluates to True: execution is treated as required
+            # 2. Subflow evaluates to False: execution is skipped
+
+            # Make this execution an entry point if none exist
+            if len(entries) == 0:
+                entries.append(execution["id"])
+
+            # Connect all existing outputs to this conditional execution
+            for i in outs:
+                links.append((i, execution["id"]))
+
+            # Process child executions recursively
+            c_ins, c_links, c_outs = _recursive_transform_flow(execution["_children"])
+
+            # Wire this execution to child entry points
+            for c_in in c_ins:
+                links.append((execution["id"], c_in))
+
+            # Preserve both existing outputs and child outputs to model both conditional paths
+            outs.extend(c_outs)
+
+            # Add child links to the overall link collection
+            links.extend(c_links)
+
+    return entries, links, outs
+
+
+def transform(
+    exec_by_flow: dict[str, list[dict[str, Any]]], realm: str
+) -> tuple[list[dict[str, Any]], list[dict[str, str]], list[dict[str, str]]]:
+    transformed_by_id: OrderedDict[str, dict[str, Any]] = OrderedDict()
+    initial_flow_steps: list[dict[str, str]] = []
+    flow_steps: list[dict[str, str]] = []
+
+    for flow_name, executions in exec_by_flow.items():
+        _parent_by_level: dict[int, str] = {}
+        _root_executions: list[dict[str, Any]] = []
+
+        # Transform executions to include parent flow/subflow relationships
+        # and create a hierarchical structure for graph processing
+        for execution in executions:
+            # Level 0 executions belong directly to the named flow
+            if execution["level"] == 0:
+                execution["_parent_flow"] = flow_name
+                _root_executions.append(execution)
+            else:
+                # Nested executions belong to their parent subflow
+                execution["_parent_subflow"] = _parent_by_level[execution["level"] - 1]
+                transformed_by_id[execution["_parent_subflow"]]["_children"].append(
+                    execution
+                )
+
+            # Track subflow parents for the next nesting level
+            if execution.get("authenticationFlow", True):
+                _parent_by_level[execution["level"]] = execution["id"]
+
+            execution["_children"] = []
+            execution["is_terminal_step"] = False  # Placeholder for terminal step flag
+            transformed_by_id[execution["id"]] = execution
+
+        # Process authentication flow structure and build execution graph
+        # Reference: https://www.keycloak.org/docs/latest/server_admin/index.html#_execution-requirements
+        entries, links, terminals = _recursive_transform_flow(_root_executions)
+
+        for entry in entries:
+            initial_flow_steps.append(
+                {
+                    "flow_name": flow_name,
+                    "execution_id": entry,
+                    "realm": realm,
+                }
+            )
+
+        for link in links:
+            flow_steps.append(
+                {
+                    "source": link[0],
+                    "target": link[1],
+                }
+            )
+
+        for node_id in terminals:
+            transformed_by_id[node_id]["is_terminal_step"] = True
+
+    return list(transformed_by_id.values()), flow_steps, initial_flow_steps
+
+
+@timeit
+def load_authenticationexecutions(
+    neo4j_session: neo4j.Session,
+    data: list[dict[str, Any]],
+    realm: str,
+    update_tag: int,
+) -> None:
+    logger.info(
+        "Loading %d Keycloak AuthenticationExecutions (%s) into Neo4j.",
+        len(data),
+        realm,
+    )
+    load(
+        neo4j_session,
+        KeycloakAuthenticationExecutionSchema(),
+        data,
+        LASTUPDATED=update_tag,
+        REALM=realm,
+    )
+
+
+def load_execution_flow(
+    neo4j_session: neo4j.Session,
+    flow_steps: list[dict[str, Any]],
+    initial_flow_steps: list[dict[str, str]],
+    realm_id: str,
+    update_tag: int,
+) -> None:
+    load_matchlinks(
+        neo4j_session,
+        ExecutionToExecutionMatchLink(),
+        flow_steps,
+        LASTUPDATED=update_tag,
+        _sub_resource_label="KeycloakRealm",
+        _sub_resource_id=realm_id,
+    )
+    load_matchlinks(
+        neo4j_session,
+        ExecutionToFlowMatchLink(),
+        initial_flow_steps,
+        LASTUPDATED=update_tag,
+        _sub_resource_label="KeycloakRealm",
+        _sub_resource_id=realm_id,
+    )
+
+
+@timeit
+def cleanup(
+    neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
+) -> None:
+    GraphJob.from_node_schema(
+        KeycloakAuthenticationExecutionSchema(), common_job_parameters
+    ).run(neo4j_session)
+    GraphJob.from_matchlink(
+        ExecutionToExecutionMatchLink(),
+        "KeycloakRealm",
+        common_job_parameters["REALM_ID"],
+        common_job_parameters["UPDATE_TAG"],
+    ).run(neo4j_session)
+    GraphJob.from_matchlink(
+        ExecutionToFlowMatchLink(),
+        "KeycloakRealm",
+        common_job_parameters["REALM_ID"],
+        common_job_parameters["UPDATE_TAG"],
+    ).run(neo4j_session)