cartography 0.107.0rc2__py3-none-any.whl → 0.108.0rc1__py3-none-any.whl

cartography/intel/github/repos.py

@@ -1,5 +1,6 @@
 import configparser
 import logging
+from collections import defaultdict
 from collections import namedtuple
 from string import Template
 from typing import Any
@@ -12,8 +13,12 @@ from packaging.requirements import InvalidRequirement
 from packaging.requirements import Requirement
 from packaging.utils import canonicalize_name
 
+from cartography.client.core.tx import load as load_data
+from cartography.graph.job import GraphJob
 from cartography.intel.github.util import fetch_all
 from cartography.intel.github.util import PaginatedGraphqlData
+from cartography.models.github.dependencies import GitHubDependencySchema
+from cartography.models.github.manifests import DependencyGraphManifestSchema
 from cartography.util import backoff_handler
 from cartography.util import retries_with_backoff
 from cartography.util import run_cleanup_job
@@ -93,6 +98,18 @@ GITHUB_ORG_REPOS_PAGINATED_GRAPHQL = """
                             text
                         }
                     }
+                    dependencyGraphManifests(first: 20) {
+                        nodes {
+                            blobPath
+                            dependencies(first: 100) {
+                                nodes {
+                                    packageName
+                                    requirements
+                                    packageManager
+                                }
+                            }
+                        }
+                    }
                 }
             }
         }
@@ -291,8 +308,10 @@ def transform(
     :param outside_collaborators: dict of repo URL to list of outside collaborators.
         See tests.data.github.repos.OUTSIDE_COLLABORATORS for data shape.
     :return: Dict containing the repos, repo->language mapping, owners->repo mapping, outside collaborators->repo
-    mapping, and Python requirements files (if any) in a repo.
+    mapping, Python requirements files (if any) in a repo, manifests from GitHub's dependency graph, and all
+    dependencies from GitHub's dependency graph.
     """
+    logger.info(f"Processing {len(repos_json)} GitHub repositories")
     transformed_repo_list: List[Dict] = []
     transformed_repo_languages: List[Dict] = []
     transformed_repo_owners: List[Dict] = []
@@ -312,6 +331,8 @@ def transform(
         "WRITE": [],
     }
     transformed_requirements_files: List[Dict] = []
+    transformed_dependencies: List[Dict] = []
+    transformed_manifests: List[Dict] = []
     for repo_object in repos_json:
         _transform_repo_languages(
             repo_object["url"],
@@ -350,6 +371,16 @@ def transform(
             repo_url,
             transformed_requirements_files,
         )
+        _transform_dependency_manifests(
+            repo_object.get("dependencyGraphManifests"),
+            repo_url,
+            transformed_manifests,
+        )
+        _transform_dependency_graph(
+            repo_object.get("dependencyGraphManifests"),
+            repo_url,
+            transformed_dependencies,
+        )
     results = {
         "repos": transformed_repo_list,
         "repo_languages": transformed_repo_languages,
@@ -357,7 +388,10 @@ def transform(
         "repo_outside_collaborators": transformed_outside_collaborators,
         "repo_direct_collaborators": transformed_direct_collaborators,
         "python_requirements": transformed_requirements_files,
+        "dependencies": transformed_dependencies,
+        "manifests": transformed_manifests,
     }
+
     return results
 
 
@@ -533,6 +567,185 @@ def _transform_setup_cfg_requirements(
     _transform_python_requirements(requirements_list, repo_url, out_requirements_files)
 
 
+def _transform_dependency_manifests(
+    dependency_manifests: Optional[Dict],
+    repo_url: str,
+    out_manifests_list: List[Dict],
+) -> None:
+    """
+    Transform GitHub dependency graph manifests into cartography manifest format.
+    :param dependency_manifests: dependencyGraphManifests from GitHub GraphQL API
+    :param repo_url: The URL of the GitHub repo
+    :param out_manifests_list: Output array to append transformed results to
+    :return: Nothing
+    """
+    if not dependency_manifests or not dependency_manifests.get("nodes"):
+        return
+
+    manifests_added = 0
+
+    for manifest in dependency_manifests["nodes"]:
+        blob_path = manifest.get("blobPath", "")
+        if not blob_path:
+            continue
+
+        # Count dependencies in this manifest
+        dependencies = manifest.get("dependencies", {})
+        dependencies_count = len(dependencies.get("nodes", []) if dependencies else [])
+
+        # Create unique manifest ID by combining repo URL and blob path
+        manifest_id = f"{repo_url}#{blob_path}"
+
+        # Extract filename from blob path
+        filename = blob_path.split("/")[-1] if blob_path else "None"
+
+        out_manifests_list.append(
+            {
+                "id": manifest_id,
+                "blob_path": blob_path,
+                "filename": filename,
+                "dependencies_count": dependencies_count,
+                "repo_url": repo_url,
+            }
+        )
+        manifests_added += 1
+
+    if manifests_added > 0:
+        repo_name = repo_url.split("/")[-1] if repo_url else "repository"
+        logger.info(f"Found {manifests_added} dependency manifests in {repo_name}")
+
+
+def _transform_dependency_graph(
+    dependency_manifests: Optional[Dict],
+    repo_url: str,
+    out_dependencies_list: List[Dict],
+) -> None:
+    """
+    Transform GitHub dependency graph manifests into cartography dependency format.
+    :param dependency_manifests: dependencyGraphManifests from GitHub GraphQL API
+    :param repo_url: The URL of the GitHub repo
+    :param out_dependencies_list: Output array to append transformed results to
+    :return: Nothing
+    """
+    if not dependency_manifests or not dependency_manifests.get("nodes"):
+        return
+
+    dependencies_added = 0
+
+    for manifest in dependency_manifests["nodes"]:
+        dependencies = manifest.get("dependencies", {})
+        if not dependencies or not dependencies.get("nodes"):
+            continue
+
+        manifest_path = manifest.get("blobPath", "")
+
+        for dep in dependencies["nodes"]:
+            package_name = dep.get("packageName")
+            if not package_name:
+                continue
+
+            requirements = dep.get("requirements", "")
+            package_manager = dep.get("packageManager", "").upper()
+
+            # Extract version from requirements string if available
+            pinned_version = _extract_version_from_requirements(requirements)
+
+            # Create ecosystem-specific canonical name
+            canonical_name = _canonicalize_dependency_name(
+                package_name, package_manager
+            )
+
+            # Create ecosystem identifier
+            ecosystem = package_manager.lower() if package_manager else "unknown"
+
+            # Create simple dependency ID using canonical name and version
+            # This allows the same dependency to be shared across multiple repos
+            dependency_id = (
+                f"{canonical_name}|{pinned_version}"
+                if pinned_version
+                else canonical_name
+            )
+
+            # Normalize requirements field (prefer None over empty string)
+            normalized_requirements = requirements if requirements else None
+
+            # Create manifest ID for the HAS_DEP relationship
+            manifest_id = f"{repo_url}#{manifest_path}"
+
+            out_dependencies_list.append(
+                {
+                    "id": dependency_id,
+                    "name": canonical_name,
+                    "original_name": package_name,  # Keep original for reference
+                    "version": pinned_version,
+                    "requirements": normalized_requirements,
+                    "ecosystem": ecosystem,
+                    "package_manager": package_manager,
+                    "manifest_path": manifest_path,
+                    "manifest_id": manifest_id,
+                    "repo_url": repo_url,
+                    # Add separate fields for easier querying
+                    "repo_name": repo_url.split("/")[-1] if repo_url else "",
+                    "manifest_file": (
+                        manifest_path.split("/")[-1] if manifest_path else ""
+                    ),
+                }
+            )
+            dependencies_added += 1
+
+    if dependencies_added > 0:
+        repo_name = repo_url.split("/")[-1] if repo_url else "repository"
+        logger.info(f"Found {dependencies_added} dependencies in {repo_name}")
+
+
+def _extract_version_from_requirements(requirements: Optional[str]) -> Optional[str]:
+    """
+    Extract a pinned version from a requirements string if it exists.
+    Examples: "1.2.3" -> "1.2.3", "^1.2.3" -> None, ">=1.0,<2.0" -> None
+    """
+    if not requirements or not requirements.strip():
+        return None
+
+    # Handle exact version specifications (no operators)
+    if requirements and not any(
+        op in requirements for op in ["^", "~", ">", "<", "=", "*"]
+    ):
+        stripped = requirements.strip()
+        return stripped if stripped else None
+
+    # Handle == specifications
+    if "==" in requirements:
+        parts = requirements.split("==")
+        if len(parts) == 2:
+            version = parts[1].strip()
+            # Remove any trailing constraints
+            version = version.split(",")[0].split(" ")[0]
+            return version if version else None
+
+    return None
+
+
+def _canonicalize_dependency_name(name: str, package_manager: Optional[str]) -> str:
+    """
+    Canonicalize dependency names based on ecosystem conventions.
+    """
+    if not name:
+        return name
+
+    # For Python packages, use existing canonicalization
+    if package_manager in ["PIP", "CONDA"]:
+        try:
+            from packaging.utils import canonicalize_name
+
+            return str(canonicalize_name(name))
+        except ImportError:
+            # Fallback if packaging not available
+            return name.lower().replace("_", "-")
+
+    # For other ecosystems, use lowercase
+    return name.lower()
+
+
 def _transform_python_requirements(
     requirements_list: List[str],
     repo_url: str,
@@ -785,6 +998,136 @@ def load_collaborators(
         )
 
 
+@timeit
+def load_python_requirements(
+    neo4j_session: neo4j.Session,
+    update_tag: int,
+    requirements_objects: List[Dict],
+) -> None:
+    query = """
+    UNWIND $Requirements AS req
+        MERGE (lib:PythonLibrary:Dependency{id: req.id})
+        ON CREATE SET lib.firstseen = timestamp(),
+        lib.name = req.name
+        SET lib.lastupdated = $UpdateTag,
+        lib.version = req.version
+
+        WITH lib, req
+        MATCH (repo:GitHubRepository{id: req.repo_url})
+        MERGE (repo)-[r:REQUIRES]->(lib)
+        ON CREATE SET r.firstseen = timestamp()
+        SET r.lastupdated = $UpdateTag,
+        r.specifier = req.specifier
+    """
+    neo4j_session.run(
+        query,
+        Requirements=requirements_objects,
+        UpdateTag=update_tag,
+    )
+
+
+@timeit
+def load_github_dependencies(
+    neo4j_session: neo4j.Session,
+    update_tag: int,
+    dependencies: List[Dict],
+) -> None:
+    """
+    Ingest GitHub dependency data into Neo4j using the new data model
+    :param neo4j_session: Neo4J session object for server communication
+    :param update_tag: Timestamp used to determine data freshness
+    :param dependencies: List of dependency objects from GitHub's dependency graph
+    :return: Nothing
+    """
+    # Group dependencies by both repo_url and manifest_id for schema-based loading
+    dependencies_by_repo_and_manifest = defaultdict(list)
+
+    for dep in dependencies:
+        repo_url = dep["repo_url"]
+        manifest_id = dep["manifest_id"]
+        # Create a key combining both repo_url and manifest_id
+        group_key = (repo_url, manifest_id)
+        # Remove repo_url and manifest_id from the dependency object since we'll pass them as kwargs
+        dep_without_kwargs = {
+            k: v for k, v in dep.items() if k not in ["repo_url", "manifest_id"]
+        }
+        dependencies_by_repo_and_manifest[group_key].append(dep_without_kwargs)
+
+    # Load dependencies for each repository/manifest combination separately
+    for (
+        repo_url,
+        manifest_id,
+    ), group_dependencies in dependencies_by_repo_and_manifest.items():
+        load_data(
+            neo4j_session,
+            GitHubDependencySchema(),
+            group_dependencies,
+            lastupdated=update_tag,
+            repo_url=repo_url,
+            manifest_id=manifest_id,
+        )
+
+
+@timeit
+def load_github_dependency_manifests(
+    neo4j_session: neo4j.Session,
+    update_tag: int,
+    manifests: List[Dict],
+) -> None:
+    """
+    Ingest GitHub dependency manifests into Neo4j
+    """
+    manifests_by_repo = defaultdict(list)
+
+    for manifest in manifests:
+        repo_url = manifest["repo_url"]
+        manifests_by_repo[repo_url].append(manifest)
+
+    # Load manifests for each repository separately
+    for repo_url, repo_manifests in manifests_by_repo.items():
+        load_data(
+            neo4j_session,
+            DependencyGraphManifestSchema(),
+            repo_manifests,
+            lastupdated=update_tag,
+            repo_url=repo_url,
+        )
+
+
+@timeit
+def cleanup_github_dependencies(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+    repo_urls: List[str],
+) -> None:
+    # Run cleanup for each repository separately
+    for repo_url in repo_urls:
+        cleanup_params = {**common_job_parameters, "repo_url": repo_url}
+        GraphJob.from_node_schema(GitHubDependencySchema(), cleanup_params).run(
+            neo4j_session
+        )
+
+
+@timeit
+def cleanup_github_manifests(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+    repo_urls: List[str],
+) -> None:
+    """
+    Delete GitHub dependency manifests and their relationships from the graph if they were not updated in the last sync.
+    :param neo4j_session: Neo4j session
+    :param common_job_parameters: Common job parameters containing UPDATE_TAG
+    :param repo_urls: List of repository URLs to clean up manifests for
+    """
+    # Run cleanup for each repository separately
+    for repo_url in repo_urls:
+        cleanup_params = {**common_job_parameters, "repo_url": repo_url}
+        GraphJob.from_node_schema(DependencyGraphManifestSchema(), cleanup_params).run(
+            neo4j_session
+        )
+
+
 @timeit
 def load(
     neo4j_session: neo4j.Session,
@@ -823,33 +1166,15 @@ def load(
         common_job_parameters["UPDATE_TAG"],
         repo_data["python_requirements"],
     )
-
-
-@timeit
-def load_python_requirements(
-    neo4j_session: neo4j.Session,
-    update_tag: int,
-    requirements_objects: List[Dict],
-) -> None:
-    query = """
-    UNWIND $Requirements AS req
-        MERGE (lib:PythonLibrary:Dependency{id: req.id})
-        ON CREATE SET lib.firstseen = timestamp(),
-        lib.name = req.name
-        SET lib.lastupdated = $UpdateTag,
-        lib.version = req.version
-
-        WITH lib, req
-        MATCH (repo:GitHubRepository{id: req.repo_url})
-        MERGE (repo)-[r:REQUIRES]->(lib)
-        ON CREATE SET r.firstseen = timestamp()
-        SET r.lastupdated = $UpdateTag,
-        r.specifier = req.specifier
-    """
-    neo4j_session.run(
-        query,
-        Requirements=requirements_objects,
-        UpdateTag=update_tag,
+    load_github_dependency_manifests(
+        neo4j_session,
+        common_job_parameters["UPDATE_TAG"],
+        repo_data["manifests"],
+    )
+    load_github_dependencies(
+        neo4j_session,
+        common_job_parameters["UPDATE_TAG"],
+        repo_data["dependencies"],
     )
 
 
@@ -896,4 +1221,21 @@ def sync(
         )
     repo_data = transform(repos_json, direct_collabs, outside_collabs)
     load(neo4j_session, common_job_parameters, repo_data)
+
+    # Collect repository URLs that have dependencies for cleanup
+    repo_urls_with_dependencies = list(
+        {dep["repo_url"] for dep in repo_data["dependencies"]}
+    )
+    cleanup_github_dependencies(
+        neo4j_session, common_job_parameters, repo_urls_with_dependencies
+    )
+
+    # Collect repository URLs that have manifests for cleanup
+    repo_urls_with_manifests = list(
+        {manifest["repo_url"] for manifest in repo_data["manifests"]}
+    )
+    cleanup_github_manifests(
+        neo4j_session, common_job_parameters, repo_urls_with_manifests
+    )
+
     run_cleanup_job("github_repos_cleanup.json", neo4j_session, common_job_parameters)

cartography/intel/sentinelone/__init__.py

@@ -3,6 +3,7 @@ import logging
 import neo4j
 
 import cartography.intel.sentinelone.agent
+import cartography.intel.sentinelone.application
 from cartography.config import Config
 from cartography.intel.sentinelone.account import sync_accounts
 from cartography.stats import get_stats_client
@@ -39,7 +40,7 @@ def start_sentinelone_ingestion(neo4j_session: neo4j.Session, config: Config) ->
         config.sentinelone_account_ids,
     )
 
-    # Sync agents for each account
+    # Sync agents and applications for each account
     for account_id in synced_account_ids:
         # Add account-specific parameter
         common_job_parameters["S1_ACCOUNT_ID"] = account_id
@@ -49,7 +50,12 @@ def start_sentinelone_ingestion(neo4j_session: neo4j.Session, config: Config) ->
             common_job_parameters,
         )
 
-        # Clean up account-specific parameter
+        cartography.intel.sentinelone.application.sync(
+            neo4j_session,
+            common_job_parameters,
+        )
+
+        # Clean up account-specific parameters
         del common_job_parameters["S1_ACCOUNT_ID"]
 
     # Record that the sync is complete