cartography 0.104.0rc2__py3-none-any.whl → 0.123.0__py3-none-any.whl

cartography/intel/github/repos.py

@@ -1,8 +1,10 @@
 import configparser
 import logging
+from collections import defaultdict
 from collections import namedtuple
 from string import Template
 from typing import Any
+from typing import cast
 from typing import Dict
 from typing import List
 from typing import Optional
@@ -12,8 +14,13 @@ from packaging.requirements import InvalidRequirement
 from packaging.requirements import Requirement
 from packaging.utils import canonicalize_name
 
+from cartography.client.core.tx import execute_write_with_retry
+from cartography.client.core.tx import load as load_data
+from cartography.graph.job import GraphJob
 from cartography.intel.github.util import fetch_all
 from cartography.intel.github.util import PaginatedGraphqlData
+from cartography.models.github.dependencies import GitHubDependencySchema
+from cartography.models.github.manifests import DependencyGraphManifestSchema
 from cartography.util import backoff_handler
 from cartography.util import retries_with_backoff
 from cartography.util import run_cleanup_job
@@ -36,12 +43,12 @@ UserAffiliationAndRepoPermission = namedtuple(
 
 
 GITHUB_ORG_REPOS_PAGINATED_GRAPHQL = """
-    query($login: String!, $cursor: String) {
+    query($login: String!, $cursor: String, $count: Int!) {
     organization(login: $login)
         {
             url
             login
-            repositories(first: 50, after: $cursor){
+            repositories(first: $count, after: $cursor){
                 pageInfo{
                     endCursor
                     hasNextPage
@@ -93,6 +100,18 @@ GITHUB_ORG_REPOS_PAGINATED_GRAPHQL = """
                             text
                         }
                     }
+                    dependencyGraphManifests(first: 20) {
+                        nodes {
+                            blobPath
+                            dependencies(first: 100) {
+                                nodes {
+                                    packageName
+                                    requirements
+                                    packageManager
+                                }
+                            }
+                        }
+                    }
                 }
             }
         }
@@ -140,25 +159,38 @@ def _get_repo_collaborators_inner_func(
     org: str,
     api_url: str,
     token: str,
-    repo_raw_data: list[dict[str, Any]],
+    repo_raw_data: list[dict[str, Any] | None],
     affiliation: str,
-    collab_users: list[dict[str, Any]],
-    collab_permission: list[str],
 ) -> dict[str, list[UserAffiliationAndRepoPermission]]:
     result: dict[str, list[UserAffiliationAndRepoPermission]] = {}
 
     for repo in repo_raw_data:
+        # GitHub can return null repo entries. See issues #1334 and #1404.
+        if repo is None:
+            logger.info(
+                "Skipping null repository entry while fetching %s collaborators.",
+                affiliation,
+            )
+            continue
         repo_name = repo["name"]
         repo_url = repo["url"]
 
-        if (
-            affiliation == "OUTSIDE" and repo["outsideCollaborators"]["totalCount"] == 0
-        ) or (
-            affiliation == "DIRECT" and repo["directCollaborators"]["totalCount"] == 0
-        ):
-            # repo has no collabs of the affiliation type we're looking for, so don't waste time making an API call
-            result[repo_url] = []
-            continue
+        # Guard against None when collaborator fields are not accessible due to permissions.
+        direct_info = repo.get("directCollaborators")
+        outside_info = repo.get("outsideCollaborators")
+
+        if affiliation == "OUTSIDE":
+            total_outside = 0 if not outside_info else outside_info.get("totalCount", 0)
+            if total_outside == 0:
+                # No outside collaborators or not permitted to view; skip API calls for this repo.
+                result[repo_url] = []
+                continue
+        else:  # DIRECT
+            total_direct = 0 if not direct_info else direct_info.get("totalCount", 0)
+            if total_direct == 0:
+                # No direct collaborators or not permitted to view; skip API calls for this repo.
+                result[repo_url] = []
+                continue
 
         logger.info(f"Loading {affiliation} collaborators for repo {repo_name}.")
         collaborators = _get_repo_collaborators(
@@ -169,6 +201,9 @@ def _get_repo_collaborators_inner_func(
             affiliation,
         )
 
+        collab_users: List[dict[str, Any]] = []
+        collab_permission: List[str] = []
+
         # nodes and edges are expected to always be present given that we only call for them if totalCount is > 0
         # however sometimes GitHub returns None, as in issue 1334 and 1404.
         for collab in collaborators.nodes or []:
@@ -186,7 +221,7 @@ def _get_repo_collaborators_inner_func(
 
 
 def _get_repo_collaborators_for_multiple_repos(
-    repo_raw_data: list[dict[str, Any]],
+    repo_raw_data: list[dict[str, Any] | None],
     affiliation: str,
     org: str,
     api_url: str,
@@ -205,8 +240,6 @@ def _get_repo_collaborators_for_multiple_repos(
     logger.info(
         f'Retrieving repo collaborators for affiliation "{affiliation}" on org "{org}".',
     )
-    collab_users: List[dict[str, Any]] = []
-    collab_permission: List[str] = []
 
     result: dict[str, list[UserAffiliationAndRepoPermission]] = retries_with_backoff(
         _get_repo_collaborators_inner_func,
@@ -219,8 +252,6 @@ def _get_repo_collaborators_for_multiple_repos(
         token=token,
         repo_raw_data=repo_raw_data,
         affiliation=affiliation,
-        collab_users=collab_users,
-        collab_permission=collab_permission,
     )
     return result
 
@@ -257,7 +288,7 @@ def _get_repo_collaborators(
 
 
 @timeit
-def get(token: str, api_url: str, organization: str) -> List[Dict]:
+def get(token: str, api_url: str, organization: str) -> List[Optional[Dict]]:
     """
     Retrieve a list of repos from a Github organization as described in
     https://docs.github.com/en/graphql/reference/objects#repository.
@@ -265,6 +296,8 @@ def get(token: str, api_url: str, organization: str) -> List[Dict]:
     :param api_url: The Github v4 API endpoint as string.
     :param organization: The name of the target Github organization as string.
     :return: A list of dicts representing repos. See tests.data.github.repos for data shape.
+        Note: The list may contain None entries per GraphQL spec when resolvers error
+        (permissions, rate limits, transient issues). See issues #1334 and #1404.
     """
     # TODO: link the Github organization to the repositories
     repos, _ = fetch_all(
@@ -273,12 +306,17 @@ def get(token: str, api_url: str, organization: str) -> List[Dict]:
         organization,
         GITHUB_ORG_REPOS_PAGINATED_GRAPHQL,
         "repositories",
+        count=50,
     )
-    return repos.nodes
+    # Cast is needed because GitHub's GraphQL RepositoryConnection.nodes is typed [Repository] (not [Repository!])
+    # per GraphQL spec, allowing null entries when resolvers error (permissions, rate limits, transient issues).
+    # See https://github.com/cartography-cncf/cartography/issues/1334
+    # and https://github.com/cartography-cncf/cartography/issues/1404
+    return cast(List[Optional[Dict]], repos.nodes)
 
 
 def transform(
-    repos_json: List[Dict],
+    repos_json: List[Optional[Dict]],
     direct_collaborators: dict[str, List[UserAffiliationAndRepoPermission]],
     outside_collaborators: dict[str, List[UserAffiliationAndRepoPermission]],
 ) -> Dict:
@@ -291,8 +329,10 @@ def transform(
     :param outside_collaborators: dict of repo URL to list of outside collaborators.
         See tests.data.github.repos.OUTSIDE_COLLABORATORS for data shape.
     :return: Dict containing the repos, repo->language mapping, owners->repo mapping, outside collaborators->repo
-    mapping, and Python requirements files (if any) in a repo.
+    mapping, Python requirements files (if any) in a repo, manifests from GitHub's dependency graph, and all
+    dependencies from GitHub's dependency graph.
     """
+    logger.info(f"Processing {len(repos_json)} GitHub repositories")
     transformed_repo_list: List[Dict] = []
     transformed_repo_languages: List[Dict] = []
     transformed_repo_owners: List[Dict] = []
@@ -312,7 +352,13 @@ def transform(
         "WRITE": [],
     }
     transformed_requirements_files: List[Dict] = []
+    transformed_dependencies: List[Dict] = []
+    transformed_manifests: List[Dict] = []
     for repo_object in repos_json:
+        # GitHub can return null repo entries. See issues #1334 and #1404.
+        if repo_object is None:
+            logger.debug("Skipping null repository entry during transformation.")
+            continue
         _transform_repo_languages(
             repo_object["url"],
             repo_object,
@@ -350,6 +396,16 @@ def transform(
             repo_url,
             transformed_requirements_files,
         )
+        _transform_dependency_manifests(
+            repo_object.get("dependencyGraphManifests"),
+            repo_url,
+            transformed_manifests,
+        )
+        _transform_dependency_graph(
+            repo_object.get("dependencyGraphManifests"),
+            repo_url,
+            transformed_dependencies,
+        )
     results = {
         "repos": transformed_repo_list,
         "repo_languages": transformed_repo_languages,
@@ -357,7 +413,10 @@ def transform(
         "repo_outside_collaborators": transformed_outside_collaborators,
         "repo_direct_collaborators": transformed_direct_collaborators,
         "python_requirements": transformed_requirements_files,
+        "dependencies": transformed_dependencies,
+        "manifests": transformed_manifests,
     }
+
     return results
 
 
@@ -371,9 +430,16 @@ def _create_default_branch_id(repo_url: str, default_branch_ref_id: str) -> str:
 
 def _create_git_url_from_ssh_url(ssh_url: str) -> str:
     """
-    Return a git:// URL from the given ssh_url
+    Convert SSH URL to git:// URL.
+    Example:
+        git@github.com:cartography-cncf/cartography.git
+        -> git://github.com/cartography-cncf/cartography.git
     """
-    return ssh_url.replace("/", ":").replace("git@", "git://")
+    # Remove the user part (e.g., "git@")
+    _, host_and_path = ssh_url.split("@", 1)
+    # Replace first ':' (separating host and repo) with '/'
+    host, path = host_and_path.split(":", 1)
+    return f"git://{host}/{path}"
 
 
 def _transform_repo_objects(input_repo_object: Dict, out_repo_list: List[Dict]) -> None:
@@ -533,6 +599,153 @@ def _transform_setup_cfg_requirements(
     _transform_python_requirements(requirements_list, repo_url, out_requirements_files)
 
 
+def _transform_dependency_manifests(
+    dependency_manifests: Optional[Dict],
+    repo_url: str,
+    out_manifests_list: List[Dict],
+) -> None:
+    """
+    Transform GitHub dependency graph manifests into cartography manifest format.
+    :param dependency_manifests: dependencyGraphManifests from GitHub GraphQL API
+    :param repo_url: The URL of the GitHub repo
+    :param out_manifests_list: Output array to append transformed results to
+    :return: Nothing
+    """
+    if not dependency_manifests or not dependency_manifests.get("nodes"):
+        return
+
+    manifests_added = 0
+
+    for manifest in dependency_manifests["nodes"]:
+        blob_path = manifest.get("blobPath", "")
+        if not blob_path:
+            continue
+
+        # Count dependencies in this manifest
+        dependencies = manifest.get("dependencies", {})
+        dependencies_count = len(dependencies.get("nodes", []) if dependencies else [])
+
+        # Create unique manifest ID by combining repo URL and blob path
+        manifest_id = f"{repo_url}#{blob_path}"
+
+        # Extract filename from blob path
+        filename = blob_path.split("/")[-1] if blob_path else "None"
+
+        out_manifests_list.append(
+            {
+                "id": manifest_id,
+                "blob_path": blob_path,
+                "filename": filename,
+                "dependencies_count": dependencies_count,
+                "repo_url": repo_url,
+            }
+        )
+        manifests_added += 1
+
+    if manifests_added > 0:
+        repo_name = repo_url.split("/")[-1] if repo_url else "repository"
+        logger.info(f"Found {manifests_added} dependency manifests in {repo_name}")
+
+
+def _transform_dependency_graph(
+    dependency_manifests: Optional[Dict],
+    repo_url: str,
+    out_dependencies_list: List[Dict],
+) -> None:
+    """
+    Transform GitHub dependency graph manifests into cartography dependency format.
+    :param dependency_manifests: dependencyGraphManifests from GitHub GraphQL API
+    :param repo_url: The URL of the GitHub repo
+    :param out_dependencies_list: Output array to append transformed results to
+    :return: Nothing
+    """
+    if not dependency_manifests or not dependency_manifests.get("nodes"):
+        return
+
+    dependencies_added = 0
+
+    for manifest in dependency_manifests["nodes"]:
+        dependencies = manifest.get("dependencies", {})
+        if not dependencies or not dependencies.get("nodes"):
+            continue
+
+        manifest_path = manifest.get("blobPath", "")
+
+        for dep in dependencies["nodes"]:
+            package_name = dep.get("packageName")
+            if not package_name:
+                continue
+
+            requirements = dep.get("requirements", "")
+            package_manager = dep.get("packageManager", "").upper()
+
+            # Create ecosystem-specific canonical name
+            canonical_name = _canonicalize_dependency_name(
+                package_name, package_manager
+            )
+
+            # Create ecosystem identifier
+            ecosystem = package_manager.lower() if package_manager else "unknown"
+
+            # Create simple dependency ID using canonical name and requirements
+            # This allows the same dependency to be shared across multiple repos
+            requirements_for_id = (requirements or "").strip()
+            dependency_id = (
+                f"{canonical_name}|{requirements_for_id}"
+                if requirements_for_id
+                else canonical_name
+            )
+
+            # Normalize requirements field (prefer None over empty string)
+            normalized_requirements = requirements if requirements else None
+
+            # Create manifest ID for the HAS_DEP relationship
+            manifest_id = f"{repo_url}#{manifest_path}"
+
+            out_dependencies_list.append(
+                {
+                    "id": dependency_id,
+                    "name": canonical_name,
+                    "original_name": package_name,  # Keep original for reference
+                    "requirements": normalized_requirements,
+                    "ecosystem": ecosystem,
+                    "package_manager": package_manager,
+                    "manifest_path": manifest_path,
+                    "manifest_id": manifest_id,
+                    "repo_url": repo_url,
+                    "manifest_file": (
+                        manifest_path.split("/")[-1] if manifest_path else ""
+                    ),
+                }
+            )
+            dependencies_added += 1
+
+    if dependencies_added > 0:
+        repo_name = repo_url.split("/")[-1] if repo_url else "repository"
+        logger.info(f"Found {dependencies_added} dependencies in {repo_name}")
+
+
+def _canonicalize_dependency_name(name: str, package_manager: Optional[str]) -> str:
+    """
+    Canonicalize dependency names based on ecosystem conventions.
+    """
+    if not name:
+        return name
+
+    # For Python packages, use existing canonicalization
+    if package_manager in ["PIP", "CONDA"]:
+        try:
+            from packaging.utils import canonicalize_name
+
+            return str(canonicalize_name(name))
+        except ImportError:
+            # Fallback if packaging not available
+            return name.lower().replace("_", "-")
+
+    # For other ecosystems, use lowercase
+    return name.lower()
+
+
 def _transform_python_requirements(
     requirements_list: List[str],
     repo_url: str,
@@ -667,11 +880,15 @@ def load_github_repos(
     ON CREATE SET r.firstseen = timestamp()
     SET r.lastupdated = r.UpdateTag
     """
-    neo4j_session.run(
-        ingest_repo,
-        RepoData=repo_data,
-        UpdateTag=update_tag,
-    )
+
+    def _ingest_repos_tx(tx: neo4j.Transaction) -> None:
+        tx.run(
+            ingest_repo,
+            RepoData=repo_data,
+            UpdateTag=update_tag,
+        ).consume()
+
+    execute_write_with_retry(neo4j_session, _ingest_repos_tx)
 
 
 @timeit
@@ -701,11 +918,14 @@ def load_github_languages(
         ON CREATE SET r.firstseen = timestamp()
         SET r.lastupdated = $UpdateTag"""
 
-    neo4j_session.run(
-        ingest_languages,
-        Languages=repo_languages,
-        UpdateTag=update_tag,
-    )
+    def _ingest_languages_tx(tx: neo4j.Transaction) -> None:
+        tx.run(
+            ingest_languages,
+            Languages=repo_languages,
+            UpdateTag=update_tag,
+        ).consume()
+
+    execute_write_with_retry(neo4j_session, _ingest_languages_tx)
 
 
 @timeit
@@ -721,31 +941,43 @@ def load_github_owners(
     :param repo_owners: list of owner to repo mappings
     :return: Nothing
     """
-    for owner in repo_owners:
-        ingest_owner_template = Template(
-            """
-            MERGE (user:$account_type{id: $Id})
-            ON CREATE SET user.firstseen = timestamp()
-            SET user.username = $UserName,
-            user.lastupdated = $UpdateTag
-            WITH user
-
-            MATCH (repo:GitHubRepository{id: $RepoId})
-            MERGE (user)<-[r:OWNER]-(repo)
-            ON CREATE SET r.firstseen = timestamp()
-            SET r.lastupdated = $UpdateTag""",
-        )
+    ingest_owner_template = Template(
+        """
+        MERGE (user:$account_type{id: $Id})
+        ON CREATE SET user.firstseen = timestamp()
+        SET user.username = $UserName,
+        user.lastupdated = $UpdateTag
+        WITH user
+
+        MATCH (repo:GitHubRepository{id: $RepoId})
+        MERGE (user)<-[r:OWNER]-(repo)
+        ON CREATE SET r.firstseen = timestamp()
+        SET r.lastupdated = $UpdateTag""",
+    )
 
-        account_type = {"User": "GitHubUser", "Organization": "GitHubOrganization"}
+    account_type = {"User": "GitHubUser", "Organization": "GitHubOrganization"}
 
-        neo4j_session.run(
+    def _ingest_owner_tx(
+        tx: neo4j.Transaction,
+        owner_record: Dict,
+        owner_label: str,
+    ) -> None:
+        tx.run(
             ingest_owner_template.safe_substitute(
-                account_type=account_type[owner["type"]],
+                account_type=owner_label,
             ),
-            Id=owner["owner_id"],
-            UserName=owner["owner"],
-            RepoId=owner["repo_id"],
+            Id=owner_record["owner_id"],
+            UserName=owner_record["owner"],
+            RepoId=owner_record["repo_id"],
             UpdateTag=update_tag,
+        ).consume()
+
+    for owner in repo_owners:
+        execute_write_with_retry(
+            neo4j_session,
+            _ingest_owner_tx,
+            owner,
+            account_type[owner["type"]],
         )
 
 
@@ -776,12 +1008,159 @@ def load_collaborators(
     SET o.lastupdated = $UpdateTag
     """,
     )
-    for collab_type in collaborators.keys():
-        relationship_label = f"{affiliation}_COLLAB_{collab_type}"
-        neo4j_session.run(
+
+    def _ingest_collaborators_tx(
+        tx: neo4j.Transaction,
+        relationship_label: str,
+        collaborator_data: List[Dict],
+    ) -> None:
+        tx.run(
             query.safe_substitute(rel_label=relationship_label),
-            UserData=collaborators[collab_type],
+            UserData=collaborator_data,
             UpdateTag=update_tag,
+        ).consume()
+
+    for collab_type, collab_data in collaborators.items():
+        relationship_label = f"{affiliation}_COLLAB_{collab_type}"
+        execute_write_with_retry(
+            neo4j_session,
+            _ingest_collaborators_tx,
+            relationship_label,
+            collab_data,
+        )
+
+
+@timeit
+def load_python_requirements(
+    neo4j_session: neo4j.Session,
+    update_tag: int,
+    requirements_objects: List[Dict],
+) -> None:
+    query = """
+    UNWIND $Requirements AS req
+        MERGE (lib:PythonLibrary:Dependency{id: req.id})
+        ON CREATE SET lib.firstseen = timestamp(),
+        lib.name = req.name
+        SET lib.lastupdated = $UpdateTag,
+        lib.version = req.version
+
+        WITH lib, req
+        MATCH (repo:GitHubRepository{id: req.repo_url})
+        MERGE (repo)-[r:REQUIRES]->(lib)
+        ON CREATE SET r.firstseen = timestamp()
+        SET r.lastupdated = $UpdateTag,
+        r.specifier = req.specifier
+    """
+
+    def _ingest_requirements_tx(tx: neo4j.Transaction) -> None:
+        tx.run(
+            query,
+            Requirements=requirements_objects,
+            UpdateTag=update_tag,
+        ).consume()
+
+    execute_write_with_retry(neo4j_session, _ingest_requirements_tx)
+
+
+@timeit
+def load_github_dependencies(
+    neo4j_session: neo4j.Session,
+    update_tag: int,
+    dependencies: List[Dict],
+) -> None:
+    """
+    Ingest GitHub dependency data into Neo4j using the new data model
+    :param neo4j_session: Neo4J session object for server communication
+    :param update_tag: Timestamp used to determine data freshness
+    :param dependencies: List of dependency objects from GitHub's dependency graph
+    :return: Nothing
+    """
+    # Group dependencies by both repo_url and manifest_id for schema-based loading
+    dependencies_by_repo_and_manifest = defaultdict(list)
+
+    for dep in dependencies:
+        repo_url = dep["repo_url"]
+        manifest_id = dep["manifest_id"]
+        # Create a key combining both repo_url and manifest_id
+        group_key = (repo_url, manifest_id)
+        # Remove repo_url and manifest_id from the dependency object since we'll pass them as kwargs
+        dep_without_kwargs = {
+            k: v for k, v in dep.items() if k not in ["repo_url", "manifest_id"]
+        }
+        dependencies_by_repo_and_manifest[group_key].append(dep_without_kwargs)
+
+    # Load dependencies for each repository/manifest combination separately
+    for (
+        repo_url,
+        manifest_id,
+    ), group_dependencies in dependencies_by_repo_and_manifest.items():
+        load_data(
+            neo4j_session,
+            GitHubDependencySchema(),
+            group_dependencies,
+            lastupdated=update_tag,
+            repo_url=repo_url,
+            manifest_id=manifest_id,
+        )
+
+
+@timeit
+def load_github_dependency_manifests(
+    neo4j_session: neo4j.Session,
+    update_tag: int,
+    manifests: List[Dict],
+) -> None:
+    """
+    Ingest GitHub dependency manifests into Neo4j
+    """
+    manifests_by_repo = defaultdict(list)
+
+    for manifest in manifests:
+        repo_url = manifest["repo_url"]
+        manifests_by_repo[repo_url].append(manifest)
+
+    # Load manifests for each repository separately
+    for repo_url, repo_manifests in manifests_by_repo.items():
+        load_data(
+            neo4j_session,
+            DependencyGraphManifestSchema(),
+            repo_manifests,
+            lastupdated=update_tag,
+            repo_url=repo_url,
+        )
+
+
+@timeit
+def cleanup_github_dependencies(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+    repo_urls: List[str],
+) -> None:
+    # Run cleanup for each repository separately
+    for repo_url in repo_urls:
+        cleanup_params = {**common_job_parameters, "repo_url": repo_url}
+        GraphJob.from_node_schema(GitHubDependencySchema(), cleanup_params).run(
+            neo4j_session
+        )
+
+
+@timeit
+def cleanup_github_manifests(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict[str, Any],
+    repo_urls: List[str],
+) -> None:
+    """
+    Delete GitHub dependency manifests and their relationships from the graph if they were not updated in the last sync.
+    :param neo4j_session: Neo4j session
+    :param common_job_parameters: Common job parameters containing UPDATE_TAG
+    :param repo_urls: List of repository URLs to clean up manifests for
+    """
+    # Run cleanup for each repository separately
+    for repo_url in repo_urls:
+        cleanup_params = {**common_job_parameters, "repo_url": repo_url}
+        GraphJob.from_node_schema(DependencyGraphManifestSchema(), cleanup_params).run(
+            neo4j_session
         )
 
 
@@ -823,33 +1202,15 @@ def load(
         common_job_parameters["UPDATE_TAG"],
         repo_data["python_requirements"],
     )
-
-
-@timeit
-def load_python_requirements(
-    neo4j_session: neo4j.Session,
-    update_tag: int,
-    requirements_objects: List[Dict],
-) -> None:
-    query = """
-    UNWIND $Requirements AS req
-        MERGE (lib:PythonLibrary:Dependency{id: req.id})
-        ON CREATE SET lib.firstseen = timestamp(),
-        lib.name = req.name
-        SET lib.lastupdated = $UpdateTag,
-        lib.version = req.version
-
-        WITH lib, req
-        MATCH (repo:GitHubRepository{id: req.repo_url})
-        MERGE (repo)-[r:REQUIRES]->(lib)
-        ON CREATE SET r.firstseen = timestamp()
-        SET r.lastupdated = $UpdateTag,
-        r.specifier = req.specifier
-    """
-    neo4j_session.run(
-        query,
-        Requirements=requirements_objects,
-        UpdateTag=update_tag,
+    load_github_dependency_manifests(
+        neo4j_session,
+        common_job_parameters["UPDATE_TAG"],
+        repo_data["manifests"],
+    )
+    load_github_dependencies(
+        neo4j_session,
+        common_job_parameters["UPDATE_TAG"],
+        repo_data["dependencies"],
     )
 
 
@@ -896,4 +1257,21 @@ def sync(
         )
     repo_data = transform(repos_json, direct_collabs, outside_collabs)
     load(neo4j_session, common_job_parameters, repo_data)
+
+    # Collect repository URLs that have dependencies for cleanup
+    repo_urls_with_dependencies = list(
+        {dep["repo_url"] for dep in repo_data["dependencies"]}
+    )
+    cleanup_github_dependencies(
+        neo4j_session, common_job_parameters, repo_urls_with_dependencies
+    )
+
+    # Collect repository URLs that have manifests for cleanup
+    repo_urls_with_manifests = list(
+        {manifest["repo_url"] for manifest in repo_data["manifests"]}
+    )
+    cleanup_github_manifests(
+        neo4j_session, common_job_parameters, repo_urls_with_manifests
+    )
+
     run_cleanup_job("github_repos_cleanup.json", neo4j_session, common_job_parameters)