cartography 0.107.0rc3__tar.gz → 0.108.0rc2__tar.gz

{cartography-0.107.0rc3 → cartography-0.108.0rc2}/.github/workflows/ossf-scorecard.yml

@@ -57,6 +57,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@181d5eefc20863364f96762470ba6f862bdef56b # v3.29.2
+        uses: github/codeql-action/upload-sarif@d6bbdef45e766d081b84a2def353b0055f728d3e # v3.29.3
         with:
           sarif_file: results.sarif

{cartography-0.107.0rc3 → cartography-0.108.0rc2}/.github/workflows/sphinx.yml

@@ -31,7 +31,7 @@ jobs:
         with:
           python-version: "3.10"
       - name: Install uv
-        uses: astral-sh/setup-uv@bd01e18f51369d5a26f1651c3cb451d3417e3bba # v6.3.1
+        uses: astral-sh/setup-uv@7edac99f961f18b581bbd960d59d049f04c0002f # v6.4.1
         with:
           enable-cache: true
           cache-dependency-glob: "uv.lock"

{cartography-0.107.0rc3 → cartography-0.108.0rc2}/.github/workflows/test_suite.yml

@@ -40,7 +40,7 @@ jobs:
         with:
           python-version: "3.10"
       - name: Install uv
-        uses: astral-sh/setup-uv@bd01e18f51369d5a26f1651c3cb451d3417e3bba # v6.3.1
+        uses: astral-sh/setup-uv@7edac99f961f18b581bbd960d59d049f04c0002f # v6.4.1
         with:
           enable-cache: true
           cache-dependency-glob: "uv.lock"
@@ -72,7 +72,7 @@ jobs:
         with:
           python-version: "3.10"
       - name: Install uv
-        uses: astral-sh/setup-uv@bd01e18f51369d5a26f1651c3cb451d3417e3bba # v6.3.1
+        uses: astral-sh/setup-uv@7edac99f961f18b581bbd960d59d049f04c0002f # v6.4.1
         with:
           enable-cache: true
           cache-dependency-glob: "uv.lock"

{cartography-0.107.0rc3 → cartography-0.108.0rc2}/.gitignore

@@ -24,3 +24,4 @@ poetry.lock
 .gitconfig
 .python-version
 .python_history
+github_config.json

{cartography-0.107.0rc3 → cartography-0.108.0rc2}/AGENTS.md

@@ -754,7 +754,7 @@ MOCK_USERS_RESPONSE = {
 
 ### Unit Tests
 
-Test your transform functions in `tests/unit/cartography/intel/your_service/`:
+(Optional) Test your transform functions in `tests/unit/cartography/intel/your_service/`:
 
 ```python
 # tests/unit/cartography/intel/your_service/test_users.py
@@ -783,6 +783,10 @@ def test_transform_users():
 
 Test actual Neo4j loading in `tests/integration/cartography/intel/your_service/`:
 
+**Key Principle: Test outcomes, not implementation details.**
+
+Focus on verifying that data is written to the graph as expected, rather than testing internal function parameters or implementation details. Mock external dependencies (APIs, databases) when necessary, but avoid brittle parameter testing.
+
 ```python
 # tests/integration/cartography/intel/your_service/test_users.py
 from unittest.mock import patch
@@ -813,7 +817,8 @@ def test_sync_users(mock_api, neo4j_session):
         {"UPDATE_TAG": TEST_UPDATE_TAG, "TENANT_ID": TEST_TENANT_ID},
     )
 
-    # Assert - Use check_nodes() instead of raw Neo4j queries
+    # ✅ DO: Test outcomes - verify data is written to the graph as expected
+    # Assert - Use check_nodes() instead of raw Neo4j queries.
     expected_nodes = {
         ("user-123", "alice@example.com"),
         ("user-456", "bob@example.com"),
@@ -826,7 +831,8 @@ def test_sync_users(mock_api, neo4j_session):
     }
     assert check_nodes(neo4j_session, "YourServiceTenant", ["id"]) == expected_tenant_nodes
 
-    # Assert - Use check_rels() instead of raw Neo4j queries for relationships
+    # Assert relationships are created correctly.
+    # Use check_rels() instead of raw Neo4j queries for relationships
     expected_rels = {
         ("user-123", TEST_TENANT_ID),
         ("user-456", TEST_TENANT_ID),
@@ -843,8 +849,35 @@ def test_sync_users(mock_api, neo4j_session):
         )
         == expected_rels
     )
+
+    # ✅ DO: Mock external dependencies when needed
+    # mock_api.return_value = MOCK_USERS_RESPONSE  # Good - provides test data
+    # (Prefer the decorator style though)
+
+    # ❌ DON'T: Test brittle implementation details
+    # mock_api.assert_called_once_with("fake-api-key", TEST_TENANT_ID)  # Brittle!
+    # mock_api.assert_called_with(specific_params)  # Brittle!
 ```
 
+**What to Test:**
+- ✅ **Outcomes**: Nodes created with correct properties
+- ✅ **Outcomes**: Relationships created between expected nodes
+
+**What NOT to Test:**
+- ❌ **Implementation**: Function parameters passed to mocks (brittle!)
+- ❌ **Implementation**: Internal function call order
+- ❌ **Implementation**: Mock call counts unless absolutely necessary
+
+**When to Mock:**
+- ✅ External APIs (AWS, Azure, third-party services) - provide test data
+- ✅ Database connections - avoid real connections
+- ✅ Network calls - avoid real network requests
+
+**When NOT to Mock:**
+- ❌ Internal Cartography functions
+- ❌ Data transformation logic
+- ❌ The function that is being tested
+
 ## 📚 Common Patterns and Examples {#common-patterns}
 
 ### Pattern 1: Simple Service with Users (LastPass Style)
@@ -933,6 +966,51 @@ Represents a user in Your Service.
 - User connected to human: `(:YourServiceUser)<-[:IDENTITY_YOUR_SERVICE]-(:Human)`
 ```
 
+### Multiple Intel Modules Modifying the Same Node Type
+
+It is possible (and encouraged) for more than one intel module to modify the same node type. However, there are two distinct patterns for this:
+
+**Simple Relationship Pattern**: When data type A only refers to data type B by an ID without providing additional properties about B, we can just define a relationship schema. This way when A is loaded, the relationship schema performs a `MATCH` to find and connect to existing nodes of type B.
+
+For example, when an RDS instance refers to EC2 security groups by ID, we create a relationship from the RDS instance to the security group nodes, since the RDS API doesn't provide additional properties about the security groups beyond their IDs.
+
+```python
+# RDS Instance refers to Security Groups by ID only
+@dataclass(frozen=True)
+class RDSInstanceToSecurityGroupRel(CartographyRelSchema):
+    target_node_label: str = "EC2SecurityGroup"
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher({
+        "id": PropertyRef("SecurityGroupId"),  # Just the ID, no additional properties
+    })
+    direction: LinkDirection = LinkDirection.OUTWARD
+    rel_label: str = "MEMBER_OF_EC2_SECURITY_GROUP"
+    properties: RDSInstanceToSecurityGroupRelProperties = RDSInstanceToSecurityGroupRelProperties()
+```
+
+**Composite Node Pattern**: When a data type A refers to another data type B and offers additional fields about B that B doesn't have itself, we should define a composite node schema. This composite node would be named "`BASchema`" to denote that it's a "`B`" object as known by an "`A`" object. When loaded, the composite node schema targets the same node label as the primary `B` schema, allowing the loading system to perform a `MERGE` operation that combines properties from both sources.
+
+For example, in the AWS EC2 module, we have both `EBSVolumeSchema` (from the EBS API) and `EBSVolumeInstanceSchema` (from the EC2 Instance API). The EC2 Instance API provides additional properties about EBS volumes that the EBS API doesn't have, such as `deleteontermination`. Both schemas target the same `EBSVolume` node label, allowing the node to accumulate properties from both sources.
+
+```python
+# EC2 Instance provides additional properties about EBS Volumes
+@dataclass(frozen=True)
+class EBSVolumeInstanceProperties(CartographyNodeProperties):
+    id: PropertyRef = PropertyRef("VolumeId")
+    arn: PropertyRef = PropertyRef("Arn", extra_index=True)
+    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
+    # Additional property that EBS API doesn't have
+    deleteontermination: PropertyRef = PropertyRef("DeleteOnTermination")
+
+@dataclass(frozen=True)
+class EBSVolumeInstanceSchema(CartographyNodeSchema):
+    label: str = "EBSVolume"  # Same label as EBSVolumeSchema
+    properties: EBSVolumeInstanceProperties = EBSVolumeInstanceProperties()
+    sub_resource_relationship: EBSVolumeToAWSAccountRel = EBSVolumeToAWSAccountRel()
+    # ... other relationships
+```
+
+The key distinction is whether the referring module provides additional properties about the target entity. If it does, use a composite node schema. If it only provides IDs, use a simple relationship schema.
+
 ## 🎯 Final Checklist
 
 Before submitting your module:

{cartography-0.107.0rc3 → cartography-0.108.0rc2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cartography
-Version: 0.107.0rc3
+Version: 0.108.0rc2
 Summary: Explore assets and their relationships across your technical infrastructure.
 Maintainer: Cartography Contributors
 License: apache2
@@ -82,14 +82,14 @@ You can learn more about the story behind Cartography in our [presentation at BS
 
 ## Supported platforms
 - [Airbyte](https://cartography-cncf.github.io/cartography/modules/airbyte/index.html) - Organization, Workspace, User, Source, Destination, Connection, Tag, Stream
-- [Amazon Web Services](https://cartography-cncf.github.io/cartography/modules/aws/index.html) - ACM, API Gateway, CloudWatch, CodeBuild, Config, EC2, ECS, ECR, Elasticsearch, Elastic Kubernetes Service (EKS), DynamoDB, IAM, Inspector, KMS, Lambda, RDS, Redshift, Route53, S3, Secrets Manager(Secret Versions), Security Hub, SQS, SSM, STS, Tags
+- [Amazon Web Services](https://cartography-cncf.github.io/cartography/modules/aws/index.html) - ACM, API Gateway, CloudWatch, CodeBuild, Config, EC2, ECS, ECR, Elasticsearch, Elastic Kubernetes Service (EKS), DynamoDB, GuardDuty, IAM, Inspector, KMS, Lambda, RDS, Redshift, Route53, S3, Secrets Manager(Secret Versions), Security Hub, SQS, SSM, STS, Tags
 - [Anthropic](https://cartography-cncf.github.io/cartography/modules/anthropic/index.html) - Organization, ApiKey, User, Workspace
 - [BigFix](https://cartography-cncf.github.io/cartography/modules/bigfix/index.html) - Computers
 - [Cloudflare](https://cartography-cncf.github.io/cartography/modules/cloudflare/index.html) - Account, Role, Member, Zone, DNSRecord
 - [Crowdstrike Falcon](https://cartography-cncf.github.io/cartography/modules/crowdstrike/index.html) - Hosts, Spotlight vulnerabilities, CVEs
 - [DigitalOcean](https://cartography-cncf.github.io/cartography/modules/digitalocean/index.html)
 - [Duo](https://cartography-cncf.github.io/cartography/modules/duo/index.html) - Users, Groups, Endpoints
-- [GitHub](https://cartography-cncf.github.io/cartography/modules/github/index.html) - repos, branches, users, teams
+- [GitHub](https://cartography-cncf.github.io/cartography/modules/github/index.html) - repos, branches, users, teams, dependency graph manifests, dependencies
 - [Google Cloud Platform](https://cartography-cncf.github.io/cartography/modules/gcp/index.html) - Cloud Resource Manager, Compute, DNS, Storage, Google Kubernetes Engine
 - [Google GSuite](https://cartography-cncf.github.io/cartography/modules/gsuite/index.html) - users, groups
 - [Kandji](https://cartography-cncf.github.io/cartography/modules/kandji/index.html) - Devices

{cartography-0.107.0rc3 → cartography-0.108.0rc2}/README.md

@@ -22,14 +22,14 @@ You can learn more about the story behind Cartography in our [presentation at BS
 
 ## Supported platforms
 - [Airbyte](https://cartography-cncf.github.io/cartography/modules/airbyte/index.html) - Organization, Workspace, User, Source, Destination, Connection, Tag, Stream
-- [Amazon Web Services](https://cartography-cncf.github.io/cartography/modules/aws/index.html) - ACM, API Gateway, CloudWatch, CodeBuild, Config, EC2, ECS, ECR, Elasticsearch, Elastic Kubernetes Service (EKS), DynamoDB, IAM, Inspector, KMS, Lambda, RDS, Redshift, Route53, S3, Secrets Manager(Secret Versions), Security Hub, SQS, SSM, STS, Tags
+- [Amazon Web Services](https://cartography-cncf.github.io/cartography/modules/aws/index.html) - ACM, API Gateway, CloudWatch, CodeBuild, Config, EC2, ECS, ECR, Elasticsearch, Elastic Kubernetes Service (EKS), DynamoDB, GuardDuty, IAM, Inspector, KMS, Lambda, RDS, Redshift, Route53, S3, Secrets Manager(Secret Versions), Security Hub, SQS, SSM, STS, Tags
 - [Anthropic](https://cartography-cncf.github.io/cartography/modules/anthropic/index.html) - Organization, ApiKey, User, Workspace
 - [BigFix](https://cartography-cncf.github.io/cartography/modules/bigfix/index.html) - Computers
 - [Cloudflare](https://cartography-cncf.github.io/cartography/modules/cloudflare/index.html) - Account, Role, Member, Zone, DNSRecord
 - [Crowdstrike Falcon](https://cartography-cncf.github.io/cartography/modules/crowdstrike/index.html) - Hosts, Spotlight vulnerabilities, CVEs
 - [DigitalOcean](https://cartography-cncf.github.io/cartography/modules/digitalocean/index.html)
 - [Duo](https://cartography-cncf.github.io/cartography/modules/duo/index.html) - Users, Groups, Endpoints
-- [GitHub](https://cartography-cncf.github.io/cartography/modules/github/index.html) - repos, branches, users, teams
+- [GitHub](https://cartography-cncf.github.io/cartography/modules/github/index.html) - repos, branches, users, teams, dependency graph manifests, dependencies
 - [Google Cloud Platform](https://cartography-cncf.github.io/cartography/modules/gcp/index.html) - Cloud Resource Manager, Compute, DNS, Storage, Google Kubernetes Engine
 - [Google GSuite](https://cartography-cncf.github.io/cartography/modules/gsuite/index.html) - users, groups
 - [Kandji](https://cartography-cncf.github.io/cartography/modules/kandji/index.html) - Devices

{cartography-0.107.0rc3 → cartography-0.108.0rc2}/cartography/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.107.0rc3'
-__version_tuple__ = version_tuple = (0, 107, 0, 'rc3')
+__version__ = version = '0.108.0rc2'
+__version_tuple__ = version_tuple = (0, 108, 0, 'rc2')

{cartography-0.107.0rc3 → cartography-0.108.0rc2}/cartography/cli.py

@@ -264,6 +264,16 @@ class CLI:
                 " If not specified, cartography by default will run all AWS sync modules available."
             ),
         )
+        parser.add_argument(
+            "--aws-guardduty-severity-threshold",
+            type=str,
+            default=None,
+            help=(
+                "GuardDuty severity threshold filter. Only findings at or above this severity level will be synced. "
+                "Valid values: LOW, MEDIUM, HIGH, CRITICAL. If not specified, all findings (except archived) will be synced. "
+                "Example: 'HIGH' will sync only HIGH and CRITICAL findings, filtering out LOW and MEDIUM severity findings."
+            ),
+        )
         parser.add_argument(
             "--analysis-job-directory",
             type=str,

{cartography-0.107.0rc3 → cartography-0.108.0rc2}/cartography/config.py

@@ -53,6 +53,9 @@ class Config:
     :param entra_client_secret: Client Secret for connecting in a Service Principal Authentication approach. Optional.
     :type aws_requested_syncs: str
     :param aws_requested_syncs: Comma-separated list of AWS resources to sync. Optional.
+    :type aws_guardduty_severity_threshold: str
+    :param aws_guardduty_severity_threshold: GuardDuty severity threshold filter. Only findings at or above this
+        severity level will be synced. Valid values: LOW, MEDIUM, HIGH, CRITICAL. Optional.
     :type analysis_job_directory: str
     :param analysis_job_directory: Path to a directory tree containing analysis jobs to run. Optional.
     :type oci_sync_all_profiles: bool
@@ -185,6 +188,7 @@ class Config:
         entra_client_id=None,
         entra_client_secret=None,
         aws_requested_syncs=None,
+        aws_guardduty_severity_threshold=None,
         analysis_job_directory=None,
         oci_sync_all_profiles=None,
         okta_org_id=None,
@@ -268,6 +272,7 @@ class Config:
         self.entra_client_id = entra_client_id
         self.entra_client_secret = entra_client_secret
         self.aws_requested_syncs = aws_requested_syncs
+        self.aws_guardduty_severity_threshold = aws_guardduty_severity_threshold
         self.analysis_job_directory = analysis_job_directory
         self.oci_sync_all_profiles = oci_sync_all_profiles
         self.okta_org_id = okta_org_id

{cartography-0.107.0rc3 → cartography-0.108.0rc2}/cartography/data/indexes.cypher

@@ -81,8 +81,6 @@ CREATE INDEX IF NOT EXISTS FOR (n:DODroplet) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:DODroplet) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:DOProject) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:DOProject) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:EBSSnapshot) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:EBSSnapshot) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:EC2KeyPair) ON (n.keyfingerprint);
 CREATE INDEX IF NOT EXISTS FOR (n:EC2ReservedInstance) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:EC2ReservedInstance) ON (n.lastupdated);
@@ -156,14 +154,8 @@ CREATE INDEX IF NOT EXISTS FOR (n:GSuiteUser) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:Ip) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:Ip) ON (n.ip);
 CREATE INDEX IF NOT EXISTS FOR (n:Ip) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:IpPermissionInbound) ON (n.ruleid);
-CREATE INDEX IF NOT EXISTS FOR (n:IpPermissionInbound) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:IpPermissionsEgress) ON (n.ruleid);
-CREATE INDEX IF NOT EXISTS FOR (n:IpPermissionsEgress) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:IpRange) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:IpRange) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:IpRule) ON (n.ruleid);
-CREATE INDEX IF NOT EXISTS FOR (n:IpRule) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:JamfComputerGroup) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:JamfComputerGroup) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:KMSKey) ON (n.id);

{cartography-0.107.0rc3 → cartography-0.108.0rc2}/cartography/data/jobs/cleanup/github_repos_cleanup.json

@@ -19,6 +19,7 @@
     "iterative": true,
     "iterationsize": 100
   },
+
   {
     "query": "MATCH (:GitHubBranch)-[r:BRANCH]->(:GitHubRepository) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
     "iterative": true,
@@ -39,6 +40,7 @@
     "iterative": true,
     "iterationsize": 100
   },
+
   {
     "query": "MATCH (:GitHubUser)-[r:OUTSIDE_COLLAB_ADMIN]->(:GitHubRepository) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
     "iterative": true,

{cartography-0.107.0rc3 → cartography-0.108.0rc2}/cartography/intel/aws/__init__.py

@@ -310,6 +310,7 @@ def start_aws_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
     common_job_parameters = {
         "UPDATE_TAG": config.update_tag,
         "permission_relationships_file": config.permission_relationships_file,
+        "aws_guardduty_severity_threshold": config.aws_guardduty_severity_threshold,
         "aws_cloudtrail_management_events_lookback_hours": config.aws_cloudtrail_management_events_lookback_hours,
     }
     try:

{cartography-0.107.0rc3 → cartography-0.108.0rc2}/cartography/intel/aws/cloudtrail.py

@@ -26,13 +26,25 @@ def get_cloudtrail_trails(
     )
 
     trails = client.describe_trails()["trailList"]
-
-    # CloudTrail multi-region trails are shown in list_trails,
-    # but the get_trail call only works in the home region
     trails_filtered = [trail for trail in trails if trail.get("HomeRegion") == region]
+
     return trails_filtered
 
 
+def transform_cloudtrail_trails(
+    trails: List[Dict[str, Any]], region: str
+) -> List[Dict[str, Any]]:
+    """
+    Transform CloudTrail trail data for ingestion
+    """
+    for trail in trails:
+        arn = trail.get("CloudWatchLogsLogGroupArn")
+        if arn:
+            trail["CloudWatchLogsLogGroupArn"] = arn.split(":*")[0]
+
+    return trails
+
+
 @timeit
 def load_cloudtrail_trails(
     neo4j_session: neo4j.Session,
@@ -79,7 +91,8 @@ def sync(
         logger.info(
             f"Syncing CloudTrail for region '{region}' in account '{current_aws_account_id}'.",
         )
-        trails = get_cloudtrail_trails(boto3_session, region)
+        trails_filtered = get_cloudtrail_trails(boto3_session, region)
+        trails = transform_cloudtrail_trails(trails_filtered, region)
 
         load_cloudtrail_trails(
             neo4j_session,