cartography 0.118.0__tar.gz → 0.119.0__tar.gz

{cartography-0.118.0 → cartography-0.119.0}/.github/workflows/ossf-scorecard.yml

@@ -49,7 +49,7 @@ jobs:
       # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
       # format to the repository Actions tab.
       - name: "Upload artifact"
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
         with:
           name: SARIF file
           path: results.sarif
@@ -57,6 +57,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@f443b600d91635bebf5b0d9ebc620189c0d6fba5 # v3.29.5
+        uses: github/codeql-action/upload-sarif@4e94bd11f71e507f7f87df81788dff88d1dacbfb # v3.29.5
         with:
           sarif_file: results.sarif

{cartography-0.118.0 → cartography-0.119.0}/.github/workflows/publish-to-ghcr-and-pypi.yml

@@ -19,7 +19,7 @@ jobs:
         with:
           python-version: "3.10"
       - name: Install uv
-        uses: astral-sh/setup-uv@3259c6206f993105e3a61b142c2d97bf4b9ef83d # v7.1.0
+        uses: astral-sh/setup-uv@85856786d1ce8acfbcc2f13a5f3fbd6b938f9f41 # v7.1.2
         with:
           enable-cache: true
           cache-dependency-glob: "uv.lock"
@@ -42,7 +42,7 @@ jobs:
     steps:
       # See: https://docs.docker.com/build/ci/github-actions/multi-platform/#build-and-load-multi-platform-images
       - name: Set up Docker
-        uses: docker/setup-docker-action@b60f85385d03ac8acfca6d9996982511d8620a19 # v4.3.0
+        uses: docker/setup-docker-action@3fb92d6d9c634363128c8cce4bc3b2826526370a # v4.4.0
         with:
           daemon-config: |
             {

{cartography-0.118.0 → cartography-0.119.0}/.github/workflows/sphinx.yml

@@ -31,7 +31,7 @@ jobs:
         with:
           python-version: "3.10"
       - name: Install uv
-        uses: astral-sh/setup-uv@3259c6206f993105e3a61b142c2d97bf4b9ef83d # v7.1.0
+        uses: astral-sh/setup-uv@85856786d1ce8acfbcc2f13a5f3fbd6b938f9f41 # v7.1.2
         with:
           enable-cache: true
           cache-dependency-glob: "uv.lock"

{cartography-0.118.0 → cartography-0.119.0}/.github/workflows/test_suite.yml

@@ -40,7 +40,7 @@ jobs:
         with:
           python-version: "3.10"
       - name: Install uv
-        uses: astral-sh/setup-uv@3259c6206f993105e3a61b142c2d97bf4b9ef83d # v7.1.0
+        uses: astral-sh/setup-uv@85856786d1ce8acfbcc2f13a5f3fbd6b938f9f41 # v7.1.2
         with:
           enable-cache: true
           cache-dependency-glob: "uv.lock"
@@ -72,7 +72,7 @@ jobs:
         with:
           python-version: "3.10"
       - name: Install uv
-        uses: astral-sh/setup-uv@3259c6206f993105e3a61b142c2d97bf4b9ef83d # v7.1.0
+        uses: astral-sh/setup-uv@85856786d1ce8acfbcc2f13a5f3fbd6b938f9f41 # v7.1.2
         with:
           enable-cache: true
           cache-dependency-glob: "uv.lock"

{cartography-0.118.0 → cartography-0.119.0}/AGENTS.md

@@ -28,13 +28,14 @@ This guide teaches you how to write intel modules for Cartography using the mode
 5. @Advanced Node Schema Properties
 6. @One-to-Many Relationships
 7. @MatchLinks: Connecting Existing Nodes
-8. @Configuration and Credentials
-9. @Error Handling
-10. @Testing Your Module
-11. @Refactoring Legacy Code to Data Model
-12. @Common Patterns and Examples
-13. @Troubleshooting Guide
-14. @Quick Reference
+8. @Ontology Integration: Mapping Users and Devices
+9. @Configuration and Credentials
+10. @Error Handling
+11. @Testing Your Module
+12. @Refactoring Legacy Code to Data Model
+13. @Common Patterns and Examples
+14. @Troubleshooting Guide
+15. @Quick Reference
 
 ## 🚀 Quick Start: Copy an Existing Module {#quick-start}
 
@@ -767,6 +768,373 @@ def cleanup(neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any])
     ).run(neo4j_session)
 ```
 
+## 🌐 Ontology Integration: Mapping Users and Devices {#ontology-integration}
+
+Cartography includes an **Ontology system** that creates normalized `User` and `Device` nodes that aggregate data from multiple sources. This provides a unified view of identity and device management across your infrastructure.
+
+### Overview of Ontology System
+
+The Ontology system works by:
+1. **Creating canonical nodes**: `(:User:Ontology)` and `(:Device:Ontology)` nodes that represent unified entities
+2. **Mapping source data**: Your module's user/device nodes get connected to these canonical nodes
+3. **Enabling unified queries**: Users can query across all systems through common ontology nodes
+
+### When to Use Ontology Integration
+
+Add ontology integration to your module when it manages:
+- **Users/Identities**: Service accounts, human users, admin accounts
+- **Devices/Assets**: Computers, phones, tablets, IoT devices, virtual machines
+
+**Examples of modules that should integrate:**
+- Identity providers (Okta, Azure AD, Duo)
+- Device management (Kandji, Jamf, CrowdStrike)
+- Infrastructure (AWS EC2, Azure VMs, GCP instances)
+- Security tools (endpoint protection, mobile device management)
+
+### Step 1: Add Ontology Mapping Configuration
+
+Create mapping configurations in `cartography/models/ontology/mapping/data/`:
+
+#### For User Entities
+
+```python
+# cartography/models/ontology/mapping/data/users.py
+from cartography.models.ontology.mapping.specs import OntologyFieldMapping
+from cartography.models.ontology.mapping.specs import OntologyMapping
+from cartography.models.ontology.mapping.specs import OntologyNodeMapping
+
+# Add your mapping to the file
+your_service_mapping = OntologyMapping(
+    module_name="your_service",
+    nodes=[
+        OntologyNodeMapping(
+            node_label="YourServiceUser",  # Your node label
+            fields=[
+                # Map your node fields to ontology fields
+                OntologyFieldMapping(ontology_field="email", node_field="email", required=True),  # Required field
+                OntologyFieldMapping(ontology_field="username", node_field="username"),
+                OntologyFieldMapping(ontology_field="fullname", node_field="display_name"),
+                OntologyFieldMapping(ontology_field="firstname", node_field="first_name"),
+                OntologyFieldMapping(ontology_field="lastname", node_field="last_name"),
+            ],
+        ),
+    ],
+)
+```
+
+#### For Device Entities
+
+```python
+# cartography/models/ontology/mapping/data/devices.py
+from cartography.models.ontology.mapping.specs import OntologyFieldMapping
+from cartography.models.ontology.mapping.specs import OntologyMapping
+from cartography.models.ontology.mapping.specs import OntologyNodeMapping
+from cartography.models.ontology.mapping.specs import OntologyRelMapping
+
+# Add your mapping to the file
+your_service_mapping = OntologyMapping(
+    module_name="your_service",
+    nodes=[
+        OntologyNodeMapping(
+            node_label="YourServiceDevice",  # Your node label
+            fields=[
+                # Map your node fields to ontology fields
+                OntologyFieldMapping(ontology_field="hostname", node_field="device_name", required=True),  # Required field
+                OntologyFieldMapping(ontology_field="os", node_field="operating_system"),
+                OntologyFieldMapping(ontology_field="os_version", node_field="os_version"),
+                OntologyFieldMapping(ontology_field="model", node_field="device_model"),
+                OntologyFieldMapping(ontology_field="platform", node_field="platform"),
+                OntologyFieldMapping(ontology_field="serial_number", node_field="serial"),
+            ],
+        ),
+    ],
+    # Optional: Add relationship mappings to connect Users to Devices
+    rels=[
+        OntologyRelMapping(
+            __comment__="Link Device to User based on YourServiceUser-YourServiceDevice ownership",
+            query="""
+                MATCH (u:User)-[:HAS_ACCOUNT]->(:YourServiceUser)-[:OWNS]->(:YourServiceDevice)<-[:OBSERVED_AS]-(d:Device)
+                MERGE (u)-[r:OWNS]->(d)
+                ON CREATE SET r.firstseen = timestamp()
+                SET r.lastupdated = $UPDATE_TAG
+            """,
+            interative=False,
+        ),
+    ],
+)
+```
+
+### Step 2: Add Ontology Relationship to Your Node Schema
+
+Update your node schema to include the relationship to ontology nodes.
+
+#### For User Nodes
+
+Add `UserAccount` label and relationship to the canonical `User` node:
+
+```python
+from cartography.models.core.nodes import ExtraNodeLabels
+
+@dataclass(frozen=True)
+class YourServiceUserSchema(CartographyNodeSchema):
+    label: str = "YourServiceUser"
+    # Add UserAccount label so ontology can find and link to this node
+    extra_node_labels: ExtraNodeLabels = ExtraNodeLabels(["UserAccount"])
+    properties: YourServiceUserNodeProperties = YourServiceUserNodeProperties()
+    sub_resource_relationship: YourServiceTenantToUserRel = YourServiceTenantToUserRel()
+```
+
+#### For Device Nodes
+
+Add the relationship to the canonical `Device` ontology node:
+
+```python
+from cartography.models.core.relationships import OtherRelationships
+
+# Define relationship to ontology Device node
+@dataclass(frozen=True)
+class YourServiceDeviceToDeviceRelProperties(CartographyRelProperties):
+    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
+
+@dataclass(frozen=True)
+class YourServiceDeviceToDeviceRel(CartographyRelSchema):
+    target_node_label: str = "Device"
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher({
+        "hostname": PropertyRef("device_name"),  # Match on hostname field
+    })
+    direction: LinkDirection = LinkDirection.INWARD
+    rel_label: str = "OBSERVED_AS"
+    properties: YourServiceDeviceToDeviceRelProperties = YourServiceDeviceToDeviceRelProperties()
+
+@dataclass(frozen=True)
+class YourServiceDeviceSchema(CartographyNodeSchema):
+    label: str = "YourServiceDevice"
+    properties: YourServiceDeviceNodeProperties = YourServiceDeviceNodeProperties()
+    sub_resource_relationship: YourServiceTenantToDeviceRel = YourServiceTenantToDeviceRel()
+    # Add the relationship to ontology Device nodes
+    other_relationships: OtherRelationships = OtherRelationships([
+        YourServiceDeviceToDeviceRel(),
+    ])
+```
+
+### Step 3: Understanding Ontology Field Mappings
+
+#### Required Fields
+
+The `required` parameter in `OntologyFieldMapping` serves two critical purposes:
+
+**1. Data Quality Control**: When `required=True`, source nodes that lack this field (i.e., the field is `None` or missing) will be completely excluded from ontology node creation. This ensures only complete, usable data creates ontology nodes.
+
+**2. Primary Identifier Validation**: Fields used as primary identifiers **must** be marked as required to ensure ontology nodes can always be properly identified and matched across data sources.
+
+```python
+# ✅ DO: Mark primary identifiers as required
+OntologyFieldMapping(ontology_field="email", node_field="email", required=True),        # Users
+OntologyFieldMapping(ontology_field="hostname", node_field="device_name", required=True), # Devices
+
+# ✅ DO: Mark optional fields as not required (default)
+OntologyFieldMapping(ontology_field="firstname", node_field="first_name"),  # Optional field
+```
+
+**Example**: If a `DuoUser` node has no email address and email is marked as `required=True`, no corresponding `User` ontology node will be created for that record.
+
+#### Common User Fields
+
+The ontology `User` node supports these fields:
+
+| Ontology Field | Purpose | Required? | Example Source Fields |
+|---------------|---------|-----------|---------------------|
+| `email` | Primary identifier | **Yes** | `email`, `mail`, `email_address` |
+| `username` | Login name | No | `username`, `login`, `user_name` |
+| `fullname` | Complete name | No | `name`, `display_name`, `full_name` |
+| `firstname` | First name | No | `first_name`, `given_name`, `fname` |
+| `lastname` | Last name | No | `last_name`, `family_name`, `surname` |
+
+#### Common Device Fields
+
+The ontology `Device` node supports these fields:
+
+| Ontology Field | Purpose | Required? | Example Source Fields |
+|---------------|---------|-----------|---------------------|
+| `hostname` | Primary identifier | **Yes** | `hostname`, `device_name`, `name` |
+| `os` | Operating system | No | `os`, `operating_system`, `os_family` |
+| `os_version` | OS version | No | `os_version`, `version`, `build` |
+| `model` | Device model | No | `model`, `device_model`, `hardware_model` |
+| `platform` | Platform type | No | `platform`, `platform_name`, `arch` |
+| `serial_number` | Serial number | No | `serial_number`, `serial`, `device_serial` |
+
+### Step 4: Update Module Registration
+
+Ensure your mappings are imported and available to the ontology system:
+
+```python
+# cartography/models/ontology/mapping/data/users.py
+# Add to the end of the file after all mappings:
+ALL_USER_MAPPINGS = [
+    # ... existing mappings ...
+    your_service_mapping,  # Add your mapping here
+]
+```
+
+```python
+# cartography/models/ontology/mapping/data/devices.py
+# Add to the end of the file after all mappings:
+ALL_DEVICE_MAPPINGS = [
+    # ... existing mappings ...
+    your_service_mapping,  # Add your mapping here
+]
+```
+
+### Step 5: Testing Ontology Integration
+
+Test that your ontology integration works correctly:
+
+```python
+# tests/integration/cartography/intel/your_service/test_users.py
+def test_ontology_integration(neo4j_session):
+    # Run your module sync
+    your_service.users.sync(neo4j_session, ...)
+
+    # Run ontology sync to create User nodes
+    import cartography.intel.ontology.users
+    cartography.intel.ontology.users.sync(
+        neo4j_session,
+        source_of_truth=["your_service"],
+        update_tag=TEST_UPDATE_TAG,
+        common_job_parameters={"UPDATE_TAG": TEST_UPDATE_TAG},
+    )
+
+    # Verify User ontology nodes were created
+    result = neo4j_session.run("""
+        MATCH (u:User:Ontology)-[:HAS_ACCOUNT]->(ua:YourServiceUser)
+        RETURN u.email, ua.email
+    """)
+    users = result.data()
+    assert len(users) > 0
+    assert users[0]["u.email"] == users[0]["ua.email"]
+```
+
+### Step 6: Handle Complex Relationships
+
+For services that have user-device relationships, add relationship mappings:
+
+```python
+# In your device mapping
+rels=[
+    OntologyRelMapping(
+        __comment__="Connect users to their devices",
+        query="""
+            MATCH (u:User)-[:HAS_ACCOUNT]->(:YourServiceUser)-[:OWNS]->(:YourServiceDevice)<-[:OBSERVED_AS]-(d:Device)
+            MERGE (u)-[r:OWNS]->(d)
+            ON CREATE SET r.firstseen = timestamp()
+            SET r.lastupdated = $UPDATE_TAG
+        """,
+        interative=False,
+    ),
+]
+```
+
+### Best Practices for Ontology Integration
+
+#### 1. Choose the Right Primary Identifier and Mark as Required
+- **Users**: Use `email` as the primary identifier when available (most reliable across systems) and mark as `required=True`
+- **Devices**: Use `hostname` as the primary identifier when available and mark as `required=True`
+
+```python
+# ✅ DO: Primary identifiers must be required
+OntologyFieldMapping(ontology_field="email", node_field="email", required=True),
+OntologyFieldMapping(ontology_field="hostname", node_field="device_name", required=True),
+```
+
+#### 2. Use Required Fields Strategically
+- **Mark as required**: Only fields that are absolutely essential for the ontology node to be useful
+- **Leave optional**: Fields that provide additional context but aren't essential
+- **Consider impact**: Required fields filter out entire records if missing
+
+```python
+# ✅ DO: Strategic use of required flag
+fields=[
+    OntologyFieldMapping(ontology_field="email", node_field="email", required=True),        # Must have
+    OntologyFieldMapping(ontology_field="username", node_field="username"),                 # Nice to have
+    OntologyFieldMapping(ontology_field="fullname", node_field="display_name"),            # Nice to have
+]
+```
+
+#### 3. Handle Missing Data Gracefully
+```python
+# In your transform function, handle optional ontology fields
+def transform_users(api_data):
+    return [
+        {
+            "email": user["email"],  # Required - let it fail if missing
+            "username": user.get("username"),  # Optional - use .get()
+            "display_name": user.get("full_name") or user.get("name"),  # Fallback logic
+            "first_name": user.get("firstName"),  # Optional
+            "last_name": user.get("lastName"),   # Optional
+        }
+        for user in api_data["users"]
+    ]
+```
+
+#### 3. Consider Multiple Device Types
+Some services have multiple device types. Map each type separately:
+
+```python
+your_service_mapping = OntologyMapping(
+    module_name="your_service",
+    nodes=[
+        OntologyNodeMapping(
+            node_label="YourServiceComputer",
+            fields=[
+                OntologyFieldMapping(ontology_field="hostname", node_field="computer_name"),
+                OntologyFieldMapping(ontology_field="os", node_field="operating_system"),
+            ],
+        ),
+        OntologyNodeMapping(
+            node_label="YourServiceMobileDevice",
+            fields=[
+                OntologyFieldMapping(ontology_field="hostname", node_field="device_name"),
+                OntologyFieldMapping(ontology_field="model", node_field="model"),
+            ],
+        ),
+    ],
+)
+```
+
+### Troubleshooting Ontology Integration
+
+#### Common Issues
+
+**Issue**: Ontology nodes not created
+**Solution**: Verify that:
+- Your mapping is registered in `ALL_USER_MAPPINGS` or `ALL_DEVICE_MAPPINGS`
+- Your source nodes have the correct labels (`UserAccount` for users)
+- Field mappings match your actual node properties
+- **Check required fields**: Ensure source nodes have all required fields populated (not `None`)
+
+**Issue**: Fewer ontology nodes created than expected
+**Solution**: Check if:
+- **Required fields are missing**: Source nodes lacking required fields are filtered out completely
+- Required fields are marked appropriately (primary identifiers should be required)
+- Your source data has the expected completeness
+
+**Issue**: Relationships not created between ontology and source nodes
+**Solution**: Check that:
+- The ontology field used for matching has data
+- Your source nodes are loaded before running ontology sync
+- The target node matcher uses the correct field names
+
+**Issue**: User-Device relationships not working
+**Solution**: Ensure that:
+- Both user and device ontology integrations are working
+- Your relationship query correctly traverses the path between source nodes
+- The relationship query includes proper `UPDATE_TAG` handling
+
+**Issue**: Unit tests failing on required field validation
+**Solution**: Ensure that:
+- Primary identifier fields (`email` for users, `hostname` for devices) are marked as `required=True`
+- The field names match exactly between your ontology mapping and node properties
+
 ## ⚙️ Configuration and Credentials {#configuration}
 
 ### Adding CLI Arguments

{cartography-0.118.0 → cartography-0.119.0}/Dockerfile

@@ -1,5 +1,5 @@
 # Base image
-FROM python:3.10.19-slim@sha256:b1e484d92b32414c67cca8c7e2105e2e31d197855546b6dd71c337889aacdf23 AS base
+FROM python:3.10.19-slim@sha256:e0c4fae70d550834a40f6c3e0326e02cfe239c2351d922e1fb1577a3c6ebde02 AS base
 # Default to ''. Overridden with a specific version specifier e.g. '==0.98.0' by build args or from GitHub actions.
 ARG VERSION_SPECIFIER
 # the UID and GID to run cartography as

{cartography-0.118.0 → cartography-0.119.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cartography
-Version: 0.118.0
+Version: 0.119.0
 Summary: Explore assets and their relationships across your technical infrastructure.
 Maintainer: Cartography Contributors
 License-Expression: Apache-2.0

{cartography-0.118.0 → cartography-0.119.0}/cartography/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.118.0'
-__version_tuple__ = version_tuple = (0, 118, 0)
+__version__ = version = '0.119.0'
+__version_tuple__ = version_tuple = (0, 119, 0)
 
-__commit_id__ = commit_id = 'g3e9cd1a2d'
+__commit_id__ = commit_id = 'g8b3daf9eb'

{cartography-0.118.0 → cartography-0.119.0}/cartography/cli.py

@@ -730,6 +730,26 @@ class CLI:
                 "Required if you are using the Trivy module. Ignored otherwise."
             ),
         )
+        parser.add_argument(
+            "--ontology-users-source",
+            type=str,
+            default=None,
+            help=(
+                "Comma-separated list of sources of truth for user data in the ontology. "
+                "'User' nodes will only be created for users that exist in one of the sources. "
+                "Required if you are using the ontology module. Ignored otherwise."
+            ),
+        )
+        parser.add_argument(
+            "--ontology-devices-source",
+            type=str,
+            default=None,
+            help=(
+                "Comma-separated list of sources of truth for client computer data in the ontology. "
+                "'Device' nodes will only be created for groups that exist in one of the sources. "
+                "Required if you are using the ontology module. Ignored otherwise."
+            ),
+        )
         parser.add_argument(
             "--trivy-results-dir",
             type=str,

{cartography-0.118.0 → cartography-0.119.0}/cartography/client/core/tx.py

@@ -249,6 +249,7 @@ def load_graph_data(
     neo4j_session: neo4j.Session,
     query: str,
     dict_list: List[Dict[str, Any]],
+    batch_size: int = 10000,
     **kwargs,
 ) -> None:
     """
@@ -257,10 +258,13 @@ def load_graph_data(
     :param query: The Neo4j write query to run. This query is not meant to be handwritten, rather it should be generated
     with cartography.graph.querybuilder.build_ingestion_query().
     :param dict_list: The data to load to the graph represented as a list of dicts.
+    :param batch_size: The number of items to process per transaction. Defaults to 10000.
     :param kwargs: Allows additional keyword args to be supplied to the Neo4j query.
     :return: None
     """
-    for data_batch in batch(dict_list, size=10000):
+    if batch_size <= 0:
+        raise ValueError(f"batch_size must be greater than 0, got {batch_size}")
+    for data_batch in batch(dict_list, size=batch_size):
         neo4j_session.write_transaction(
             write_list_of_dicts_tx,
             query,
@@ -316,6 +320,7 @@ def load(
     neo4j_session: neo4j.Session,
     node_schema: CartographyNodeSchema,
     dict_list: List[Dict[str, Any]],
+    batch_size: int = 10000,
     **kwargs,
 ) -> None:
     """
@@ -324,21 +329,27 @@ def load(
     :param neo4j_session: The Neo4j session
     :param node_schema: The CartographyNodeSchema object to create indexes for and generate a query.
     :param dict_list: The data to load to the graph represented as a list of dicts.
+    :param batch_size: The number of items to process per transaction. Defaults to 10000.
     :param kwargs: Allows additional keyword args to be supplied to the Neo4j query.
     :return: None
     """
+    if batch_size <= 0:
+        raise ValueError(f"batch_size must be greater than 0, got {batch_size}")
     if len(dict_list) == 0:
         # If there is no data to load, save some time.
         return
     ensure_indexes(neo4j_session, node_schema)
     ingestion_query = build_ingestion_query(node_schema)
-    load_graph_data(neo4j_session, ingestion_query, dict_list, **kwargs)
+    load_graph_data(
+        neo4j_session, ingestion_query, dict_list, batch_size=batch_size, **kwargs
+    )
 
 
 def load_matchlinks(
     neo4j_session: neo4j.Session,
     rel_schema: CartographyRelSchema,
     dict_list: list[dict[str, Any]],
+    batch_size: int = 10000,
     **kwargs,
 ) -> None:
     """
@@ -347,9 +358,12 @@ def load_matchlinks(
     :param rel_schema: The CartographyRelSchema object to generate a query.
     :param dict_list: The data to load to the graph represented as a list of dicts. The dicts must contain the source and
     target node ids.
+    :param batch_size: The number of items to process per transaction. Defaults to 10000.
     :param kwargs: Allows additional keyword args to be supplied to the Neo4j query.
     :return: None
     """
+    if batch_size <= 0:
+        raise ValueError(f"batch_size must be greater than 0, got {batch_size}")
     if len(dict_list) == 0:
         # If there is no data to load, save some time.
         return
@@ -369,4 +383,6 @@ def load_matchlinks(
     ensure_indexes_for_matchlinks(neo4j_session, rel_schema)
     matchlink_query = build_matchlink_query(rel_schema)
     logger.debug(f"Matchlink query: {matchlink_query}")
-    load_graph_data(neo4j_session, matchlink_query, dict_list, **kwargs)
+    load_graph_data(
+        neo4j_session, matchlink_query, dict_list, batch_size=batch_size, **kwargs
+    )

{cartography-0.118.0 → cartography-0.119.0}/cartography/config.py

@@ -161,6 +161,11 @@ class Config:
     :param trivy_s3_bucket: The S3 bucket name containing Trivy scan results. Optional.
     :type trivy_s3_prefix: str
     :param trivy_s3_prefix: The S3 prefix path containing Trivy scan results. Optional.
+    :type ontology_users_source: str
+    :param ontology_users_source: Comma-separated list of sources of truth for user data in the ontology. Optional.
+    :type ontology_devices_source: str
+    :param ontology_devices_source: Comma-separated list of sources of truth for client computers data in the ontology.
+        Optional.
     :type trivy_results_dir: str
     :param trivy_results_dir: Local directory containing Trivy scan results. Optional.
     :type scaleway_access_key: str
@@ -266,6 +271,8 @@ class Config:
         airbyte_api_url=None,
         trivy_s3_bucket=None,
         trivy_s3_prefix=None,
+        ontology_users_source=None,
+        ontology_devices_source=None,
         trivy_results_dir=None,
         scaleway_access_key=None,
         scaleway_secret_key=None,
@@ -359,6 +366,8 @@ class Config:
         self.airbyte_api_url = airbyte_api_url
         self.trivy_s3_bucket = trivy_s3_bucket
         self.trivy_s3_prefix = trivy_s3_prefix
+        self.ontology_users_source = ontology_users_source
+        self.ontology_devices_source = ontology_devices_source
         self.trivy_results_dir = trivy_results_dir
         self.scaleway_access_key = scaleway_access_key
         self.scaleway_secret_key = scaleway_secret_key

{cartography-0.118.0 → cartography-0.119.0}/cartography/data/indexes.cypher

@@ -102,12 +102,6 @@ CREATE INDEX IF NOT EXISTS FOR (n:GCPVpc) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:GCPVpc) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:GitHubRepository) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:GitHubRepository) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:GSuiteGroup) ON (n.email);
-CREATE INDEX IF NOT EXISTS FOR (n:GSuiteGroup) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:GSuiteGroup) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:GSuiteUser) ON (n.email);
-CREATE INDEX IF NOT EXISTS FOR (n:GSuiteUser) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:GSuiteUser) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:Ip) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:Ip) ON (n.ip);
 CREATE INDEX IF NOT EXISTS FOR (n:Ip) ON (n.lastupdated);

{cartography-0.118.0 → cartography-0.119.0}/cartography/graph/job.py

@@ -125,11 +125,13 @@ class GraphJob:
         }
 
     @classmethod
-    def from_json(cls, blob: str, short_name: Optional[str] = None) -> "GraphJob":
+    def from_json(
+        cls, blob: Union[str, dict], short_name: Optional[str] = None
+    ) -> "GraphJob":
         """
-        Create a job from a JSON blob.
+        Create a job from a JSON dict or blob.
         """
-        data: Dict = json.loads(blob)
+        data = json.loads(blob) if isinstance(blob, str) else blob
         statements = _get_statements_from_json(data, short_name)
         name = data["name"]
         return cls(name, statements, short_name)
@@ -242,12 +244,12 @@ class GraphJob:
     def run_from_json(
         cls,
         neo4j_session: neo4j.Session,
-        blob: str,
+        blob: Union[str, dict],
         parameters: Dict,
         short_name: Optional[str] = None,
     ) -> None:
         """
-        Run a job from a JSON blob. This will deserialize the job and execute all statements sequentially.
+        Run a job from a JSON dict or blob. This will deserialize the job and execute all statements sequentially.
         """
         if not parameters:
             parameters = {}