cartography 0.104.0rc3__py3-none-any.whl → 0.106.0__py3-none-any.whl

cartography/models/trivy/fix.py

@@ -0,0 +1,66 @@
+from dataclasses import dataclass
+
+from cartography.models.core.common import PropertyRef
+from cartography.models.core.nodes import CartographyNodeProperties
+from cartography.models.core.nodes import CartographyNodeSchema
+from cartography.models.core.nodes import ExtraNodeLabels
+from cartography.models.core.relationships import CartographyRelProperties
+from cartography.models.core.relationships import CartographyRelSchema
+from cartography.models.core.relationships import LinkDirection
+from cartography.models.core.relationships import make_target_node_matcher
+from cartography.models.core.relationships import OtherRelationships
+from cartography.models.core.relationships import TargetNodeMatcher
+
+
+@dataclass(frozen=True)
+class TrivyFixNodeProperties(CartographyNodeProperties):
+    id: PropertyRef = PropertyRef("id")
+    version: PropertyRef = PropertyRef("FixedVersion")
+    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+class TrivyFixToPackageRelProperties(CartographyRelProperties):
+    version: PropertyRef = PropertyRef("FixedVersion")
+    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+class TrivyFixToPackage(CartographyRelSchema):
+    target_node_label: str = "Package"
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
+        {"id": PropertyRef("PackageId")},
+    )
+    direction: LinkDirection = LinkDirection.INWARD
+    rel_label: str = "SHOULD_UPDATE_TO"
+    properties: TrivyFixToPackageRelProperties = TrivyFixToPackageRelProperties()
+
+
+@dataclass(frozen=True)
+class TrivyFixToFindingRelProperties(CartographyRelProperties):
+    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+class TrivyFixToFinding(CartographyRelSchema):
+    target_node_label: str = "TrivyImageFinding"
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
+        {"id": PropertyRef("FindingId")},
+    )
+    direction: LinkDirection = LinkDirection.OUTWARD
+    rel_label: str = "APPLIES_TO"
+    properties: TrivyFixToFindingRelProperties = TrivyFixToFindingRelProperties()
+
+
+@dataclass(frozen=True)
+class TrivyFixSchema(CartographyNodeSchema):
+    label: str = "TrivyFix"
+    scoped_cleanup: bool = False
+    extra_node_labels: ExtraNodeLabels = ExtraNodeLabels(["Fix"])
+    properties: TrivyFixNodeProperties = TrivyFixNodeProperties()
+    other_relationships: OtherRelationships = OtherRelationships(
+        [
+            TrivyFixToPackage(),
+            TrivyFixToFinding(),
+        ],
+    )

cartography/models/trivy/package.py

@@ -0,0 +1,71 @@
+from dataclasses import dataclass
+
+from cartography.models.core.common import PropertyRef
+from cartography.models.core.nodes import CartographyNodeProperties
+from cartography.models.core.nodes import CartographyNodeSchema
+from cartography.models.core.nodes import ExtraNodeLabels
+from cartography.models.core.relationships import CartographyRelProperties
+from cartography.models.core.relationships import CartographyRelSchema
+from cartography.models.core.relationships import LinkDirection
+from cartography.models.core.relationships import make_target_node_matcher
+from cartography.models.core.relationships import OtherRelationships
+from cartography.models.core.relationships import TargetNodeMatcher
+
+
+@dataclass(frozen=True)
+class TrivyPackageNodeProperties(CartographyNodeProperties):
+    id: PropertyRef = PropertyRef("id")
+    installed_version: PropertyRef = PropertyRef("InstalledVersion")
+    name: PropertyRef = PropertyRef("PkgName")
+    version: PropertyRef = PropertyRef("InstalledVersion")
+    class_name: PropertyRef = PropertyRef("Class")
+    type: PropertyRef = PropertyRef("Type")
+    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+class TrivyPackageToImageRelProperties(CartographyRelProperties):
+    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+class TrivyPackageToImage(CartographyRelSchema):
+    target_node_label: str = "ECRImage"
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
+        {"id": PropertyRef("ImageDigest")},
+    )
+    direction: LinkDirection = LinkDirection.OUTWARD
+    rel_label: str = "DEPLOYED"
+    properties: TrivyPackageToImageRelProperties = TrivyPackageToImageRelProperties()
+
+
+@dataclass(frozen=True)
+class TrivyPackageToFindingRelProperties(CartographyRelProperties):
+    lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+class TrivyPackageToFinding(CartographyRelSchema):
+    target_node_label: str = "TrivyImageFinding"
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
+        {"id": PropertyRef("FindingId")},
+    )
+    direction: LinkDirection = LinkDirection.INWARD
+    rel_label: str = "AFFECTS"
+    properties: TrivyPackageToFindingRelProperties = (
+        TrivyPackageToFindingRelProperties()
+    )
+
+
+@dataclass(frozen=True)
+class TrivyPackageSchema(CartographyNodeSchema):
+    label: str = "Package"
+    scoped_cleanup: bool = False
+    extra_node_labels: ExtraNodeLabels = ExtraNodeLabels(["TrivyPackage"])
+    properties: TrivyPackageNodeProperties = TrivyPackageNodeProperties()
+    other_relationships: OtherRelationships = OtherRelationships(
+        [
+            TrivyPackageToImage(),
+            TrivyPackageToFinding(),
+        ],
+    )

cartography/sync.py

@@ -13,6 +13,7 @@ import neo4j.exceptions
 from neo4j import GraphDatabase
 from statsd import StatsClient
 
+import cartography.intel.airbyte
 import cartography.intel.analysis
 import cartography.intel.anthropic
 import cartography.intel.aws
@@ -28,15 +29,19 @@ import cartography.intel.entra
 import cartography.intel.gcp
 import cartography.intel.github
 import cartography.intel.gsuite
+import cartography.intel.jamf
 import cartography.intel.kandji
 import cartography.intel.kubernetes
 import cartography.intel.lastpass
 import cartography.intel.oci
 import cartography.intel.okta
 import cartography.intel.openai
+import cartography.intel.pagerduty
+import cartography.intel.scaleway
 import cartography.intel.semgrep
 import cartography.intel.snipeit
 import cartography.intel.tailscale
+import cartography.intel.trivy
 from cartography.config import Config
 from cartography.stats import set_stats_client
 from cartography.util import STATUS_FAILURE
@@ -48,6 +53,7 @@ logger = logging.getLogger(__name__)
 TOP_LEVEL_MODULES = OrderedDict(
     {  # preserve order so that the default sync always runs `analysis` at the very end
         "create-indexes": cartography.intel.create_indexes.run,
+        "airbyte": cartography.intel.airbyte.start_airbyte_ingestion,
         "anthropic": cartography.intel.anthropic.start_anthropic_ingestion,
         "aws": cartography.intel.aws.start_aws_ingestion,
         "azure": cartography.intel.azure.start_azure_ingestion,
@@ -67,9 +73,13 @@ TOP_LEVEL_MODULES = OrderedDict(
         "lastpass": cartography.intel.lastpass.start_lastpass_ingestion,
         "bigfix": cartography.intel.bigfix.start_bigfix_ingestion,
         "duo": cartography.intel.duo.start_duo_ingestion,
+        "scaleway": cartography.intel.scaleway.start_scaleway_ingestion,
         "semgrep": cartography.intel.semgrep.start_semgrep_ingestion,
         "snipeit": cartography.intel.snipeit.start_snipeit_ingestion,
         "tailscale": cartography.intel.tailscale.start_tailscale_ingestion,
+        "jamf": cartography.intel.jamf.start_jamf_ingestion,
+        "pagerduty": cartography.intel.pagerduty.start_pagerduty_ingestion,
+        "trivy": cartography.intel.trivy.start_trivy_ingestion,
         "analysis": cartography.intel.analysis.run,
     }
 )
@@ -201,10 +211,6 @@ class Sync:
         return available_modules
 
 
-# Used to avoid repeatedly calling Sync.list_intel_modules()
-TOP_LEVEL_MODULES = Sync.list_intel_modules()
-
-
 def run_with_config(sync: Sync, config: Union[Config, argparse.Namespace]) -> int:
     """
     Execute the cartography.sync.Sync.run method with parameters built from the given configuration object.

cartography/util.py

@@ -5,6 +5,7 @@ from functools import partial
 from functools import wraps
 from importlib.resources import open_binary
 from importlib.resources import read_text
+from itertools import islice
 from string import Template
 from typing import Any
 from typing import Awaitable
@@ -37,6 +38,7 @@ STATUS_SUCCESS = 0
 STATUS_FAILURE = 1
 STATUS_KEYBOARD_INTERRUPT = 130
 DEFAULT_BATCH_SIZE = 1000
+DEFAULT_MAX_PAGES = 10000
 
 
 def run_analysis_job(
@@ -208,26 +210,28 @@ def aws_paginate(
     client: boto3.client,
     method_name: str,
     object_name: str,
+    max_pages: int | None = DEFAULT_MAX_PAGES,
     **kwargs: Any,
-) -> List[Dict]:
+) -> Iterable[Dict]:
     """
     Helper method for boilerplate boto3 pagination
     The **kwargs will be forwarded to the paginator
     """
     paginator = client.get_paginator(method_name)
-    items = []
-    i = 0
     for i, page in enumerate(paginator.paginate(**kwargs), start=1):
         if i % 100 == 0:
             logger.info(f"fetching page number {i}")
         if object_name in page:
-            items.extend(page[object_name])
+            items = page[object_name]
+            yield from items
         else:
             logger.warning(
                 f"""aws_paginate: Key "{object_name}" is not present, check if this is a typo.
 If not, then the AWS datatype somehow does not have this key.""",
             )
-    return items
+        if max_pages is not None and i >= max_pages:
+            logger.warning(f"Reached max batch size of {max_pages} pages")
+            break
 
 
 AWSGetFunc = TypeVar("AWSGetFunc", bound=Callable[..., Iterable])
@@ -353,17 +357,18 @@ def camel_to_snake(name: str) -> str:
     return re.sub(r"(?<!^)(?=[A-Z])", "_", name).lower()
 
 
-def batch(items: Iterable, size: int = DEFAULT_BATCH_SIZE) -> List[List]:
+def batch(items: Iterable, size: int = DEFAULT_BATCH_SIZE) -> Iterable[List[Any]]:
     """
-    Takes an Iterable of items and returns a list of lists of the same items,
+    Takes an Iterable of items and returns a Generator of lists of the same items,
      batched into chunks of the provided `size`.
 
     Use:
     x = [1,2,3,4,5,6,7,8]
-    batch(x, size=3) -> [[1, 2, 3], [4, 5, 6], [7, 8]]
+    batch(x, size=3) -> Iterator yielding [1, 2, 3], [4, 5, 6], [7, 8]
     """
-    items = list(items)
-    return [items[i : i + size] for i in range(0, len(items), size)]
+    it = iter(items)
+    while chunk := list(islice(it, size)):
+        yield chunk
 
 
 def is_throttling_exception(exc: Exception) -> bool:

{cartography-0.104.0rc3.dist-info → cartography-0.106.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cartography
-Version: 0.104.0rc3
+Version: 0.106.0
 Summary: Explore assets and their relationships across your technical infrastructure.
 Maintainer: Cartography Contributors
 License: apache2
@@ -55,6 +55,7 @@ Requires-Dist: python-dateutil
 Requires-Dist: xmltodict
 Requires-Dist: duo-client
 Requires-Dist: cloudflare<5.0.0,>=4.1.0
+Requires-Dist: scaleway>=2.9.0
 Dynamic: license-file
 
 ![Cartography](docs/root/images/logo-horizontal.png)
@@ -80,7 +81,7 @@ You can learn more about the story behind Cartography in our [presentation at BS
 
 
 ## Supported platforms
-- [Amazon Web Services](https://cartography-cncf.github.io/cartography/modules/aws/index.html) - API Gateway, CloudWatch, Config, EC2, ECS, ECR, Elasticsearch, Elastic Kubernetes Service (EKS), DynamoDB, IAM, Inspector, KMS, Lambda, RDS, Redshift, Route53, S3, Secrets Manager(Secret Versions), Security Hub, SQS, SSM, STS, Tags
+- [Amazon Web Services](https://cartography-cncf.github.io/cartography/modules/aws/index.html) - ACM, API Gateway, CloudWatch, CodeBuild, Config, EC2, ECS, ECR, Elasticsearch, Elastic Kubernetes Service (EKS), DynamoDB, IAM, Inspector, KMS, Lambda, RDS, Redshift, Route53, S3, Secrets Manager(Secret Versions), Security Hub, SQS, SSM, STS, Tags
 - [Google Cloud Platform](https://cartography-cncf.github.io/cartography/modules/gcp/index.html) - Cloud Resource Manager, Compute, DNS, Storage, Google Kubernetes Engine
 - [Google GSuite](https://cartography-cncf.github.io/cartography/modules/gsuite/index.html) - users, groups
 - [Oracle Cloud Infrastructure](https://cartography-cncf.github.io/cartography/modules/oci/index.html) - IAM
@@ -102,6 +103,9 @@ You can learn more about the story behind Cartography in our [presentation at BS
 - [Cloudflare](https://cartography-cncf.github.io/cartography/modules/cloudflare/index.html) - Account, Role, Member, Zone, DNSRecord
 - [OpenAI](https://cartography-cncf.github.io/cartography/modules/openai/index.html) - Organization, AdminApiKey, User, Project, ServiceAccount, ApiKey
 - [Anthropic](https://cartography-cncf.github.io/cartography/modules/anthropic/index.html) - Organization, ApiKey, User, Workspace
+- [Airbyte](https://cartography-cncf.github.io/cartography/modules/airbyte/index.html) - Organization, Workspace, User, Source, Destination, Connection, Tag, Stream
+- [Trivy Scanner](https://cartography-cncf.github.io/cartography/modules/trivy/index.html) - AWS ECR Images
+- [Scaleway](https://cartography-cncf.github.io/cartography/modules/scaleway/index.html) - Projects, IAM, Local Storage, Instances
 
 
 ## Philosophy