cartography 0.105.0__py3-none-any.whl → 0.106.0rc2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cartography might be problematic. Click here for more details.

Files changed (53) hide show
  1. cartography/_version.py +2 -2
  2. cartography/client/core/tx.py +62 -0
  3. cartography/data/indexes.cypher +0 -34
  4. cartography/graph/cleanupbuilder.py +47 -0
  5. cartography/graph/job.py +42 -0
  6. cartography/graph/querybuilder.py +136 -2
  7. cartography/graph/statement.py +1 -1
  8. cartography/intel/aws/ecs.py +228 -380
  9. cartography/intel/aws/efs.py +261 -0
  10. cartography/intel/aws/identitycenter.py +14 -3
  11. cartography/intel/aws/inspector.py +96 -53
  12. cartography/intel/aws/rds.py +2 -1
  13. cartography/intel/aws/resources.py +2 -0
  14. cartography/intel/entra/__init__.py +11 -0
  15. cartography/intel/entra/applications.py +366 -0
  16. cartography/intel/kubernetes/__init__.py +30 -14
  17. cartography/intel/kubernetes/clusters.py +86 -0
  18. cartography/intel/kubernetes/namespaces.py +59 -57
  19. cartography/intel/kubernetes/pods.py +140 -77
  20. cartography/intel/kubernetes/secrets.py +95 -45
  21. cartography/intel/kubernetes/services.py +131 -67
  22. cartography/intel/kubernetes/util.py +125 -14
  23. cartography/models/aws/ecs/__init__.py +0 -0
  24. cartography/models/aws/ecs/clusters.py +64 -0
  25. cartography/models/aws/ecs/container_definitions.py +93 -0
  26. cartography/models/aws/ecs/container_instances.py +84 -0
  27. cartography/models/aws/ecs/containers.py +80 -0
  28. cartography/models/aws/ecs/services.py +117 -0
  29. cartography/models/aws/ecs/task_definitions.py +97 -0
  30. cartography/models/aws/ecs/tasks.py +110 -0
  31. cartography/models/aws/efs/__init__.py +0 -0
  32. cartography/models/aws/efs/access_point.py +77 -0
  33. cartography/models/aws/efs/file_system.py +60 -0
  34. cartography/models/aws/efs/mount_target.py +79 -0
  35. cartography/models/core/common.py +1 -0
  36. cartography/models/core/relationships.py +44 -0
  37. cartography/models/entra/app_role_assignment.py +115 -0
  38. cartography/models/entra/application.py +47 -0
  39. cartography/models/kubernetes/__init__.py +0 -0
  40. cartography/models/kubernetes/clusters.py +26 -0
  41. cartography/models/kubernetes/containers.py +108 -0
  42. cartography/models/kubernetes/namespaces.py +51 -0
  43. cartography/models/kubernetes/pods.py +80 -0
  44. cartography/models/kubernetes/secrets.py +79 -0
  45. cartography/models/kubernetes/services.py +108 -0
  46. cartography/util.py +15 -10
  47. {cartography-0.105.0.dist-info → cartography-0.106.0rc2.dist-info}/METADATA +1 -1
  48. {cartography-0.105.0.dist-info → cartography-0.106.0rc2.dist-info}/RECORD +52 -29
  49. cartography/data/jobs/cleanup/kubernetes_import_cleanup.json +0 -70
  50. {cartography-0.105.0.dist-info → cartography-0.106.0rc2.dist-info}/WHEEL +0 -0
  51. {cartography-0.105.0.dist-info → cartography-0.106.0rc2.dist-info}/entry_points.txt +0 -0
  52. {cartography-0.105.0.dist-info → cartography-0.106.0rc2.dist-info}/licenses/LICENSE +0 -0
  53. {cartography-0.105.0.dist-info → cartography-0.106.0rc2.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,26 @@
1
+ from dataclasses import dataclass
2
+
3
+ from cartography.models.core.common import PropertyRef
4
+ from cartography.models.core.nodes import CartographyNodeProperties
5
+ from cartography.models.core.nodes import CartographyNodeSchema
6
+
7
+
8
+ @dataclass(frozen=True)
9
+ class KubernetesClusterNodeProperties(CartographyNodeProperties):
10
+ id: PropertyRef = PropertyRef("id")
11
+ name: PropertyRef = PropertyRef("name", extra_index=True)
12
+ creation_timestamp: PropertyRef = PropertyRef("creation_timestamp")
13
+ external_id: PropertyRef = PropertyRef("external_id", extra_index=True)
14
+ version: PropertyRef = PropertyRef("git_version")
15
+ version_major: PropertyRef = PropertyRef("version_major")
16
+ version_minor: PropertyRef = PropertyRef("version_minor")
17
+ go_version: PropertyRef = PropertyRef("go_version")
18
+ compiler: PropertyRef = PropertyRef("compiler")
19
+ platform: PropertyRef = PropertyRef("platform")
20
+ lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
21
+
22
+
23
+ @dataclass(frozen=True)
24
+ class KubernetesClusterSchema(CartographyNodeSchema):
25
+ label: str = "KubernetesCluster"
26
+ properties: KubernetesClusterNodeProperties = KubernetesClusterNodeProperties()
@@ -0,0 +1,108 @@
1
+ from dataclasses import dataclass
2
+
3
+ from cartography.models.core.common import PropertyRef
4
+ from cartography.models.core.nodes import CartographyNodeProperties
5
+ from cartography.models.core.nodes import CartographyNodeSchema
6
+ from cartography.models.core.relationships import CartographyRelProperties
7
+ from cartography.models.core.relationships import CartographyRelSchema
8
+ from cartography.models.core.relationships import LinkDirection
9
+ from cartography.models.core.relationships import make_target_node_matcher
10
+ from cartography.models.core.relationships import OtherRelationships
11
+ from cartography.models.core.relationships import TargetNodeMatcher
12
+
13
+
14
+ @dataclass(frozen=True)
15
+ class KubernetesContainerNodeProperties(CartographyNodeProperties):
16
+ id: PropertyRef = PropertyRef("uid")
17
+ name: PropertyRef = PropertyRef("name", extra_index=True)
18
+ image: PropertyRef = PropertyRef("image", extra_index=True)
19
+ namespace: PropertyRef = PropertyRef("namespace", extra_index=True)
20
+ cluster_name: PropertyRef = PropertyRef(
21
+ "CLUSTER_NAME", set_in_kwargs=True, extra_index=True
22
+ )
23
+ image_pull_policy: PropertyRef = PropertyRef("image_pull_policy")
24
+ status_image_id: PropertyRef = PropertyRef("status_image_id")
25
+ status_image_sha: PropertyRef = PropertyRef("status_image_sha")
26
+ status_ready: PropertyRef = PropertyRef("status_ready")
27
+ status_started: PropertyRef = PropertyRef("status_started")
28
+ status_state: PropertyRef = PropertyRef("status_state")
29
+ lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
30
+
31
+
32
+ @dataclass(frozen=True)
33
+ class KubernetesContainerToKubernetesNamespaceRelProperties(CartographyRelProperties):
34
+ lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
35
+
36
+
37
+ @dataclass(frozen=True)
38
+ class KubernetesContainerToKubernetesPodRelProperties(CartographyRelProperties):
39
+ lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
40
+
41
+
42
+ @dataclass(frozen=True)
43
+ # (:KubernetesContainer)<-[:CONTAINS]-(:KubernetesNamespace)
44
+ class KubernetesContainerToKubernetesNamespaceRel(CartographyRelSchema):
45
+ target_node_label: str = "KubernetesNamespace"
46
+ target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
47
+ {
48
+ "cluster_name": PropertyRef("CLUSTER_NAME", set_in_kwargs=True),
49
+ "name": PropertyRef("namespace"),
50
+ }
51
+ )
52
+ direction: LinkDirection = LinkDirection.INWARD
53
+ rel_label: str = "CONTAINS"
54
+ properties: KubernetesContainerToKubernetesNamespaceRelProperties = (
55
+ KubernetesContainerToKubernetesNamespaceRelProperties()
56
+ )
57
+
58
+
59
+ @dataclass(frozen=True)
60
+ # (:KubernetesContainer)<-[:CONTAINS]-(:KubernetesPod)
61
+ class KubernetesContainerToKubernetesPodRel(CartographyRelSchema):
62
+ target_node_label: str = "KubernetesPod"
63
+ target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
64
+ {
65
+ "cluster_name": PropertyRef("CLUSTER_NAME", set_in_kwargs=True),
66
+ "namespace": PropertyRef("namespace"),
67
+ "id": PropertyRef("pod_id"),
68
+ }
69
+ )
70
+ direction: LinkDirection = LinkDirection.INWARD
71
+ rel_label: str = "CONTAINS"
72
+ properties: KubernetesContainerToKubernetesPodRelProperties = (
73
+ KubernetesContainerToKubernetesPodRelProperties()
74
+ )
75
+
76
+
77
+ @dataclass(frozen=True)
78
+ class KubernetesContainerToKubernetesClusterRelProperties(CartographyRelProperties):
79
+ lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
80
+
81
+
82
+ @dataclass(frozen=True)
83
+ # (:KubernetesContainer)<-[:RESOURCE]-(:KubernetesCluster)
84
+ class KubernetesContainerToKubernetesClusterRel(CartographyRelSchema):
85
+ target_node_label: str = "KubernetesCluster"
86
+ target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
87
+ {"id": PropertyRef("CLUSTER_ID", set_in_kwargs=True)}
88
+ )
89
+ direction: LinkDirection = LinkDirection.INWARD
90
+ rel_label: str = "RESOURCE"
91
+ properties: KubernetesContainerToKubernetesClusterRelProperties = (
92
+ KubernetesContainerToKubernetesClusterRelProperties()
93
+ )
94
+
95
+
96
+ @dataclass(frozen=True)
97
+ class KubernetesContainerSchema(CartographyNodeSchema):
98
+ label: str = "KubernetesContainer"
99
+ properties: KubernetesContainerNodeProperties = KubernetesContainerNodeProperties()
100
+ sub_resource_relationship: KubernetesContainerToKubernetesClusterRel = (
101
+ KubernetesContainerToKubernetesClusterRel()
102
+ )
103
+ other_relationships: OtherRelationships = OtherRelationships(
104
+ [
105
+ KubernetesContainerToKubernetesNamespaceRel(),
106
+ KubernetesContainerToKubernetesPodRel(),
107
+ ]
108
+ )
@@ -0,0 +1,51 @@
1
+ from dataclasses import dataclass
2
+
3
+ from cartography.models.core.common import PropertyRef
4
+ from cartography.models.core.nodes import CartographyNodeProperties
5
+ from cartography.models.core.nodes import CartographyNodeSchema
6
+ from cartography.models.core.relationships import CartographyRelProperties
7
+ from cartography.models.core.relationships import CartographyRelSchema
8
+ from cartography.models.core.relationships import LinkDirection
9
+ from cartography.models.core.relationships import make_target_node_matcher
10
+ from cartography.models.core.relationships import TargetNodeMatcher
11
+
12
+
13
+ @dataclass(frozen=True)
14
+ class KubernetesNamespaceNodeProperties(CartographyNodeProperties):
15
+ id: PropertyRef = PropertyRef("uid")
16
+ name: PropertyRef = PropertyRef("name", extra_index=True)
17
+ creation_timestamp: PropertyRef = PropertyRef("creation_timestamp")
18
+ deletion_timestamp: PropertyRef = PropertyRef("deletion_timestamp")
19
+ status_phase: PropertyRef = PropertyRef("status_phase")
20
+ cluster_name: PropertyRef = PropertyRef(
21
+ "cluster_name", set_in_kwargs=True, extra_index=True
22
+ )
23
+ lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
24
+
25
+
26
+ @dataclass(frozen=True)
27
+ class KubernetesNamespaceToKubernetesClusterRelProperties(CartographyRelProperties):
28
+ lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
29
+
30
+
31
+ @dataclass(frozen=True)
32
+ # (:KubernetesNamespace)<-[:RESOURCE]-(:KubernetesCluster)
33
+ class KubernetesNamespaceToKubernetesClusterRel(CartographyRelSchema):
34
+ target_node_label: str = "KubernetesCluster"
35
+ target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
36
+ {"id": PropertyRef("CLUSTER_ID", set_in_kwargs=True)},
37
+ )
38
+ direction: LinkDirection = LinkDirection.INWARD
39
+ rel_label: str = "RESOURCE"
40
+ properties: KubernetesNamespaceToKubernetesClusterRelProperties = (
41
+ KubernetesNamespaceToKubernetesClusterRelProperties()
42
+ )
43
+
44
+
45
+ @dataclass(frozen=True)
46
+ class KubernetesNamespaceSchema(CartographyNodeSchema):
47
+ label: str = "KubernetesNamespace"
48
+ properties: KubernetesNamespaceNodeProperties = KubernetesNamespaceNodeProperties()
49
+ sub_resource_relationship: KubernetesNamespaceToKubernetesClusterRel = (
50
+ KubernetesNamespaceToKubernetesClusterRel()
51
+ )
@@ -0,0 +1,80 @@
1
+ from dataclasses import dataclass
2
+
3
+ from cartography.models.core.common import PropertyRef
4
+ from cartography.models.core.nodes import CartographyNodeProperties
5
+ from cartography.models.core.nodes import CartographyNodeSchema
6
+ from cartography.models.core.relationships import CartographyRelProperties
7
+ from cartography.models.core.relationships import CartographyRelSchema
8
+ from cartography.models.core.relationships import LinkDirection
9
+ from cartography.models.core.relationships import make_target_node_matcher
10
+ from cartography.models.core.relationships import OtherRelationships
11
+ from cartography.models.core.relationships import TargetNodeMatcher
12
+
13
+
14
+ @dataclass(frozen=True)
15
+ class KubernetesPodNodeProperties(CartographyNodeProperties):
16
+ id: PropertyRef = PropertyRef("uid")
17
+ name: PropertyRef = PropertyRef("name", extra_index=True)
18
+ status_phase: PropertyRef = PropertyRef("status_phase")
19
+ creation_timestamp: PropertyRef = PropertyRef("creation_timestamp")
20
+ deletion_timestamp: PropertyRef = PropertyRef("deletion_timestamp")
21
+ namespace: PropertyRef = PropertyRef("namespace", extra_index=True)
22
+ labels: PropertyRef = PropertyRef("labels")
23
+ cluster_name: PropertyRef = PropertyRef(
24
+ "CLUSTER_NAME", set_in_kwargs=True, extra_index=True
25
+ )
26
+ node: PropertyRef = PropertyRef("node")
27
+ lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
28
+
29
+
30
+ @dataclass(frozen=True)
31
+ class KubernetesPodToKubernetesNamespaceRelProperties(CartographyRelProperties):
32
+ lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
33
+
34
+
35
+ @dataclass(frozen=True)
36
+ # (:KubernetesPod)<-[:CONTAINS]-(:KubernetesNamespace)
37
+ class KubernetesPodToKubernetesNamespaceRel(CartographyRelSchema):
38
+ target_node_label: str = "KubernetesNamespace"
39
+ target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
40
+ {
41
+ "cluster_name": PropertyRef("CLUSTER_NAME", set_in_kwargs=True),
42
+ "name": PropertyRef("namespace"),
43
+ }
44
+ )
45
+ direction: LinkDirection = LinkDirection.INWARD
46
+ rel_label: str = "CONTAINS"
47
+ properties: KubernetesPodToKubernetesNamespaceRelProperties = (
48
+ KubernetesPodToKubernetesNamespaceRelProperties()
49
+ )
50
+
51
+
52
+ @dataclass(frozen=True)
53
+ class KubernetesPodToKubernetesClusterRelProperties(CartographyRelProperties):
54
+ lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
55
+
56
+
57
+ @dataclass(frozen=True)
58
+ # (:KubernetesPod)<-[:RESOURCE]-(:KubernetesCluster)
59
+ class KubernetesPodToKubernetesClusterRel(CartographyRelSchema):
60
+ target_node_label: str = "KubernetesCluster"
61
+ target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
62
+ {"id": PropertyRef("CLUSTER_ID", set_in_kwargs=True)}
63
+ )
64
+ direction: LinkDirection = LinkDirection.INWARD
65
+ rel_label: str = "RESOURCE"
66
+ properties: KubernetesPodToKubernetesClusterRelProperties = (
67
+ KubernetesPodToKubernetesClusterRelProperties()
68
+ )
69
+
70
+
71
+ @dataclass(frozen=True)
72
+ class KubernetesPodSchema(CartographyNodeSchema):
73
+ label: str = "KubernetesPod"
74
+ properties: KubernetesPodNodeProperties = KubernetesPodNodeProperties()
75
+ sub_resource_relationship: KubernetesPodToKubernetesClusterRel = (
76
+ KubernetesPodToKubernetesClusterRel()
77
+ )
78
+ other_relationships: OtherRelationships = OtherRelationships(
79
+ [KubernetesPodToKubernetesNamespaceRel()]
80
+ )
@@ -0,0 +1,79 @@
1
+ from dataclasses import dataclass
2
+
3
+ from cartography.models.core.common import PropertyRef
4
+ from cartography.models.core.nodes import CartographyNodeProperties
5
+ from cartography.models.core.nodes import CartographyNodeSchema
6
+ from cartography.models.core.relationships import CartographyRelProperties
7
+ from cartography.models.core.relationships import CartographyRelSchema
8
+ from cartography.models.core.relationships import LinkDirection
9
+ from cartography.models.core.relationships import make_target_node_matcher
10
+ from cartography.models.core.relationships import OtherRelationships
11
+ from cartography.models.core.relationships import TargetNodeMatcher
12
+
13
+
14
+ @dataclass(frozen=True)
15
+ class KubernetesSecretNodeProperties(CartographyNodeProperties):
16
+ id: PropertyRef = PropertyRef("uid")
17
+ name: PropertyRef = PropertyRef("name", extra_index=True)
18
+ creation_timestamp: PropertyRef = PropertyRef("creation_timestamp")
19
+ deletion_timestamp: PropertyRef = PropertyRef("deletion_timestamp")
20
+ namespace: PropertyRef = PropertyRef("namespace", extra_index=True)
21
+ owner_references: PropertyRef = PropertyRef("owner_references")
22
+ type: PropertyRef = PropertyRef("type")
23
+ cluster_name: PropertyRef = PropertyRef(
24
+ "CLUSTER_NAME", set_in_kwargs=True, extra_index=True
25
+ )
26
+ lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
27
+
28
+
29
+ @dataclass(frozen=True)
30
+ class KubernetesSecretToKubernetesNamespaceRelProperties(CartographyRelProperties):
31
+ lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
32
+
33
+
34
+ @dataclass(frozen=True)
35
+ # (:KubernetesSecret)<-[:CONTAINS]-(:KubernetesNamespace)
36
+ class KubernetesSecretToKubernetesNamespaceRel(CartographyRelSchema):
37
+ target_node_label: str = "KubernetesNamespace"
38
+ target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
39
+ {
40
+ "cluster_name": PropertyRef("CLUSTER_NAME", set_in_kwargs=True),
41
+ "name": PropertyRef("namespace"),
42
+ }
43
+ )
44
+ direction: LinkDirection = LinkDirection.INWARD
45
+ rel_label: str = "CONTAINS"
46
+ properties: KubernetesSecretToKubernetesNamespaceRelProperties = (
47
+ KubernetesSecretToKubernetesNamespaceRelProperties()
48
+ )
49
+
50
+
51
+ @dataclass(frozen=True)
52
+ class KubernetesSecretToKubernetesClusterRelProperties(CartographyRelProperties):
53
+ lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
54
+
55
+
56
+ @dataclass(frozen=True)
57
+ # (:KubernetesSecret)<-[:RESOURCE]-(:KubernetesCluster)
58
+ class KubernetesSecretToKubernetesClusterRel(CartographyRelSchema):
59
+ target_node_label: str = "KubernetesCluster"
60
+ target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
61
+ {"id": PropertyRef("CLUSTER_ID", set_in_kwargs=True)}
62
+ )
63
+ direction: LinkDirection = LinkDirection.INWARD
64
+ rel_label: str = "RESOURCE"
65
+ properties: KubernetesSecretToKubernetesClusterRelProperties = (
66
+ KubernetesSecretToKubernetesClusterRelProperties()
67
+ )
68
+
69
+
70
+ @dataclass(frozen=True)
71
+ class KubernetesSecretSchema(CartographyNodeSchema):
72
+ label: str = "KubernetesSecret"
73
+ properties: KubernetesSecretNodeProperties = KubernetesSecretNodeProperties()
74
+ sub_resource_relationship: KubernetesSecretToKubernetesClusterRel = (
75
+ KubernetesSecretToKubernetesClusterRel()
76
+ )
77
+ other_relationships: OtherRelationships = OtherRelationships(
78
+ [KubernetesSecretToKubernetesNamespaceRel()]
79
+ )
@@ -0,0 +1,108 @@
1
+ from dataclasses import dataclass
2
+
3
+ from cartography.models.core.common import PropertyRef
4
+ from cartography.models.core.nodes import CartographyNodeProperties
5
+ from cartography.models.core.nodes import CartographyNodeSchema
6
+ from cartography.models.core.relationships import CartographyRelProperties
7
+ from cartography.models.core.relationships import CartographyRelSchema
8
+ from cartography.models.core.relationships import LinkDirection
9
+ from cartography.models.core.relationships import make_target_node_matcher
10
+ from cartography.models.core.relationships import OtherRelationships
11
+ from cartography.models.core.relationships import TargetNodeMatcher
12
+
13
+
14
+ @dataclass(frozen=True)
15
+ class KubernetesServiceNodeProperties(CartographyNodeProperties):
16
+ id: PropertyRef = PropertyRef("uid")
17
+ name: PropertyRef = PropertyRef("name", extra_index=True)
18
+ creation_timestamp: PropertyRef = PropertyRef("creation_timestamp")
19
+ deletion_timestamp: PropertyRef = PropertyRef("deletion_timestamp")
20
+ namespace: PropertyRef = PropertyRef("namespace", extra_index=True)
21
+ selector: PropertyRef = PropertyRef("selector")
22
+ type: PropertyRef = PropertyRef("type")
23
+ cluster_ip: PropertyRef = PropertyRef("cluster_ip")
24
+ load_balancer_ip: PropertyRef = PropertyRef("load_balancer_ip")
25
+ load_balancer_ingress: PropertyRef = PropertyRef("load_balancer_ingress")
26
+ cluster_name: PropertyRef = PropertyRef(
27
+ "CLUSTER_NAME", set_in_kwargs=True, extra_index=True
28
+ )
29
+ lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
30
+
31
+
32
+ @dataclass(frozen=True)
33
+ class KubernetesServiceToKubernetesClusterRelProperties(CartographyRelProperties):
34
+ lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
35
+
36
+
37
+ @dataclass(frozen=True)
38
+ # (:KubernetesService)<-[:RESOURCE]-(:KubernetesCluster)
39
+ class KubernetesServiceToKubernetesClusterRel(CartographyRelSchema):
40
+ target_node_label: str = "KubernetesCluster"
41
+ target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
42
+ {"id": PropertyRef("CLUSTER_ID", set_in_kwargs=True)}
43
+ )
44
+ direction: LinkDirection = LinkDirection.INWARD
45
+ rel_label: str = "RESOURCE"
46
+ properties: KubernetesServiceToKubernetesClusterRelProperties = (
47
+ KubernetesServiceToKubernetesClusterRelProperties()
48
+ )
49
+
50
+
51
+ @dataclass(frozen=True)
52
+ class KubernetesServiceToKubernetesNamespaceRelProperties(CartographyRelProperties):
53
+ lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
54
+
55
+
56
+ @dataclass(frozen=True)
57
+ # (:KubernetesService)<-[:CONTAINS]-(:KubernetesNamespace)
58
+ class KubernetesServiceToKubernetesNamespaceRel(CartographyRelSchema):
59
+ target_node_label: str = "KubernetesNamespace"
60
+ target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
61
+ {
62
+ "cluster_name": PropertyRef("CLUSTER_NAME", set_in_kwargs=True),
63
+ "name": PropertyRef("namespace"),
64
+ }
65
+ )
66
+ direction: LinkDirection = LinkDirection.INWARD
67
+ rel_label: str = "CONTAINS"
68
+ properties: KubernetesServiceToKubernetesNamespaceRelProperties = (
69
+ KubernetesServiceToKubernetesNamespaceRelProperties()
70
+ )
71
+
72
+
73
+ @dataclass(frozen=True)
74
+ class KubernetesServiceToKubernetesPodRelProperties(CartographyRelProperties):
75
+ lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
76
+
77
+
78
+ @dataclass(frozen=True)
79
+ # (:KubernetesService)-[:TARGET]->(:KubernetesPod)
80
+ class KubernetesServiceToKubernetesPodRel(CartographyRelSchema):
81
+ target_node_label: str = "KubernetesPod"
82
+ target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
83
+ {
84
+ "cluster_name": PropertyRef("CLUSTER_NAME", set_in_kwargs=True),
85
+ "namespace": PropertyRef("namespace"),
86
+ "id": PropertyRef("pod_ids", one_to_many=True),
87
+ }
88
+ )
89
+ direction: LinkDirection = LinkDirection.OUTWARD
90
+ rel_label: str = "TARGETS"
91
+ properties: KubernetesServiceToKubernetesPodRelProperties = (
92
+ KubernetesServiceToKubernetesPodRelProperties()
93
+ )
94
+
95
+
96
+ @dataclass(frozen=True)
97
+ class KubernetesServiceSchema(CartographyNodeSchema):
98
+ label: str = "KubernetesService"
99
+ properties: KubernetesServiceNodeProperties = KubernetesServiceNodeProperties()
100
+ sub_resource_relationship: KubernetesServiceToKubernetesClusterRel = (
101
+ KubernetesServiceToKubernetesClusterRel()
102
+ )
103
+ other_relationships: OtherRelationships = OtherRelationships(
104
+ [
105
+ KubernetesServiceToKubernetesNamespaceRel(),
106
+ KubernetesServiceToKubernetesPodRel(),
107
+ ]
108
+ )
cartography/util.py CHANGED
@@ -5,6 +5,7 @@ from functools import partial
5
5
  from functools import wraps
6
6
  from importlib.resources import open_binary
7
7
  from importlib.resources import read_text
8
+ from itertools import islice
8
9
  from string import Template
9
10
  from typing import Any
10
11
  from typing import Awaitable
@@ -37,6 +38,7 @@ STATUS_SUCCESS = 0
37
38
  STATUS_FAILURE = 1
38
39
  STATUS_KEYBOARD_INTERRUPT = 130
39
40
  DEFAULT_BATCH_SIZE = 1000
41
+ DEFAULT_MAX_PAGES = 10000
40
42
 
41
43
 
42
44
  def run_analysis_job(
@@ -208,26 +210,28 @@ def aws_paginate(
208
210
  client: boto3.client,
209
211
  method_name: str,
210
212
  object_name: str,
213
+ max_pages: int | None = DEFAULT_MAX_PAGES,
211
214
  **kwargs: Any,
212
- ) -> List[Dict]:
215
+ ) -> Iterable[Dict]:
213
216
  """
214
217
  Helper method for boilerplate boto3 pagination
215
218
  The **kwargs will be forwarded to the paginator
216
219
  """
217
220
  paginator = client.get_paginator(method_name)
218
- items = []
219
- i = 0
220
221
  for i, page in enumerate(paginator.paginate(**kwargs), start=1):
221
222
  if i % 100 == 0:
222
223
  logger.info(f"fetching page number {i}")
223
224
  if object_name in page:
224
- items.extend(page[object_name])
225
+ items = page[object_name]
226
+ yield from items
225
227
  else:
226
228
  logger.warning(
227
229
  f"""aws_paginate: Key "{object_name}" is not present, check if this is a typo.
228
230
  If not, then the AWS datatype somehow does not have this key.""",
229
231
  )
230
- return items
232
+ if max_pages is not None and i >= max_pages:
233
+ logger.warning(f"Reached max batch size of {max_pages} pages")
234
+ break
231
235
 
232
236
 
233
237
  AWSGetFunc = TypeVar("AWSGetFunc", bound=Callable[..., Iterable])
@@ -353,17 +357,18 @@ def camel_to_snake(name: str) -> str:
353
357
  return re.sub(r"(?<!^)(?=[A-Z])", "_", name).lower()
354
358
 
355
359
 
356
- def batch(items: Iterable, size: int = DEFAULT_BATCH_SIZE) -> List[List]:
360
+ def batch(items: Iterable, size: int = DEFAULT_BATCH_SIZE) -> Iterable[List[Any]]:
357
361
  """
358
- Takes an Iterable of items and returns a list of lists of the same items,
362
+ Takes an Iterable of items and returns a Generator of lists of the same items,
359
363
  batched into chunks of the provided `size`.
360
364
 
361
365
  Use:
362
366
  x = [1,2,3,4,5,6,7,8]
363
- batch(x, size=3) -> [[1, 2, 3], [4, 5, 6], [7, 8]]
367
+ batch(x, size=3) -> Iterator yielding [1, 2, 3], [4, 5, 6], [7, 8]
364
368
  """
365
- items = list(items)
366
- return [items[i : i + size] for i in range(0, len(items), size)]
369
+ it = iter(items)
370
+ while chunk := list(islice(it, size)):
371
+ yield chunk
367
372
 
368
373
 
369
374
  def is_throttling_exception(exc: Exception) -> bool:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: cartography
3
- Version: 0.105.0
3
+ Version: 0.106.0rc2
4
4
  Summary: Explore assets and their relationships across your technical infrastructure.
5
5
  Maintainer: Cartography Contributors
6
6
  License: apache2