cartography 0.111.0rc1__py3-none-any.whl → 0.113.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cartography might be problematic. Click here for more details.
- cartography/_version.py +2 -2
- cartography/cli.py +57 -0
- cartography/config.py +24 -0
- cartography/data/indexes.cypher +0 -6
- cartography/data/jobs/analysis/keycloak_inheritance.json +30 -0
- cartography/intel/aws/apigateway.py +128 -17
- cartography/intel/aws/apigatewayv2.py +116 -0
- cartography/intel/aws/ec2/instances.py +3 -1
- cartography/intel/aws/ec2/network_interfaces.py +1 -1
- cartography/intel/aws/ec2/vpc_peerings.py +262 -125
- cartography/intel/aws/resources.py +2 -0
- cartography/intel/azure/__init__.py +35 -32
- cartography/intel/azure/subscription.py +2 -2
- cartography/intel/azure/tenant.py +39 -30
- cartography/intel/azure/util/credentials.py +49 -174
- cartography/intel/entra/__init__.py +47 -1
- cartography/intel/entra/applications.py +220 -170
- cartography/intel/entra/groups.py +41 -22
- cartography/intel/entra/ou.py +28 -20
- cartography/intel/entra/users.py +24 -18
- cartography/intel/gcp/__init__.py +32 -11
- cartography/intel/gcp/compute.py +47 -12
- cartography/intel/gcp/dns.py +82 -169
- cartography/intel/gcp/iam.py +66 -54
- cartography/intel/gcp/storage.py +75 -159
- cartography/intel/github/repos.py +19 -10
- cartography/intel/github/util.py +12 -0
- cartography/intel/keycloak/__init__.py +153 -0
- cartography/intel/keycloak/authenticationexecutions.py +322 -0
- cartography/intel/keycloak/authenticationflows.py +77 -0
- cartography/intel/keycloak/clients.py +187 -0
- cartography/intel/keycloak/groups.py +126 -0
- cartography/intel/keycloak/identityproviders.py +94 -0
- cartography/intel/keycloak/organizations.py +163 -0
- cartography/intel/keycloak/realms.py +61 -0
- cartography/intel/keycloak/roles.py +202 -0
- cartography/intel/keycloak/scopes.py +73 -0
- cartography/intel/keycloak/users.py +70 -0
- cartography/intel/keycloak/util.py +47 -0
- cartography/intel/kubernetes/__init__.py +26 -0
- cartography/intel/kubernetes/eks.py +402 -0
- cartography/intel/kubernetes/rbac.py +133 -0
- cartography/models/aws/apigateway/apigatewayintegration.py +79 -0
- cartography/models/aws/apigateway/apigatewaymethod.py +74 -0
- cartography/models/aws/apigatewayv2/__init__.py +0 -0
- cartography/models/aws/apigatewayv2/apigatewayv2.py +53 -0
- cartography/models/aws/ec2/vpc_peering.py +157 -0
- cartography/models/azure/principal.py +44 -0
- cartography/models/azure/tenant.py +20 -0
- cartography/models/gcp/dns.py +109 -0
- cartography/models/gcp/iam.py +3 -0
- cartography/models/gcp/storage/__init__.py +0 -0
- cartography/models/gcp/storage/bucket.py +119 -0
- cartography/models/keycloak/__init__.py +0 -0
- cartography/models/keycloak/authenticationexecution.py +160 -0
- cartography/models/keycloak/authenticationflow.py +54 -0
- cartography/models/keycloak/client.py +177 -0
- cartography/models/keycloak/group.py +101 -0
- cartography/models/keycloak/identityprovider.py +89 -0
- cartography/models/keycloak/organization.py +116 -0
- cartography/models/keycloak/organizationdomain.py +73 -0
- cartography/models/keycloak/realm.py +173 -0
- cartography/models/keycloak/role.py +126 -0
- cartography/models/keycloak/scope.py +73 -0
- cartography/models/keycloak/user.py +51 -0
- cartography/models/kubernetes/clusterrolebindings.py +40 -0
- cartography/models/kubernetes/groups.py +107 -0
- cartography/models/kubernetes/oidc.py +51 -0
- cartography/models/kubernetes/rolebindings.py +40 -0
- cartography/models/kubernetes/users.py +105 -0
- cartography/sync.py +2 -0
- cartography/util.py +10 -0
- {cartography-0.111.0rc1.dist-info → cartography-0.113.0.dist-info}/METADATA +9 -5
- {cartography-0.111.0rc1.dist-info → cartography-0.113.0.dist-info}/RECORD +78 -41
- cartography/data/jobs/cleanup/aws_import_vpc_peering_cleanup.json +0 -45
- cartography/data/jobs/cleanup/gcp_dns_cleanup.json +0 -29
- cartography/data/jobs/cleanup/gcp_storage_bucket_cleanup.json +0 -29
- {cartography-0.111.0rc1.dist-info → cartography-0.113.0.dist-info}/WHEEL +0 -0
- {cartography-0.111.0rc1.dist-info → cartography-0.113.0.dist-info}/entry_points.txt +0 -0
- {cartography-0.111.0rc1.dist-info → cartography-0.113.0.dist-info}/licenses/LICENSE +0 -0
- {cartography-0.111.0rc1.dist-info → cartography-0.113.0.dist-info}/top_level.txt +0 -0
cartography/_version.py
CHANGED
|
@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
|
|
|
28
28
|
commit_id: COMMIT_ID
|
|
29
29
|
__commit_id__: COMMIT_ID
|
|
30
30
|
|
|
31
|
-
__version__ = version = '0.
|
|
32
|
-
__version_tuple__ = version_tuple = (0,
|
|
31
|
+
__version__ = version = '0.113.0'
|
|
32
|
+
__version_tuple__ = version_tuple = (0, 113, 0)
|
|
33
33
|
|
|
34
34
|
__commit_id__ = commit_id = None
|
cartography/cli.py
CHANGED
|
@@ -234,6 +234,11 @@ class CLI:
|
|
|
234
234
|
"The name of environment variable containing Azure Client Secret for Service Principal Authentication."
|
|
235
235
|
),
|
|
236
236
|
)
|
|
237
|
+
parser.add_argument(
|
|
238
|
+
"--azure-subscription-id",
|
|
239
|
+
type=str,
|
|
240
|
+
help="The Azure Subscription ID to sync.",
|
|
241
|
+
)
|
|
237
242
|
parser.add_argument(
|
|
238
243
|
"--entra-tenant-id",
|
|
239
244
|
type=str,
|
|
@@ -394,6 +399,12 @@ class CLI:
|
|
|
394
399
|
"The path to kubeconfig file specifying context to access K8s cluster(s)."
|
|
395
400
|
),
|
|
396
401
|
)
|
|
402
|
+
parser.add_argument(
|
|
403
|
+
"--managed-kubernetes",
|
|
404
|
+
default=None,
|
|
405
|
+
type=str,
|
|
406
|
+
help=("Type of managed Kubernetes service (e.g., 'eks'). Optional."),
|
|
407
|
+
)
|
|
397
408
|
parser.add_argument(
|
|
398
409
|
"--nist-cve-url",
|
|
399
410
|
type=str,
|
|
@@ -762,6 +773,42 @@ class CLI:
|
|
|
762
773
|
"Required if you are using the SentinelOne intel module. Ignored otherwise."
|
|
763
774
|
),
|
|
764
775
|
)
|
|
776
|
+
parser.add_argument(
|
|
777
|
+
"--keycloak-client-id",
|
|
778
|
+
type=str,
|
|
779
|
+
default=None,
|
|
780
|
+
help=(
|
|
781
|
+
"The Keycloak client ID to sync. "
|
|
782
|
+
"Required if you are using the Keycloak intel module. Ignored otherwise."
|
|
783
|
+
),
|
|
784
|
+
)
|
|
785
|
+
parser.add_argument(
|
|
786
|
+
"--keycloak-client-secret-env-var",
|
|
787
|
+
type=str,
|
|
788
|
+
default="KEYCLOAK_CLIENT_SECRET",
|
|
789
|
+
help=(
|
|
790
|
+
"The name of an environment variable containing the Keycloak client secret. "
|
|
791
|
+
"Required if you are using the Keycloak intel module. Ignored otherwise."
|
|
792
|
+
),
|
|
793
|
+
)
|
|
794
|
+
parser.add_argument(
|
|
795
|
+
"--keycloak-url",
|
|
796
|
+
type=str,
|
|
797
|
+
help=(
|
|
798
|
+
"The base URL for the Keycloak instance. "
|
|
799
|
+
"Required if you are using the Keycloak intel module. Ignored otherwise. "
|
|
800
|
+
),
|
|
801
|
+
)
|
|
802
|
+
parser.add_argument(
|
|
803
|
+
"--keycloak-realm",
|
|
804
|
+
type=str,
|
|
805
|
+
default="master",
|
|
806
|
+
help=(
|
|
807
|
+
"The Keycloak realm used for authentication (note: all available realms will be synced). "
|
|
808
|
+
"Should be `master` (default value) in most of the cases. "
|
|
809
|
+
"Required if you are using the Keycloak intel module. Ignored otherwise. "
|
|
810
|
+
),
|
|
811
|
+
)
|
|
765
812
|
|
|
766
813
|
return parser
|
|
767
814
|
|
|
@@ -1133,6 +1180,16 @@ class CLI:
|
|
|
1133
1180
|
else:
|
|
1134
1181
|
config.sentinelone_api_token = None
|
|
1135
1182
|
|
|
1183
|
+
if config.keycloak_client_secret_env_var:
|
|
1184
|
+
logger.debug(
|
|
1185
|
+
f"Reading Client Secret for Keycloak from environment variable {config.keycloak_client_secret_env_var}",
|
|
1186
|
+
)
|
|
1187
|
+
config.keycloak_client_secret = os.environ.get(
|
|
1188
|
+
config.keycloak_client_secret_env_var
|
|
1189
|
+
)
|
|
1190
|
+
else:
|
|
1191
|
+
config.keycloak_client_secret = None
|
|
1192
|
+
|
|
1136
1193
|
# Run cartography
|
|
1137
1194
|
try:
|
|
1138
1195
|
return cartography.sync.run_with_config(self.sync, config)
|
cartography/config.py
CHANGED
|
@@ -45,6 +45,8 @@ class Config:
|
|
|
45
45
|
:param azure_client_id: Client Id for connecting in a Service Principal Authentication approach. Optional.
|
|
46
46
|
:type azure_client_secret: str
|
|
47
47
|
:param azure_client_secret: Client Secret for connecting in a Service Principal Authentication approach. Optional.
|
|
48
|
+
:type azure_subscription_id: str | None
|
|
49
|
+
:param azure_subscription_id: The Azure Subscription ID to sync.
|
|
48
50
|
:type entra_tenant_id: str
|
|
49
51
|
:param entra_tenant_id: Tenant Id for connecting in a Service Principal Authentication approach. Optional.
|
|
50
52
|
:type entra_client_id: str
|
|
@@ -92,6 +94,8 @@ class Config:
|
|
|
92
94
|
:param statsd_port: If statsd_enabled is True, send metrics to this port on statsd_host. Optional.
|
|
93
95
|
:type: k8s_kubeconfig: str
|
|
94
96
|
:param k8s_kubeconfig: Path to kubeconfig file for kubernetes cluster(s). Optional
|
|
97
|
+
:type: managed_kubernetes: str
|
|
98
|
+
:param managed_kubernetes: Type of managed Kubernetes service (e.g., "eks"). Optional.
|
|
95
99
|
:type: pagerduty_api_key: str
|
|
96
100
|
:param pagerduty_api_key: API authentication key for pagerduty. Optional.
|
|
97
101
|
:type: pagerduty_request_timeout: int
|
|
@@ -166,6 +170,14 @@ class Config:
|
|
|
166
170
|
:param sentinelone_api_token: SentinelOne API token for authentication. Optional.
|
|
167
171
|
:type sentinelone_account_ids: list[str]
|
|
168
172
|
:param sentinelone_account_ids: List of SentinelOne account IDs to sync. Optional.
|
|
173
|
+
:type keycloak_client_id: str
|
|
174
|
+
:param keycloak_client_id: Keycloak client ID for API authentication. Optional.
|
|
175
|
+
:type keycloak_client_secret: str
|
|
176
|
+
:param keycloak_client_secret: Keycloak client secret for API authentication. Optional.
|
|
177
|
+
:type keycloak_realm: str
|
|
178
|
+
:param keycloak_realm: Keycloak realm for authentication (all realms will be synced). Optional.
|
|
179
|
+
:type keycloak_url: str
|
|
180
|
+
:param keycloak_url: Keycloak base URL, e.g. https://keycloak.example.com. Optional.
|
|
169
181
|
"""
|
|
170
182
|
|
|
171
183
|
def __init__(
|
|
@@ -186,6 +198,7 @@ class Config:
|
|
|
186
198
|
azure_tenant_id=None,
|
|
187
199
|
azure_client_id=None,
|
|
188
200
|
azure_client_secret=None,
|
|
201
|
+
azure_subscription_id: str | None = None,
|
|
189
202
|
entra_tenant_id=None,
|
|
190
203
|
entra_client_id=None,
|
|
191
204
|
entra_client_secret=None,
|
|
@@ -206,6 +219,7 @@ class Config:
|
|
|
206
219
|
kandji_tenant_id=None,
|
|
207
220
|
kandji_token=None,
|
|
208
221
|
k8s_kubeconfig=None,
|
|
222
|
+
managed_kubernetes=None,
|
|
209
223
|
statsd_enabled=False,
|
|
210
224
|
statsd_prefix=None,
|
|
211
225
|
statsd_host=None,
|
|
@@ -252,6 +266,10 @@ class Config:
|
|
|
252
266
|
sentinelone_api_url=None,
|
|
253
267
|
sentinelone_api_token=None,
|
|
254
268
|
sentinelone_account_ids=None,
|
|
269
|
+
keycloak_client_id=None,
|
|
270
|
+
keycloak_client_secret=None,
|
|
271
|
+
keycloak_realm=None,
|
|
272
|
+
keycloak_url=None,
|
|
255
273
|
):
|
|
256
274
|
self.neo4j_uri = neo4j_uri
|
|
257
275
|
self.neo4j_user = neo4j_user
|
|
@@ -271,6 +289,7 @@ class Config:
|
|
|
271
289
|
self.azure_tenant_id = azure_tenant_id
|
|
272
290
|
self.azure_client_id = azure_client_id
|
|
273
291
|
self.azure_client_secret = azure_client_secret
|
|
292
|
+
self.azure_subscription_id = azure_subscription_id
|
|
274
293
|
self.entra_tenant_id = entra_tenant_id
|
|
275
294
|
self.entra_client_id = entra_client_id
|
|
276
295
|
self.entra_client_secret = entra_client_secret
|
|
@@ -291,6 +310,7 @@ class Config:
|
|
|
291
310
|
self.kandji_tenant_id = kandji_tenant_id
|
|
292
311
|
self.kandji_token = kandji_token
|
|
293
312
|
self.k8s_kubeconfig = k8s_kubeconfig
|
|
313
|
+
self.managed_kubernetes = managed_kubernetes
|
|
294
314
|
self.statsd_enabled = statsd_enabled
|
|
295
315
|
self.statsd_prefix = statsd_prefix
|
|
296
316
|
self.statsd_host = statsd_host
|
|
@@ -337,3 +357,7 @@ class Config:
|
|
|
337
357
|
self.sentinelone_api_url = sentinelone_api_url
|
|
338
358
|
self.sentinelone_api_token = sentinelone_api_token
|
|
339
359
|
self.sentinelone_account_ids = sentinelone_account_ids
|
|
360
|
+
self.keycloak_client_id = keycloak_client_id
|
|
361
|
+
self.keycloak_client_secret = keycloak_client_secret
|
|
362
|
+
self.keycloak_realm = keycloak_realm
|
|
363
|
+
self.keycloak_url = keycloak_url
|
cartography/data/indexes.cypher
CHANGED
|
@@ -29,8 +29,6 @@ CREATE INDEX IF NOT EXISTS FOR (n:AWSIpv4CidrBlock) ON (n.id);
|
|
|
29
29
|
CREATE INDEX IF NOT EXISTS FOR (n:AWSIpv4CidrBlock) ON (n.lastupdated);
|
|
30
30
|
CREATE INDEX IF NOT EXISTS FOR (n:AWSIpv6CidrBlock) ON (n.id);
|
|
31
31
|
CREATE INDEX IF NOT EXISTS FOR (n:AWSIpv6CidrBlock) ON (n.lastupdated);
|
|
32
|
-
CREATE INDEX IF NOT EXISTS FOR (n:AWSPeeringConnection) ON (n.id);
|
|
33
|
-
CREATE INDEX IF NOT EXISTS FOR (n:AWSPeeringConnection) ON (n.lastupdated);
|
|
34
32
|
CREATE INDEX IF NOT EXISTS FOR (n:AWSPolicy) ON (n.id);
|
|
35
33
|
CREATE INDEX IF NOT EXISTS FOR (n:AWSPolicy) ON (n.name);
|
|
36
34
|
CREATE INDEX IF NOT EXISTS FOR (n:AWSPolicy) ON (n.lastupdated);
|
|
@@ -102,10 +100,6 @@ CREATE INDEX IF NOT EXISTS FOR (n:ESDomain) ON (n.arn);
|
|
|
102
100
|
CREATE INDEX IF NOT EXISTS FOR (n:ESDomain) ON (n.id);
|
|
103
101
|
CREATE INDEX IF NOT EXISTS FOR (n:ESDomain) ON (n.name);
|
|
104
102
|
CREATE INDEX IF NOT EXISTS FOR (n:ESDomain) ON (n.lastupdated);
|
|
105
|
-
CREATE INDEX IF NOT EXISTS FOR (n:GCPDNSZone) ON (n.id);
|
|
106
|
-
CREATE INDEX IF NOT EXISTS FOR (n:GCPDNSZone) ON (n.lastupdated);
|
|
107
|
-
CREATE INDEX IF NOT EXISTS FOR (n:GCPRecordSet) ON (n.id);
|
|
108
|
-
CREATE INDEX IF NOT EXISTS FOR (n:GCPRecordSet) ON (n.lastupdated);
|
|
109
103
|
CREATE INDEX IF NOT EXISTS FOR (n:GCPFolder) ON (n.id);
|
|
110
104
|
CREATE INDEX IF NOT EXISTS FOR (n:GCPFolder) ON (n.lastupdated);
|
|
111
105
|
CREATE INDEX IF NOT EXISTS FOR (n:GCPForwardingRule) ON (n.id);
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
{
|
|
2
|
+
"statements": [
|
|
3
|
+
{
|
|
4
|
+
"__comment__": "Inherit group memberships from subgroups to parent groups",
|
|
5
|
+
"query": "MATCH (u:KeycloakUser)-[:MEMBER_OF]->(g:KeycloakGroup)-[:SUBGROUP_OF*1..5]->(pg:KeycloakGroup) MERGE (u)-[r:INHERITED_MEMBER_OF]->(pg) ON CREATE SET r.firstseen = $UPDATE_TAG SET r.lastupdated = $UPDATE_TAG",
|
|
6
|
+
"iterative": false
|
|
7
|
+
},
|
|
8
|
+
{
|
|
9
|
+
"__comment__": "Assign roles to users based on group memberships",
|
|
10
|
+
"query": "MATCH (u:KeycloakUser)-[:MEMBER_OF|INHERITED_MEMBER_OR]->(g:KeycloakGroup)-[:GRANTS]->(r:KeycloakRole) MERGE (u)-[r0:ASSUME_ROLE]-(r) ON CREATE SET r0.firstseen = $UPDATE_TAG SET r0.lastupdated = $UPDATE_TAG",
|
|
11
|
+
"iterative": false
|
|
12
|
+
},
|
|
13
|
+
{
|
|
14
|
+
"__comment__": "Propagate role grants to composite roles",
|
|
15
|
+
"query": "MATCH (r:KeycloakRole)-[:INCLUDES*1..5]->(c:KeycloakRole)-[:GRANTS]->(s:KeycloakScope) MERGE (r)-[r0:INDIRECT_GRANTS]-(s) ON CREATE SET r0.firstseen = $UPDATE_TAG SET r0.lastupdated = $UPDATE_TAG",
|
|
16
|
+
"iterative": false
|
|
17
|
+
},
|
|
18
|
+
{
|
|
19
|
+
"__comment__": "Identify legitimate scopes for users based on assumed roles",
|
|
20
|
+
"query": "MATCH (u:KeycloakUser)-[:ASSUME_ROLE]-(:KeycloakRole)-[:GRANTS|INDIRECT_GRANTS]->(s:KeycloakScope) MERGE (u)-[r:ASSUME_SCOPE]->(s) ON CREATE SET r.firstseen = $UPDATE_TAG SET r.lastupdated = $UPDATE_TAG",
|
|
21
|
+
"iterative": false
|
|
22
|
+
},
|
|
23
|
+
{
|
|
24
|
+
"__comment__": "Assign assumed scopes to users for orphan scopes (scopes not granted by any role)",
|
|
25
|
+
"query": "MATCH (s:KeycloakScope)<-[:RESOURCE]-(r:KeycloakRealm) MATCH (u:KeycloakUser)<-[:RESOURCE]-(r) WHERE NOT (s)<-[:GRANTS|INDIRECT_GRANTS]-(:KeycloakRole) MERGE (u)-[r0:ASSUME_SCOPE]->(s) SET r0.firstseen = $UPDATE_TAG SET r0.lastupdated = $UPDATE_TAG",
|
|
26
|
+
"iterative": false
|
|
27
|
+
}
|
|
28
|
+
],
|
|
29
|
+
"name": "Keycloak inheritance analysis"
|
|
30
|
+
}
|
|
@@ -22,6 +22,10 @@ from cartography.models.aws.apigateway.apigatewaycertificate import (
|
|
|
22
22
|
from cartography.models.aws.apigateway.apigatewaydeployment import (
|
|
23
23
|
APIGatewayDeploymentSchema,
|
|
24
24
|
)
|
|
25
|
+
from cartography.models.aws.apigateway.apigatewayintegration import (
|
|
26
|
+
APIGatewayIntegrationSchema,
|
|
27
|
+
)
|
|
28
|
+
from cartography.models.aws.apigateway.apigatewaymethod import APIGatewayMethodSchema
|
|
25
29
|
from cartography.models.aws.apigateway.apigatewayresource import (
|
|
26
30
|
APIGatewayResourceSchema,
|
|
27
31
|
)
|
|
@@ -84,7 +88,7 @@ def get_rest_api_details(
|
|
|
84
88
|
boto3_session: boto3.session.Session,
|
|
85
89
|
rest_apis: List[Dict],
|
|
86
90
|
region: str,
|
|
87
|
-
) -> List[Tuple[Any, Any, Any, Any, Any]]:
|
|
91
|
+
) -> List[Tuple[Any, Any, Any, Any, Any, Any, Any]]:
|
|
88
92
|
"""
|
|
89
93
|
Iterates over all API Gateway REST APIs.
|
|
90
94
|
"""
|
|
@@ -94,13 +98,19 @@ def get_rest_api_details(
|
|
|
94
98
|
stages = get_rest_api_stages(api, client)
|
|
95
99
|
# clientcertificate id is given by the api stage
|
|
96
100
|
certificate = get_rest_api_client_certificate(stages, client)
|
|
97
|
-
resources =
|
|
101
|
+
resources, methods, integrations = get_rest_api_resources_methods_integrations(
|
|
102
|
+
api,
|
|
103
|
+
client,
|
|
104
|
+
)
|
|
98
105
|
policy = get_rest_api_policy(api, client)
|
|
99
|
-
apis.append(
|
|
106
|
+
apis.append(
|
|
107
|
+
(api["id"], stages, certificate, resources, methods, integrations, policy)
|
|
108
|
+
)
|
|
100
109
|
return apis
|
|
101
110
|
|
|
102
111
|
|
|
103
112
|
@timeit
|
|
113
|
+
@aws_handle_regions
|
|
104
114
|
def get_rest_api_stages(api: Dict, client: botocore.client.BaseClient) -> Any:
|
|
105
115
|
"""
|
|
106
116
|
Gets the REST API Stage Resources.
|
|
@@ -142,17 +152,44 @@ def get_rest_api_client_certificate(
|
|
|
142
152
|
|
|
143
153
|
|
|
144
154
|
@timeit
|
|
145
|
-
|
|
155
|
+
@aws_handle_regions
|
|
156
|
+
def get_rest_api_resources_methods_integrations(
|
|
157
|
+
api: Dict, client: botocore.client.BaseClient
|
|
158
|
+
) -> Tuple[List[Any], List[Dict], List[Dict]]:
|
|
146
159
|
"""
|
|
147
160
|
Gets the collection of Resource resources.
|
|
148
161
|
"""
|
|
149
162
|
resources: List[Any] = []
|
|
163
|
+
methods: List[Any] = []
|
|
164
|
+
integrations: List[Any] = []
|
|
165
|
+
|
|
150
166
|
paginator = client.get_paginator("get_resources")
|
|
151
167
|
response_iterator = paginator.paginate(restApiId=api["id"])
|
|
152
168
|
for page in response_iterator:
|
|
153
|
-
|
|
169
|
+
page_resources = page["items"]
|
|
170
|
+
resources.extend(page_resources)
|
|
171
|
+
|
|
172
|
+
for resource in page_resources:
|
|
173
|
+
resource_id = resource["id"]
|
|
174
|
+
resource_methods = resource.get("resourceMethods", {})
|
|
175
|
+
|
|
176
|
+
for http_method, method in resource_methods.items():
|
|
177
|
+
method["resourceId"] = resource_id
|
|
178
|
+
method["apiId"] = api["id"]
|
|
179
|
+
method["httpMethod"] = http_method
|
|
180
|
+
methods.append(method)
|
|
181
|
+
integration = client.get_integration(
|
|
182
|
+
restApiId=api["id"],
|
|
183
|
+
resourceId=resource_id,
|
|
184
|
+
httpMethod=http_method,
|
|
185
|
+
)
|
|
186
|
+
integration["resourceId"] = resource_id
|
|
187
|
+
integration["apiId"] = api["id"]
|
|
188
|
+
integration["integrationHttpMethod"] = integration.get("httpMethod")
|
|
189
|
+
integration["httpMethod"] = http_method
|
|
190
|
+
integrations.append(integration)
|
|
154
191
|
|
|
155
|
-
return resources
|
|
192
|
+
return resources, methods, integrations
|
|
156
193
|
|
|
157
194
|
|
|
158
195
|
@timeit
|
|
@@ -248,16 +285,27 @@ def transform_apigateway_certificates(
|
|
|
248
285
|
|
|
249
286
|
|
|
250
287
|
def transform_rest_api_details(
|
|
251
|
-
stages_certificate_resources: List[Tuple[Any, Any, Any, Any, Any]],
|
|
252
|
-
) -> Tuple[List[Dict], List[Dict], List[Dict]]:
|
|
288
|
+
stages_certificate_resources: List[Tuple[Any, Any, Any, Any, Any, Any, Any]],
|
|
289
|
+
) -> Tuple[List[Dict], List[Dict], List[Dict], List[Dict], List[Dict]]:
|
|
253
290
|
"""
|
|
254
|
-
Transform Stage, Client Certificate, and
|
|
291
|
+
Transform Stage, Client Certificate, Resource, Method and Integration data for ingestion
|
|
255
292
|
"""
|
|
256
293
|
stages: List[Dict] = []
|
|
257
294
|
certificates: List[Dict] = []
|
|
258
295
|
resources: List[Dict] = []
|
|
296
|
+
methods: List[Dict] = []
|
|
297
|
+
integrations: List[Dict] = []
|
|
298
|
+
|
|
299
|
+
for (
|
|
300
|
+
api_id,
|
|
301
|
+
stage,
|
|
302
|
+
certificate,
|
|
303
|
+
resource,
|
|
304
|
+
method_list,
|
|
305
|
+
integration_list,
|
|
306
|
+
_,
|
|
307
|
+
) in stages_certificate_resources:
|
|
259
308
|
|
|
260
|
-
for api_id, stage, certificate, resource, _ in stages_certificate_resources:
|
|
261
309
|
if len(stage) > 0:
|
|
262
310
|
for s in stage:
|
|
263
311
|
s["apiId"] = api_id
|
|
@@ -279,7 +327,32 @@ def transform_rest_api_details(
|
|
|
279
327
|
r["apiId"] = api_id
|
|
280
328
|
resources.extend(resource)
|
|
281
329
|
|
|
282
|
-
|
|
330
|
+
if len(method_list) > 0:
|
|
331
|
+
for method in method_list:
|
|
332
|
+
method["id"] = (
|
|
333
|
+
f"{method['apiId']}/{method['resourceId']}/{method['httpMethod']}"
|
|
334
|
+
)
|
|
335
|
+
method["authorizationType"] = method.get("authorizationType")
|
|
336
|
+
method["authorizerId"] = method.get("authorizerId")
|
|
337
|
+
method["requestValidatorId"] = method.get("requestValidatorId")
|
|
338
|
+
method["operationName"] = method.get("operationName")
|
|
339
|
+
method["apiKeyRequired"] = method.get("apiKeyRequired", False)
|
|
340
|
+
methods.extend(method_list)
|
|
341
|
+
|
|
342
|
+
if len(integration_list) > 0:
|
|
343
|
+
for integration in integration_list:
|
|
344
|
+
if not integration.get("id"):
|
|
345
|
+
integration["id"] = (
|
|
346
|
+
f"{integration['apiId']}/{integration['resourceId']}/{integration['httpMethod']}"
|
|
347
|
+
)
|
|
348
|
+
integration["type"] = integration.get("type")
|
|
349
|
+
integration["uri"] = integration.get("uri")
|
|
350
|
+
integration["connectionType"] = integration.get("connectionType")
|
|
351
|
+
integration["connectionId"] = integration.get("connectionId")
|
|
352
|
+
integration["credentials"] = integration.get("credentials")
|
|
353
|
+
integrations.extend(integration_list)
|
|
354
|
+
|
|
355
|
+
return stages, certificates, resources, methods, integrations
|
|
283
356
|
|
|
284
357
|
|
|
285
358
|
def transform_apigateway_deployments(
|
|
@@ -304,15 +377,17 @@ def transform_apigateway_deployments(
|
|
|
304
377
|
@timeit
|
|
305
378
|
def load_rest_api_details(
|
|
306
379
|
neo4j_session: neo4j.Session,
|
|
307
|
-
|
|
380
|
+
stages_certificate_resources_methods_integrations: List[
|
|
381
|
+
Tuple[Any, Any, Any, Any, Any, Any, Any]
|
|
382
|
+
],
|
|
308
383
|
aws_account_id: str,
|
|
309
384
|
update_tag: int,
|
|
310
385
|
) -> None:
|
|
311
386
|
"""
|
|
312
387
|
Transform and load Stage, Client Certificate, and Resource data
|
|
313
388
|
"""
|
|
314
|
-
stages, certificates, resources = transform_rest_api_details(
|
|
315
|
-
|
|
389
|
+
stages, certificates, resources, methods, integrations = transform_rest_api_details(
|
|
390
|
+
stages_certificate_resources_methods_integrations,
|
|
316
391
|
)
|
|
317
392
|
|
|
318
393
|
load(
|
|
@@ -339,6 +414,22 @@ def load_rest_api_details(
|
|
|
339
414
|
AWS_ID=aws_account_id,
|
|
340
415
|
)
|
|
341
416
|
|
|
417
|
+
load(
|
|
418
|
+
neo4j_session,
|
|
419
|
+
APIGatewayMethodSchema(),
|
|
420
|
+
methods,
|
|
421
|
+
lastupdated=update_tag,
|
|
422
|
+
AWS_ID=aws_account_id,
|
|
423
|
+
)
|
|
424
|
+
|
|
425
|
+
load(
|
|
426
|
+
neo4j_session,
|
|
427
|
+
APIGatewayIntegrationSchema(),
|
|
428
|
+
integrations,
|
|
429
|
+
lastupdated=update_tag,
|
|
430
|
+
AWS_ID=aws_account_id,
|
|
431
|
+
)
|
|
432
|
+
|
|
342
433
|
|
|
343
434
|
@timeit
|
|
344
435
|
def load_apigateway_deployments(
|
|
@@ -432,6 +523,18 @@ def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
|
|
|
432
523
|
)
|
|
433
524
|
cleanup_job.run(neo4j_session)
|
|
434
525
|
|
|
526
|
+
cleanup_job = GraphJob.from_node_schema(
|
|
527
|
+
APIGatewayMethodSchema(),
|
|
528
|
+
common_job_parameters,
|
|
529
|
+
)
|
|
530
|
+
cleanup_job.run(neo4j_session)
|
|
531
|
+
|
|
532
|
+
cleanup_job = GraphJob.from_node_schema(
|
|
533
|
+
APIGatewayIntegrationSchema(),
|
|
534
|
+
common_job_parameters,
|
|
535
|
+
)
|
|
536
|
+
cleanup_job.run(neo4j_session)
|
|
537
|
+
|
|
435
538
|
|
|
436
539
|
@timeit
|
|
437
540
|
def sync_apigateway_rest_apis(
|
|
@@ -442,7 +545,7 @@ def sync_apigateway_rest_apis(
|
|
|
442
545
|
aws_update_tag: int,
|
|
443
546
|
) -> None:
|
|
444
547
|
rest_apis = get_apigateway_rest_apis(boto3_session, region)
|
|
445
|
-
|
|
548
|
+
stages_certificate_resources_methods_integrations = get_rest_api_details(
|
|
446
549
|
boto3_session,
|
|
447
550
|
rest_apis,
|
|
448
551
|
region,
|
|
@@ -450,7 +553,15 @@ def sync_apigateway_rest_apis(
|
|
|
450
553
|
|
|
451
554
|
# Extract policies and transform the data
|
|
452
555
|
policies = []
|
|
453
|
-
for
|
|
556
|
+
for (
|
|
557
|
+
api_id,
|
|
558
|
+
_,
|
|
559
|
+
_,
|
|
560
|
+
_,
|
|
561
|
+
_,
|
|
562
|
+
_,
|
|
563
|
+
policy,
|
|
564
|
+
) in stages_certificate_resources_methods_integrations:
|
|
454
565
|
parsed_policy = parse_policy(api_id, policy)
|
|
455
566
|
if parsed_policy is not None:
|
|
456
567
|
policies.append(parsed_policy)
|
|
@@ -484,7 +595,7 @@ def sync_apigateway_rest_apis(
|
|
|
484
595
|
)
|
|
485
596
|
load_rest_api_details(
|
|
486
597
|
neo4j_session,
|
|
487
|
-
|
|
598
|
+
stages_certificate_resources_methods_integrations,
|
|
488
599
|
current_aws_account_id,
|
|
489
600
|
aws_update_tag,
|
|
490
601
|
)
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from typing import Any
|
|
3
|
+
|
|
4
|
+
import boto3
|
|
5
|
+
import neo4j
|
|
6
|
+
|
|
7
|
+
from cartography.client.core.tx import load
|
|
8
|
+
from cartography.graph.job import GraphJob
|
|
9
|
+
from cartography.models.aws.apigatewayv2.apigatewayv2 import APIGatewayV2APISchema
|
|
10
|
+
from cartography.util import aws_handle_regions
|
|
11
|
+
from cartography.util import timeit
|
|
12
|
+
|
|
13
|
+
logger = logging.getLogger(__name__)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@timeit
|
|
17
|
+
@aws_handle_regions
|
|
18
|
+
def get_apigatewayv2_apis(
|
|
19
|
+
boto3_session: boto3.session.Session,
|
|
20
|
+
region: str,
|
|
21
|
+
) -> list[dict[str, Any]]:
|
|
22
|
+
client = boto3_session.client("apigatewayv2", region_name=region)
|
|
23
|
+
paginator = client.get_paginator("get_apis")
|
|
24
|
+
apis: list[dict[str, Any]] = []
|
|
25
|
+
for page in paginator.paginate():
|
|
26
|
+
apis.extend(page.get("Items", []))
|
|
27
|
+
return apis
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def transform_apigatewayv2_apis(apis: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
31
|
+
transformed: list[dict[str, Any]] = []
|
|
32
|
+
for api in apis:
|
|
33
|
+
transformed.append(
|
|
34
|
+
{
|
|
35
|
+
"id": api.get("ApiId"),
|
|
36
|
+
"name": api.get("Name"),
|
|
37
|
+
"protocoltype": api.get("ProtocolType"),
|
|
38
|
+
"routeselectionexpression": api.get("RouteSelectionExpression"),
|
|
39
|
+
"apikeyselectionexpression": api.get("ApiKeySelectionExpression"),
|
|
40
|
+
"apiendpoint": api.get("ApiEndpoint"),
|
|
41
|
+
"version": api.get("Version"),
|
|
42
|
+
"createddate": api.get("CreatedDate"),
|
|
43
|
+
"description": api.get("Description"),
|
|
44
|
+
},
|
|
45
|
+
)
|
|
46
|
+
return transformed
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@timeit
|
|
50
|
+
def load_apigatewayv2_apis(
|
|
51
|
+
neo4j_session: neo4j.Session,
|
|
52
|
+
data: list[dict[str, Any]],
|
|
53
|
+
region: str,
|
|
54
|
+
current_aws_account_id: str,
|
|
55
|
+
update_tag: int,
|
|
56
|
+
) -> None:
|
|
57
|
+
load(
|
|
58
|
+
neo4j_session,
|
|
59
|
+
APIGatewayV2APISchema(),
|
|
60
|
+
data,
|
|
61
|
+
lastupdated=update_tag,
|
|
62
|
+
AWS_ID=current_aws_account_id,
|
|
63
|
+
region=region,
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@timeit
|
|
68
|
+
def cleanup(
|
|
69
|
+
neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
|
|
70
|
+
) -> None:
|
|
71
|
+
GraphJob.from_node_schema(
|
|
72
|
+
APIGatewayV2APISchema(),
|
|
73
|
+
common_job_parameters,
|
|
74
|
+
).run(neo4j_session)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
@timeit
|
|
78
|
+
def sync_apigatewayv2_apis(
|
|
79
|
+
neo4j_session: neo4j.Session,
|
|
80
|
+
boto3_session: boto3.session.Session,
|
|
81
|
+
region: str,
|
|
82
|
+
current_aws_account_id: str,
|
|
83
|
+
aws_update_tag: int,
|
|
84
|
+
) -> None:
|
|
85
|
+
apis = get_apigatewayv2_apis(boto3_session, region)
|
|
86
|
+
transformed = transform_apigatewayv2_apis(apis)
|
|
87
|
+
load_apigatewayv2_apis(
|
|
88
|
+
neo4j_session,
|
|
89
|
+
transformed,
|
|
90
|
+
region,
|
|
91
|
+
current_aws_account_id,
|
|
92
|
+
aws_update_tag,
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
@timeit
|
|
97
|
+
def sync(
|
|
98
|
+
neo4j_session: neo4j.Session,
|
|
99
|
+
boto3_session: boto3.session.Session,
|
|
100
|
+
regions: list[str],
|
|
101
|
+
current_aws_account_id: str,
|
|
102
|
+
update_tag: int,
|
|
103
|
+
common_job_parameters: dict[str, Any],
|
|
104
|
+
) -> None:
|
|
105
|
+
for region in regions:
|
|
106
|
+
logger.info(
|
|
107
|
+
f"Syncing AWS APIGatewayV2 APIs for region '{region}' in account '{current_aws_account_id}'.",
|
|
108
|
+
)
|
|
109
|
+
sync_apigatewayv2_apis(
|
|
110
|
+
neo4j_session,
|
|
111
|
+
boto3_session,
|
|
112
|
+
region,
|
|
113
|
+
current_aws_account_id,
|
|
114
|
+
update_tag,
|
|
115
|
+
)
|
|
116
|
+
cleanup(neo4j_session, common_job_parameters)
|
|
@@ -163,7 +163,9 @@ def transform_ec2_instances(
|
|
|
163
163
|
"MacAddress": network_interface["MacAddress"],
|
|
164
164
|
"Description": network_interface["Description"],
|
|
165
165
|
"PrivateDnsName": network_interface.get("PrivateDnsName"),
|
|
166
|
-
"PrivateIpAddress": network_interface
|
|
166
|
+
"PrivateIpAddress": network_interface.get(
|
|
167
|
+
"PrivateIpAddress"
|
|
168
|
+
),
|
|
167
169
|
"InstanceId": instance_id,
|
|
168
170
|
"SubnetId": subnet_id,
|
|
169
171
|
"GroupId": security_group["GroupId"],
|
|
@@ -91,7 +91,7 @@ def transform_network_interface_data(
|
|
|
91
91
|
"InterfaceType": network_interface["InterfaceType"],
|
|
92
92
|
"MacAddress": network_interface["MacAddress"],
|
|
93
93
|
"PrivateDnsName": network_interface.get("PrivateDnsName"),
|
|
94
|
-
"PrivateIpAddress": network_interface
|
|
94
|
+
"PrivateIpAddress": network_interface.get("PrivateIpAddress"),
|
|
95
95
|
"PublicIp": network_interface.get("Association", {}).get("PublicIp"),
|
|
96
96
|
"RequesterId": network_interface.get("RequesterId"),
|
|
97
97
|
"RequesterManaged": network_interface["RequesterManaged"],
|