cartography 0.93.0rc1__py3-none-any.whl → 0.123.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cartography/__main__.py +1 -2
- cartography/_version.py +34 -0
- cartography/cli.py +903 -225
- cartography/client/aws/__init__.py +19 -0
- cartography/client/aws/ecr.py +51 -0
- cartography/client/core/tx.py +400 -27
- cartography/config.py +215 -10
- cartography/data/azure_permission_relationships.yaml +20 -0
- cartography/data/gcp_permission_relationships.yaml +21 -0
- cartography/data/indexes.cypher +1 -200
- cartography/data/jobs/analysis/aws_ec2_asset_exposure.json +17 -2
- cartography/data/jobs/analysis/aws_ec2_keypair_analysis.json +2 -2
- cartography/data/jobs/analysis/gcp_compute_asset_inet_exposure.json +1 -1
- cartography/data/jobs/analysis/keycloak_inheritance.json +30 -0
- cartography/data/jobs/cleanup/crowdstrike_import_cleanup.json +0 -5
- cartography/data/jobs/cleanup/gcp_compute_vpc_cleanup.json +0 -12
- cartography/data/jobs/cleanup/github_repos_cleanup.json +27 -0
- cartography/data/jobs/scoped_analysis/aws_ec2_iaminstanceprofile.json +15 -0
- cartography/data/jobs/scoped_analysis/semgrep_sca_risk_analysis.json +13 -13
- cartography/driftdetect/__main__.py +1 -2
- cartography/driftdetect/add_shortcut.py +10 -2
- cartography/driftdetect/cli.py +72 -75
- cartography/driftdetect/detect_deviations.py +7 -3
- cartography/driftdetect/get_states.py +20 -8
- cartography/driftdetect/model.py +5 -5
- cartography/driftdetect/serializers.py +8 -6
- cartography/driftdetect/storage.py +2 -2
- cartography/graph/cleanupbuilder.py +255 -35
- cartography/graph/job.py +104 -20
- cartography/graph/querybuilder.py +689 -91
- cartography/graph/statement.py +49 -36
- cartography/intel/airbyte/__init__.py +105 -0
- cartography/intel/airbyte/connections.py +120 -0
- cartography/intel/airbyte/destinations.py +81 -0
- cartography/intel/airbyte/organizations.py +59 -0
- cartography/intel/airbyte/sources.py +78 -0
- cartography/intel/airbyte/tags.py +64 -0
- cartography/intel/airbyte/users.py +106 -0
- cartography/intel/airbyte/util.py +122 -0
- cartography/intel/airbyte/workspaces.py +63 -0
- cartography/intel/analysis.py +4 -1
- cartography/intel/anthropic/__init__.py +62 -0
- cartography/intel/anthropic/apikeys.py +72 -0
- cartography/intel/anthropic/users.py +75 -0
- cartography/intel/anthropic/util.py +51 -0
- cartography/intel/anthropic/workspaces.py +95 -0
- cartography/intel/aws/__init__.py +137 -59
- cartography/intel/aws/acm.py +124 -0
- cartography/intel/aws/apigateway.py +482 -217
- cartography/intel/aws/apigatewayv2.py +116 -0
- cartography/intel/aws/cloudtrail.py +105 -0
- cartography/intel/aws/cloudtrail_management_events.py +962 -0
- cartography/intel/aws/cloudwatch.py +239 -0
- cartography/intel/aws/codebuild.py +132 -0
- cartography/intel/aws/cognito.py +201 -0
- cartography/intel/aws/config.py +63 -23
- cartography/intel/aws/dynamodb.py +108 -40
- cartography/intel/aws/ec2/__init__.py +2 -2
- cartography/intel/aws/ec2/auto_scaling_groups.py +254 -189
- cartography/intel/aws/ec2/elastic_ip_addresses.py +44 -14
- cartography/intel/aws/ec2/images.py +74 -39
- cartography/intel/aws/ec2/instances.py +262 -137
- cartography/intel/aws/ec2/internet_gateways.py +44 -13
- cartography/intel/aws/ec2/key_pairs.py +72 -39
- cartography/intel/aws/ec2/launch_templates.py +143 -66
- cartography/intel/aws/ec2/load_balancer_v2s.py +119 -45
- cartography/intel/aws/ec2/load_balancers.py +165 -147
- cartography/intel/aws/ec2/network_acls.py +233 -0
- cartography/intel/aws/ec2/network_interfaces.py +150 -87
- cartography/intel/aws/ec2/reserved_instances.py +48 -17
- cartography/intel/aws/ec2/route_tables.py +327 -0
- cartography/intel/aws/ec2/security_groups.py +189 -121
- cartography/intel/aws/ec2/snapshots.py +93 -91
- cartography/intel/aws/ec2/subnets.py +70 -58
- cartography/intel/aws/ec2/tgw.py +111 -39
- cartography/intel/aws/ec2/util.py +1 -1
- cartography/intel/aws/ec2/volumes.py +69 -41
- cartography/intel/aws/ec2/vpc.py +157 -116
- cartography/intel/aws/ec2/vpc_peerings.py +317 -121
- cartography/intel/aws/ecr.py +336 -93
- cartography/intel/aws/ecr_image_layers.py +923 -0
- cartography/intel/aws/ecs.py +310 -403
- cartography/intel/aws/efs.py +261 -0
- cartography/intel/aws/eks.py +55 -29
- cartography/intel/aws/elasticache.py +130 -83
- cartography/intel/aws/elasticsearch.py +70 -24
- cartography/intel/aws/emr.py +61 -23
- cartography/intel/aws/eventbridge.py +164 -0
- cartography/intel/aws/glue.py +181 -0
- cartography/intel/aws/guardduty.py +443 -0
- cartography/intel/aws/iam.py +978 -464
- cartography/intel/aws/iam_instance_profiles.py +73 -0
- cartography/intel/aws/identitycenter.py +847 -0
- cartography/intel/aws/inspector.py +330 -133
- cartography/intel/aws/kms.py +235 -209
- cartography/intel/aws/lambda_function.py +328 -176
- cartography/intel/aws/organizations.py +40 -19
- cartography/intel/aws/permission_relationships.py +144 -68
- cartography/intel/aws/rds.py +467 -412
- cartography/intel/aws/redshift.py +116 -50
- cartography/intel/aws/resourcegroupstaggingapi.py +198 -82
- cartography/intel/aws/resources.py +80 -42
- cartography/intel/aws/route53.py +419 -318
- cartography/intel/aws/s3.py +489 -96
- cartography/intel/aws/s3accountpublicaccessblock.py +157 -0
- cartography/intel/aws/secretsmanager.py +217 -40
- cartography/intel/aws/securityhub.py +23 -10
- cartography/intel/aws/sns.py +226 -0
- cartography/intel/aws/sqs.py +74 -96
- cartography/intel/aws/ssm.py +142 -33
- cartography/intel/aws/util/arns.py +7 -7
- cartography/intel/aws/util/common.py +31 -4
- cartography/intel/azure/__init__.py +259 -46
- cartography/intel/azure/aks.py +175 -0
- cartography/intel/azure/app_service.py +105 -0
- cartography/intel/azure/compute.py +141 -120
- cartography/intel/azure/container_instances.py +95 -0
- cartography/intel/azure/cosmosdb.py +706 -519
- cartography/intel/azure/data_factory.py +85 -0
- cartography/intel/azure/data_factory_dataset.py +128 -0
- cartography/intel/azure/data_factory_linked_service.py +119 -0
- cartography/intel/azure/data_factory_pipeline.py +142 -0
- cartography/intel/azure/data_lake.py +124 -0
- cartography/intel/azure/event_grid.py +94 -0
- cartography/intel/azure/functions.py +124 -0
- cartography/intel/azure/load_balancers.py +263 -0
- cartography/intel/azure/logic_apps.py +101 -0
- cartography/intel/azure/monitor.py +105 -0
- cartography/intel/azure/network.py +467 -0
- cartography/intel/azure/permission_relationships.py +466 -0
- cartography/intel/azure/rbac.py +309 -0
- cartography/intel/azure/resource_groups.py +82 -0
- cartography/intel/azure/security_center.py +106 -0
- cartography/intel/azure/sql.py +436 -392
- cartography/intel/azure/storage.py +467 -335
- cartography/intel/azure/subscription.py +49 -55
- cartography/intel/azure/tenant.py +46 -28
- cartography/intel/azure/util/common.py +13 -0
- cartography/intel/azure/util/credentials.py +58 -143
- cartography/intel/azure/util/tag.py +41 -0
- cartography/intel/bigfix/__init__.py +2 -2
- cartography/intel/bigfix/computers.py +93 -65
- cartography/intel/cloudflare/__init__.py +74 -0
- cartography/intel/cloudflare/accounts.py +57 -0
- cartography/intel/cloudflare/dnsrecords.py +64 -0
- cartography/intel/cloudflare/members.py +75 -0
- cartography/intel/cloudflare/roles.py +65 -0
- cartography/intel/cloudflare/zones.py +64 -0
- cartography/intel/create_indexes.py +5 -3
- cartography/intel/crowdstrike/__init__.py +26 -12
- cartography/intel/crowdstrike/endpoints.py +17 -45
- cartography/intel/crowdstrike/spotlight.py +13 -5
- cartography/intel/cve/__init__.py +91 -26
- cartography/intel/cve/feed.py +77 -56
- cartography/intel/digitalocean/__init__.py +22 -13
- cartography/intel/digitalocean/compute.py +75 -108
- cartography/intel/digitalocean/management.py +44 -80
- cartography/intel/digitalocean/platform.py +48 -43
- cartography/intel/dns.py +41 -12
- cartography/intel/duo/__init__.py +21 -16
- cartography/intel/duo/api_host.py +14 -9
- cartography/intel/duo/endpoints.py +50 -45
- cartography/intel/duo/groups.py +18 -14
- cartography/intel/duo/phones.py +37 -34
- cartography/intel/duo/tokens.py +26 -23
- cartography/intel/duo/users.py +54 -50
- cartography/intel/duo/web_authn_credentials.py +30 -25
- cartography/intel/entra/__init__.py +160 -0
- cartography/intel/entra/app_role_assignments.py +284 -0
- cartography/intel/entra/applications.py +182 -0
- cartography/intel/entra/federation/__init__.py +0 -0
- cartography/intel/entra/federation/aws_identity_center.py +77 -0
- cartography/intel/entra/groups.py +198 -0
- cartography/intel/entra/ou.py +136 -0
- cartography/intel/entra/service_principals.py +217 -0
- cartography/intel/entra/users.py +259 -0
- cartography/intel/gcp/__init__.py +381 -175
- cartography/intel/gcp/bigtable_app_profile.py +101 -0
- cartography/intel/gcp/bigtable_backup.py +91 -0
- cartography/intel/gcp/bigtable_cluster.py +93 -0
- cartography/intel/gcp/bigtable_instance.py +86 -0
- cartography/intel/gcp/bigtable_table.py +87 -0
- cartography/intel/gcp/cai.py +292 -0
- cartography/intel/gcp/clients.py +112 -0
- cartography/intel/gcp/compute.py +521 -325
- cartography/intel/gcp/crm/__init__.py +0 -0
- cartography/intel/gcp/crm/folders.py +114 -0
- cartography/intel/gcp/crm/orgs.py +70 -0
- cartography/intel/gcp/crm/projects.py +120 -0
- cartography/intel/gcp/dns.py +134 -179
- cartography/intel/gcp/gke.py +100 -107
- cartography/intel/gcp/iam.py +262 -0
- cartography/intel/gcp/permission_relationships.py +394 -0
- cartography/intel/gcp/policy_bindings.py +225 -0
- cartography/intel/gcp/storage.py +103 -158
- cartography/intel/github/__init__.py +66 -27
- cartography/intel/github/commits.py +423 -0
- cartography/intel/github/repos.py +871 -160
- cartography/intel/github/teams.py +386 -53
- cartography/intel/github/users.py +214 -49
- cartography/intel/github/util.py +50 -35
- cartography/intel/googleworkspace/__init__.py +193 -0
- cartography/intel/googleworkspace/devices.py +254 -0
- cartography/intel/googleworkspace/groups.py +568 -0
- cartography/intel/googleworkspace/oauth_apps.py +259 -0
- cartography/intel/googleworkspace/tenant.py +85 -0
- cartography/intel/googleworkspace/users.py +138 -0
- cartography/intel/gsuite/__init__.py +101 -42
- cartography/intel/gsuite/groups.py +291 -0
- cartography/intel/gsuite/users.py +142 -0
- cartography/intel/jamf/__init__.py +19 -1
- cartography/intel/jamf/computers.py +37 -8
- cartography/intel/jamf/util.py +7 -2
- cartography/intel/kandji/__init__.py +6 -3
- cartography/intel/kandji/devices.py +40 -10
- cartography/intel/keycloak/__init__.py +153 -0
- cartography/intel/keycloak/authenticationexecutions.py +322 -0
- cartography/intel/keycloak/authenticationflows.py +77 -0
- cartography/intel/keycloak/clients.py +187 -0
- cartography/intel/keycloak/groups.py +126 -0
- cartography/intel/keycloak/identityproviders.py +94 -0
- cartography/intel/keycloak/organizations.py +163 -0
- cartography/intel/keycloak/realms.py +61 -0
- cartography/intel/keycloak/roles.py +202 -0
- cartography/intel/keycloak/scopes.py +73 -0
- cartography/intel/keycloak/users.py +70 -0
- cartography/intel/keycloak/util.py +47 -0
- cartography/intel/kubernetes/__init__.py +60 -14
- cartography/intel/kubernetes/clusters.py +86 -0
- cartography/intel/kubernetes/eks.py +402 -0
- cartography/intel/kubernetes/namespaces.py +60 -55
- cartography/intel/kubernetes/pods.py +171 -75
- cartography/intel/kubernetes/rbac.py +597 -0
- cartography/intel/kubernetes/secrets.py +95 -45
- cartography/intel/kubernetes/services.py +131 -63
- cartography/intel/kubernetes/util.py +142 -14
- cartography/intel/lastpass/__init__.py +2 -2
- cartography/intel/lastpass/users.py +23 -12
- cartography/intel/oci/__init__.py +44 -11
- cartography/intel/oci/iam.py +157 -47
- cartography/intel/oci/organizations.py +16 -7
- cartography/intel/oci/utils.py +71 -25
- cartography/intel/okta/__init__.py +66 -15
- cartography/intel/okta/applications.py +57 -25
- cartography/intel/okta/awssaml.py +105 -41
- cartography/intel/okta/factors.py +19 -5
- cartography/intel/okta/groups.py +61 -31
- cartography/intel/okta/organization.py +8 -2
- cartography/intel/okta/origins.py +9 -3
- cartography/intel/okta/roles.py +20 -7
- cartography/intel/okta/users.py +31 -10
- cartography/intel/okta/utils.py +6 -4
- cartography/intel/ontology/__init__.py +44 -0
- cartography/intel/ontology/devices.py +54 -0
- cartography/intel/ontology/users.py +54 -0
- cartography/intel/ontology/utils.py +176 -0
- cartography/intel/openai/__init__.py +86 -0
- cartography/intel/openai/adminapikeys.py +89 -0
- cartography/intel/openai/apikeys.py +96 -0
- cartography/intel/openai/projects.py +97 -0
- cartography/intel/openai/serviceaccounts.py +82 -0
- cartography/intel/openai/users.py +75 -0
- cartography/intel/openai/util.py +45 -0
- cartography/intel/pagerduty/__init__.py +8 -7
- cartography/intel/pagerduty/escalation_policies.py +31 -12
- cartography/intel/pagerduty/schedules.py +21 -8
- cartography/intel/pagerduty/services.py +18 -7
- cartography/intel/pagerduty/teams.py +13 -5
- cartography/intel/pagerduty/users.py +6 -2
- cartography/intel/pagerduty/vendors.py +6 -2
- cartography/intel/scaleway/__init__.py +127 -0
- cartography/intel/scaleway/iam/__init__.py +0 -0
- cartography/intel/scaleway/iam/apikeys.py +71 -0
- cartography/intel/scaleway/iam/applications.py +71 -0
- cartography/intel/scaleway/iam/groups.py +71 -0
- cartography/intel/scaleway/iam/users.py +71 -0
- cartography/intel/scaleway/instances/__init__.py +0 -0
- cartography/intel/scaleway/instances/flexibleips.py +86 -0
- cartography/intel/scaleway/instances/instances.py +92 -0
- cartography/intel/scaleway/projects.py +79 -0
- cartography/intel/scaleway/storage/__init__.py +0 -0
- cartography/intel/scaleway/storage/snapshots.py +86 -0
- cartography/intel/scaleway/storage/volumes.py +84 -0
- cartography/intel/scaleway/utils.py +37 -0
- cartography/intel/semgrep/__init__.py +30 -5
- cartography/intel/semgrep/dependencies.py +255 -0
- cartography/intel/semgrep/deployment.py +69 -0
- cartography/intel/semgrep/findings.py +157 -117
- cartography/intel/sentinelone/__init__.py +75 -0
- cartography/intel/sentinelone/account.py +140 -0
- cartography/intel/sentinelone/agent.py +139 -0
- cartography/intel/sentinelone/api.py +124 -0
- cartography/intel/sentinelone/application.py +248 -0
- cartography/intel/sentinelone/cve.py +119 -0
- cartography/intel/sentinelone/utils.py +28 -0
- cartography/intel/slack/__init__.py +78 -0
- cartography/intel/slack/channels.py +80 -0
- cartography/intel/slack/groups.py +90 -0
- cartography/intel/slack/teams.py +65 -0
- cartography/intel/slack/users.py +57 -0
- cartography/intel/slack/utils.py +29 -0
- cartography/intel/snipeit/__init__.py +44 -0
- cartography/intel/snipeit/asset.py +80 -0
- cartography/intel/snipeit/user.py +78 -0
- cartography/intel/snipeit/util.py +40 -0
- cartography/intel/spacelift/__init__.py +161 -0
- cartography/intel/spacelift/account.py +73 -0
- cartography/intel/spacelift/ec2_ownership.py +280 -0
- cartography/intel/spacelift/runs.py +463 -0
- cartography/intel/spacelift/spaces.py +112 -0
- cartography/intel/spacelift/stacks.py +119 -0
- cartography/intel/spacelift/util.py +122 -0
- cartography/intel/spacelift/workerpools.py +131 -0
- cartography/intel/spacelift/workers.py +128 -0
- cartography/intel/tailscale/__init__.py +77 -0
- cartography/intel/tailscale/acls.py +146 -0
- cartography/intel/tailscale/devices.py +127 -0
- cartography/intel/tailscale/postureintegrations.py +81 -0
- cartography/intel/tailscale/tailnets.py +76 -0
- cartography/intel/tailscale/users.py +80 -0
- cartography/intel/tailscale/utils.py +132 -0
- cartography/intel/trivy/__init__.py +272 -0
- cartography/intel/trivy/scanner.py +386 -0
- cartography/models/airbyte/__init__.py +0 -0
- cartography/models/airbyte/connection.py +138 -0
- cartography/models/airbyte/destination.py +75 -0
- cartography/models/airbyte/organization.py +19 -0
- cartography/models/airbyte/source.py +75 -0
- cartography/models/airbyte/stream.py +74 -0
- cartography/models/airbyte/tag.py +69 -0
- cartography/models/airbyte/user.py +115 -0
- cartography/models/airbyte/workspace.py +46 -0
- cartography/models/anthropic/__init__.py +0 -0
- cartography/models/anthropic/apikey.py +94 -0
- cartography/models/anthropic/organization.py +19 -0
- cartography/models/anthropic/user.py +52 -0
- cartography/models/anthropic/workspace.py +90 -0
- cartography/models/aws/acm/__init__.py +0 -0
- cartography/models/aws/acm/certificate.py +75 -0
- cartography/models/aws/apigateway/__init__.py +0 -0
- cartography/models/aws/apigateway/apigateway.py +51 -0
- cartography/models/aws/apigateway/apigatewaycertificate.py +72 -0
- cartography/models/aws/apigateway/apigatewaydeployment.py +74 -0
- cartography/models/aws/apigateway/apigatewayintegration.py +79 -0
- cartography/models/aws/apigateway/apigatewaymethod.py +74 -0
- cartography/models/aws/apigateway/apigatewayresource.py +70 -0
- cartography/models/aws/apigateway/apigatewaystage.py +75 -0
- cartography/models/aws/apigatewayv2/__init__.py +0 -0
- cartography/models/aws/apigatewayv2/apigatewayv2.py +53 -0
- cartography/models/aws/cloudtrail/__init__.py +0 -0
- cartography/models/aws/cloudtrail/management_events.py +153 -0
- cartography/models/aws/cloudtrail/trail.py +106 -0
- cartography/models/aws/cloudwatch/__init__.py +0 -0
- cartography/models/aws/cloudwatch/log_metric_filter.py +79 -0
- cartography/models/aws/cloudwatch/loggroup.py +52 -0
- cartography/models/aws/cloudwatch/metric_alarm.py +53 -0
- cartography/models/aws/codebuild/__init__.py +0 -0
- cartography/models/aws/codebuild/project.py +49 -0
- cartography/models/aws/cognito/__init__.py +0 -0
- cartography/models/aws/cognito/identity_pool.py +70 -0
- cartography/models/aws/cognito/user_pool.py +47 -0
- cartography/models/aws/dynamodb/gsi.py +30 -22
- cartography/models/aws/dynamodb/tables.py +27 -17
- cartography/models/aws/ec2/auto_scaling_groups.py +224 -0
- cartography/models/aws/ec2/images.py +36 -34
- cartography/models/aws/ec2/instances.py +85 -38
- cartography/models/aws/ec2/keypair.py +59 -0
- cartography/models/aws/ec2/keypair_instance.py +76 -0
- cartography/models/aws/ec2/launch_configurations.py +59 -0
- cartography/models/aws/ec2/launch_template_versions.py +48 -38
- cartography/models/aws/ec2/launch_templates.py +21 -17
- cartography/models/aws/ec2/load_balancer_listeners.py +72 -0
- cartography/models/aws/ec2/load_balancers.py +112 -0
- cartography/models/aws/ec2/network_acl_rules.py +106 -0
- cartography/models/aws/ec2/network_acls.py +95 -0
- cartography/models/aws/ec2/networkinterface_instance.py +52 -39
- cartography/models/aws/ec2/networkinterfaces.py +57 -37
- cartography/models/aws/ec2/privateip_networkinterface.py +32 -22
- cartography/models/aws/ec2/reservations.py +18 -14
- cartography/models/aws/ec2/route_table_associations.py +97 -0
- cartography/models/aws/ec2/route_tables.py +128 -0
- cartography/models/aws/ec2/routes.py +85 -0
- cartography/models/aws/ec2/security_group_rules.py +109 -0
- cartography/models/aws/ec2/security_groups.py +90 -0
- cartography/models/aws/ec2/securitygroup_instance.py +29 -20
- cartography/models/aws/ec2/securitygroup_networkinterface.py +24 -15
- cartography/models/aws/ec2/snapshots.py +58 -0
- cartography/models/aws/ec2/subnet_instance.py +26 -19
- cartography/models/aws/ec2/subnet_networkinterface.py +42 -31
- cartography/models/aws/ec2/subnets.py +65 -0
- cartography/models/aws/ec2/volumes.py +67 -40
- cartography/models/aws/ec2/vpc.py +46 -0
- cartography/models/aws/ec2/vpc_cidr.py +102 -0
- cartography/models/aws/ec2/vpc_peering.py +157 -0
- cartography/models/aws/ecr/__init__.py +0 -0
- cartography/models/aws/ecr/image.py +146 -0
- cartography/models/aws/ecr/image_layer.py +107 -0
- cartography/models/aws/ecr/repository.py +72 -0
- cartography/models/aws/ecr/repository_image.py +95 -0
- cartography/models/aws/ecs/__init__.py +0 -0
- cartography/models/aws/ecs/clusters.py +64 -0
- cartography/models/aws/ecs/container_definitions.py +93 -0
- cartography/models/aws/ecs/container_instances.py +84 -0
- cartography/models/aws/ecs/containers.py +101 -0
- cartography/models/aws/ecs/services.py +134 -0
- cartography/models/aws/ecs/task_definitions.py +135 -0
- cartography/models/aws/ecs/tasks.py +134 -0
- cartography/models/aws/efs/__init__.py +0 -0
- cartography/models/aws/efs/access_point.py +77 -0
- cartography/models/aws/efs/file_system.py +60 -0
- cartography/models/aws/efs/mount_target.py +79 -0
- cartography/models/aws/eks/clusters.py +23 -21
- cartography/models/aws/elasticache/__init__.py +0 -0
- cartography/models/aws/elasticache/cluster.py +65 -0
- cartography/models/aws/elasticache/topic.py +67 -0
- cartography/models/aws/emr.py +32 -30
- cartography/models/aws/eventbridge/__init__.py +0 -0
- cartography/models/aws/eventbridge/rule.py +77 -0
- cartography/models/aws/eventbridge/target.py +71 -0
- cartography/models/aws/glue/__init__.py +0 -0
- cartography/models/aws/glue/connection.py +51 -0
- cartography/models/aws/glue/job.py +69 -0
- cartography/models/aws/guardduty/__init__.py +1 -0
- cartography/models/aws/guardduty/detectors.py +50 -0
- cartography/models/aws/guardduty/findings.py +121 -0
- cartography/models/aws/iam/__init__.py +0 -0
- cartography/models/aws/iam/access_key.py +103 -0
- cartography/models/aws/iam/account_role.py +24 -0
- cartography/models/aws/iam/federated_principal.py +60 -0
- cartography/models/aws/iam/group.py +60 -0
- cartography/models/aws/iam/group_membership.py +27 -0
- cartography/models/aws/iam/inline_policy.py +78 -0
- cartography/models/aws/iam/instanceprofile.py +76 -0
- cartography/models/aws/iam/managed_policy.py +51 -0
- cartography/models/aws/iam/policy_statement.py +57 -0
- cartography/models/aws/iam/role.py +83 -0
- cartography/models/aws/iam/root_principal.py +52 -0
- cartography/models/aws/iam/service_principal.py +30 -0
- cartography/models/aws/iam/sts_assumerole_allow.py +38 -0
- cartography/models/aws/iam/user.py +59 -0
- cartography/models/aws/identitycenter/__init__.py +0 -0
- cartography/models/aws/identitycenter/awsidentitycenter.py +49 -0
- cartography/models/aws/identitycenter/awspermissionset.py +162 -0
- cartography/models/aws/identitycenter/awssogroup.py +70 -0
- cartography/models/aws/identitycenter/awsssouser.py +110 -0
- cartography/models/aws/inspector/findings.py +124 -58
- cartography/models/aws/inspector/packages.py +18 -42
- cartography/models/aws/kms/__init__.py +0 -0
- cartography/models/aws/kms/aliases.py +86 -0
- cartography/models/aws/kms/grants.py +65 -0
- cartography/models/aws/kms/keys.py +88 -0
- cartography/models/aws/lambda_function/__init__.py +0 -0
- cartography/models/aws/lambda_function/alias.py +74 -0
- cartography/models/aws/lambda_function/event_source_mapping.py +88 -0
- cartography/models/aws/lambda_function/lambda_function.py +91 -0
- cartography/models/aws/lambda_function/layer.py +72 -0
- cartography/models/aws/rds/__init__.py +0 -0
- cartography/models/aws/rds/cluster.py +91 -0
- cartography/models/aws/rds/event_subscription.py +146 -0
- cartography/models/aws/rds/instance.py +156 -0
- cartography/models/aws/rds/snapshot.py +108 -0
- cartography/models/aws/rds/subnet_group.py +101 -0
- cartography/models/aws/route53/__init__.py +0 -0
- cartography/models/aws/route53/dnsrecord.py +235 -0
- cartography/models/aws/route53/nameserver.py +63 -0
- cartography/models/aws/route53/subzone.py +40 -0
- cartography/models/aws/route53/zone.py +47 -0
- cartography/models/aws/s3/__init__.py +0 -0
- cartography/models/aws/s3/account_public_access_block.py +51 -0
- cartography/models/aws/s3/notification.py +24 -0
- cartography/models/aws/secretsmanager/__init__.py +0 -0
- cartography/models/aws/secretsmanager/secret.py +106 -0
- cartography/models/aws/secretsmanager/secret_version.py +114 -0
- cartography/models/aws/sns/__init__.py +0 -0
- cartography/models/aws/sns/topic.py +50 -0
- cartography/models/aws/sns/topic_subscription.py +74 -0
- cartography/models/aws/sqs/__init__.py +0 -0
- cartography/models/aws/sqs/queue.py +89 -0
- cartography/models/aws/ssm/instance_information.py +51 -39
- cartography/models/aws/ssm/instance_patch.py +32 -26
- cartography/models/aws/ssm/parameters.py +84 -0
- cartography/models/azure/__init__.py +0 -0
- cartography/models/azure/aks_cluster.py +54 -0
- cartography/models/azure/aks_nodepool.py +54 -0
- cartography/models/azure/app_service.py +59 -0
- cartography/models/azure/container_instance.py +57 -0
- cartography/models/azure/cosmosdb/__init__.py +0 -0
- cartography/models/azure/cosmosdb/account.py +77 -0
- cartography/models/azure/cosmosdb/accountfailoverpolicy.py +77 -0
- cartography/models/azure/cosmosdb/cassandrakeyspace.py +82 -0
- cartography/models/azure/cosmosdb/cassandratable.py +81 -0
- cartography/models/azure/cosmosdb/corspolicy.py +74 -0
- cartography/models/azure/cosmosdb/dblocation.py +120 -0
- cartography/models/azure/cosmosdb/mongodbcollection.py +82 -0
- cartography/models/azure/cosmosdb/mongodbdatabase.py +78 -0
- cartography/models/azure/cosmosdb/privateendpointconnection.py +81 -0
- cartography/models/azure/cosmosdb/sqlcontainer.py +88 -0
- cartography/models/azure/cosmosdb/sqldatabase.py +78 -0
- cartography/models/azure/cosmosdb/tableresource.py +76 -0
- cartography/models/azure/cosmosdb/virtualnetworkrule.py +78 -0
- cartography/models/azure/data_factory/__init__.py +0 -0
- cartography/models/azure/data_factory/data_factory.py +51 -0
- cartography/models/azure/data_factory/data_factory_dataset.py +94 -0
- cartography/models/azure/data_factory/data_factory_linked_service.py +78 -0
- cartography/models/azure/data_factory/data_factory_pipeline.py +93 -0
- cartography/models/azure/data_lake_filesystem.py +51 -0
- cartography/models/azure/event_grid_topic.py +57 -0
- cartography/models/azure/function_app.py +59 -0
- cartography/models/azure/load_balancer/__init__.py +0 -0
- cartography/models/azure/load_balancer/load_balancer.py +49 -0
- cartography/models/azure/load_balancer/load_balancer_backend_pool.py +73 -0
- cartography/models/azure/load_balancer/load_balancer_frontend_ip.py +75 -0
- cartography/models/azure/load_balancer/load_balancer_inbound_nat_rule.py +78 -0
- cartography/models/azure/load_balancer/load_balancer_rule.py +108 -0
- cartography/models/azure/logic_apps.py +56 -0
- cartography/models/azure/monitor.py +54 -0
- cartography/models/azure/network_interface.py +112 -0
- cartography/models/azure/network_security_group.py +50 -0
- cartography/models/azure/permission_relationships.py +60 -0
- cartography/models/azure/principal.py +41 -0
- cartography/models/azure/public_ip_address.py +50 -0
- cartography/models/azure/rbac.py +268 -0
- cartography/models/azure/resource_groups.py +52 -0
- cartography/models/azure/security_center.py +50 -0
- cartography/models/azure/sql/__init__.py +0 -0
- cartography/models/azure/sql/databasethreatdetectionpolicy.py +85 -0
- cartography/models/azure/sql/elasticpool.py +77 -0
- cartography/models/azure/sql/failovergroup.py +73 -0
- cartography/models/azure/sql/recoverabledatabase.py +75 -0
- cartography/models/azure/sql/replicationlink.py +81 -0
- cartography/models/azure/sql/restorabledroppeddatabase.py +82 -0
- cartography/models/azure/sql/restorepoint.py +74 -0
- cartography/models/azure/sql/serveradadministrator.py +74 -0
- cartography/models/azure/sql/serverdnsalias.py +71 -0
- cartography/models/azure/sql/sqldatabase.py +85 -0
- cartography/models/azure/sql/sqlserver.py +50 -0
- cartography/models/azure/sql/transparentdataencryption.py +76 -0
- cartography/models/azure/storage/__init__.py +0 -0
- cartography/models/azure/storage/account.py +59 -0
- cartography/models/azure/storage/blobcontainer.py +85 -0
- cartography/models/azure/storage/blobservice.py +71 -0
- cartography/models/azure/storage/fileservice.py +71 -0
- cartography/models/azure/storage/fileshare.py +82 -0
- cartography/models/azure/storage/queue.py +71 -0
- cartography/models/azure/storage/queueservice.py +73 -0
- cartography/models/azure/storage/table.py +72 -0
- cartography/models/azure/storage/tableservice.py +73 -0
- cartography/models/azure/subnet.py +101 -0
- cartography/models/azure/subscription.py +47 -0
- cartography/models/azure/tags/__init__.py +0 -0
- cartography/models/azure/tags/storage_tag.py +40 -0
- cartography/models/azure/tags/tag.py +37 -0
- cartography/models/azure/tenant.py +17 -0
- cartography/models/azure/virtual_network.py +49 -0
- cartography/models/azure/vm/__init__.py +0 -0
- cartography/models/azure/vm/datadisk.py +80 -0
- cartography/models/azure/vm/disk.py +55 -0
- cartography/models/azure/vm/snapshot.py +56 -0
- cartography/models/azure/vm/virtualmachine.py +59 -0
- cartography/models/bigfix/bigfix_computer.py +42 -38
- cartography/models/bigfix/bigfix_root.py +3 -3
- cartography/models/cloudflare/__init__.py +0 -0
- cartography/models/cloudflare/account.py +25 -0
- cartography/models/cloudflare/dnsrecord.py +55 -0
- cartography/models/cloudflare/member.py +86 -0
- cartography/models/cloudflare/role.py +44 -0
- cartography/models/cloudflare/zone.py +59 -0
- cartography/models/core/common.py +53 -2
- cartography/models/core/nodes.py +20 -4
- cartography/models/core/relationships.py +58 -6
- cartography/models/crowdstrike/__init__.py +0 -0
- cartography/models/crowdstrike/hosts.py +51 -0
- cartography/models/cve/cve.py +34 -32
- cartography/models/cve/cve_feed.py +6 -6
- cartography/models/digitalocean/__init__.py +0 -0
- cartography/models/digitalocean/account.py +21 -0
- cartography/models/digitalocean/droplet.py +58 -0
- cartography/models/digitalocean/project.py +48 -0
- cartography/models/duo/api_host.py +3 -3
- cartography/models/duo/endpoint.py +43 -41
- cartography/models/duo/group.py +14 -14
- cartography/models/duo/phone.py +27 -27
- cartography/models/duo/token.py +16 -16
- cartography/models/duo/user.py +50 -44
- cartography/models/duo/web_authn_credential.py +27 -19
- cartography/models/entra/__init__.py +0 -0
- cartography/models/entra/app_role_assignment.py +115 -0
- cartography/models/entra/application.py +49 -0
- cartography/models/entra/entra_user_to_aws_sso.py +41 -0
- cartography/models/entra/group.py +117 -0
- cartography/models/entra/ou.py +48 -0
- cartography/models/entra/service_principal.py +104 -0
- cartography/models/entra/tenant.py +39 -0
- cartography/models/entra/user.py +90 -0
- cartography/models/gcp/__init__.py +0 -0
- cartography/models/gcp/bigtable/__init__.py +0 -0
- cartography/models/gcp/bigtable/app_profile.py +94 -0
- cartography/models/gcp/bigtable/backup.py +91 -0
- cartography/models/gcp/bigtable/cluster.py +73 -0
- cartography/models/gcp/bigtable/instance.py +52 -0
- cartography/models/gcp/bigtable/table.py +69 -0
- cartography/models/gcp/compute/__init__.py +0 -0
- cartography/models/gcp/compute/subnet.py +74 -0
- cartography/models/gcp/compute/vpc.py +50 -0
- cartography/models/gcp/crm/__init__.py +0 -0
- cartography/models/gcp/crm/folders.py +98 -0
- cartography/models/gcp/crm/organizations.py +21 -0
- cartography/models/gcp/crm/projects.py +100 -0
- cartography/models/gcp/dns.py +109 -0
- cartography/models/gcp/gke.py +69 -0
- cartography/models/gcp/iam.py +73 -0
- cartography/models/gcp/permission_relationships.py +61 -0
- cartography/models/gcp/policy_bindings.py +93 -0
- cartography/models/gcp/storage/__init__.py +0 -0
- cartography/models/gcp/storage/bucket.py +119 -0
- cartography/models/github/commits.py +63 -0
- cartography/models/github/dependencies.py +73 -0
- cartography/models/github/manifests.py +49 -0
- cartography/models/github/orgs.py +27 -0
- cartography/models/github/teams.py +74 -22
- cartography/models/github/users.py +149 -0
- cartography/models/googleworkspace/__init__.py +0 -0
- cartography/models/googleworkspace/device.py +132 -0
- cartography/models/googleworkspace/group.py +382 -0
- cartography/models/googleworkspace/oauth_app.py +124 -0
- cartography/models/googleworkspace/tenant.py +30 -0
- cartography/models/googleworkspace/user.py +113 -0
- cartography/models/gsuite/__init__.py +0 -0
- cartography/models/gsuite/group.py +218 -0
- cartography/models/gsuite/tenant.py +29 -0
- cartography/models/gsuite/user.py +107 -0
- cartography/models/kandji/device.py +22 -17
- cartography/models/kandji/tenant.py +6 -4
- cartography/models/keycloak/__init__.py +0 -0
- cartography/models/keycloak/authenticationexecution.py +160 -0
- cartography/models/keycloak/authenticationflow.py +54 -0
- cartography/models/keycloak/client.py +179 -0
- cartography/models/keycloak/group.py +101 -0
- cartography/models/keycloak/identityprovider.py +89 -0
- cartography/models/keycloak/organization.py +116 -0
- cartography/models/keycloak/organizationdomain.py +73 -0
- cartography/models/keycloak/realm.py +173 -0
- cartography/models/keycloak/role.py +126 -0
- cartography/models/keycloak/scope.py +73 -0
- cartography/models/keycloak/user.py +55 -0
- cartography/models/kubernetes/__init__.py +0 -0
- cartography/models/kubernetes/clusterrolebindings.py +138 -0
- cartography/models/kubernetes/clusterroles.py +52 -0
- cartography/models/kubernetes/clusters.py +26 -0
- cartography/models/kubernetes/containers.py +133 -0
- cartography/models/kubernetes/groups.py +107 -0
- cartography/models/kubernetes/namespaces.py +51 -0
- cartography/models/kubernetes/oidc.py +51 -0
- cartography/models/kubernetes/pods.py +80 -0
- cartography/models/kubernetes/rolebindings.py +159 -0
- cartography/models/kubernetes/roles.py +76 -0
- cartography/models/kubernetes/secrets.py +79 -0
- cartography/models/kubernetes/serviceaccounts.py +77 -0
- cartography/models/kubernetes/services.py +108 -0
- cartography/models/kubernetes/users.py +105 -0
- cartography/models/lastpass/tenant.py +3 -3
- cartography/models/lastpass/user.py +36 -28
- cartography/models/ontology/__init__.py +0 -0
- cartography/models/ontology/device.py +137 -0
- cartography/models/ontology/mapping/__init__.py +76 -0
- cartography/models/ontology/mapping/data/__init__.py +0 -0
- cartography/models/ontology/mapping/data/apikeys.py +93 -0
- cartography/models/ontology/mapping/data/computeinstance.py +95 -0
- cartography/models/ontology/mapping/data/containers.py +88 -0
- cartography/models/ontology/mapping/data/databases.py +182 -0
- cartography/models/ontology/mapping/data/devices.py +194 -0
- cartography/models/ontology/mapping/data/thirdpartyapps.py +140 -0
- cartography/models/ontology/mapping/data/useraccounts.py +416 -0
- cartography/models/ontology/mapping/data/users.py +63 -0
- cartography/models/ontology/mapping/specs.py +85 -0
- cartography/models/ontology/user.py +51 -0
- cartography/models/openai/__init__.py +0 -0
- cartography/models/openai/adminapikey.py +94 -0
- cartography/models/openai/apikey.py +88 -0
- cartography/models/openai/organization.py +17 -0
- cartography/models/openai/project.py +89 -0
- cartography/models/openai/serviceaccount.py +50 -0
- cartography/models/openai/user.py +53 -0
- cartography/models/scaleway/__init__.py +0 -0
- cartography/models/scaleway/iam/__init__.py +0 -0
- cartography/models/scaleway/iam/apikey.py +100 -0
- cartography/models/scaleway/iam/application.py +52 -0
- cartography/models/scaleway/iam/group.py +95 -0
- cartography/models/scaleway/iam/user.py +64 -0
- cartography/models/scaleway/instance/__init__.py +0 -0
- cartography/models/scaleway/instance/flexibleip.py +52 -0
- cartography/models/scaleway/instance/instance.py +120 -0
- cartography/models/scaleway/organization.py +19 -0
- cartography/models/scaleway/project.py +48 -0
- cartography/models/scaleway/storage/__init__.py +0 -0
- cartography/models/scaleway/storage/snapshot.py +78 -0
- cartography/models/scaleway/storage/volume.py +51 -0
- cartography/models/semgrep/dependencies.py +102 -0
- cartography/models/semgrep/deployment.py +5 -5
- cartography/models/semgrep/findings.py +58 -40
- cartography/models/semgrep/locations.py +27 -21
- cartography/models/sentinelone/__init__.py +1 -0
- cartography/models/sentinelone/account.py +40 -0
- cartography/models/sentinelone/agent.py +50 -0
- cartography/models/sentinelone/application.py +44 -0
- cartography/models/sentinelone/application_version.py +96 -0
- cartography/models/sentinelone/cve.py +73 -0
- cartography/models/slack/__init__.py +0 -0
- cartography/models/slack/channels.py +92 -0
- cartography/models/slack/group.py +129 -0
- cartography/models/slack/team.py +22 -0
- cartography/models/slack/user.py +62 -0
- cartography/models/snipeit/__init__.py +0 -0
- cartography/models/snipeit/asset.py +92 -0
- cartography/models/snipeit/tenant.py +19 -0
- cartography/models/snipeit/user.py +60 -0
- cartography/models/spacelift/__init__.py +0 -0
- cartography/models/spacelift/cloudtrailevent.py +120 -0
- cartography/models/spacelift/run.py +162 -0
- cartography/models/spacelift/space.py +131 -0
- cartography/models/spacelift/spaceliftaccount.py +31 -0
- cartography/models/spacelift/spaceliftgitcommit.py +157 -0
- cartography/models/spacelift/stack.py +96 -0
- cartography/models/spacelift/user.py +63 -0
- cartography/models/spacelift/worker.py +97 -0
- cartography/models/spacelift/workerpool.py +90 -0
- cartography/models/tailscale/__init__.py +0 -0
- cartography/models/tailscale/device.py +96 -0
- cartography/models/tailscale/group.py +86 -0
- cartography/models/tailscale/postureintegration.py +58 -0
- cartography/models/tailscale/tag.py +102 -0
- cartography/models/tailscale/tailnet.py +29 -0
- cartography/models/tailscale/user.py +57 -0
- cartography/models/trivy/__init__.py +0 -0
- cartography/models/trivy/findings.py +66 -0
- cartography/models/trivy/fix.py +66 -0
- cartography/models/trivy/package.py +71 -0
- cartography/rules/README.md +1 -0
- cartography/rules/__init__.py +0 -0
- cartography/rules/cli.py +261 -0
- cartography/rules/data/__init__.py +0 -0
- cartography/rules/data/rules/__init__.py +46 -0
- cartography/rules/data/rules/cloud_security_product_deactivated.py +49 -0
- cartography/rules/data/rules/compute_instance_exposed.py +51 -0
- cartography/rules/data/rules/database_instance_exposed.py +53 -0
- cartography/rules/data/rules/delegation_boundary_modifiable.py +90 -0
- cartography/rules/data/rules/identity_administration_privileges.py +100 -0
- cartography/rules/data/rules/inactive_user_active_accounts.py +48 -0
- cartography/rules/data/rules/malicious_npm_dependencies_shai_hulud.py +2222 -0
- cartography/rules/data/rules/mfa_missing.py +46 -0
- cartography/rules/data/rules/object_storage_public.py +100 -0
- cartography/rules/data/rules/policy_administration_privileges.py +104 -0
- cartography/rules/data/rules/unmanaged_accounts.py +43 -0
- cartography/rules/data/rules/workload_identity_admin_capabilities.py +193 -0
- cartography/rules/formatters.py +108 -0
- cartography/rules/runners.py +216 -0
- cartography/rules/spec/__init__.py +0 -0
- cartography/rules/spec/model.py +267 -0
- cartography/rules/spec/result.py +38 -0
- cartography/stats.py +4 -4
- cartography/sync.py +137 -31
- cartography/util.py +187 -77
- cartography-0.123.0.dist-info/METADATA +230 -0
- cartography-0.123.0.dist-info/RECORD +856 -0
- {cartography-0.93.0rc1.dist-info → cartography-0.123.0.dist-info}/WHEEL +1 -1
- {cartography-0.93.0rc1.dist-info → cartography-0.123.0.dist-info}/entry_points.txt +1 -0
- {cartography-0.93.0rc1.dist-info → cartography-0.123.0.dist-info/licenses}/LICENSE +1 -1
- cartography/data/jobs/analysis/aws_ec2_iaminstance.json +0 -10
- cartography/data/jobs/analysis/aws_ec2_iaminstanceprofile.json +0 -10
- cartography/data/jobs/cleanup/aws_apigateway_details.json +0 -10
- cartography/data/jobs/cleanup/aws_dns_cleanup.json +0 -65
- cartography/data/jobs/cleanup/aws_import_account_access_key_cleanup.json +0 -17
- cartography/data/jobs/cleanup/aws_import_apigateway_cleanup.json +0 -45
- cartography/data/jobs/cleanup/aws_import_ec2_security_groupinfo_cleanup.json +0 -24
- cartography/data/jobs/cleanup/aws_import_groups_cleanup.json +0 -13
- cartography/data/jobs/cleanup/aws_import_lambda_cleanup.json +0 -50
- cartography/data/jobs/cleanup/aws_import_principals_cleanup.json +0 -30
- cartography/data/jobs/cleanup/aws_import_rds_clusters_cleanup.json +0 -23
- cartography/data/jobs/cleanup/aws_import_rds_instances_cleanup.json +0 -47
- cartography/data/jobs/cleanup/aws_import_rds_snapshots_cleanup.json +0 -23
- cartography/data/jobs/cleanup/aws_import_roles_cleanup.json +0 -13
- cartography/data/jobs/cleanup/aws_import_secrets_cleanup.json +0 -8
- cartography/data/jobs/cleanup/aws_import_snapshots_cleanup.json +0 -30
- cartography/data/jobs/cleanup/aws_import_users_cleanup.json +0 -8
- cartography/data/jobs/cleanup/aws_import_vpc_cleanup.json +0 -23
- cartography/data/jobs/cleanup/aws_import_vpc_peering_cleanup.json +0 -45
- cartography/data/jobs/cleanup/aws_kms_details.json +0 -10
- cartography/data/jobs/cleanup/azure_cosmosdb_cassandra_keyspace_cleanup.json +0 -25
- cartography/data/jobs/cleanup/azure_cosmosdb_cors_details.json +0 -15
- cartography/data/jobs/cleanup/azure_cosmosdb_mongodb_database_cleanup.json +0 -25
- cartography/data/jobs/cleanup/azure_cosmosdb_sql_database_cleanup.json +0 -25
- cartography/data/jobs/cleanup/azure_cosmosdb_table_resources_cleanup.json +0 -15
- cartography/data/jobs/cleanup/azure_database_account_cleanup.json +0 -85
- cartography/data/jobs/cleanup/azure_import_disks_cleanup.json +0 -15
- cartography/data/jobs/cleanup/azure_import_snapshots_cleanup.json +0 -15
- cartography/data/jobs/cleanup/azure_import_virtual_machines_cleanup.json +0 -25
- cartography/data/jobs/cleanup/azure_sql_server_cleanup.json +0 -125
- cartography/data/jobs/cleanup/azure_storage_account_cleanup.json +0 -95
- cartography/data/jobs/cleanup/azure_subscriptions_cleanup.json +0 -14
- cartography/data/jobs/cleanup/azure_tenant_cleanup.json +0 -9
- cartography/data/jobs/cleanup/crxcavator_import_cleanup.json +0 -18
- cartography/data/jobs/cleanup/gcp_compute_vpc_subnet_cleanup.json +0 -35
- cartography/data/jobs/cleanup/gcp_crm_folder_cleanup.json +0 -23
- cartography/data/jobs/cleanup/gcp_crm_organization_cleanup.json +0 -17
- cartography/data/jobs/cleanup/gcp_crm_project_cleanup.json +0 -23
- cartography/data/jobs/cleanup/gcp_dns_cleanup.json +0 -29
- cartography/data/jobs/cleanup/gcp_gke_cluster_cleanup.json +0 -17
- cartography/data/jobs/cleanup/gcp_storage_bucket_cleanup.json +0 -29
- cartography/data/jobs/cleanup/github_users_cleanup.json +0 -23
- cartography/data/jobs/cleanup/gsuite_ingest_groups_cleanup.json +0 -23
- cartography/data/jobs/cleanup/gsuite_ingest_users_cleanup.json +0 -11
- cartography/data/jobs/cleanup/kubernetes_import_cleanup.json +0 -70
- cartography/intel/crxcavator/__init__.py +0 -44
- cartography/intel/crxcavator/crxcavator.py +0 -329
- cartography/intel/gcp/crm.py +0 -302
- cartography/intel/gsuite/api.py +0 -284
- cartography/models/aws/ec2/keypairs.py +0 -64
- cartography-0.93.0rc1.dist-info/METADATA +0 -55
- cartography-0.93.0rc1.dist-info/NOTICE +0 -4
- cartography-0.93.0rc1.dist-info/RECORD +0 -341
- /cartography/data/jobs/{analysis → scoped_analysis}/aws_s3acl_analysis.json +0 -0
- {cartography-0.93.0rc1.dist-info → cartography-0.123.0.dist-info}/top_level.txt +0 -0
cartography/intel/aws/s3.py
CHANGED
|
@@ -16,7 +16,9 @@ from botocore.exceptions import ClientError
|
|
|
16
16
|
from botocore.exceptions import EndpointConnectionError
|
|
17
17
|
from policyuniverse.policy import Policy
|
|
18
18
|
|
|
19
|
+
from cartography.client.core.tx import run_write_query
|
|
19
20
|
from cartography.stats import get_stats_client
|
|
21
|
+
from cartography.util import aws_handle_regions
|
|
20
22
|
from cartography.util import merge_module_sync_metadata
|
|
21
23
|
from cartography.util import run_analysis_job
|
|
22
24
|
from cartography.util import run_cleanup_job
|
|
@@ -30,16 +32,20 @@ stat_handler = get_stats_client(__name__)
|
|
|
30
32
|
|
|
31
33
|
@timeit
|
|
32
34
|
def get_s3_bucket_list(boto3_session: boto3.session.Session) -> List[Dict]:
|
|
33
|
-
client = boto3_session.client(
|
|
35
|
+
client = boto3_session.client("s3")
|
|
34
36
|
# NOTE no paginator available for this operation
|
|
35
37
|
buckets = client.list_buckets()
|
|
36
|
-
for bucket in buckets[
|
|
38
|
+
for bucket in buckets["Buckets"]:
|
|
37
39
|
try:
|
|
38
|
-
bucket[
|
|
40
|
+
bucket["Region"] = client.get_bucket_location(Bucket=bucket["Name"])[
|
|
41
|
+
"LocationConstraint"
|
|
42
|
+
]
|
|
39
43
|
except ClientError as e:
|
|
40
44
|
if _is_common_exception(e, bucket):
|
|
41
|
-
bucket[
|
|
42
|
-
logger.warning(
|
|
45
|
+
bucket["Region"] = None
|
|
46
|
+
logger.warning(
|
|
47
|
+
"skipping bucket='{}' due to exception.".format(bucket["Name"]),
|
|
48
|
+
)
|
|
43
49
|
continue
|
|
44
50
|
else:
|
|
45
51
|
raise
|
|
@@ -48,9 +54,9 @@ def get_s3_bucket_list(boto3_session: boto3.session.Session) -> List[Dict]:
|
|
|
48
54
|
|
|
49
55
|
@timeit
|
|
50
56
|
def get_s3_bucket_details(
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
) -> Generator[Tuple[str, Dict, Dict, Dict, Dict, Dict], None, None]:
|
|
57
|
+
boto3_session: boto3.session.Session,
|
|
58
|
+
bucket_data: Dict,
|
|
59
|
+
) -> Generator[Tuple[str, Dict, Dict, Dict, Dict, Dict, Dict], None, None]:
|
|
54
60
|
"""
|
|
55
61
|
Iterates over all S3 buckets. Yields bucket name (string), S3 bucket policies (JSON), ACLs (JSON),
|
|
56
62
|
default encryption policy (JSON), Versioning (JSON), and Public Access Block (JSON)
|
|
@@ -58,32 +64,56 @@ def get_s3_bucket_details(
|
|
|
58
64
|
# a local store for s3 clients so that we may re-use clients for an AWS region
|
|
59
65
|
s3_regional_clients: Dict[Any, Any] = {}
|
|
60
66
|
|
|
61
|
-
BucketDetail = Tuple[
|
|
67
|
+
BucketDetail = Tuple[
|
|
68
|
+
str,
|
|
69
|
+
Dict[str, Any],
|
|
70
|
+
Dict[str, Any],
|
|
71
|
+
Dict[str, Any],
|
|
72
|
+
Dict[str, Any],
|
|
73
|
+
Dict[str, Any],
|
|
74
|
+
Dict[str, Any],
|
|
75
|
+
Dict[str, Any],
|
|
76
|
+
]
|
|
62
77
|
|
|
63
78
|
async def _get_bucket_detail(bucket: Dict[str, Any]) -> BucketDetail:
|
|
64
79
|
# Note: bucket['Region'] is sometimes None because
|
|
65
80
|
# client.get_bucket_location() does not return a location constraint for buckets
|
|
66
81
|
# in us-east-1 region
|
|
67
|
-
client = s3_regional_clients.get(bucket[
|
|
82
|
+
client = s3_regional_clients.get(bucket["Region"])
|
|
68
83
|
if not client:
|
|
69
|
-
client = boto3_session.client(
|
|
70
|
-
s3_regional_clients[bucket[
|
|
84
|
+
client = boto3_session.client("s3", bucket["Region"])
|
|
85
|
+
s3_regional_clients[bucket["Region"]] = client
|
|
71
86
|
(
|
|
72
87
|
acl,
|
|
73
88
|
policy,
|
|
74
89
|
encryption,
|
|
75
90
|
versioning,
|
|
76
91
|
public_access_block,
|
|
92
|
+
bucket_ownership_controls,
|
|
93
|
+
bucket_logging,
|
|
77
94
|
) = await asyncio.gather(
|
|
78
95
|
to_asynchronous(get_acl, bucket, client),
|
|
79
96
|
to_asynchronous(get_policy, bucket, client),
|
|
80
97
|
to_asynchronous(get_encryption, bucket, client),
|
|
81
98
|
to_asynchronous(get_versioning, bucket, client),
|
|
82
99
|
to_asynchronous(get_public_access_block, bucket, client),
|
|
100
|
+
to_asynchronous(get_bucket_ownership_controls, bucket, client),
|
|
101
|
+
to_asynchronous(get_bucket_logging, bucket, client),
|
|
102
|
+
)
|
|
103
|
+
return (
|
|
104
|
+
bucket["Name"],
|
|
105
|
+
acl,
|
|
106
|
+
policy,
|
|
107
|
+
encryption,
|
|
108
|
+
versioning,
|
|
109
|
+
public_access_block,
|
|
110
|
+
bucket_ownership_controls,
|
|
111
|
+
bucket_logging,
|
|
83
112
|
)
|
|
84
|
-
return bucket['Name'], acl, policy, encryption, versioning, public_access_block
|
|
85
113
|
|
|
86
|
-
bucket_details = to_synchronous(
|
|
114
|
+
bucket_details = to_synchronous(
|
|
115
|
+
*[_get_bucket_detail(bucket) for bucket in bucket_data["Buckets"]],
|
|
116
|
+
)
|
|
87
117
|
yield from bucket_details
|
|
88
118
|
|
|
89
119
|
|
|
@@ -94,7 +124,7 @@ def get_policy(bucket: Dict, client: botocore.client.BaseClient) -> Optional[Dic
|
|
|
94
124
|
"""
|
|
95
125
|
policy = None
|
|
96
126
|
try:
|
|
97
|
-
policy = client.get_bucket_policy(Bucket=bucket[
|
|
127
|
+
policy = client.get_bucket_policy(Bucket=bucket["Name"])
|
|
98
128
|
except ClientError as e:
|
|
99
129
|
if _is_common_exception(e, bucket):
|
|
100
130
|
pass
|
|
@@ -114,7 +144,7 @@ def get_acl(bucket: Dict, client: botocore.client.BaseClient) -> Optional[Dict]:
|
|
|
114
144
|
"""
|
|
115
145
|
acl = None
|
|
116
146
|
try:
|
|
117
|
-
acl = client.get_bucket_acl(Bucket=bucket[
|
|
147
|
+
acl = client.get_bucket_acl(Bucket=bucket["Name"])
|
|
118
148
|
except ClientError as e:
|
|
119
149
|
if _is_common_exception(e, bucket):
|
|
120
150
|
pass
|
|
@@ -134,7 +164,7 @@ def get_encryption(bucket: Dict, client: botocore.client.BaseClient) -> Optional
|
|
|
134
164
|
"""
|
|
135
165
|
encryption = None
|
|
136
166
|
try:
|
|
137
|
-
encryption = client.get_bucket_encryption(Bucket=bucket[
|
|
167
|
+
encryption = client.get_bucket_encryption(Bucket=bucket["Name"])
|
|
138
168
|
except ClientError as e:
|
|
139
169
|
if _is_common_exception(e, bucket):
|
|
140
170
|
pass
|
|
@@ -154,7 +184,7 @@ def get_versioning(bucket: Dict, client: botocore.client.BaseClient) -> Optional
|
|
|
154
184
|
"""
|
|
155
185
|
versioning = None
|
|
156
186
|
try:
|
|
157
|
-
versioning = client.get_bucket_versioning(Bucket=bucket[
|
|
187
|
+
versioning = client.get_bucket_versioning(Bucket=bucket["Name"])
|
|
158
188
|
except ClientError as e:
|
|
159
189
|
if _is_common_exception(e, bucket):
|
|
160
190
|
pass
|
|
@@ -168,13 +198,16 @@ def get_versioning(bucket: Dict, client: botocore.client.BaseClient) -> Optional
|
|
|
168
198
|
|
|
169
199
|
|
|
170
200
|
@timeit
|
|
171
|
-
def get_public_access_block(
|
|
201
|
+
def get_public_access_block(
|
|
202
|
+
bucket: Dict,
|
|
203
|
+
client: botocore.client.BaseClient,
|
|
204
|
+
) -> Optional[Dict]:
|
|
172
205
|
"""
|
|
173
206
|
Gets the S3 bucket public access block configuration.
|
|
174
207
|
"""
|
|
175
208
|
public_access_block = None
|
|
176
209
|
try:
|
|
177
|
-
public_access_block = client.get_public_access_block(Bucket=bucket[
|
|
210
|
+
public_access_block = client.get_public_access_block(Bucket=bucket["Name"])
|
|
178
211
|
except ClientError as e:
|
|
179
212
|
if _is_common_exception(e, bucket):
|
|
180
213
|
pass
|
|
@@ -188,6 +221,54 @@ def get_public_access_block(bucket: Dict, client: botocore.client.BaseClient) ->
|
|
|
188
221
|
return public_access_block
|
|
189
222
|
|
|
190
223
|
|
|
224
|
+
@timeit
|
|
225
|
+
def get_bucket_ownership_controls(
|
|
226
|
+
bucket: Dict, client: botocore.client.BaseClient
|
|
227
|
+
) -> Optional[Dict]:
|
|
228
|
+
"""
|
|
229
|
+
Gets the S3 object ownership controls configuration.
|
|
230
|
+
"""
|
|
231
|
+
bucket_ownership_controls = None
|
|
232
|
+
try:
|
|
233
|
+
bucket_ownership_controls = client.get_bucket_ownership_controls(
|
|
234
|
+
Bucket=bucket["Name"]
|
|
235
|
+
)
|
|
236
|
+
except ClientError as e:
|
|
237
|
+
if _is_common_exception(e, bucket):
|
|
238
|
+
pass
|
|
239
|
+
else:
|
|
240
|
+
raise
|
|
241
|
+
except EndpointConnectionError:
|
|
242
|
+
logger.warning(
|
|
243
|
+
f"Failed to retrieve S3 bucket ownership controls for {bucket['Name']}"
|
|
244
|
+
" - Could not connect to the endpoint URL",
|
|
245
|
+
)
|
|
246
|
+
return bucket_ownership_controls
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
@timeit
|
|
250
|
+
@aws_handle_regions
|
|
251
|
+
def get_bucket_logging(
|
|
252
|
+
bucket: Dict, client: botocore.client.BaseClient
|
|
253
|
+
) -> Optional[Dict]:
|
|
254
|
+
"""
|
|
255
|
+
Gets the S3 bucket logging status configuration.
|
|
256
|
+
"""
|
|
257
|
+
bucket_logging = None
|
|
258
|
+
try:
|
|
259
|
+
bucket_logging = client.get_bucket_logging(Bucket=bucket["Name"])
|
|
260
|
+
except ClientError as e:
|
|
261
|
+
if _is_common_exception(e, bucket):
|
|
262
|
+
pass
|
|
263
|
+
else:
|
|
264
|
+
raise
|
|
265
|
+
except EndpointConnectionError:
|
|
266
|
+
logger.warning(
|
|
267
|
+
f"Failed to retrieve S3 bucket logging status for {bucket['Name']} - Could not connect to the endpoint URL",
|
|
268
|
+
)
|
|
269
|
+
return bucket_logging
|
|
270
|
+
|
|
271
|
+
|
|
191
272
|
@timeit
|
|
192
273
|
def _is_common_exception(e: Exception, bucket: Dict) -> bool:
|
|
193
274
|
error_msg = "Failed to retrieve S3 bucket detail"
|
|
@@ -207,26 +288,37 @@ def _is_common_exception(e: Exception, bucket: Dict) -> bool:
|
|
|
207
288
|
logger.warning(f"{error_msg} for {bucket['Name']} - EndpointConnectionError")
|
|
208
289
|
return True
|
|
209
290
|
elif "ServerSideEncryptionConfigurationNotFoundError" in e.args[0]:
|
|
210
|
-
logger.warning(
|
|
291
|
+
logger.warning(
|
|
292
|
+
f"{error_msg} for {bucket['Name']} - ServerSideEncryptionConfigurationNotFoundError",
|
|
293
|
+
)
|
|
211
294
|
return True
|
|
212
295
|
elif "InvalidToken" in e.args[0]:
|
|
213
296
|
logger.warning(f"{error_msg} for {bucket['Name']} - InvalidToken")
|
|
214
297
|
return True
|
|
215
298
|
elif "NoSuchPublicAccessBlockConfiguration" in e.args[0]:
|
|
216
|
-
logger.warning(
|
|
299
|
+
logger.warning(
|
|
300
|
+
f"{error_msg} for {bucket['Name']} - NoSuchPublicAccessBlockConfiguration",
|
|
301
|
+
)
|
|
217
302
|
return True
|
|
218
303
|
elif "IllegalLocationConstraintException" in e.args[0]:
|
|
219
|
-
logger.warning(
|
|
304
|
+
logger.warning(
|
|
305
|
+
f"{error_msg} for {bucket['Name']} - IllegalLocationConstraintException",
|
|
306
|
+
)
|
|
307
|
+
return True
|
|
308
|
+
elif "OwnershipControlsNotFoundError" in e.args[0]:
|
|
309
|
+
logger.warning(
|
|
310
|
+
f"{error_msg} for {bucket['Name']} - OwnershipControlsNotFoundError"
|
|
311
|
+
)
|
|
220
312
|
return True
|
|
221
313
|
return False
|
|
222
314
|
|
|
223
315
|
|
|
224
316
|
@timeit
|
|
225
317
|
def _load_s3_acls(
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
318
|
+
neo4j_session: neo4j.Session,
|
|
319
|
+
acls: List[Dict[str, Any]],
|
|
320
|
+
aws_account_id: str,
|
|
321
|
+
update_tag: int,
|
|
230
322
|
) -> None:
|
|
231
323
|
"""
|
|
232
324
|
Ingest S3 ACL into neo4j.
|
|
@@ -243,7 +335,8 @@ def _load_s3_acls(
|
|
|
243
335
|
SET r.lastupdated = $UpdateTag
|
|
244
336
|
"""
|
|
245
337
|
|
|
246
|
-
|
|
338
|
+
run_write_query(
|
|
339
|
+
neo4j_session,
|
|
247
340
|
ingest_acls,
|
|
248
341
|
acls=acls,
|
|
249
342
|
UpdateTag=update_tag,
|
|
@@ -252,14 +345,19 @@ def _load_s3_acls(
|
|
|
252
345
|
# implement the acl permission
|
|
253
346
|
# https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#permissions
|
|
254
347
|
run_analysis_job(
|
|
255
|
-
|
|
348
|
+
"aws_s3acl_analysis.json",
|
|
256
349
|
neo4j_session,
|
|
257
|
-
{
|
|
350
|
+
{"AWS_ID": aws_account_id},
|
|
351
|
+
package="cartography.data.jobs.scoped_analysis",
|
|
258
352
|
)
|
|
259
353
|
|
|
260
354
|
|
|
261
355
|
@timeit
|
|
262
|
-
def _load_s3_policies(
|
|
356
|
+
def _load_s3_policies(
|
|
357
|
+
neo4j_session: neo4j.Session,
|
|
358
|
+
policies: List[Dict],
|
|
359
|
+
update_tag: int,
|
|
360
|
+
) -> None:
|
|
263
361
|
"""
|
|
264
362
|
Ingest S3 policy results into neo4j.
|
|
265
363
|
"""
|
|
@@ -272,7 +370,8 @@ def _load_s3_policies(neo4j_session: neo4j.Session, policies: List[Dict], update
|
|
|
272
370
|
s.lastupdated = $UpdateTag
|
|
273
371
|
"""
|
|
274
372
|
|
|
275
|
-
|
|
373
|
+
run_write_query(
|
|
374
|
+
neo4j_session,
|
|
276
375
|
ingest_policies,
|
|
277
376
|
policies=policies,
|
|
278
377
|
UpdateTag=update_tag,
|
|
@@ -281,7 +380,9 @@ def _load_s3_policies(neo4j_session: neo4j.Session, policies: List[Dict], update
|
|
|
281
380
|
|
|
282
381
|
@timeit
|
|
283
382
|
def _load_s3_policy_statements(
|
|
284
|
-
neo4j_session: neo4j.Session,
|
|
383
|
+
neo4j_session: neo4j.Session,
|
|
384
|
+
statements: List[Dict],
|
|
385
|
+
update_tag: int,
|
|
285
386
|
) -> None:
|
|
286
387
|
ingest_policy_statement = """
|
|
287
388
|
UNWIND $Statements as statement_data
|
|
@@ -303,15 +404,20 @@ def _load_s3_policy_statements(
|
|
|
303
404
|
MERGE (bucket)-[r:POLICY_STATEMENT]->(statement)
|
|
304
405
|
SET r.lastupdated = $UpdateTag
|
|
305
406
|
"""
|
|
306
|
-
|
|
407
|
+
run_write_query(
|
|
408
|
+
neo4j_session,
|
|
307
409
|
ingest_policy_statement,
|
|
308
410
|
Statements=statements,
|
|
309
411
|
UpdateTag=update_tag,
|
|
310
|
-
)
|
|
412
|
+
)
|
|
311
413
|
|
|
312
414
|
|
|
313
415
|
@timeit
|
|
314
|
-
def _load_s3_encryption(
|
|
416
|
+
def _load_s3_encryption(
|
|
417
|
+
neo4j_session: neo4j.Session,
|
|
418
|
+
encryption_configs: List[Dict],
|
|
419
|
+
update_tag: int,
|
|
420
|
+
) -> None:
|
|
315
421
|
"""
|
|
316
422
|
Ingest S3 default encryption results into neo4j.
|
|
317
423
|
"""
|
|
@@ -325,7 +431,8 @@ def _load_s3_encryption(neo4j_session: neo4j.Session, encryption_configs: List[D
|
|
|
325
431
|
s.lastupdated = $UpdateTag
|
|
326
432
|
"""
|
|
327
433
|
|
|
328
|
-
|
|
434
|
+
run_write_query(
|
|
435
|
+
neo4j_session,
|
|
329
436
|
ingest_encryption,
|
|
330
437
|
encryption_configs=encryption_configs,
|
|
331
438
|
UpdateTag=update_tag,
|
|
@@ -333,7 +440,11 @@ def _load_s3_encryption(neo4j_session: neo4j.Session, encryption_configs: List[D
|
|
|
333
440
|
|
|
334
441
|
|
|
335
442
|
@timeit
|
|
336
|
-
def _load_s3_versioning(
|
|
443
|
+
def _load_s3_versioning(
|
|
444
|
+
neo4j_session: neo4j.Session,
|
|
445
|
+
versioning_configs: List[Dict],
|
|
446
|
+
update_tag: int,
|
|
447
|
+
) -> None:
|
|
337
448
|
"""
|
|
338
449
|
Ingest S3 versioning results into neo4j.
|
|
339
450
|
"""
|
|
@@ -345,7 +456,8 @@ def _load_s3_versioning(neo4j_session: neo4j.Session, versioning_configs: List[D
|
|
|
345
456
|
s.lastupdated = $UpdateTag
|
|
346
457
|
"""
|
|
347
458
|
|
|
348
|
-
|
|
459
|
+
run_write_query(
|
|
460
|
+
neo4j_session,
|
|
349
461
|
ingest_versioning,
|
|
350
462
|
versioning_configs=versioning_configs,
|
|
351
463
|
UpdateTag=update_tag,
|
|
@@ -371,19 +483,70 @@ def _load_s3_public_access_block(
|
|
|
371
483
|
s.lastupdated = $UpdateTag
|
|
372
484
|
"""
|
|
373
485
|
|
|
374
|
-
|
|
486
|
+
run_write_query(
|
|
487
|
+
neo4j_session,
|
|
375
488
|
ingest_public_access_block,
|
|
376
489
|
public_access_block_configs=public_access_block_configs,
|
|
377
490
|
UpdateTag=update_tag,
|
|
378
491
|
)
|
|
379
492
|
|
|
380
493
|
|
|
494
|
+
@timeit
|
|
495
|
+
def _load_bucket_ownership_controls(
|
|
496
|
+
neo4j_session: neo4j.Session,
|
|
497
|
+
bucket_ownership_controls_configs: List[Dict],
|
|
498
|
+
update_tag: int,
|
|
499
|
+
) -> None:
|
|
500
|
+
"""
|
|
501
|
+
Ingest S3 BucketOwnershipControls results into neo4j.
|
|
502
|
+
"""
|
|
503
|
+
ingest_bucket_ownership_controls = """
|
|
504
|
+
UNWIND $bucket_ownership_controls_configs AS bucket_ownership_controls
|
|
505
|
+
MATCH (s:S3Bucket) where s.name = bucket_ownership_controls.bucket
|
|
506
|
+
SET s.object_ownership = bucket_ownership_controls.object_ownership,
|
|
507
|
+
s.lastupdated = $UpdateTag
|
|
508
|
+
"""
|
|
509
|
+
|
|
510
|
+
run_write_query(
|
|
511
|
+
neo4j_session,
|
|
512
|
+
ingest_bucket_ownership_controls,
|
|
513
|
+
bucket_ownership_controls_configs=bucket_ownership_controls_configs,
|
|
514
|
+
UpdateTag=update_tag,
|
|
515
|
+
)
|
|
516
|
+
|
|
517
|
+
|
|
518
|
+
@timeit
|
|
519
|
+
def _load_bucket_logging(
|
|
520
|
+
neo4j_session: neo4j.Session,
|
|
521
|
+
bucket_logging_configs: List[Dict],
|
|
522
|
+
update_tag: int,
|
|
523
|
+
) -> None:
|
|
524
|
+
"""
|
|
525
|
+
Ingest S3 bucket logging status configuration into neo4j.
|
|
526
|
+
"""
|
|
527
|
+
# Load basic logging status
|
|
528
|
+
ingest_bucket_logging = """
|
|
529
|
+
UNWIND $bucket_logging_configs AS bucket_logging
|
|
530
|
+
MATCH (bucket:S3Bucket{name: bucket_logging.bucket})
|
|
531
|
+
SET bucket.logging_enabled = bucket_logging.logging_enabled,
|
|
532
|
+
bucket.logging_target_bucket = bucket_logging.target_bucket,
|
|
533
|
+
bucket.lastupdated = $update_tag
|
|
534
|
+
"""
|
|
535
|
+
run_write_query(
|
|
536
|
+
neo4j_session,
|
|
537
|
+
ingest_bucket_logging,
|
|
538
|
+
bucket_logging_configs=bucket_logging_configs,
|
|
539
|
+
update_tag=update_tag,
|
|
540
|
+
)
|
|
541
|
+
|
|
542
|
+
|
|
381
543
|
def _set_default_values(neo4j_session: neo4j.Session, aws_account_id: str) -> None:
|
|
382
544
|
set_defaults = """
|
|
383
545
|
MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(s:S3Bucket) where s.anonymous_actions IS NULL
|
|
384
546
|
SET s.anonymous_access = false, s.anonymous_actions = []
|
|
385
547
|
"""
|
|
386
|
-
|
|
548
|
+
run_write_query(
|
|
549
|
+
neo4j_session,
|
|
387
550
|
set_defaults,
|
|
388
551
|
AWS_ID=aws_account_id,
|
|
389
552
|
)
|
|
@@ -392,7 +555,8 @@ def _set_default_values(neo4j_session: neo4j.Session, aws_account_id: str) -> No
|
|
|
392
555
|
MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(s:S3Bucket) where s.default_encryption IS NULL
|
|
393
556
|
SET s.default_encryption = false
|
|
394
557
|
"""
|
|
395
|
-
|
|
558
|
+
run_write_query(
|
|
559
|
+
neo4j_session,
|
|
396
560
|
set_encryption_defaults,
|
|
397
561
|
AWS_ID=aws_account_id,
|
|
398
562
|
)
|
|
@@ -400,7 +564,9 @@ def _set_default_values(neo4j_session: neo4j.Session, aws_account_id: str) -> No
|
|
|
400
564
|
|
|
401
565
|
@timeit
|
|
402
566
|
def load_s3_details(
|
|
403
|
-
neo4j_session: neo4j.Session,
|
|
567
|
+
neo4j_session: neo4j.Session,
|
|
568
|
+
s3_details_iter: Generator[Any, Any, Any],
|
|
569
|
+
aws_account_id: str,
|
|
404
570
|
update_tag: int,
|
|
405
571
|
) -> None:
|
|
406
572
|
"""
|
|
@@ -412,7 +578,18 @@ def load_s3_details(
|
|
|
412
578
|
encryption_configs: List[Dict] = []
|
|
413
579
|
versioning_configs: List[Dict] = []
|
|
414
580
|
public_access_block_configs: List[Dict] = []
|
|
415
|
-
|
|
581
|
+
bucket_ownership_controls_configs: List[Dict] = []
|
|
582
|
+
bucket_logging_configs: List[Dict] = []
|
|
583
|
+
for (
|
|
584
|
+
bucket,
|
|
585
|
+
acl,
|
|
586
|
+
policy,
|
|
587
|
+
encryption,
|
|
588
|
+
versioning,
|
|
589
|
+
public_access_block,
|
|
590
|
+
bucket_ownership_controls,
|
|
591
|
+
bucket_logging,
|
|
592
|
+
) in s3_details_iter:
|
|
416
593
|
parsed_acls = parse_acl(acl, bucket, aws_account_id)
|
|
417
594
|
if parsed_acls is not None:
|
|
418
595
|
acls.extend(parsed_acls)
|
|
@@ -428,15 +605,26 @@ def load_s3_details(
|
|
|
428
605
|
parsed_versioning = parse_versioning(bucket, versioning)
|
|
429
606
|
if parsed_versioning is not None:
|
|
430
607
|
versioning_configs.append(parsed_versioning)
|
|
431
|
-
parsed_public_access_block = parse_public_access_block(
|
|
608
|
+
parsed_public_access_block = parse_public_access_block(
|
|
609
|
+
bucket,
|
|
610
|
+
public_access_block,
|
|
611
|
+
)
|
|
432
612
|
if parsed_public_access_block is not None:
|
|
433
613
|
public_access_block_configs.append(parsed_public_access_block)
|
|
614
|
+
parsed_bucket_ownership_controls = parse_bucket_ownership_controls(
|
|
615
|
+
bucket, bucket_ownership_controls
|
|
616
|
+
)
|
|
617
|
+
if parsed_bucket_ownership_controls is not None:
|
|
618
|
+
bucket_ownership_controls_configs.append(parsed_bucket_ownership_controls)
|
|
619
|
+
parsed_bucket_logging = parse_bucket_logging(bucket, bucket_logging)
|
|
620
|
+
if parsed_bucket_logging is not None:
|
|
621
|
+
bucket_logging_configs.append(parsed_bucket_logging)
|
|
434
622
|
|
|
435
623
|
# cleanup existing policy properties set on S3 Buckets
|
|
436
624
|
run_cleanup_job(
|
|
437
|
-
|
|
625
|
+
"aws_s3_details.json",
|
|
438
626
|
neo4j_session,
|
|
439
|
-
{
|
|
627
|
+
{"UPDATE_TAG": update_tag, "AWS_ID": aws_account_id},
|
|
440
628
|
)
|
|
441
629
|
|
|
442
630
|
_load_s3_acls(neo4j_session, acls, aws_account_id, update_tag)
|
|
@@ -446,6 +634,10 @@ def load_s3_details(
|
|
|
446
634
|
_load_s3_encryption(neo4j_session, encryption_configs, update_tag)
|
|
447
635
|
_load_s3_versioning(neo4j_session, versioning_configs, update_tag)
|
|
448
636
|
_load_s3_public_access_block(neo4j_session, public_access_block_configs, update_tag)
|
|
637
|
+
_load_bucket_ownership_controls(
|
|
638
|
+
neo4j_session, bucket_ownership_controls_configs, update_tag
|
|
639
|
+
)
|
|
640
|
+
_load_bucket_logging(neo4j_session, bucket_logging_configs, update_tag)
|
|
449
641
|
_set_default_values(neo4j_session, aws_account_id)
|
|
450
642
|
|
|
451
643
|
|
|
@@ -492,7 +684,7 @@ def parse_policy(bucket: str, policyDict: Optional[Dict]) -> Optional[Dict]:
|
|
|
492
684
|
if policyDict is None:
|
|
493
685
|
return None
|
|
494
686
|
# get just the policy element and convert to JSON because boto3 returns this as string
|
|
495
|
-
policy = Policy(json.loads(policyDict[
|
|
687
|
+
policy = Policy(json.loads(policyDict["Policy"]))
|
|
496
688
|
if policy.is_internet_accessible():
|
|
497
689
|
return {
|
|
498
690
|
"bucket": bucket,
|
|
@@ -512,7 +704,7 @@ def parse_policy_statements(bucket: str, policyDict: Policy) -> List[Dict]:
|
|
|
512
704
|
if policyDict is None:
|
|
513
705
|
return None
|
|
514
706
|
|
|
515
|
-
policy = json.loads(policyDict[
|
|
707
|
+
policy = json.loads(policyDict["Policy"])
|
|
516
708
|
statements = []
|
|
517
709
|
stmt_index = 1
|
|
518
710
|
for s in policy["Statement"]:
|
|
@@ -544,8 +736,12 @@ def parse_policy_statements(bucket: str, policyDict: Policy) -> List[Dict]:
|
|
|
544
736
|
|
|
545
737
|
|
|
546
738
|
@timeit
|
|
547
|
-
def parse_acl(
|
|
548
|
-
|
|
739
|
+
def parse_acl(
|
|
740
|
+
acl: Optional[Dict],
|
|
741
|
+
bucket: str,
|
|
742
|
+
aws_account_id: str,
|
|
743
|
+
) -> Optional[List[Dict]]:
|
|
744
|
+
"""Parses the AWS ACL object and returns a dict of the relevant data"""
|
|
549
745
|
# ACL JSON looks like
|
|
550
746
|
# ...metadata...
|
|
551
747
|
# {
|
|
@@ -570,47 +766,47 @@ def parse_acl(acl: Optional[Dict], bucket: str, aws_account_id: str) -> Optional
|
|
|
570
766
|
if acl is None:
|
|
571
767
|
return None
|
|
572
768
|
acl_list: List[Dict] = []
|
|
573
|
-
for grant in acl[
|
|
769
|
+
for grant in acl["Grants"]:
|
|
574
770
|
parsed_acl = None
|
|
575
|
-
if grant[
|
|
771
|
+
if grant["Grantee"]["Type"] == "CanonicalUser":
|
|
576
772
|
parsed_acl = {
|
|
577
773
|
"bucket": bucket,
|
|
578
|
-
"owner": acl[
|
|
579
|
-
"ownerid": acl[
|
|
580
|
-
"type": grant[
|
|
581
|
-
"displayname": grant[
|
|
582
|
-
"granteeid": grant[
|
|
774
|
+
"owner": acl["Owner"].get("DisplayName"),
|
|
775
|
+
"ownerid": acl["Owner"].get("ID"),
|
|
776
|
+
"type": grant["Grantee"]["Type"],
|
|
777
|
+
"displayname": grant["Grantee"].get("DisplayName"),
|
|
778
|
+
"granteeid": grant["Grantee"].get("ID"),
|
|
583
779
|
"uri": None,
|
|
584
|
-
"permission": grant.get(
|
|
780
|
+
"permission": grant.get("Permission"),
|
|
585
781
|
}
|
|
586
|
-
elif grant[
|
|
782
|
+
elif grant["Grantee"]["Type"] == "Group":
|
|
587
783
|
parsed_acl = {
|
|
588
784
|
"bucket": bucket,
|
|
589
|
-
"owner": acl[
|
|
590
|
-
"ownerid": acl[
|
|
591
|
-
"type": grant[
|
|
785
|
+
"owner": acl["Owner"].get("DisplayName"),
|
|
786
|
+
"ownerid": acl["Owner"].get("ID"),
|
|
787
|
+
"type": grant["Grantee"]["Type"],
|
|
592
788
|
"displayname": None,
|
|
593
789
|
"granteeid": None,
|
|
594
|
-
"uri": grant[
|
|
595
|
-
"permission": grant.get(
|
|
790
|
+
"uri": grant["Grantee"].get("URI"),
|
|
791
|
+
"permission": grant.get("Permission"),
|
|
596
792
|
}
|
|
597
793
|
else:
|
|
598
|
-
logger.warning("Unexpected grant type: %s", grant[
|
|
794
|
+
logger.warning("Unexpected grant type: %s", grant["Grantee"]["Type"])
|
|
599
795
|
continue
|
|
600
796
|
|
|
601
797
|
# TODO this can be replaced with a string join
|
|
602
798
|
id_data = "{}:{}:{}:{}:{}:{}:{}:{}".format(
|
|
603
799
|
aws_account_id,
|
|
604
|
-
parsed_acl[
|
|
605
|
-
parsed_acl[
|
|
606
|
-
parsed_acl[
|
|
607
|
-
parsed_acl[
|
|
608
|
-
parsed_acl[
|
|
609
|
-
parsed_acl[
|
|
610
|
-
parsed_acl[
|
|
800
|
+
parsed_acl["owner"],
|
|
801
|
+
parsed_acl["ownerid"],
|
|
802
|
+
parsed_acl["type"],
|
|
803
|
+
parsed_acl["displayname"],
|
|
804
|
+
parsed_acl["granteeid"],
|
|
805
|
+
parsed_acl["uri"],
|
|
806
|
+
parsed_acl["permission"],
|
|
611
807
|
)
|
|
612
808
|
|
|
613
|
-
parsed_acl[
|
|
809
|
+
parsed_acl["id"] = hashlib.sha256(id_data.encode("utf8")).hexdigest()
|
|
614
810
|
acl_list.append(parsed_acl)
|
|
615
811
|
|
|
616
812
|
return acl_list
|
|
@@ -618,7 +814,7 @@ def parse_acl(acl: Optional[Dict], bucket: str, aws_account_id: str) -> Optional
|
|
|
618
814
|
|
|
619
815
|
@timeit
|
|
620
816
|
def parse_encryption(bucket: str, encryption: Optional[Dict]) -> Optional[Dict]:
|
|
621
|
-
"""
|
|
817
|
+
"""Parses the S3 default encryption object and returns a dict of the relevant data"""
|
|
622
818
|
# Encryption object JSON looks like:
|
|
623
819
|
# {
|
|
624
820
|
# 'ServerSideEncryptionConfiguration': {
|
|
@@ -635,27 +831,29 @@ def parse_encryption(bucket: str, encryption: Optional[Dict]) -> Optional[Dict]:
|
|
|
635
831
|
# }
|
|
636
832
|
if encryption is None:
|
|
637
833
|
return None
|
|
638
|
-
_ssec = encryption.get(
|
|
834
|
+
_ssec = encryption.get("ServerSideEncryptionConfiguration", {})
|
|
639
835
|
# Rules is a list, but only one rule ever exists
|
|
640
836
|
try:
|
|
641
|
-
rule = _ssec.get(
|
|
837
|
+
rule = _ssec.get("Rules", []).pop()
|
|
642
838
|
except IndexError:
|
|
643
839
|
return None
|
|
644
|
-
algorithm = rule.get(
|
|
840
|
+
algorithm = rule.get("ApplyServerSideEncryptionByDefault", {}).get("SSEAlgorithm")
|
|
645
841
|
if not algorithm:
|
|
646
842
|
return None
|
|
647
843
|
return {
|
|
648
844
|
"bucket": bucket,
|
|
649
845
|
"default_encryption": True,
|
|
650
846
|
"encryption_algorithm": algorithm,
|
|
651
|
-
"encryption_key_id": rule.get("ApplyServerSideEncryptionByDefault", {}).get(
|
|
652
|
-
|
|
847
|
+
"encryption_key_id": rule.get("ApplyServerSideEncryptionByDefault", {}).get(
|
|
848
|
+
"KMSMasterKeyID",
|
|
849
|
+
),
|
|
850
|
+
"bucket_key_enabled": rule.get("BucketKeyEnabled"),
|
|
653
851
|
}
|
|
654
852
|
|
|
655
853
|
|
|
656
854
|
@timeit
|
|
657
855
|
def parse_versioning(bucket: str, versioning: Optional[Dict]) -> Optional[Dict]:
|
|
658
|
-
"""
|
|
856
|
+
"""Parses the S3 versioning object and returns a dict of the relevant data"""
|
|
659
857
|
# Versioning object JSON looks like:
|
|
660
858
|
# {
|
|
661
859
|
# 'Status': 'Enabled'|'Suspended',
|
|
@@ -671,8 +869,11 @@ def parse_versioning(bucket: str, versioning: Optional[Dict]) -> Optional[Dict]:
|
|
|
671
869
|
|
|
672
870
|
|
|
673
871
|
@timeit
|
|
674
|
-
def parse_public_access_block(
|
|
675
|
-
|
|
872
|
+
def parse_public_access_block(
|
|
873
|
+
bucket: str,
|
|
874
|
+
public_access_block: Optional[Dict],
|
|
875
|
+
) -> Optional[Dict]:
|
|
876
|
+
"""Parses the S3 public access block object and returns a dict of the relevant data"""
|
|
676
877
|
# Versioning object JSON looks like:
|
|
677
878
|
# {
|
|
678
879
|
# 'PublicAccessBlockConfiguration': {
|
|
@@ -695,7 +896,129 @@ def parse_public_access_block(bucket: str, public_access_block: Optional[Dict])
|
|
|
695
896
|
|
|
696
897
|
|
|
697
898
|
@timeit
|
|
698
|
-
def
|
|
899
|
+
def parse_bucket_ownership_controls(
|
|
900
|
+
bucket: str, bucket_ownership_controls: Optional[Dict]
|
|
901
|
+
) -> Optional[Dict]:
|
|
902
|
+
"""Parses the S3 bucket ownership controls object and returns a dict of the relevant data"""
|
|
903
|
+
# Versioning object JSON looks like:
|
|
904
|
+
# {
|
|
905
|
+
# 'OwnershipControls': {
|
|
906
|
+
# 'Rules': [
|
|
907
|
+
# {
|
|
908
|
+
# 'ObjectOwnership': 'BucketOwnerPreferred'|'ObjectWriter'|'BucketOwnerEnforced'
|
|
909
|
+
# },
|
|
910
|
+
# ]
|
|
911
|
+
# }
|
|
912
|
+
# }
|
|
913
|
+
if bucket_ownership_controls is None:
|
|
914
|
+
return None
|
|
915
|
+
return {
|
|
916
|
+
"bucket": bucket,
|
|
917
|
+
"object_ownership": bucket_ownership_controls.get("OwnershipControls", {})
|
|
918
|
+
.get("Rules", [{}])[0]
|
|
919
|
+
.get("ObjectOwnership"),
|
|
920
|
+
}
|
|
921
|
+
|
|
922
|
+
|
|
923
|
+
def parse_bucket_logging(bucket: str, bucket_logging: Optional[Dict]) -> Optional[Dict]:
|
|
924
|
+
"""Parses the S3 bucket logging status configuration and returns a dict of the relevant data"""
|
|
925
|
+
# Logging status object JSON looks like:
|
|
926
|
+
# {
|
|
927
|
+
# 'LoggingEnabled': {
|
|
928
|
+
# 'TargetBucket': 'string',
|
|
929
|
+
# 'TargetGrants': [
|
|
930
|
+
# {
|
|
931
|
+
# 'Grantee': {
|
|
932
|
+
# 'DisplayName': 'string',
|
|
933
|
+
# 'EmailAddress': 'string',
|
|
934
|
+
# 'ID': 'string',
|
|
935
|
+
# 'Type': 'CanonicalUser'|'AmazonCustomerByEmail'|'Group',
|
|
936
|
+
# 'URI': 'string'
|
|
937
|
+
# },
|
|
938
|
+
# 'Permission': 'FULL_CONTROL'|'READ'|'WRITE'
|
|
939
|
+
# },
|
|
940
|
+
# ],
|
|
941
|
+
# 'TargetPrefix': 'string',
|
|
942
|
+
# 'TargetObjectKeyFormat': {
|
|
943
|
+
# 'SimplePrefix': {},
|
|
944
|
+
# 'PartitionedPrefix': {
|
|
945
|
+
# 'PartitionDateSource': 'EventTime'|'DeliveryTime'
|
|
946
|
+
# }
|
|
947
|
+
# }
|
|
948
|
+
# }
|
|
949
|
+
# }
|
|
950
|
+
# Or empty dict {} if logging is not enabled
|
|
951
|
+
if bucket_logging is None:
|
|
952
|
+
return None
|
|
953
|
+
|
|
954
|
+
logging_config = bucket_logging.get("LoggingEnabled", {})
|
|
955
|
+
if not logging_config:
|
|
956
|
+
return {
|
|
957
|
+
"bucket": bucket,
|
|
958
|
+
"logging_enabled": False,
|
|
959
|
+
"target_bucket": None,
|
|
960
|
+
}
|
|
961
|
+
|
|
962
|
+
return {
|
|
963
|
+
"bucket": bucket,
|
|
964
|
+
"logging_enabled": True,
|
|
965
|
+
"target_bucket": logging_config.get("TargetBucket"),
|
|
966
|
+
}
|
|
967
|
+
|
|
968
|
+
|
|
969
|
+
@timeit
|
|
970
|
+
def parse_notification_configuration(
|
|
971
|
+
bucket: str, notification_config: Optional[Dict]
|
|
972
|
+
) -> List[Dict]:
|
|
973
|
+
"""
|
|
974
|
+
Parse S3 bucket notification configuration to extract SNS topic notifications.
|
|
975
|
+
Returns a list of notification configurations.
|
|
976
|
+
"""
|
|
977
|
+
if not notification_config or "TopicConfigurations" not in notification_config:
|
|
978
|
+
return []
|
|
979
|
+
|
|
980
|
+
notifications = []
|
|
981
|
+
for topic_config in notification_config.get("TopicConfigurations", []):
|
|
982
|
+
notification = {
|
|
983
|
+
"bucket": bucket,
|
|
984
|
+
"TopicArn": topic_config["TopicArn"],
|
|
985
|
+
}
|
|
986
|
+
notifications.append(notification)
|
|
987
|
+
return notifications
|
|
988
|
+
|
|
989
|
+
|
|
990
|
+
@timeit
|
|
991
|
+
def _load_s3_notifications(
|
|
992
|
+
neo4j_session: neo4j.Session,
|
|
993
|
+
notifications: List[Dict],
|
|
994
|
+
update_tag: int,
|
|
995
|
+
) -> None:
|
|
996
|
+
"""
|
|
997
|
+
Ingest S3 bucket to SNS topic notification relationships into neo4j.
|
|
998
|
+
"""
|
|
999
|
+
ingest_notifications = """
|
|
1000
|
+
UNWIND $notifications AS notification
|
|
1001
|
+
MATCH (bucket:S3Bucket{name: notification.bucket})
|
|
1002
|
+
MATCH (topic:SNSTopic{arn: notification.TopicArn})
|
|
1003
|
+
MERGE (bucket)-[r:NOTIFIES]->(topic)
|
|
1004
|
+
ON CREATE SET r.firstseen = timestamp()
|
|
1005
|
+
SET r.lastupdated = $UpdateTag
|
|
1006
|
+
"""
|
|
1007
|
+
run_write_query(
|
|
1008
|
+
neo4j_session,
|
|
1009
|
+
ingest_notifications,
|
|
1010
|
+
notifications=notifications,
|
|
1011
|
+
UpdateTag=update_tag,
|
|
1012
|
+
)
|
|
1013
|
+
|
|
1014
|
+
|
|
1015
|
+
@timeit
|
|
1016
|
+
def load_s3_buckets(
|
|
1017
|
+
neo4j_session: neo4j.Session,
|
|
1018
|
+
data: Dict,
|
|
1019
|
+
current_aws_account_id: str,
|
|
1020
|
+
aws_update_tag: int,
|
|
1021
|
+
) -> None:
|
|
699
1022
|
ingest_bucket = """
|
|
700
1023
|
MERGE (bucket:S3Bucket{id:$BucketName})
|
|
701
1024
|
ON CREATE SET bucket.firstseen = timestamp(), bucket.creationdate = $CreationDate
|
|
@@ -714,7 +1037,8 @@ def load_s3_buckets(neo4j_session: neo4j.Session, data: Dict, current_aws_accoun
|
|
|
714
1037
|
|
|
715
1038
|
for bucket in data["Buckets"]:
|
|
716
1039
|
arn = "arn:aws:s3:::" + bucket["Name"]
|
|
717
|
-
|
|
1040
|
+
run_write_query(
|
|
1041
|
+
neo4j_session,
|
|
718
1042
|
ingest_bucket,
|
|
719
1043
|
BucketName=bucket["Name"],
|
|
720
1044
|
BucketRegion=bucket["Region"],
|
|
@@ -726,35 +1050,104 @@ def load_s3_buckets(neo4j_session: neo4j.Session, data: Dict, current_aws_accoun
|
|
|
726
1050
|
|
|
727
1051
|
|
|
728
1052
|
@timeit
|
|
729
|
-
def cleanup_s3_buckets(
|
|
730
|
-
|
|
1053
|
+
def cleanup_s3_buckets(
|
|
1054
|
+
neo4j_session: neo4j.Session,
|
|
1055
|
+
common_job_parameters: Dict,
|
|
1056
|
+
) -> None:
|
|
1057
|
+
run_cleanup_job(
|
|
1058
|
+
"aws_import_s3_buckets_cleanup.json",
|
|
1059
|
+
neo4j_session,
|
|
1060
|
+
common_job_parameters,
|
|
1061
|
+
)
|
|
731
1062
|
|
|
732
1063
|
|
|
733
1064
|
@timeit
|
|
734
|
-
def cleanup_s3_bucket_acl_and_policy(
|
|
735
|
-
|
|
1065
|
+
def cleanup_s3_bucket_acl_and_policy(
|
|
1066
|
+
neo4j_session: neo4j.Session,
|
|
1067
|
+
common_job_parameters: Dict,
|
|
1068
|
+
) -> None:
|
|
1069
|
+
run_cleanup_job(
|
|
1070
|
+
"aws_import_s3_acl_cleanup.json",
|
|
1071
|
+
neo4j_session,
|
|
1072
|
+
common_job_parameters,
|
|
1073
|
+
)
|
|
1074
|
+
|
|
1075
|
+
|
|
1076
|
+
@timeit
|
|
1077
|
+
@aws_handle_regions
|
|
1078
|
+
def _sync_s3_notifications(
|
|
1079
|
+
neo4j_session: neo4j.Session,
|
|
1080
|
+
boto3_session: boto3.session.Session,
|
|
1081
|
+
bucket_data: Dict,
|
|
1082
|
+
update_tag: int,
|
|
1083
|
+
) -> None:
|
|
1084
|
+
"""
|
|
1085
|
+
Sync S3 bucket notification configurations to Neo4j.
|
|
1086
|
+
"""
|
|
1087
|
+
logger.info("Syncing S3 bucket notifications")
|
|
1088
|
+
s3_client = boto3_session.client("s3")
|
|
1089
|
+
notifications = []
|
|
1090
|
+
|
|
1091
|
+
for bucket in bucket_data["Buckets"]:
|
|
1092
|
+
try:
|
|
1093
|
+
notification_config = s3_client.get_bucket_notification_configuration(
|
|
1094
|
+
Bucket=bucket["Name"]
|
|
1095
|
+
)
|
|
1096
|
+
parsed_notifications = parse_notification_configuration(
|
|
1097
|
+
bucket["Name"], notification_config
|
|
1098
|
+
)
|
|
1099
|
+
notifications.extend(parsed_notifications)
|
|
1100
|
+
logger.debug(
|
|
1101
|
+
f"Found {len(parsed_notifications)} notifications for bucket {bucket['Name']}"
|
|
1102
|
+
)
|
|
1103
|
+
except ClientError as e:
|
|
1104
|
+
logger.warning(
|
|
1105
|
+
f"Failed to retrieve notification configuration for bucket {bucket['Name']}: {e}"
|
|
1106
|
+
)
|
|
1107
|
+
continue
|
|
1108
|
+
|
|
1109
|
+
logger.info(f"Loading {len(notifications)} S3 bucket notifications into Neo4j")
|
|
1110
|
+
_load_s3_notifications(neo4j_session, notifications, update_tag)
|
|
736
1111
|
|
|
737
1112
|
|
|
738
1113
|
@timeit
|
|
739
1114
|
def sync(
|
|
740
|
-
neo4j_session: neo4j.Session,
|
|
741
|
-
|
|
1115
|
+
neo4j_session: neo4j.Session,
|
|
1116
|
+
boto3_session: boto3.session.Session,
|
|
1117
|
+
regions: List[str],
|
|
1118
|
+
current_aws_account_id: str,
|
|
1119
|
+
update_tag: int,
|
|
1120
|
+
common_job_parameters: Dict,
|
|
742
1121
|
) -> None:
|
|
743
|
-
|
|
744
|
-
|
|
1122
|
+
"""
|
|
1123
|
+
Sync S3 buckets and their configurations to Neo4j.
|
|
1124
|
+
This includes:
|
|
1125
|
+
1. Basic bucket information
|
|
1126
|
+
2. ACLs and policies
|
|
1127
|
+
3. Notification configurations
|
|
1128
|
+
"""
|
|
1129
|
+
logger.info("Syncing S3 for account '%s'", current_aws_account_id)
|
|
745
1130
|
|
|
1131
|
+
bucket_data = get_s3_bucket_list(boto3_session)
|
|
746
1132
|
load_s3_buckets(neo4j_session, bucket_data, current_aws_account_id, update_tag)
|
|
747
1133
|
cleanup_s3_buckets(neo4j_session, common_job_parameters)
|
|
748
1134
|
|
|
749
1135
|
acl_and_policy_data_iter = get_s3_bucket_details(boto3_session, bucket_data)
|
|
750
|
-
load_s3_details(
|
|
1136
|
+
load_s3_details(
|
|
1137
|
+
neo4j_session,
|
|
1138
|
+
acl_and_policy_data_iter,
|
|
1139
|
+
current_aws_account_id,
|
|
1140
|
+
update_tag,
|
|
1141
|
+
)
|
|
751
1142
|
cleanup_s3_bucket_acl_and_policy(neo4j_session, common_job_parameters)
|
|
752
1143
|
|
|
1144
|
+
_sync_s3_notifications(neo4j_session, boto3_session, bucket_data, update_tag)
|
|
1145
|
+
|
|
753
1146
|
merge_module_sync_metadata(
|
|
754
1147
|
neo4j_session,
|
|
755
|
-
group_type=
|
|
1148
|
+
group_type="AWSAccount",
|
|
756
1149
|
group_id=current_aws_account_id,
|
|
757
|
-
synced_type=
|
|
1150
|
+
synced_type="S3Bucket",
|
|
758
1151
|
update_tag=update_tag,
|
|
759
1152
|
stat_handler=stat_handler,
|
|
760
1153
|
)
|