specweave 0.18.1 → 0.20.0

package/plugins/specweave-kafka/terraform/aws-msk/variables.tf

@@ -0,0 +1,235 @@
+# AWS MSK Terraform Module - Variables
+
+variable "environment" {
+  description = "Environment name (dev, staging, prod)"
+  type        = string
+}
+
+variable "cluster_name" {
+  description = "MSK cluster name"
+  type        = string
+}
+
+variable "kafka_version" {
+  description = "Kafka version (e.g., 3.6.0, 3.7.0)"
+  type        = string
+  default     = "3.7.0"
+}
+
+variable "number_of_broker_nodes" {
+  description = "Number of broker nodes (must be multiple of AZ count)"
+  type        = number
+  default     = 3
+
+  validation {
+    condition     = var.number_of_broker_nodes >= 3
+    error_message = "Number of broker nodes must be at least 3."
+  }
+}
+
+variable "broker_instance_type" {
+  description = "MSK broker instance type (null = auto-select)"
+  type        = string
+  default     = null
+}
+
+variable "workload_profile" {
+  description = "Workload profile (high-throughput, balanced, low-latency)"
+  type        = string
+  default     = "balanced"
+}
+
+variable "storage_per_broker_gb" {
+  description = "EBS storage per broker in GB (null = auto-calculate)"
+  type        = number
+  default     = null
+}
+
+variable "enable_provisioned_throughput" {
+  description = "Enable EBS provisioned throughput"
+  type        = bool
+  default     = false
+}
+
+variable "provisioned_throughput_mbps" {
+  description = "Provisioned throughput in MB/s (if enabled)"
+  type        = number
+  default     = 250
+}
+
+variable "vpc_id" {
+  description = "VPC ID for MSK cluster"
+  type        = string
+}
+
+variable "subnet_ids" {
+  description = "List of subnet IDs (spread across AZs)"
+  type        = list(string)
+}
+
+variable "allowed_cidr_blocks" {
+  description = "CIDR blocks allowed to connect to MSK"
+  type        = list(string)
+}
+
+variable "enable_public_access" {
+  description = "Enable public access to MSK cluster"
+  type        = bool
+  default     = false
+}
+
+variable "encryption_in_transit_client_broker" {
+  description = "Encryption in transit (TLS, TLS_PLAINTEXT, PLAINTEXT)"
+  type        = string
+  default     = "TLS"
+
+  validation {
+    condition     = contains(["TLS", "TLS_PLAINTEXT", "PLAINTEXT"], var.encryption_in_transit_client_broker)
+    error_message = "Must be TLS, TLS_PLAINTEXT, or PLAINTEXT."
+  }
+}
+
+variable "kms_key_arn" {
+  description = "KMS key ARN for encryption at rest"
+  type        = string
+  default     = null
+}
+
+variable "enable_iam_auth" {
+  description = "Enable IAM authentication"
+  type        = bool
+  default     = true
+}
+
+variable "enable_scram_auth" {
+  description = "Enable SASL/SCRAM authentication"
+  type        = bool
+  default     = false
+}
+
+variable "tls_ca_arns" {
+  description = "TLS certificate authority ARNs"
+  type        = list(string)
+  default     = []
+}
+
+variable "allow_unauthenticated" {
+  description = "Allow unauthenticated access"
+  type        = bool
+  default     = false
+}
+
+variable "enable_cloudwatch_logs" {
+  description = "Enable CloudWatch Logs"
+  type        = bool
+  default     = true
+}
+
+variable "cloudwatch_log_retention_days" {
+  description = "CloudWatch log retention in days"
+  type        = number
+  default     = 7
+}
+
+variable "enable_firehose_logs" {
+  description = "Enable Kinesis Firehose logging"
+  type        = bool
+  default     = false
+}
+
+variable "firehose_delivery_stream_name" {
+  description = "Firehose delivery stream name"
+  type        = string
+  default     = null
+}
+
+variable "enable_s3_logs" {
+  description = "Enable S3 logging"
+  type        = bool
+  default     = false
+}
+
+variable "sns_alarm_topic_arn" {
+  description = "SNS topic ARN for CloudWatch alarms"
+  type        = string
+  default     = null
+}
+
+# Kafka Configuration
+variable "auto_create_topics_enable" {
+  description = "Auto create topics on first produce"
+  type        = bool
+  default     = false
+}
+
+variable "default_replication_factor" {
+  description = "Default replication factor"
+  type        = number
+  default     = 3
+}
+
+variable "min_insync_replicas" {
+  description = "Minimum in-sync replicas"
+  type        = number
+  default     = 2
+}
+
+variable "log_retention_hours" {
+  description = "Log retention in hours"
+  type        = number
+  default     = 168
+}
+
+variable "log_segment_bytes" {
+  description = "Log segment size in bytes"
+  type        = number
+  default     = 1073741824
+}
+
+variable "compression_type" {
+  description = "Compression type (lz4, gzip, snappy, zstd)"
+  type        = string
+  default     = "lz4"
+}
+
+variable "num_io_threads" {
+  description = "Number of I/O threads"
+  type        = number
+  default     = 8
+}
+
+variable "num_network_threads" {
+  description = "Number of network threads"
+  type        = number
+  default     = 5
+}
+
+variable "num_replica_fetchers" {
+  description = "Number of replica fetcher threads"
+  type        = number
+  default     = 2
+}
+
+variable "socket_request_max_bytes" {
+  description = "Socket request max bytes"
+  type        = number
+  default     = 104857600
+}
+
+variable "unclean_leader_election_enable" {
+  description = "Allow unclean leader election"
+  type        = bool
+  default     = false
+}
+
+variable "delete_topic_enable" {
+  description = "Enable topic deletion"
+  type        = bool
+  default     = true
+}
+
+variable "tags" {
+  description = "Additional tags for all resources"
+  type        = map(string)
+  default     = {}
+}

package/plugins/specweave-kafka/terraform/azure-event-hubs/main.tf

@@ -0,0 +1,281 @@
+# Azure Event Hubs (Kafka API) - Terraform Module
+#
+# Provisions Azure Event Hubs namespace with Kafka protocol support.
+# Event Hubs is Azure's managed Kafka-compatible service.
+
+terraform {
+  required_version = ">= 1.5.0"
+
+  required_providers {
+    azurerm = {
+      source  = "hashicorp/azurerm"
+      version = "~> 3.0"
+    }
+  }
+}
+
+locals {
+  namespace_name = "${var.environment}-${var.namespace_name}"
+
+  common_tags = merge(var.tags, {
+    Environment = var.environment
+    ManagedBy   = "Terraform"
+    Service     = "EventHubs"
+  })
+}
+
+# ========================================
+# Event Hubs Namespace
+# ========================================
+
+resource "azurerm_eventhub_namespace" "main" {
+  name                = local.namespace_name
+  location            = var.location
+  resource_group_name = var.resource_group_name
+  sku                 = var.sku
+  capacity            = var.capacity
+
+  # Kafka support
+  kafka_enabled = true
+
+  # Network configuration
+  public_network_access_enabled = var.public_network_access_enabled
+  minimum_tls_version           = "1.2"
+
+  # Auto-inflate (Standard/Premium only)
+  auto_inflate_enabled     = var.auto_inflate_enabled
+  maximum_throughput_units = var.auto_inflate_enabled ? var.maximum_throughput_units : null
+
+  # Zone redundancy (Premium only)
+  zone_redundant = var.sku == "Premium" ? var.zone_redundant : false
+
+  # Identity for managed identity auth
+  identity {
+    type = "SystemAssigned"
+  }
+
+  # Network rules
+  dynamic "network_rulesets" {
+    for_each = length(var.allowed_ip_addresses) > 0 || length(var.allowed_subnet_ids) > 0 ? [1] : []
+    content {
+      default_action = "Deny"
+      trusted_service_access_enabled = true
+
+      dynamic "ip_rule" {
+        for_each = var.allowed_ip_addresses
+        content {
+          ip_mask = ip_rule.value
+        }
+      }
+
+      dynamic "virtual_network_rule" {
+        for_each = var.allowed_subnet_ids
+        content {
+          subnet_id = virtual_network_rule.value
+        }
+      }
+    }
+  }
+
+  tags = merge(local.common_tags, {
+    Name = local.namespace_name
+  })
+}
+
+# ========================================
+# Event Hubs (Topics)
+# ========================================
+
+resource "azurerm_eventhub" "topics" {
+  for_each = { for topic in var.event_hubs : topic.name => topic }
+
+  name                = each.value.name
+  namespace_name      = azurerm_eventhub_namespace.main.name
+  resource_group_name = var.resource_group_name
+  partition_count     = each.value.partition_count
+  message_retention   = each.value.message_retention_days
+
+  # Capture to storage (optional)
+  dynamic "capture_description" {
+    for_each = try(each.value.capture_enabled, false) ? [1] : []
+    content {
+      enabled             = true
+      encoding            = "Avro"
+      interval_in_seconds = 300
+      size_limit_in_bytes = 314572800
+
+      destination {
+        name                = "EventHubArchive.AzureBlockBlob"
+        archive_name_format = "{Namespace}/{EventHub}/{PartitionId}/{Year}/{Month}/{Day}/{Hour}/{Minute}/{Second}"
+        blob_container_name = each.value.capture_container_name
+        storage_account_id  = each.value.capture_storage_account_id
+      }
+    }
+  }
+}
+
+# ========================================
+# Consumer Groups
+# ========================================
+
+resource "azurerm_eventhub_consumer_group" "groups" {
+  for_each = {
+    for item in flatten([
+      for topic in var.event_hubs : [
+        for group in try(topic.consumer_groups, []) : {
+          topic = topic.name
+          group = group
+        }
+      ]
+    ]) : "${item.topic}-${item.group}" => item
+  }
+
+  name                = each.value.group
+  namespace_name      = azurerm_eventhub_namespace.main.name
+  eventhub_name       = azurerm_eventhub.topics[each.value.topic].name
+  resource_group_name = var.resource_group_name
+}
+
+# ========================================
+# Authorization Rules (SAS Keys)
+# ========================================
+
+resource "azurerm_eventhub_namespace_authorization_rule" "listen" {
+  count = var.create_listen_authorization_rule ? 1 : 0
+
+  name                = "${local.namespace_name}-listen"
+  namespace_name      = azurerm_eventhub_namespace.main.name
+  resource_group_name = var.resource_group_name
+
+  listen = true
+  send   = false
+  manage = false
+}
+
+resource "azurerm_eventhub_namespace_authorization_rule" "send" {
+  count = var.create_send_authorization_rule ? 1 : 0
+
+  name                = "${local.namespace_name}-send"
+  namespace_name      = azurerm_eventhub_namespace.main.name
+  resource_group_name = var.resource_group_name
+
+  listen = false
+  send   = true
+  manage = false
+}
+
+resource "azurerm_eventhub_namespace_authorization_rule" "manage" {
+  count = var.create_manage_authorization_rule ? 1 : 0
+
+  name                = "${local.namespace_name}-manage"
+  namespace_name      = azurerm_eventhub_namespace.main.name
+  resource_group_name = var.resource_group_name
+
+  listen = true
+  send   = true
+  manage = true
+}
+
+# ========================================
+# Diagnostic Settings
+# ========================================
+
+resource "azurerm_monitor_diagnostic_setting" "namespace" {
+  count = var.enable_diagnostic_logs ? 1 : 0
+
+  name                       = "${local.namespace_name}-diagnostics"
+  target_resource_id         = azurerm_eventhub_namespace.main.id
+  log_analytics_workspace_id = var.log_analytics_workspace_id
+
+  dynamic "enabled_log" {
+    for_each = [
+      "OperationalLogs",
+      "AutoScaleLogs",
+      "KafkaCoordinatorLogs",
+      "KafkaUserErrorLogs",
+      "EventHubVNetConnectionEvent",
+      "CustomerManagedKeyUserLogs"
+    ]
+    content {
+      category = enabled_log.value
+    }
+  }
+
+  metric {
+    category = "AllMetrics"
+  }
+}
+
+# ========================================
+# Private Endpoint (Premium SKU only)
+# ========================================
+
+resource "azurerm_private_endpoint" "namespace" {
+  count = var.sku == "Premium" && var.enable_private_endpoint ? 1 : 0
+
+  name                = "${local.namespace_name}-pe"
+  location            = var.location
+  resource_group_name = var.resource_group_name
+  subnet_id           = var.private_endpoint_subnet_id
+
+  private_service_connection {
+    name                           = "${local.namespace_name}-psc"
+    private_connection_resource_id = azurerm_eventhub_namespace.main.id
+    is_manual_connection           = false
+    subresource_names              = ["namespace"]
+  }
+
+  tags = local.common_tags
+}
+
+# ========================================
+# Alerts
+# ========================================
+
+resource "azurerm_monitor_metric_alert" "throttled_requests" {
+  count = var.enable_alerts ? 1 : 0
+
+  name                = "${local.namespace_name}-throttled-requests"
+  resource_group_name = var.resource_group_name
+  scopes              = [azurerm_eventhub_namespace.main.id]
+  description         = "Alert when throttled requests exceed threshold"
+  severity            = 2
+
+  criteria {
+    metric_namespace = "Microsoft.EventHub/namespaces"
+    metric_name      = "ThrottledRequests"
+    aggregation      = "Total"
+    operator         = "GreaterThan"
+    threshold        = 100
+  }
+
+  action {
+    action_group_id = var.action_group_id
+  }
+
+  tags = local.common_tags
+}
+
+resource "azurerm_monitor_metric_alert" "server_errors" {
+  count = var.enable_alerts ? 1 : 0
+
+  name                = "${local.namespace_name}-server-errors"
+  resource_group_name = var.resource_group_name
+  scopes              = [azurerm_eventhub_namespace.main.id]
+  description         = "Alert when server errors occur"
+  severity            = 1
+
+  criteria {
+    metric_namespace = "Microsoft.EventHub/namespaces"
+    metric_name      = "ServerErrors"
+    aggregation      = "Total"
+    operator         = "GreaterThan"
+    threshold        = 10
+  }
+
+  action {
+    action_group_id = var.action_group_id
+  }
+
+  tags = local.common_tags
+}

package/plugins/specweave-kafka/terraform/azure-event-hubs/outputs.tf

@@ -0,0 +1,118 @@
+# Azure Event Hubs Terraform Module - Outputs
+
+output "namespace_id" {
+  description = "Event Hubs namespace ID"
+  value       = azurerm_eventhub_namespace.main.id
+}
+
+output "namespace_name" {
+  description = "Event Hubs namespace name"
+  value       = azurerm_eventhub_namespace.main.name
+}
+
+output "kafka_endpoint" {
+  description = "Kafka endpoint (bootstrap servers)"
+  value       = "${azurerm_eventhub_namespace.main.name}.servicebus.windows.net:9093"
+}
+
+output "default_primary_connection_string" {
+  description = "Default primary connection string"
+  value       = azurerm_eventhub_namespace.main.default_primary_connection_string
+  sensitive   = true
+}
+
+output "default_primary_key" {
+  description = "Default primary key"
+  value       = azurerm_eventhub_namespace.main.default_primary_key
+  sensitive   = true
+}
+
+output "listen_connection_string" {
+  description = "Listen-only connection string"
+  value       = var.create_listen_authorization_rule ? azurerm_eventhub_namespace_authorization_rule.listen[0].primary_connection_string : null
+  sensitive   = true
+}
+
+output "send_connection_string" {
+  description = "Send-only connection string"
+  value       = var.create_send_authorization_rule ? azurerm_eventhub_namespace_authorization_rule.send[0].primary_connection_string : null
+  sensitive   = true
+}
+
+output "manage_connection_string" {
+  description = "Manage connection string"
+  value       = var.create_manage_authorization_rule ? azurerm_eventhub_namespace_authorization_rule.manage[0].primary_connection_string : null
+  sensitive   = true
+}
+
+output "event_hub_ids" {
+  description = "Map of Event Hub names to IDs"
+  value       = { for k, v in azurerm_eventhub.topics : k => v.id }
+}
+
+output "consumer_group_ids" {
+  description = "Map of consumer group names to IDs"
+  value       = { for k, v in azurerm_eventhub_consumer_group.groups : k => v.id }
+}
+
+output "connection_examples" {
+  description = "Example connection configurations for Kafka API"
+  value = {
+    java = <<-EOT
+      Properties props = new Properties();
+      props.put("bootstrap.servers", "${azurerm_eventhub_namespace.main.name}.servicebus.windows.net:9093");
+      props.put("security.protocol", "SASL_SSL");
+      props.put("sasl.mechanism", "PLAIN");
+      props.put("sasl.jaas.config", "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"$$ConnectionString\" password=\"${var.create_send_authorization_rule ? "CONNECTION_STRING_HERE" : "USE_DEFAULT_CONNECTION_STRING"}\";");
+      props.put("group.id", "my-consumer-group");
+    EOT
+
+    nodejs = <<-EOT
+      const { Kafka } = require('kafkajs');
+
+      const kafka = new Kafka({
+        clientId: 'my-app',
+        brokers: ['${azurerm_eventhub_namespace.main.name}.servicebus.windows.net:9093'],
+        ssl: true,
+        sasl: {
+          mechanism: 'plain',
+          username: '$$ConnectionString',
+          password: 'CONNECTION_STRING_HERE'
+        }
+      });
+    EOT
+
+    python = <<-EOT
+      from kafka import KafkaProducer, KafkaConsumer
+
+      producer = KafkaProducer(
+          bootstrap_servers='${azurerm_eventhub_namespace.main.name}.servicebus.windows.net:9093',
+          security_protocol='SASL_SSL',
+          sasl_mechanism='PLAIN',
+          sasl_plain_username='$$ConnectionString',
+          sasl_plain_password='CONNECTION_STRING_HERE'
+      )
+    EOT
+
+    cli = <<-EOT
+      # Using kcat
+      kcat -b ${azurerm_eventhub_namespace.main.name}.servicebus.windows.net:9093 \\
+        -X security.protocol=SASL_SSL \\
+        -X sasl.mechanism=PLAIN \\
+        -X sasl.username='$$ConnectionString' \\
+        -X sasl.password='CONNECTION_STRING_HERE' \\
+        -L
+    EOT
+  }
+}
+
+output "capacity_info" {
+  description = "Capacity and throughput information"
+  value = {
+    sku                      = var.sku
+    capacity                 = var.capacity
+    auto_inflate_enabled     = var.auto_inflate_enabled
+    maximum_throughput_units = var.maximum_throughput_units
+    zone_redundant           = var.zone_redundant
+  }
+}