specweave 0.18.1 → 0.20.1

package/plugins/specweave-kafka/terraform/apache-kafka/variables.tf

@@ -0,0 +1,156 @@
+# Apache Kafka Terraform Module - Variables
+
+variable "environment" {
+  description = "Environment name (dev, staging, prod)"
+  type        = string
+}
+
+variable "cluster_name" {
+  description = "Kafka cluster name"
+  type        = string
+}
+
+variable "kafka_version" {
+  description = "Apache Kafka version (3.6.0+)"
+  type        = string
+  default     = "3.7.0"
+
+  validation {
+    condition     = can(regex("^3\\.[6-9]\\.", var.kafka_version))
+    error_message = "Kafka version must be 3.6.0 or higher for KRaft support."
+  }
+}
+
+variable "broker_count" {
+  description = "Number of Kafka brokers (min 3 for KRaft quorum)"
+  type        = number
+  default     = 3
+
+  validation {
+    condition     = var.broker_count >= 3
+    error_message = "Broker count must be at least 3 for KRaft mode."
+  }
+}
+
+variable "instance_type" {
+  description = "EC2 instance type for brokers (null = auto-select based on workload_profile)"
+  type        = string
+  default     = null
+}
+
+variable "workload_profile" {
+  description = "Workload profile for auto-sizing (high-throughput, balanced, low-latency)"
+  type        = string
+  default     = "balanced"
+
+  validation {
+    condition     = contains(["high-throughput", "balanced", "low-latency"], var.workload_profile)
+    error_message = "Workload profile must be: high-throughput, balanced, or low-latency."
+  }
+}
+
+variable "data_disk_count" {
+  description = "Number of data disks per broker (RAID 0)"
+  type        = number
+  default     = 2
+}
+
+variable "disk_size_gb" {
+  description = "Size of each data disk in GB"
+  type        = number
+  default     = 1000
+}
+
+variable "disk_type" {
+  description = "EBS volume type (gp3 or io2)"
+  type        = string
+  default     = "gp3"
+
+  validation {
+    condition     = contains(["gp3", "io2"], var.disk_type)
+    error_message = "Disk type must be gp3 or io2."
+  }
+}
+
+variable "disk_iops" {
+  description = "Disk IOPS (for io2 only)"
+  type        = number
+  default     = null
+}
+
+variable "disk_throughput" {
+  description = "Disk throughput in MB/s (for gp3 only, max 1000)"
+  type        = number
+  default     = 500
+}
+
+variable "heap_size_gb" {
+  description = "JVM heap size in GB (null = auto-calculate)"
+  type        = number
+  default     = null
+}
+
+variable "vpc_id" {
+  description = "VPC ID for Kafka cluster"
+  type        = string
+}
+
+variable "subnet_ids" {
+  description = "List of subnet IDs (spread across AZs)"
+  type        = list(string)
+
+  validation {
+    condition     = length(var.subnet_ids) >= 3
+    error_message = "At least 3 subnets required for high availability."
+  }
+}
+
+variable "key_name" {
+  description = "SSH key pair name for EC2 instances"
+  type        = string
+}
+
+variable "allowed_cidr_blocks" {
+  description = "CIDR blocks allowed to connect to Kafka (clients)"
+  type        = list(string)
+}
+
+variable "monitoring_cidr_blocks" {
+  description = "CIDR blocks allowed to access JMX/Prometheus metrics"
+  type        = list(string)
+  default     = []
+}
+
+variable "route53_zone_id" {
+  description = "Route53 hosted zone ID for DNS records"
+  type        = string
+}
+
+variable "domain" {
+  description = "Domain for DNS records (e.g., kafka.example.com)"
+  type        = string
+}
+
+variable "kms_key_id" {
+  description = "KMS key ID for encryption (null = use default AWS key)"
+  type        = string
+  default     = null
+}
+
+variable "ami_id" {
+  description = "Custom AMI ID (null = use latest Amazon Linux 2)"
+  type        = string
+  default     = null
+}
+
+variable "sns_alarm_topic_arn" {
+  description = "SNS topic ARN for CloudWatch alarms"
+  type        = string
+  default     = null
+}
+
+variable "tags" {
+  description = "Additional tags for all resources"
+  type        = map(string)
+  default     = {}
+}

package/plugins/specweave-kafka/terraform/aws-msk/main.tf

@@ -0,0 +1,362 @@
+# AWS MSK (Managed Streaming for Kafka) - Terraform Module
+#
+# Provisions production-ready AWS MSK cluster with monitoring, encryption, and IAM auth.
+
+terraform {
+  required_version = ">= 1.5.0"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 5.0"
+    }
+  }
+}
+
+locals {
+  cluster_name = "${var.environment}-${var.cluster_name}"
+
+  # Auto-select instance type based on workload
+  broker_instance_type = var.broker_instance_type != null ? var.broker_instance_type : (
+    var.workload_profile == "high-throughput" ? "kafka.m5.4xlarge" :
+    var.workload_profile == "balanced" ? "kafka.m5.2xlarge" :
+    "kafka.m5.xlarge"
+  )
+
+  # Storage per broker
+  storage_per_broker_gb = var.storage_per_broker_gb != null ? var.storage_per_broker_gb : (
+    var.workload_profile == "high-throughput" ? 2000 :
+    var.workload_profile == "balanced" ? 1000 :
+    500
+  )
+
+  common_tags = merge(var.tags, {
+    Environment  = var.environment
+    ManagedBy    = "Terraform"
+    ClusterName  = local.cluster_name
+    Service      = "MSK"
+  })
+}
+
+# ========================================
+# MSK Cluster
+# ========================================
+
+resource "aws_msk_cluster" "main" {
+  cluster_name           = local.cluster_name
+  kafka_version          = var.kafka_version
+  number_of_broker_nodes = var.number_of_broker_nodes
+
+  broker_node_group_info {
+    instance_type   = local.broker_instance_type
+    client_subnets  = var.subnet_ids
+    security_groups = [aws_security_group.msk_cluster.id]
+
+    storage_info {
+      ebs_storage_info {
+        volume_size            = local.storage_per_broker_gb
+        provisioned_throughput {
+          enabled           = var.enable_provisioned_throughput
+          volume_throughput = var.enable_provisioned_throughput ? var.provisioned_throughput_mbps : null
+        }
+      }
+    }
+
+    connectivity_info {
+      public_access {
+        type = var.enable_public_access ? "SERVICE_PROVIDED_EIPS" : "DISABLED"
+      }
+    }
+  }
+
+  encryption_info {
+    encryption_in_transit {
+      client_broker = var.encryption_in_transit_client_broker
+      in_cluster    = true
+    }
+
+    encryption_at_rest_kms_key_arn = var.kms_key_arn
+  }
+
+  client_authentication {
+    sasl {
+      iam   = var.enable_iam_auth
+      scram = var.enable_scram_auth
+    }
+
+    tls {
+      certificate_authority_arns = var.tls_ca_arns
+    }
+
+    unauthenticated = var.allow_unauthenticated
+  }
+
+  configuration_info {
+    arn      = aws_msk_configuration.main.arn
+    revision = aws_msk_configuration.main.latest_revision
+  }
+
+  open_monitoring {
+    prometheus {
+      jmx_exporter {
+        enabled_in_broker = true
+      }
+      node_exporter {
+        enabled_in_broker = true
+      }
+    }
+  }
+
+  logging_info {
+    broker_logs {
+      cloudwatch_logs {
+        enabled   = var.enable_cloudwatch_logs
+        log_group = var.enable_cloudwatch_logs ? aws_cloudwatch_log_group.msk_broker[0].name : null
+      }
+
+      firehose {
+        enabled         = var.enable_firehose_logs
+        delivery_stream = var.enable_firehose_logs ? var.firehose_delivery_stream_name : null
+      }
+
+      s3 {
+        enabled = var.enable_s3_logs
+        bucket  = var.enable_s3_logs ? aws_s3_bucket.msk_logs[0].bucket : null
+        prefix  = var.enable_s3_logs ? "msk-logs/" : null
+      }
+    }
+  }
+
+  tags = merge(local.common_tags, {
+    Name = local.cluster_name
+  })
+}
+
+# ========================================
+# MSK Configuration
+# ========================================
+
+resource "aws_msk_configuration" "main" {
+  name              = "${local.cluster_name}-config"
+  kafka_versions    = [var.kafka_version]
+  server_properties = templatefile("${path.module}/templates/server.properties.tpl", {
+    auto_create_topics_enable           = var.auto_create_topics_enable
+    default_replication_factor          = var.default_replication_factor
+    min_insync_replicas                 = var.min_insync_replicas
+    log_retention_hours                 = var.log_retention_hours
+    log_segment_bytes                   = var.log_segment_bytes
+    compression_type                    = var.compression_type
+    num_io_threads                      = var.num_io_threads
+    num_network_threads                 = var.num_network_threads
+    num_replica_fetchers                = var.num_replica_fetchers
+    socket_request_max_bytes            = var.socket_request_max_bytes
+    unclean_leader_election_enable      = var.unclean_leader_election_enable
+    delete_topic_enable                 = var.delete_topic_enable
+  })
+
+  description = "MSK cluster configuration for ${local.cluster_name}"
+}
+
+# ========================================
+# Security Group
+# ========================================
+
+resource "aws_security_group" "msk_cluster" {
+  name_prefix = "${local.cluster_name}-msk-"
+  description = "Security group for MSK cluster"
+  vpc_id      = var.vpc_id
+
+  # Plaintext (if enabled)
+  dynamic "ingress" {
+    for_each = var.encryption_in_transit_client_broker == "PLAINTEXT" ? [1] : []
+    content {
+      description = "Kafka plaintext"
+      from_port   = 9092
+      to_port     = 9092
+      protocol    = "tcp"
+      cidr_blocks = var.allowed_cidr_blocks
+    }
+  }
+
+  # TLS
+  dynamic "ingress" {
+    for_each = contains(["TLS", "TLS_PLAINTEXT"], var.encryption_in_transit_client_broker) ? [1] : []
+    content {
+      description = "Kafka TLS"
+      from_port   = 9094
+      to_port     = 9094
+      protocol    = "tcp"
+      cidr_blocks = var.allowed_cidr_blocks
+    }
+  }
+
+  # SASL/SCRAM
+  dynamic "ingress" {
+    for_each = var.enable_scram_auth ? [1] : []
+    content {
+      description = "Kafka SASL/SCRAM"
+      from_port   = 9096
+      to_port     = 9096
+      protocol    = "tcp"
+      cidr_blocks = var.allowed_cidr_blocks
+    }
+  }
+
+  # IAM auth
+  dynamic "ingress" {
+    for_each = var.enable_iam_auth ? [1] : []
+    content {
+      description = "Kafka IAM auth"
+      from_port   = 9098
+      to_port     = 9098
+      protocol    = "tcp"
+      cidr_blocks = var.allowed_cidr_blocks
+    }
+  }
+
+  # ZooKeeper (legacy, only if not Kafka 3.0+)
+  dynamic "ingress" {
+    for_each = !startswith(var.kafka_version, "3.") ? [1] : []
+    content {
+      description = "ZooKeeper"
+      from_port   = 2181
+      to_port     = 2181
+      protocol    = "tcp"
+      cidr_blocks = var.allowed_cidr_blocks
+    }
+  }
+
+  egress {
+    description = "Allow all outbound"
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  tags = merge(local.common_tags, {
+    Name = "${local.cluster_name}-msk-sg"
+  })
+}
+
+# ========================================
+# CloudWatch Log Group
+# ========================================
+
+resource "aws_cloudwatch_log_group" "msk_broker" {
+  count = var.enable_cloudwatch_logs ? 1 : 0
+
+  name              = "/aws/msk/${local.cluster_name}"
+  retention_in_days = var.cloudwatch_log_retention_days
+  kms_key_id        = var.kms_key_arn
+
+  tags = local.common_tags
+}
+
+# ========================================
+# S3 Bucket for Logs
+# ========================================
+
+resource "aws_s3_bucket" "msk_logs" {
+  count = var.enable_s3_logs ? 1 : 0
+
+  bucket_prefix = "${local.cluster_name}-msk-logs-"
+
+  tags = merge(local.common_tags, {
+    Name = "${local.cluster_name}-msk-logs"
+  })
+}
+
+resource "aws_s3_bucket_server_side_encryption_configuration" "msk_logs" {
+  count = var.enable_s3_logs ? 1 : 0
+
+  bucket = aws_s3_bucket.msk_logs[0].id
+
+  rule {
+    apply_server_side_encryption_by_default {
+      sse_algorithm     = "aws:kms"
+      kms_master_key_id = var.kms_key_arn
+    }
+  }
+}
+
+resource "aws_s3_bucket_lifecycle_configuration" "msk_logs" {
+  count = var.enable_s3_logs ? 1 : 0
+
+  bucket = aws_s3_bucket.msk_logs[0].id
+
+  rule {
+    id     = "archive-logs"
+    status = "Enabled"
+
+    transition {
+      days          = 30
+      storage_class = "GLACIER"
+    }
+
+    expiration {
+      days = 90
+    }
+  }
+}
+
+# ========================================
+# CloudWatch Alarms
+# ========================================
+
+resource "aws_cloudwatch_metric_alarm" "cpu_utilization" {
+  alarm_name          = "${local.cluster_name}-cpu-high"
+  comparison_operator = "GreaterThanThreshold"
+  evaluation_periods  = 2
+  metric_name         = "CpuUser"
+  namespace           = "AWS/Kafka"
+  period              = 300
+  statistic           = "Average"
+  threshold           = 80
+  alarm_description   = "MSK cluster CPU usage > 80%"
+  alarm_actions       = var.sns_alarm_topic_arn != null ? [var.sns_alarm_topic_arn] : []
+
+  dimensions = {
+    "Cluster Name" = aws_msk_cluster.main.cluster_name
+  }
+
+  tags = local.common_tags
+}
+
+resource "aws_cloudwatch_metric_alarm" "under_replicated_partitions" {
+  alarm_name          = "${local.cluster_name}-under-replicated"
+  comparison_operator = "GreaterThanThreshold"
+  evaluation_periods  = 2
+  metric_name         = "UnderReplicatedPartitions"
+  namespace           = "AWS/Kafka"
+  period              = 60
+  statistic           = "Maximum"
+  threshold           = 0
+  alarm_description   = "MSK cluster has under-replicated partitions"
+  alarm_actions       = var.sns_alarm_topic_arn != null ? [var.sns_alarm_topic_arn] : []
+
+  dimensions = {
+    "Cluster Name" = aws_msk_cluster.main.cluster_name
+  }
+
+  tags = local.common_tags
+}
+
+resource "aws_cloudwatch_metric_alarm" "offline_partitions" {
+  alarm_name          = "${local.cluster_name}-offline-partitions"
+  comparison_operator = "GreaterThanThreshold"
+  evaluation_periods  = 1
+  metric_name         = "OfflinePartitionsCount"
+  namespace           = "AWS/Kafka"
+  period              = 60
+  statistic           = "Maximum"
+  threshold           = 0
+  alarm_description   = "MSK cluster has offline partitions (CRITICAL)"
+  alarm_actions       = var.sns_alarm_topic_arn != null ? [var.sns_alarm_topic_arn] : []
+
+  dimensions = {
+    "Cluster Name" = aws_msk_cluster.main.cluster_name
+  }
+
+  tags = local.common_tags
+}

package/plugins/specweave-kafka/terraform/aws-msk/outputs.tf

@@ -0,0 +1,93 @@
+# AWS MSK Terraform Module - Outputs
+
+output "cluster_arn" {
+  description = "MSK cluster ARN"
+  value       = aws_msk_cluster.main.arn
+}
+
+output "cluster_name" {
+  description = "MSK cluster name"
+  value       = aws_msk_cluster.main.cluster_name
+}
+
+output "zookeeper_connect_string" {
+  description = "ZooKeeper connection string (legacy)"
+  value       = aws_msk_cluster.main.zookeeper_connect_string
+}
+
+output "bootstrap_brokers" {
+  description = "Plaintext bootstrap brokers"
+  value       = aws_msk_cluster.main.bootstrap_brokers
+}
+
+output "bootstrap_brokers_tls" {
+  description = "TLS bootstrap brokers"
+  value       = aws_msk_cluster.main.bootstrap_brokers_tls
+}
+
+output "bootstrap_brokers_sasl_scram" {
+  description = "SASL/SCRAM bootstrap brokers"
+  value       = var.enable_scram_auth ? aws_msk_cluster.main.bootstrap_brokers_sasl_scram : null
+}
+
+output "bootstrap_brokers_sasl_iam" {
+  description = "IAM auth bootstrap brokers"
+  value       = var.enable_iam_auth ? aws_msk_cluster.main.bootstrap_brokers_sasl_iam : null
+}
+
+output "security_group_id" {
+  description = "Security group ID for MSK cluster"
+  value       = aws_security_group.msk_cluster.id
+}
+
+output "configuration_arn" {
+  description = "MSK configuration ARN"
+  value       = aws_msk_configuration.main.arn
+}
+
+output "configuration_revision" {
+  description = "MSK configuration revision"
+  value       = aws_msk_configuration.main.latest_revision
+}
+
+output "connection_examples" {
+  description = "Example connection configurations"
+  value = {
+    java_iam = <<-EOT
+      Properties props = new Properties();
+      props.put("bootstrap.servers", "${aws_msk_cluster.main.bootstrap_brokers_sasl_iam}");
+      props.put("security.protocol", "SASL_SSL");
+      props.put("sasl.mechanism", "AWS_MSK_IAM");
+      props.put("sasl.jaas.config", "software.amazon.msk.auth.iam.IAMLoginModule required;");
+      props.put("sasl.client.callback.handler.class", "software.amazon.msk.auth.iam.IAMClientCallbackHandler");
+    EOT
+
+    nodejs_iam = <<-EOT
+      const { Kafka } = require('kafkajs');
+      const { KafkaJSAWSIAMAuthenticationMechanism } = require('@jm18457/kafkajs-msk-iam-authentication-mechanism');
+
+      const kafka = new Kafka({
+        clientId: 'my-app',
+        brokers: ${jsonencode(split(",", aws_msk_cluster.main.bootstrap_brokers_sasl_iam))},
+        ssl: true,
+        sasl: {
+          mechanism: 'aws',
+          authenticationProvider: KafkaJSAWSIAMAuthenticationMechanism({
+            region: '${data.aws_region.current.name}'
+          })
+        }
+      });
+    EOT
+
+    cli_iam = <<-EOT
+      # Using kcat (requires aws-msk-iam-auth plugin)
+      kcat -b ${aws_msk_cluster.main.bootstrap_brokers_sasl_iam} \\
+        -X security.protocol=SASL_SSL \\
+        -X sasl.mechanism=AWS_MSK_IAM \\
+        -L
+    EOT
+  }
+}
+
+# Data sources
+data "aws_region" "current" {}

package/plugins/specweave-kafka/terraform/aws-msk/templates/server.properties.tpl

@@ -0,0 +1,32 @@
+# MSK Server Configuration Template
+
+# Topic Management
+auto.create.topics.enable=${auto_create_topics_enable}
+delete.topic.enable=${delete_topic_enable}
+
+# Replication
+default.replication.factor=${default_replication_factor}
+min.insync.replicas=${min_insync_replicas}
+offsets.topic.replication.factor=3
+transaction.state.log.replication.factor=3
+transaction.state.log.min.isr=2
+
+# Log Retention
+log.retention.hours=${log_retention_hours}
+log.segment.bytes=${log_segment_bytes}
+log.retention.check.interval.ms=300000
+
+# Compression
+compression.type=${compression_type}
+
+# Performance Tuning
+num.io.threads=${num_io_threads}
+num.network.threads=${num_network_threads}
+num.replica.fetchers=${num_replica_fetchers}
+socket.request.max.bytes=${socket_request_max_bytes}
+
+# Leader Election
+unclean.leader.election.enable=${unclean_leader_election_enable}
+
+# Group Coordinator
+group.coordinator.rebalance.protocols=classic,cooperative