specweave 0.18.1 → 0.20.0

package/plugins/specweave-kafka/terraform/apache-kafka/main.tf

@@ -0,0 +1,347 @@
+# Apache Kafka Cluster (KRaft Mode) - Terraform Module
+#
+# Provisions a production-ready Apache Kafka cluster using KRaft (no ZooKeeper).
+# Supports AWS, Azure, GCP with auto-scaling, monitoring, and security.
+
+terraform {
+  required_version = ">= 1.5.0"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 5.0"
+      configuration_aliases = [aws]
+    }
+  }
+}
+
+locals {
+  cluster_name = "${var.environment}-${var.cluster_name}"
+
+  # Calculate broker count (min 3 for KRaft quorum)
+  broker_count = max(3, var.broker_count)
+
+  # Determine instance type based on workload
+  instance_type = var.instance_type != null ? var.instance_type : (
+    var.workload_profile == "high-throughput" ? "r6i.4xlarge" :
+    var.workload_profile == "balanced" ? "r6i.2xlarge" :
+    "r6i.xlarge" # low-latency
+  )
+
+  # Tags for all resources
+  common_tags = merge(var.tags, {
+    Environment  = var.environment
+    ManagedBy    = "Terraform"
+    ClusterName  = local.cluster_name
+    KafkaVersion = var.kafka_version
+  })
+}
+
+# ========================================
+# EC2 Instances for Kafka Brokers
+# ========================================
+
+resource "aws_instance" "kafka_broker" {
+  count = local.broker_count
+
+  ami           = var.ami_id != null ? var.ami_id : data.aws_ami.amazon_linux_2.id
+  instance_type = local.instance_type
+  key_name      = var.key_name
+
+  subnet_id              = var.subnet_ids[count.index % length(var.subnet_ids)]
+  vpc_security_group_ids = [aws_security_group.kafka_broker.id]
+
+  iam_instance_profile = aws_iam_instance_profile.kafka_broker.name
+
+  root_block_device {
+    volume_type           = "gp3"
+    volume_size          = 100 # OS disk
+    delete_on_termination = true
+    encrypted            = true
+    kms_key_id           = var.kms_key_id
+  }
+
+  # Data disks for Kafka logs
+  dynamic "ebs_block_device" {
+    for_each = range(var.data_disk_count)
+    content {
+      device_name           = "/dev/sd${substr("fghijklmnop", ebs_block_device.key, 1)}"
+      volume_type           = var.disk_type # "gp3" or "io2"
+      volume_size           = var.disk_size_gb
+      iops                  = var.disk_type == "io2" ? var.disk_iops : null
+      throughput            = var.disk_type == "gp3" ? var.disk_throughput : null
+      delete_on_termination = false
+      encrypted            = true
+      kms_key_id           = var.kms_key_id
+    }
+  }
+
+  user_data = templatefile("${path.module}/templates/kafka-broker-init.sh.tpl", {
+    broker_id          = count.index
+    cluster_name       = local.cluster_name
+    kafka_version      = var.kafka_version
+    broker_count       = local.broker_count
+    data_disk_count    = var.data_disk_count
+    heap_size_gb       = var.heap_size_gb
+    kraft_cluster_id   = random_uuid.kraft_cluster_id.result
+    controller_quorum  = join(",", [for i in range(local.broker_count) : "${i}@kafka-broker-${i}.${var.domain}:9093"])
+    log_dirs           = join(",", [for i in range(var.data_disk_count) : "/data/kafka-logs-${i}"])
+    s3_bucket_backups  = aws_s3_bucket.kafka_backups.bucket
+    cloudwatch_namespace = "Kafka/${local.cluster_name}"
+  })
+
+  metadata_options {
+    http_endpoint = "enabled"
+    http_tokens   = "required" # IMDSv2 only
+  }
+
+  tags = merge(local.common_tags, {
+    Name     = "${local.cluster_name}-broker-${count.index}"
+    Role     = "KafkaBroker"
+    BrokerId = count.index
+  })
+}
+
+# ========================================
+# Security Group
+# ========================================
+
+resource "aws_security_group" "kafka_broker" {
+  name_prefix = "${local.cluster_name}-broker-"
+  description = "Security group for Kafka brokers"
+  vpc_id      = var.vpc_id
+
+  # Kafka client connections (SASL_SSL)
+  ingress {
+    description = "Kafka client connections (SASL_SSL)"
+    from_port   = 9092
+    to_port     = 9092
+    protocol    = "tcp"
+    cidr_blocks = var.allowed_cidr_blocks
+  }
+
+  # KRaft controller (internal)
+  ingress {
+    description = "KRaft controller (broker-to-broker)"
+    from_port   = 9093
+    to_port     = 9093
+    protocol    = "tcp"
+    self        = true
+  }
+
+  # JMX metrics
+  ingress {
+    description = "JMX metrics"
+    from_port   = 9999
+    to_port     = 9999
+    protocol    = "tcp"
+    cidr_blocks = var.monitoring_cidr_blocks
+  }
+
+  # Prometheus exporter
+  ingress {
+    description = "Prometheus JMX exporter"
+    from_port   = 7071
+    to_port     = 7071
+    protocol    = "tcp"
+    cidr_blocks = var.monitoring_cidr_blocks
+  }
+
+  # Allow all outbound
+  egress {
+    description = "Allow all outbound"
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  tags = merge(local.common_tags, {
+    Name = "${local.cluster_name}-broker-sg"
+  })
+}
+
+# ========================================
+# IAM Role for Brokers
+# ========================================
+
+resource "aws_iam_role" "kafka_broker" {
+  name_prefix = "${local.cluster_name}-broker-"
+
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [{
+      Action = "sts:AssumeRole"
+      Effect = "Allow"
+      Principal = {
+        Service = "ec2.amazonaws.com"
+      }
+    }]
+  })
+
+  tags = local.common_tags
+}
+
+resource "aws_iam_instance_profile" "kafka_broker" {
+  name_prefix = "${local.cluster_name}-broker-"
+  role        = aws_iam_role.kafka_broker.name
+}
+
+# CloudWatch Logs permissions
+resource "aws_iam_role_policy_attachment" "cloudwatch_logs" {
+  role       = aws_iam_role.kafka_broker.name
+  policy_arn = "arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy"
+}
+
+# S3 backup permissions
+resource "aws_iam_role_policy" "s3_backups" {
+  name_prefix = "s3-backups-"
+  role        = aws_iam_role.kafka_broker.name
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [{
+      Effect = "Allow"
+      Action = [
+        "s3:PutObject",
+        "s3:GetObject",
+        "s3:ListBucket"
+      ]
+      Resource = [
+        aws_s3_bucket.kafka_backups.arn,
+        "${aws_s3_bucket.kafka_backups.arn}/*"
+      ]
+    }]
+  })
+}
+
+# ========================================
+# S3 Bucket for Backups
+# ========================================
+
+resource "aws_s3_bucket" "kafka_backups" {
+  bucket_prefix = "${local.cluster_name}-backups-"
+
+  tags = merge(local.common_tags, {
+    Name = "${local.cluster_name}-backups"
+  })
+}
+
+resource "aws_s3_bucket_versioning" "kafka_backups" {
+  bucket = aws_s3_bucket.kafka_backups.id
+
+  versioning_configuration {
+    status = "Enabled"
+  }
+}
+
+resource "aws_s3_bucket_server_side_encryption_configuration" "kafka_backups" {
+  bucket = aws_s3_bucket.kafka_backups.id
+
+  rule {
+    apply_server_side_encryption_by_default {
+      sse_algorithm     = "aws:kms"
+      kms_master_key_id = var.kms_key_id
+    }
+  }
+}
+
+resource "aws_s3_bucket_lifecycle_configuration" "kafka_backups" {
+  bucket = aws_s3_bucket.kafka_backups.id
+
+  rule {
+    id     = "archive-old-backups"
+    status = "Enabled"
+
+    transition {
+      days          = 30
+      storage_class = "GLACIER"
+    }
+
+    expiration {
+      days = 90
+    }
+  }
+}
+
+# ========================================
+# Route53 DNS Records
+# ========================================
+
+resource "aws_route53_record" "kafka_broker" {
+  count = local.broker_count
+
+  zone_id = var.route53_zone_id
+  name    = "kafka-broker-${count.index}.${var.domain}"
+  type    = "A"
+  ttl     = 60
+
+  records = [aws_instance.kafka_broker[count.index].private_ip]
+}
+
+# ========================================
+# CloudWatch Alarms
+# ========================================
+
+resource "aws_cloudwatch_metric_alarm" "under_replicated_partitions" {
+  count = local.broker_count
+
+  alarm_name          = "${local.cluster_name}-broker-${count.index}-under-replicated"
+  comparison_operator = "GreaterThanThreshold"
+  evaluation_periods  = 2
+  metric_name         = "UnderReplicatedPartitions"
+  namespace           = "Kafka/${local.cluster_name}"
+  period              = 60
+  statistic           = "Maximum"
+  threshold           = 0
+  alarm_description   = "Broker ${count.index} has under-replicated partitions"
+  alarm_actions       = var.sns_alarm_topic_arn != null ? [var.sns_alarm_topic_arn] : []
+
+  dimensions = {
+    BrokerId = count.index
+  }
+
+  tags = local.common_tags
+}
+
+resource "aws_cloudwatch_metric_alarm" "offline_partitions" {
+  alarm_name          = "${local.cluster_name}-offline-partitions"
+  comparison_operator = "GreaterThanThreshold"
+  evaluation_periods  = 1
+  metric_name         = "OfflinePartitionsCount"
+  namespace           = "Kafka/${local.cluster_name}"
+  period              = 60
+  statistic           = "Maximum"
+  threshold           = 0
+  alarm_description   = "Cluster has offline partitions (CRITICAL)"
+  alarm_actions       = var.sns_alarm_topic_arn != null ? [var.sns_alarm_topic_arn] : []
+
+  tags = local.common_tags
+}
+
+# ========================================
+# Data Sources
+# ========================================
+
+data "aws_ami" "amazon_linux_2" {
+  most_recent = true
+  owners      = ["amazon"]
+
+  filter {
+    name   = "name"
+    values = ["amzn2-ami-hvm-*-x86_64-gp2"]
+  }
+
+  filter {
+    name   = "virtualization-type"
+    values = ["hvm"]
+  }
+}
+
+# ========================================
+# Random Resources
+# ========================================
+
+resource "random_uuid" "kraft_cluster_id" {
+  # KRaft cluster ID (generated once, stored in state)
+}

package/plugins/specweave-kafka/terraform/apache-kafka/outputs.tf

@@ -0,0 +1,107 @@
+# Apache Kafka Terraform Module - Outputs
+
+output "cluster_name" {
+  description = "Full Kafka cluster name"
+  value       = local.cluster_name
+}
+
+output "broker_ids" {
+  description = "List of Kafka broker IDs"
+  value       = [for i in range(local.broker_count) : i]
+}
+
+output "broker_instance_ids" {
+  description = "List of EC2 instance IDs for Kafka brokers"
+  value       = aws_instance.kafka_broker[*].id
+}
+
+output "broker_private_ips" {
+  description = "List of private IPs for Kafka brokers"
+  value       = aws_instance.kafka_broker[*].private_ip
+}
+
+output "broker_dns_names" {
+  description = "List of DNS names for Kafka brokers"
+  value       = [for i in range(local.broker_count) : "kafka-broker-${i}.${var.domain}"]
+}
+
+output "bootstrap_servers" {
+  description = "Kafka bootstrap servers connection string"
+  value       = join(",", [for i in range(local.broker_count) : "kafka-broker-${i}.${var.domain}:9092"])
+}
+
+output "bootstrap_servers_sasl_ssl" {
+  description = "Kafka bootstrap servers (SASL_SSL)"
+  value       = join(",", [for i in range(local.broker_count) : "kafka-broker-${i}.${var.domain}:9092"])
+}
+
+output "controller_quorum" {
+  description = "KRaft controller quorum endpoints"
+  value       = join(",", [for i in range(local.broker_count) : "${i}@kafka-broker-${i}.${var.domain}:9093"])
+}
+
+output "kraft_cluster_id" {
+  description = "KRaft cluster ID (UUID)"
+  value       = random_uuid.kraft_cluster_id.result
+  sensitive   = true
+}
+
+output "security_group_id" {
+  description = "Security group ID for Kafka brokers"
+  value       = aws_security_group.kafka_broker.id
+}
+
+output "iam_role_arn" {
+  description = "IAM role ARN for Kafka brokers"
+  value       = aws_iam_role.kafka_broker.arn
+}
+
+output "backup_s3_bucket" {
+  description = "S3 bucket name for Kafka backups"
+  value       = aws_s3_bucket.kafka_backups.bucket
+}
+
+output "cloudwatch_namespace" {
+  description = "CloudWatch namespace for Kafka metrics"
+  value       = "Kafka/${local.cluster_name}"
+}
+
+output "connection_examples" {
+  description = "Example connection configurations"
+  value = {
+    java = <<-EOT
+      Properties props = new Properties();
+      props.put("bootstrap.servers", "${join(",", [for i in range(local.broker_count) : "kafka-broker-${i}.${var.domain}:9092"])}");
+      props.put("security.protocol", "SASL_SSL");
+      props.put("sasl.mechanism", "SCRAM-SHA-512");
+      props.put("sasl.jaas.config", "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"user\" password=\"password\";");
+    EOT
+
+    nodejs = <<-EOT
+      const kafka = new Kafka({
+        clientId: 'my-app',
+        brokers: ${jsonencode([for i in range(local.broker_count) : "kafka-broker-${i}.${var.domain}:9092"])},
+        ssl: true,
+        sasl: {
+          mechanism: 'scram-sha-512',
+          username: 'user',
+          password: 'password'
+        }
+      });
+    EOT
+
+    cli = <<-EOT
+      kcat -b ${join(",", [for i in range(local.broker_count) : "kafka-broker-${i}.${var.domain}:9092"])} \\
+        -X security.protocol=SASL_SSL \\
+        -X sasl.mechanism=SCRAM-SHA-512 \\
+        -X sasl.username=user \\
+        -X sasl.password=password \\
+        -L
+    EOT
+  }
+}
+
+output "kafka_version" {
+  description = "Apache Kafka version deployed"
+  value       = var.kafka_version
+}

package/plugins/specweave-kafka/terraform/apache-kafka/templates/kafka-broker-init.sh.tpl

@@ -0,0 +1,216 @@
+#!/bin/bash
+# Kafka Broker Initialization Script (KRaft Mode)
+#
+# This script runs on first boot to set up a Kafka broker.
+
+set -e
+
+# ========================================
+# Variables from Terraform
+# ========================================
+
+BROKER_ID="${broker_id}"
+CLUSTER_NAME="${cluster_name}"
+KAFKA_VERSION="${kafka_version}"
+BROKER_COUNT="${broker_count}"
+DATA_DISK_COUNT="${data_disk_count}"
+HEAP_SIZE_GB="${heap_size_gb}"
+KRAFT_CLUSTER_ID="${kraft_cluster_id}"
+CONTROLLER_QUORUM="${controller_quorum}"
+LOG_DIRS="${log_dirs}"
+S3_BUCKET_BACKUPS="${s3_bucket_backups}"
+CLOUDWATCH_NAMESPACE="${cloudwatch_namespace}"
+
+# ========================================
+# System Setup
+# ========================================
+
+echo "==> Starting Kafka broker initialization (ID: $BROKER_ID)"
+
+# Update system
+yum update -y
+
+# Install required packages
+yum install -y java-17-amazon-corretto-headless wget unzip jq awscli
+
+# Install CloudWatch agent
+wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm
+rpm -U ./amazon-cloudwatch-agent.rpm
+rm -f ./amazon-cloudwatch-agent.rpm
+
+# ========================================
+# Disk Setup (RAID 0 for data disks)
+# ========================================
+
+echo "==> Setting up data disks"
+
+# Wait for disks to be attached
+for i in $(seq 1 $DATA_DISK_COUNT); do
+  DEVICE="/dev/nvme$((i))n1"
+  while [ ! -b "$DEVICE" ]; do
+    echo "Waiting for $DEVICE..."
+    sleep 5
+  done
+done
+
+# Format disks
+for i in $(seq 1 $DATA_DISK_COUNT); do
+  DEVICE="/dev/nvme$((i))n1"
+  MOUNT_POINT="/data/kafka-logs-$((i-1))"
+
+  # Format if not already formatted
+  if ! blkid "$DEVICE"; then
+    mkfs.xfs -f "$DEVICE"
+  fi
+
+  # Create mount point
+  mkdir -p "$MOUNT_POINT"
+
+  # Mount disk
+  UUID=$(blkid -s UUID -o value "$DEVICE")
+  echo "UUID=$UUID $MOUNT_POINT xfs defaults,noatime,nodiratime 0 2" >> /etc/fstab
+  mount "$MOUNT_POINT"
+
+  # Set ownership
+  chown -R kafka:kafka "$MOUNT_POINT"
+done
+
+# ========================================
+# Kafka Installation
+# ========================================
+
+echo "==> Installing Apache Kafka $KAFKA_VERSION"
+
+# Download Kafka
+cd /opt
+wget -q "https://archive.apache.org/dist/kafka/$KAFKA_VERSION/kafka_2.13-$KAFKA_VERSION.tgz"
+tar -xzf "kafka_2.13-$KAFKA_VERSION.tgz"
+ln -s "kafka_2.13-$KAFKA_VERSION" kafka
+rm -f "kafka_2.13-$KAFKA_VERSION.tgz"
+
+# Create kafka user
+useradd -r -s /bin/false kafka || true
+chown -R kafka:kafka /opt/kafka*
+
+# ========================================
+# Kafka Configuration
+# ========================================
+
+echo "==> Configuring Kafka broker"
+
+# Calculate heap size if not provided
+if [ -z "$HEAP_SIZE_GB" ]; then
+  TOTAL_MEM_GB=$(free -g | awk '/^Mem:/{print $2}')
+  HEAP_SIZE_GB=$((TOTAL_MEM_GB * 60 / 100)) # 60% of total RAM
+fi
+
+# KRaft configuration
+cat > /opt/kafka/config/kraft/server.properties <<EOF
+# Broker ID
+node.id=$BROKER_ID
+process.roles=broker,controller
+controller.quorum.voters=$CONTROLLER_QUORUM
+
+# Socket server settings
+listeners=SASL_SSL://:9092,CONTROLLER://:9093
+advertised.listeners=SASL_SSL://$(hostname -f):9092
+controller.listener.names=CONTROLLER
+listener.security.protocol.map=SASL_SSL:SASL_SSL,CONTROLLER:PLAINTEXT
+inter.broker.listener.name=SASL_SSL
+
+# Log directories
+log.dirs=$LOG_DIRS
+
+# Replication
+default.replication.factor=3
+min.insync.replicas=2
+offsets.topic.replication.factor=3
+transaction.state.log.replication.factor=3
+transaction.state.log.min.isr=2
+
+# Log retention
+log.retention.hours=168
+log.segment.bytes=1073741824
+log.retention.check.interval.ms=300000
+
+# Performance tuning
+num.network.threads=8
+num.io.threads=16
+socket.send.buffer.bytes=1048576
+socket.receive.buffer.bytes=1048576
+socket.request.max.bytes=104857600
+num.replica.fetchers=4
+
+# Compression
+compression.type=lz4
+
+# Group coordinator
+group.coordinator.rebalance.protocols=classic,cooperative
+group.coordinator.num.threads=4
+
+# SSL/TLS configuration
+ssl.keystore.location=/opt/kafka/config/kafka.keystore.jks
+ssl.keystore.password=changeme
+ssl.key.password=changeme
+ssl.truststore.location=/opt/kafka/config/kafka.truststore.jks
+ssl.truststore.password=changeme
+
+# SASL configuration
+sasl.enabled.mechanisms=SCRAM-SHA-512
+sasl.mechanism.inter.broker.protocol=SCRAM-SHA-512
+EOF
+
+# JVM settings
+cat > /opt/kafka/bin/kafka-server-start-jvm.sh <<EOF
+#!/bin/bash
+export KAFKA_HEAP_OPTS="-Xms${HEAP_SIZE_GB}g -Xmx${HEAP_SIZE_GB}g"
+export KAFKA_JVM_PERFORMANCE_OPTS="-XX:+UseG1GC -XX:MaxGCPauseMillis=20 -XX:InitiatingHeapOccupancyPercent=35 -XX:G1HeapRegionSize=16M -XX:MinMetaspaceSize=96m -XX:MaxMetaspaceSize=512m -XX:+ExplicitGCInvokesConcurrent"
+export KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=9999"
+exec /opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/kraft/server.properties
+EOF
+chmod +x /opt/kafka/bin/kafka-server-start-jvm.sh
+
+# ========================================
+# KRaft Cluster ID
+# ========================================
+
+echo "==> Formatting KRaft metadata directory"
+
+/opt/kafka/bin/kafka-storage.sh format \
+  -t "$KRAFT_CLUSTER_ID" \
+  -c /opt/kafka/config/kraft/server.properties
+
+# ========================================
+# Systemd Service
+# ========================================
+
+echo "==> Creating systemd service"
+
+cat > /etc/systemd/system/kafka.service <<EOF
+[Unit]
+Description=Apache Kafka (KRaft Mode)
+After=network.target
+
+[Service]
+Type=simple
+User=kafka
+ExecStart=/opt/kafka/bin/kafka-server-start-jvm.sh
+Restart=on-failure
+RestartSec=10s
+LimitNOFILE=100000
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+# ========================================
+# Start Kafka
+# ========================================
+
+systemctl daemon-reload
+systemctl enable kafka
+systemctl start kafka
+
+echo "==> Kafka broker started successfully (ID: $BROKER_ID)"
+echo "==> Cluster: $CLUSTER_NAME"
+echo "==> Version: $KAFKA_VERSION"